cloudagent-daemon 0.2.1__py3-none-any.whl

cloudagent_daemon/doctor.py

@@ -0,0 +1,534 @@
+"""Wave C #3 — health self-check used by the ``doctor`` CLI subcommand.
+
+Produces a single :class:`DoctorReport` dataclass + a human-readable
+renderer. Inputs come from three sources:
+
+  * **Daemon process** — ``daemon.pid`` file + PID liveness. Doesn't
+    require a running daemon to function (the doctor command is most
+    useful when a daemon is misbehaving).
+  * **On-disk state** — ``state.json`` (last_seq, processed counts)
+    and ``~/.cloudagent/agents/<id>/.meta/`` files (workspace count,
+    outbox depth, last-spawn outcome).
+  * **Live metrics** — optional, scraped from the running daemon's
+    metrics socket. Falls back gracefully when the socket isn't
+    available (e.g. pre-Wave-C-#3 daemon, or process down).
+
+All checks are stdlib-only — no httpx, no aiohttp — so the doctor
+runs on a minimal install.
+"""
+from __future__ import annotations
+
+import json
+import os
+import socket
+import time
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Iterable
+
+from cloudagent_daemon.config import DaemonState
+from cloudagent_daemon.metrics_server import DEFAULT_METRICS_SOCKET_PATH
+from cloudagent_daemon.workspace import Workspace
+
+
+# ---------------------------------------------------------------------------
+# Report dataclasses
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class AgentSnapshot:
+    """One row of the per-agent state table."""
+
+    agent_id: str
+    has_meta_dir: bool
+    last_spawn_rc: int | None
+    last_spawn_session_id: str | None
+    last_spawn_ts: str | None
+    pending_spawn_lines: int
+    pending_received_without_done: int
+
+
+@dataclass
+class DoctorReport:
+    """Top-level health summary, JSON-able + human-renderable.
+
+    All numeric / optional-string fields default to ``None`` /
+    ``False`` so a missing-data path renders as "—" rather than
+    crashing the renderer.
+    """
+
+    daemon_pid: int | None = None
+    daemon_pid_alive: bool = False
+    server_url: str | None = None
+    host_id: str | None = None
+    state_path: str | None = None
+    last_seq: int | None = None
+    processed_count: int | None = None
+    workspace_root: str | None = None
+    agents: list[AgentSnapshot] = field(default_factory=list)
+    total_pending_received_without_done: int = 0
+    profile: str | None = None
+    metrics_socket_path: str | None = None
+    metrics_available: bool = False
+    metrics_text_excerpt: list[str] = field(default_factory=list)
+    metrics_uptime_seconds: float | None = None
+    sse_last_event_age_seconds: float | None = None
+    sse_reconnects_total: int | None = None
+    sse_active_watchdog_fires_total: int | None = None
+    deliver_frames_total: int | None = None
+    spawn_completed_idle: int | None = None
+    spawn_completed_crashed: int | None = None
+    spawn_completed_timeout: int | None = None
+    spawn_fallback_send_message_total: int | None = None
+    outbox_stuck_count: int | None = None
+    agents_bound: int | None = None
+    notes: list[str] = field(default_factory=list)
+
+    def health_signals(self) -> list[tuple[str, str, str]]:
+        """Return ``[(label, status, value), ...]`` for the renderer.
+
+        Status values:
+          * ``"OK"`` — green
+          * ``"WARN"`` — yellow
+          * ``"ALERT"`` — red
+          * ``"—"`` — data missing / not applicable
+        """
+        rows: list[tuple[str, str, str]] = []
+
+        if self.daemon_pid_alive:
+            rows.append(("daemon", "OK", f"running pid={self.daemon_pid}"))
+        elif self.daemon_pid is not None:
+            rows.append(("daemon", "ALERT", f"stale pid file (pid={self.daemon_pid})"))
+        else:
+            rows.append(("daemon", "ALERT", "not running (no PID file)"))
+
+        # SSE liveness from metrics.
+        if self.sse_last_event_age_seconds is None:
+            rows.append(("sse", "—", "no metrics (daemon down or pre-Wave-C-#3)"))
+        else:
+            age = self.sse_last_event_age_seconds
+            status = (
+                "OK" if age < 60 else ("WARN" if age < 180 else "ALERT")
+            )
+            rows.append(("sse", status, f"last event {age:.0f}s ago"))
+
+        if self.total_pending_received_without_done == 0:
+            rows.append(("outbox", "OK", "0 pending"))
+        elif self.outbox_stuck_count and self.outbox_stuck_count > 0:
+            rows.append(
+                (
+                    "outbox",
+                    "ALERT",
+                    f"{self.total_pending_received_without_done} pending, "
+                    f"{self.outbox_stuck_count} STUCK",
+                )
+            )
+        else:
+            rows.append(
+                (
+                    "outbox",
+                    "WARN",
+                    f"{self.total_pending_received_without_done} pending "
+                    f"(within priority thresholds)",
+                )
+            )
+
+        # Spawn outcomes (cumulative).
+        idle = self.spawn_completed_idle or 0
+        crashed = self.spawn_completed_crashed or 0
+        timeout = self.spawn_completed_timeout or 0
+        total = idle + crashed + timeout
+        if total == 0 and not self.metrics_available:
+            rows.append(("wakes", "—", "no metrics yet"))
+        else:
+            fail = crashed + timeout
+            rate = (fail / total) if total else 0.0
+            status = (
+                "OK" if rate < 0.05
+                else ("WARN" if rate < 0.20 else "ALERT")
+            )
+            rows.append(
+                (
+                    "wakes",
+                    status,
+                    f"{total} total ({idle} idle / {crashed} crashed "
+                    f"/ {timeout} timeout)",
+                )
+            )
+
+        # Daemon uptime.
+        if self.metrics_uptime_seconds is None:
+            rows.append(("uptime", "—", "—"))
+        else:
+            s = int(self.metrics_uptime_seconds)
+            human = (
+                f"{s}s" if s < 60
+                else f"{s // 60}m {s % 60}s" if s < 3600
+                else f"{s // 3600}h {(s % 3600) // 60}m"
+            )
+            rows.append(("uptime", "OK", human))
+
+        return rows
+
+
+# ---------------------------------------------------------------------------
+# Sub-collectors
+# ---------------------------------------------------------------------------
+
+
+def _pid_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def _collect_daemon_process(report: DoctorReport, profile: str) -> None:
+    from .paths import profile_pid_path
+
+    pid_path = profile_pid_path(profile)
+    if not pid_path.exists():
+        return
+    try:
+        pid = int(pid_path.read_text().strip())
+    except (ValueError, OSError):
+        return
+    report.daemon_pid = pid
+    report.daemon_pid_alive = _pid_alive(pid)
+
+
+def _collect_state(report: DoctorReport, profile: str) -> None:
+    state_path = DaemonState.default_path(profile)
+    report.state_path = str(state_path)
+    if not state_path.exists():
+        return
+    try:
+        state = DaemonState.load_default(profile)
+    except (ValueError, OSError):
+        report.notes.append(f"state.json corrupt: {state_path}")
+        return
+    report.last_seq = state.last_seq
+    report.processed_count = len(state.processed_message_ids or [])
+
+
+def _collect_agents(report: DoctorReport, workspace: Workspace) -> None:
+    report.workspace_root = str(workspace.root)
+    if not workspace.root.exists():
+        return
+    for agent_dir in sorted(workspace.root.iterdir()):
+        if not agent_dir.is_dir():
+            continue
+        agent_id = agent_dir.name
+        meta = agent_dir / ".meta"
+        has_meta = meta.exists()
+        last_spawn_path = meta / "last-spawn.json"
+        last_rc: int | None = None
+        last_sid: str | None = None
+        last_ts: str | None = None
+        if last_spawn_path.exists():
+            try:
+                data = json.loads(last_spawn_path.read_text())
+                last_rc = data.get("exit_code")
+                last_sid = data.get("session_id")
+                last_ts = data.get("ts")
+            except (json.JSONDecodeError, OSError):
+                report.notes.append(
+                    f"last-spawn.json unreadable on {agent_id}"
+                )
+
+        pending_path = meta / "pending-spawn.jsonl"
+        line_count = 0
+        received_minus_done = 0
+        if pending_path.exists():
+            received_ids: set[str] = set()
+            done_ids: set[str] = set()
+            try:
+                for raw in pending_path.read_text(encoding="utf-8").splitlines():
+                    raw = raw.strip()
+                    if not raw:
+                        continue
+                    line_count += 1
+                    try:
+                        entry = json.loads(raw)
+                    except (TypeError, ValueError):
+                        continue
+                    msg_id = str(entry.get("msg_id") or "")
+                    if not msg_id:
+                        continue
+                    if entry.get("event") == "received":
+                        received_ids.add(msg_id)
+                    elif entry.get("event") == "done":
+                        done_ids.add(msg_id)
+            except OSError:
+                report.notes.append(
+                    f"pending-spawn.jsonl unreadable on {agent_id}"
+                )
+            received_minus_done = len(received_ids - done_ids)
+        report.agents.append(
+            AgentSnapshot(
+                agent_id=agent_id,
+                has_meta_dir=has_meta,
+                last_spawn_rc=last_rc,
+                last_spawn_session_id=last_sid,
+                last_spawn_ts=last_ts,
+                pending_spawn_lines=line_count,
+                pending_received_without_done=received_minus_done,
+            )
+        )
+        report.total_pending_received_without_done += received_minus_done
+
+
+# ---------------------------------------------------------------------------
+# Metrics fetch (best-effort)
+# ---------------------------------------------------------------------------
+
+
+def _fetch_metrics_text(socket_path: Path, *, timeout: float = 2.0) -> str | None:
+    """Connect to the metrics unix socket and return the HTTP body.
+
+    Returns ``None`` on any failure — the doctor falls back to
+    on-disk-only signals. We use the stdlib ``socket`` module (sync)
+    so the doctor can run from a plain CLI without spinning up an
+    asyncio loop just for the scrape.
+    """
+    if not socket_path.exists():
+        return None
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
+    try:
+        sock.connect(str(socket_path))
+        sock.sendall(b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n")
+        chunks: list[bytes] = []
+        deadline = time.monotonic() + timeout
+        while True:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                break
+            try:
+                sock.settimeout(remaining)
+                chunk = sock.recv(8192)
+            except socket.timeout:
+                break
+            if not chunk:
+                break
+            chunks.append(chunk)
+        raw = b"".join(chunks)
+        # Split header from body — accept either CRLF*2 or LF*2.
+        sep = b"\r\n\r\n"
+        if sep not in raw:
+            sep = b"\n\n"
+        if sep not in raw:
+            return raw.decode("utf-8", errors="replace")
+        _, body = raw.split(sep, 1)
+        return body.decode("utf-8", errors="replace")
+    except (OSError, socket.timeout):
+        return None
+    finally:
+        try:
+            sock.close()
+        except OSError:
+            pass
+
+
+def _apply_metrics(report: DoctorReport, text: str) -> None:
+    """Parse the Prometheus body and copy fields onto ``report``.
+
+    Tolerant: any unrecognised line is preserved in
+    ``metrics_text_excerpt`` (capped at 20 lines so the doctor output
+    doesn't explode). Standard counter/gauge names are mapped onto
+    typed fields for the renderer.
+    """
+    report.metrics_available = True
+    counter_idle = 0
+    counter_crashed = 0
+    counter_timeout = 0
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        # Each sample is ``name{labels?} value`` (optional ``timestamp``).
+        name_section, _, rest = line.partition(" ")
+        try:
+            value_token = rest.split()[0]
+            value_float = float(value_token)
+        except (IndexError, ValueError):
+            continue
+        if "{" in name_section:
+            name, label_part = name_section.split("{", 1)
+            label_part = label_part.rstrip("}")
+        else:
+            name = name_section
+            label_part = ""
+
+        labels: dict[str, str] = {}
+        if label_part:
+            for kv in label_part.split(","):
+                if "=" in kv:
+                    k, v = kv.split("=", 1)
+                    labels[k.strip()] = v.strip().strip('"')
+
+        if name == "cloudagent_daemon_uptime_seconds":
+            report.metrics_uptime_seconds = value_float
+        elif name == "cloudagent_daemon_sse_last_event_age_seconds":
+            report.sse_last_event_age_seconds = value_float
+        elif name == "cloudagent_daemon_sse_reconnects_total":
+            report.sse_reconnects_total = int(value_float)
+        elif name == "cloudagent_daemon_sse_active_watchdog_fires_total":
+            report.sse_active_watchdog_fires_total = int(value_float)
+        elif name == "cloudagent_daemon_deliver_frames_total":
+            report.deliver_frames_total = int(value_float)
+        elif name == "cloudagent_daemon_spawn_completed_total":
+            status = labels.get("status")
+            if status == "idle":
+                counter_idle += int(value_float)
+            elif status == "crashed":
+                counter_crashed += int(value_float)
+            elif status == "timeout":
+                counter_timeout += int(value_float)
+        elif name == "cloudagent_daemon_spawn_fallback_send_message_total":
+            report.spawn_fallback_send_message_total = int(value_float)
+        elif name == "cloudagent_daemon_outbox_stuck":
+            report.outbox_stuck_count = int(value_float)
+        elif name == "cloudagent_daemon_agents_bound":
+            report.agents_bound = int(value_float)
+    report.spawn_completed_idle = counter_idle
+    report.spawn_completed_crashed = counter_crashed
+    report.spawn_completed_timeout = counter_timeout
+
+    # Show first 20 non-blank lines as a raw excerpt for operators
+    # who want to see the wire format.
+    excerpt: list[str] = []
+    for ln in text.splitlines():
+        if not ln.strip() or ln.startswith("#"):
+            continue
+        excerpt.append(ln)
+        if len(excerpt) >= 20:
+            break
+    report.metrics_text_excerpt = excerpt
+
+
+# ---------------------------------------------------------------------------
+# Public entry points
+# ---------------------------------------------------------------------------
+
+
+def collect_report(
+    *,
+    workspace: Workspace | None = None,
+    metrics_socket_path: Path | None = None,
+    profile: str | None = None,
+) -> DoctorReport:
+    """Walk the daemon's local state + scrape live metrics. Synchronous.
+
+    Workspace + metrics socket are injectable for tests; defaults
+    match the real on-disk layout under ``~/.cloudagent``. *profile*
+    selects which profile's pid / state / config to read; default
+    profile reads the legacy paths so existing callers don't change.
+    """
+    from .paths import resolve_profile
+
+    profile_name = resolve_profile(profile)
+    report = DoctorReport()
+    report.profile = profile_name
+    workspace = workspace or Workspace(profile=profile_name)
+    metrics_socket_path = metrics_socket_path or DEFAULT_METRICS_SOCKET_PATH
+    report.metrics_socket_path = str(metrics_socket_path)
+
+    # Best-effort daemon config — only used to display server_url +
+    # host_id. Missing config is non-fatal.
+    try:
+        from cloudagent_daemon.config import DaemonConfig
+
+        cfg = DaemonConfig.load_default(profile_name)
+        report.server_url = cfg.server_url
+        report.host_id = cfg.host_id
+    except (FileNotFoundError, ValueError, OSError):
+        report.notes.append("daemon not configured (run `cloudagent-daemon login`)")
+
+    _collect_daemon_process(report, profile_name)
+    _collect_state(report, profile_name)
+    _collect_agents(report, workspace)
+    metrics_text = _fetch_metrics_text(metrics_socket_path)
+    if metrics_text is not None:
+        _apply_metrics(report, metrics_text)
+    return report
+
+
+def render_report(report: DoctorReport) -> str:
+    """Human-readable doctor output (the ``cloudagent-daemon doctor`` body)."""
+    lines: list[str] = []
+    lines.append("=" * 60)
+    lines.append("cloudagent-daemon doctor")
+    lines.append(datetime.now(UTC).isoformat())
+    lines.append("=" * 60)
+    lines.append("")
+
+    lines.append("--- identity ---")
+    lines.append(f"  profile:       {report.profile or '—'}")
+    lines.append(f"  server_url:    {report.server_url or '—'}")
+    lines.append(f"  host_id:       {report.host_id or '—'}")
+    lines.append(f"  workspace:     {report.workspace_root or '—'}")
+    lines.append(f"  state.json:    {report.state_path or '—'}")
+    if report.last_seq is not None:
+        lines.append(f"  last_seq:      {report.last_seq}")
+    if report.processed_count is not None:
+        lines.append(f"  processed_msgs:{report.processed_count}")
+    lines.append("")
+
+    lines.append("--- health ---")
+    for label, status, value in report.health_signals():
+        lines.append(f"  {label:<10} [{status:^5}] {value}")
+    if report.agents_bound is not None:
+        lines.append(f"  agents_bound[ OK   ] {report.agents_bound} (per /metrics)")
+    if report.sse_reconnects_total is not None:
+        lines.append(
+            f"  sse_reconnects  total {report.sse_reconnects_total} "
+            f"(watchdog fires: {report.sse_active_watchdog_fires_total or 0})"
+        )
+    if report.spawn_fallback_send_message_total:
+        lines.append(
+            f"  fallback_send_message: {report.spawn_fallback_send_message_total} "
+            "(1k #63 Layer B — non-zero means Layer A template slipped)"
+        )
+    lines.append("")
+
+    if report.agents:
+        lines.append("--- agents ---")
+        for a in report.agents:
+            tail = (
+                f"rc={a.last_spawn_rc} sid={a.last_spawn_session_id}"
+                if a.last_spawn_rc is not None
+                else "(no last-spawn)"
+            )
+            pending_tail = (
+                f"pending={a.pending_received_without_done}"
+                if a.pending_received_without_done
+                else ""
+            )
+            lines.append(f"  {a.agent_id}: {tail} {pending_tail}".rstrip())
+        lines.append("")
+
+    if not report.metrics_available:
+        lines.append(
+            f"--- /metrics ---  unavailable ({report.metrics_socket_path})"
+        )
+    else:
+        lines.append("--- /metrics excerpt (first 20 series) ---")
+        for ln in report.metrics_text_excerpt:
+            lines.append(f"  {ln}")
+        lines.append("")
+
+    if report.notes:
+        lines.append("--- notes ---")
+        for n in report.notes:
+            lines.append(f"  * {n}")
+
+    return "\n".join(lines) + "\n"
+
+
+__all__ = ["DoctorReport", "AgentSnapshot", "collect_report", "render_report"]