code-coordinator 0.5.46__py3-none-any.whl

coord/health/checks/deploy_lane_facts.py

@@ -0,0 +1,458 @@
+"""Per-machine facts for two of the fleet deploy-lane checks (#1806).
+
+Root cause of #1806: `fleet_deploy_lanes`'s ``~/.coord-cli-venv`` lane and
+the whole of `fleet_tui_binary` used to be gathered by ``os.stat``-ing the
+**daemon host's** filesystem
+(``coord.health.fleet_snapshot.FleetHealthRefresher._daemon_host_facts``).
+That is correct for a fact that can only exist on the daemon process itself
+(``coord-serve``'s own venv) — it is wrong for these two, whose subject is a
+path on the **operator's** machine, which is very often a different box
+from the one running ``coord-serve``. When the two differ, the daemon was
+reporting on its own, usually-absent, copy of paths that live somewhere
+else entirely — and reporting the *opposite* of the truth (#1806's live
+example: WARN "stale" while the operator's real binary was current;
+UNKNOWN "no data" while the operator's real CLI venv was on the release
+everyone believed was live).
+
+These two **machine-scope** checks report the same two raw facts (CLI-venv
+version; tui-binary mtime vs. newest ``tui/`` source mtime) from wherever
+they actually run — every agent's own ``/health`` poll, the transport #1630
+already built for exactly this "measure locally, judge centrally" split.
+`coord.health.checks.fleet_deploy_lanes` aggregates these across every
+``ctx.fleet.machines`` entry instead of trusting a single host's local
+``os.stat`` to answer a fleet-wide question.
+
+Absence is the *common* case here — most machines in a fleet have neither
+an operator's CLI venv nor a local ``tui/`` checkout+build — so both probes
+report that plainly as ``OK`` ("not present on this machine") rather than
+``UNKNOWN``/``WARN``: a box that was never meant to have either lane must
+not read as faulty just because it doesn't.
+
+A third check, ``webapp_bundle``, joins these two for the same reason and
+the same shape (#1834 lane 5, the phone-webapp bundle `coord web --dist`
+serves): the machine actually running `coord web` is very often not the
+daemon host either. It is deliberately **not** graded against the released
+wheel's version the way ``agent_venv``/``spawned_coord`` are —
+``deploy/coord-web-dist-build.timer`` publishes continuously off
+``origin/main``'s SHA (#1543), decoupled on purpose from the
+``~/.coord-venv`` release cadence, so "is it on the released version?" is
+not a well-formed question for it. What *is* well-formed, and what this
+checks — mirroring ``tui_binary`` exactly — is whether the live bundle is
+older than the ``coord/dashboard/webapp/`` source tree it claims to have
+been built from.
+"""
+
+from __future__ import annotations
+
+import subprocess
+
+from coord.dist_name import CANDIDATE_NAMES
+from coord.health.checks.agent_install import pip_show
+from coord.health.models import CheckResult, HealthContext, Severity
+from coord.health.registry import check
+from coord.health.units import expand, shorten_path
+
+# Same documented default locations `coord.health.fleet_snapshot` used to
+# stat on the daemon host — now stat'd wherever this check actually runs.
+# `health.cli_venv_python` / `health.tui_binary_path` / `health.tui_source_dir`
+# override these, same convention as `agent_venv_python`: None means "use the
+# documented default location", NOT "disable the lane".
+_DEFAULT_CLI_VENV_PYTHON = "~/.coord-cli-venv/bin/python3"
+_DEFAULT_TUI_BINARY = "~/.local/bin/coord-tui"
+# Mirrors `deploy/coord-web-dist-build.sh`'s `$LIVE_LINK` — the symlink
+# `coord-web-dist-build.timer` atomically repoints at each new release.
+_DEFAULT_WEBAPP_DIST = "~/coord-web-dist"
+
+# Cap on the tui/ source walk, mirroring the old daemon-side walk: the tree
+# is a few hundred .rs files, so anything past this is a misconfigured
+# `tui_source_dir` pointed at something huge, and a machine's own /health
+# tick must not spend its budget discovering that. A partial walk still
+# yields a *lower bound* on the newest mtime — it can only under-report
+# staleness, never fabricate it.
+_MAX_TUI_SOURCE_FILES = 5000
+# Same cap, same reasoning, for the webapp/ source walk below.
+_MAX_WEBAPP_SOURCE_FILES = 5000
+# Directories a webapp source walk must never descend into: dependency and
+# build output trees that dwarf the actual source and would either blow the
+# file cap on noise or (worse) let a stale `dist/` sitting inside a checkout
+# masquerade as "source newer than itself".
+_WEBAPP_SOURCE_SKIP_DIRS = {"node_modules", "dist", "build", "coverage"}
+
+
+def resolve_cli_venv_python(ctx: HealthContext):
+    """The interpreter for this machine's ``~/.coord-cli-venv``, if any."""
+    configured = getattr(ctx.thresholds, "cli_venv_python", None)
+    if configured:
+        return expand(configured, ctx.home)
+    return expand(_DEFAULT_CLI_VENV_PYTHON, ctx.home)
+
+
+def resolve_tui_binary_path(ctx: HealthContext):
+    """This machine's locally-built ``coord-tui`` binary, if any."""
+    configured = getattr(ctx.thresholds, "tui_binary_path", None)
+    if configured:
+        return expand(configured, ctx.home)
+    return expand(_DEFAULT_TUI_BINARY, ctx.home)
+
+
+def resolve_tui_source_dir(ctx: HealthContext):
+    """``<checkout>/tui/src`` for the first local checkout that has one.
+
+    Derived from ``ctx.checkouts`` (the same locally-existing checkouts
+    every other checkout-scope probe sees) rather than guessed relative to
+    the binary path — a `target/release/...` install path says nothing
+    reliable about where the sources live, and comparing against the
+    *wrong* tree would manufacture staleness rather than report it.
+    Configured `health.tui_source_dir` wins outright when set.
+    """
+    configured = getattr(ctx.thresholds, "tui_source_dir", None)
+    if configured:
+        return expand(configured, ctx.home)
+    for checkout in ctx.checkouts:
+        candidate = checkout.path / "tui" / "src"
+        if candidate.is_dir():
+            return candidate
+    return None
+
+
+def resolve_webapp_dist_path(ctx: HealthContext):
+    """This machine's live ``coord web --dist`` bundle, if any (#1834)."""
+    configured = getattr(ctx.thresholds, "webapp_dist_path", None)
+    if configured:
+        return expand(configured, ctx.home)
+    return expand(_DEFAULT_WEBAPP_DIST, ctx.home)
+
+
+def resolve_webapp_source_dir(ctx: HealthContext):
+    """``<checkout>/coord/dashboard/webapp/src`` for the first local checkout
+    that has one. Same convention as :func:`resolve_tui_source_dir`."""
+    configured = getattr(ctx.thresholds, "webapp_source_dir", None)
+    if configured:
+        return expand(configured, ctx.home)
+    for checkout in ctx.checkouts:
+        candidate = checkout.path / "coord" / "dashboard" / "webapp" / "src"
+        if candidate.is_dir():
+            return candidate
+    return None
+
+
+def _newest_webapp_source_mtime(source_dir):
+    """Newest mtime under *source_dir*, skipping dependency/build dirs.
+
+    Unlike :func:`_newest_rust_source_mtime` this is deliberately NOT
+    extension-filtered: a webapp source tree mixes ``.ts``/``.tsx``/``.css``/
+    ``.html`` meaningfully and filtering would only risk under-reporting
+    staleness. ``node_modules``/``dist``/``build`` are the trees that
+    actually need skipping (size, and — for ``dist``/``build`` sitting
+    inside a checkout — self-reference), so those are excluded outright.
+    """
+    newest = 0.0
+    seen = 0
+    try:
+        if not source_dir.is_dir():
+            return None
+        stack = [source_dir]
+        while stack:
+            current = stack.pop()
+            try:
+                entries = list(current.iterdir())
+            except OSError:
+                continue
+            for entry in entries:
+                name = entry.name
+                if name.startswith(".") or name in _WEBAPP_SOURCE_SKIP_DIRS:
+                    continue
+                try:
+                    if entry.is_dir():
+                        stack.append(entry)
+                        continue
+                    newest = max(newest, entry.stat().st_mtime)
+                except OSError:
+                    continue
+                seen += 1
+                if seen >= _MAX_WEBAPP_SOURCE_FILES:
+                    stack.clear()
+                    break
+    except OSError:
+        return newest or None
+    return newest or None
+
+
+def _newest_rust_source_mtime(source_dir):
+    """Newest ``*.rs`` mtime under *source_dir*, or None if there are none.
+
+    Skips ``target``/hidden directories so a ``tui_source_dir`` pointed at a
+    crate root (rather than its ``src/``) degrades to slow-but-correct
+    instead of walking a multi-GB build tree.
+    """
+    newest = 0.0
+    seen = 0
+    try:
+        if not source_dir.is_dir():
+            return None
+        stack = [source_dir]
+        while stack:
+            current = stack.pop()
+            try:
+                entries = list(current.iterdir())
+            except OSError:
+                continue
+            for entry in entries:
+                name = entry.name
+                if name.startswith(".") or name == "target":
+                    continue
+                try:
+                    if entry.is_dir():
+                        stack.append(entry)
+                        continue
+                    if not name.endswith(".rs"):
+                        continue
+                    newest = max(newest, entry.stat().st_mtime)
+                except OSError:
+                    continue
+                seen += 1
+                if seen >= _MAX_TUI_SOURCE_FILES:
+                    stack.clear()
+                    break
+    except OSError:
+        return newest or None
+    return newest or None
+
+
+@check(
+    id="cli_venv",
+    scope="machine",
+    title="cli venv",
+    order=42,
+    description=(
+        "This machine's ~/.coord-cli-venv coordinator version, when "
+        "it has one — the operator's CLI venv lane of fleet_deploy_lanes (#1806)."
+    ),
+)
+def probe_cli_venv(ctx: HealthContext) -> CheckResult:
+    python = resolve_cli_venv_python(ctx)
+    if not python.exists():
+        # The overwhelming common case: most machines are workers, not the
+        # operator's own box, and never had this venv created. That is not
+        # a fault — fleet_deploy_lanes reads "no machine reports this lane"
+        # as UNKNOWN, not this machine's absence as one.
+        return CheckResult(
+            check_id="cli_venv",
+            scope="machine",
+            severity=Severity.OK,
+            headroom="not present on this machine",
+            values={"python": str(python), "present": False, "version": None},
+        )
+
+    try:
+        fields = pip_show(python)
+    except (OSError, subprocess.SubprocessError) as exc:
+        return CheckResult(
+            check_id="cli_venv",
+            scope="machine",
+            severity=Severity.UNKNOWN,
+            headroom=f"could not run pip show ({type(exc).__name__})",
+            error=str(exc),
+            values={"python": str(python), "present": True, "version": None},
+        )
+
+    version = fields.get("Version") or None
+    if not version:
+        tried = " or ".join(CANDIDATE_NAMES)
+        return CheckResult(
+            check_id="cli_venv",
+            scope="machine",
+            severity=Severity.UNKNOWN,
+            headroom=f"neither {tried} installed in {shorten_path(str(python), str(ctx.home))}",
+            error="pip show returned nothing",
+            values={"python": str(python), "present": True, "version": None},
+        )
+
+    return CheckResult(
+        check_id="cli_venv",
+        scope="machine",
+        severity=Severity.OK,
+        headroom=f"pypi {version}",
+        values={
+            "python": str(python),
+            "present": True,
+            "version": version,
+            "editable": bool(fields.get("Editable project location")),
+        },
+    )
+
+
+@check(
+    id="tui_binary",
+    scope="machine",
+    title="tui binary",
+    order=43,
+    description=(
+        "This machine's locally-built tui/ binary vs. the tui/ source tree "
+        "it was built from — feeds the fleet_tui_binary check (#1806)."
+    ),
+)
+def probe_tui_binary(ctx: HealthContext) -> CheckResult:
+    binary_path = resolve_tui_binary_path(ctx)
+    values: dict = {"path": str(binary_path)}
+
+    try:
+        binary_mtime = binary_path.stat().st_mtime if binary_path.exists() else None
+    except OSError:
+        binary_mtime = None
+
+    if binary_mtime is None:
+        # Most machines never built a local tui/ binary — absent, not stale.
+        return CheckResult(
+            check_id="tui_binary",
+            scope="machine",
+            severity=Severity.OK,
+            headroom="not present on this machine",
+            values={**values, "present": False},
+        )
+
+    values["present"] = True
+    values["binary_mtime"] = binary_mtime
+
+    source_dir = resolve_tui_source_dir(ctx)
+    if source_dir is None:
+        return CheckResult(
+            check_id="tui_binary",
+            scope="machine",
+            severity=Severity.OK,
+            headroom="binary present (tui/ source tree not found to compare)",
+            values=values,
+        )
+
+    values["source_dir"] = str(source_dir)
+    source_mtime = _newest_rust_source_mtime(source_dir)
+    if source_mtime is None:
+        return CheckResult(
+            check_id="tui_binary",
+            scope="machine",
+            severity=Severity.OK,
+            headroom="binary present (tui/ source tree not found to compare)",
+            values=values,
+        )
+
+    values["source_mtime"] = source_mtime
+    if source_mtime > binary_mtime:
+        stale_hours = (source_mtime - binary_mtime) / 3600.0
+        return CheckResult(
+            check_id="tui_binary",
+            scope="machine",
+            severity=Severity.WARN,
+            headroom=f"binary is {stale_hours:.1f}h older than tui/ source",
+            detail="rebuild the tui/ binary — source changed since the last local build",
+            threshold="warn when source/ is newer than the built binary",
+            values=values,
+        )
+
+    return CheckResult(
+        check_id="tui_binary",
+        scope="machine",
+        severity=Severity.OK,
+        headroom="up to date with tui/ source",
+        values=values,
+    )
+
+
+@check(
+    id="webapp_bundle",
+    scope="machine",
+    title="webapp bundle",
+    order=47,
+    description=(
+        "This machine's live `coord web --dist` bundle vs. the "
+        "coord/dashboard/webapp/ source tree it was supposedly built from — "
+        "feeds fleet_webapp_bundle (#1834 lane 5)."
+    ),
+)
+def probe_webapp_bundle(ctx: HealthContext) -> CheckResult:
+    """The webapp analogue of :func:`probe_tui_binary`: same shape, same
+    fail-soft-to-absent convention, different subject.
+
+    Not graded against ``OWN_VERSION`` or the released wheel — see this
+    module's docstring — so this only ever reports staleness relative to the
+    ``webapp/`` source tree, never a version.
+    """
+    dist_path = resolve_webapp_dist_path(ctx)
+    values: dict = {"path": str(dist_path)}
+
+    try:
+        present = dist_path.exists()
+    except OSError:
+        present = False
+
+    if not present:
+        # Most machines never run `coord web --dist` — absent, not stale.
+        return CheckResult(
+            check_id="webapp_bundle",
+            scope="machine",
+            severity=Severity.OK,
+            headroom="not present on this machine",
+            values={**values, "present": False},
+        )
+
+    values["present"] = True
+    try:
+        resolved = dist_path.resolve()
+        # `coord-web-dist-build.sh` names each release directory after the
+        # SHA it built — the symlink target's basename doubles as "which
+        # commit this machine is actually serving" whenever that convention
+        # holds, and is simply absent (not fabricated) when it doesn't.
+        values["sha"] = resolved.name
+    except OSError:
+        resolved = dist_path
+
+    try:
+        dist_mtime = resolved.stat().st_mtime
+    except OSError:
+        dist_mtime = None
+    values["dist_mtime"] = dist_mtime
+
+    source_dir = resolve_webapp_source_dir(ctx)
+    if source_dir is None or dist_mtime is None:
+        return CheckResult(
+            check_id="webapp_bundle",
+            scope="machine",
+            severity=Severity.OK,
+            headroom="bundle present (webapp/ source tree not found to compare)",
+            values=values,
+        )
+
+    values["source_dir"] = str(source_dir)
+    source_mtime = _newest_webapp_source_mtime(source_dir)
+    if source_mtime is None:
+        return CheckResult(
+            check_id="webapp_bundle",
+            scope="machine",
+            severity=Severity.OK,
+            headroom="bundle present (webapp/ source tree not found to compare)",
+            values=values,
+        )
+
+    values["source_mtime"] = source_mtime
+    if source_mtime > dist_mtime:
+        stale_hours = (source_mtime - dist_mtime) / 3600.0
+        return CheckResult(
+            check_id="webapp_bundle",
+            scope="machine",
+            severity=Severity.WARN,
+            headroom=f"bundle is {stale_hours:.1f}h older than webapp/ source",
+            detail=(
+                "coord-web-dist-build.timer has not published since this "
+                "source changed — check `systemctl --user status "
+                "coord-web-dist-build.timer` on this machine"
+            ),
+            threshold="warn when webapp/ source is newer than the live bundle",
+            values=values,
+        )
+
+    return CheckResult(
+        check_id="webapp_bundle",
+        scope="machine",
+        severity=Severity.OK,
+        headroom="up to date with webapp/ source",
+        values=values,
+    )

coord/health/checks/disk.py

@@ -0,0 +1,99 @@
+"""Disk headroom per configured mount point (#1628).
+
+The 2026-07-30 incident that motivated this milestone: elitebook's ``/home``
+reached **0 bytes free** and every worker on that machine failed in a way
+that looked like a coordinator bug.  Nothing was watching the one number
+that would have said so a day earlier.
+
+Thresholds are expressed as percent *free* — headroom — rather than percent
+used, because "how much is left" is the question, and because the rendered
+line still shows used% for the operator who thinks in those terms.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+
+from coord.health.models import CheckResult, HealthContext, Severity
+from coord.health.registry import check
+from coord.health.units import expand, human_bytes
+
+
+@check(
+    id="disk",
+    scope="machine",
+    title="disk",
+    order=10,
+    description="Free space on the filesystems the fleet writes to.",
+)
+def probe_disk(ctx: HealthContext) -> list[CheckResult]:
+    """One result per distinct filesystem behind ``thresholds.disk_paths``.
+
+    Paths that don't exist are skipped silently (``/home`` is not a separate
+    mount everywhere).  Paths that resolve to a filesystem already reported
+    are skipped too — on a single-root machine ``/``, ``/home`` and
+    ``~/.coord`` are the same device, and three identical CRIT lines is noise
+    that trains an operator to skim.
+    """
+    th = ctx.thresholds
+    results: list[CheckResult] = []
+    seen_devices: set[int] = set()
+
+    for raw in th.disk_paths:
+        path = expand(raw, ctx.home)
+        try:
+            device = os.stat(path).st_dev
+        except OSError:
+            continue  # not present on this machine — not a finding
+        if device in seen_devices:
+            continue
+        seen_devices.add(device)
+
+        try:
+            usage = shutil.disk_usage(str(path))
+        except OSError as exc:
+            results.append(
+                CheckResult(
+                    check_id="disk",
+                    scope="machine",
+                    subject=str(raw),
+                    severity=Severity.UNKNOWN,
+                    headroom=f"could not stat filesystem: {exc}",
+                    error=str(exc),
+                )
+            )
+            continue
+
+        total = float(usage.total)
+        free_pct = (usage.free / total * 100.0) if total > 0 else 0.0
+        used_pct = 100.0 - free_pct
+
+        if free_pct < th.disk_crit_free_pct:
+            severity = Severity.CRIT
+        elif free_pct < th.disk_warn_free_pct:
+            severity = Severity.WARN
+        else:
+            severity = Severity.OK
+
+        results.append(
+            CheckResult(
+                check_id="disk",
+                scope="machine",
+                subject=str(raw),
+                severity=severity,
+                headroom=f"{used_pct:.0f}% used ({human_bytes(usage.free)} free)",
+                threshold=f"crit at {100.0 - th.disk_crit_free_pct:.0f}%",
+                values={
+                    "path": str(path),
+                    "total_bytes": usage.total,
+                    "free_bytes": usage.free,
+                    "used_bytes": usage.used,
+                    "free_pct": round(free_pct, 2),
+                    "used_pct": round(used_pct, 2),
+                    "warn_free_pct": th.disk_warn_free_pct,
+                    "crit_free_pct": th.disk_crit_free_pct,
+                },
+            )
+        )
+    return results

coord/health/checks/fleet_board.py

@@ -0,0 +1,89 @@
+"""Fleet-scope: the board daemon's own ``/board`` latency + payload size (#1630, #1597).
+
+A 5.3MB ``/board`` payload already broke ``coord report-result`` once (#1336)
+by blowing a 5s prefetch timeout — this check exists so that regression shows
+up as a fleet-health row instead of a mysterious timeout report weeks later.
+Like the other fleet checks in this package, the daemon's ``/board`` handler
+(``coord.serve_app``) is the thing that actually measures its own latency and
+serialized body size on each rebuild; this probe only judges the numbers it's
+handed.
+"""
+
+from __future__ import annotations
+
+from coord.health.models import CheckResult, HealthContext, Severity
+from coord.health.registry import check
+from coord.health.units import human_bytes
+
+# #1336: the 5.3MB payload that broke a 5s prefetch. Warn well before that.
+_PAYLOAD_WARN_BYTES = 2 * 1024 * 1024
+_PAYLOAD_CRIT_BYTES = 5 * 1024 * 1024
+_LATENCY_WARN_MS = 1500.0
+_LATENCY_CRIT_MS = 4000.0
+
+
+@check(
+    id="fleet_board_latency",
+    scope="fleet",
+    title="board latency+size",
+    order=20,
+    description="The daemon's own /board rebuild latency and serialized payload size.",
+)
+def probe_board_latency(ctx: HealthContext) -> CheckResult:
+    if ctx.fleet is None:
+        return CheckResult(
+            check_id="fleet_board_latency",
+            scope="fleet",
+            severity=Severity.UNKNOWN,
+            headroom="no fleet snapshot (fleet checks only run on the daemon)",
+        )
+
+    dh = ctx.fleet.daemon_host or {}
+    latency_ms = dh.get("board_latency_ms")
+    payload_bytes = dh.get("board_payload_bytes")
+
+    if latency_ms is None and payload_bytes is None:
+        return CheckResult(
+            check_id="fleet_board_latency",
+            scope="fleet",
+            severity=Severity.UNKNOWN,
+            headroom="no /board build measured yet",
+        )
+
+    severity = Severity.OK
+    reasons: list[str] = []
+    if payload_bytes is not None:
+        if payload_bytes >= _PAYLOAD_CRIT_BYTES:
+            severity = Severity.CRIT
+            reasons.append(f"payload {human_bytes(payload_bytes)}")
+        elif payload_bytes >= _PAYLOAD_WARN_BYTES:
+            severity = Severity.WARN
+            reasons.append(f"payload {human_bytes(payload_bytes)}")
+    if latency_ms is not None:
+        if latency_ms >= _LATENCY_CRIT_MS:
+            severity = Severity.CRIT
+            reasons.append(f"latency {latency_ms:.0f}ms")
+        elif latency_ms >= _LATENCY_WARN_MS and severity is not Severity.CRIT:
+            severity = Severity.WARN
+            reasons.append(f"latency {latency_ms:.0f}ms")
+
+    parts = []
+    if latency_ms is not None:
+        parts.append(f"{latency_ms:.0f}ms")
+    if payload_bytes is not None:
+        parts.append(human_bytes(payload_bytes))
+    headroom = " / ".join(parts) if parts else "no data"
+    if severity is not Severity.OK:
+        headroom += f" ({', '.join(reasons)})"
+
+    return CheckResult(
+        check_id="fleet_board_latency",
+        scope="fleet",
+        severity=severity,
+        headroom=headroom,
+        threshold=(
+            f"warn at {human_bytes(_PAYLOAD_WARN_BYTES)}/{_LATENCY_WARN_MS:.0f}ms, "
+            f"crit at {human_bytes(_PAYLOAD_CRIT_BYTES)}/{_LATENCY_CRIT_MS:.0f}ms"
+        ),
+        values={"latency_ms": latency_ms, "payload_bytes": payload_bytes},
+    )