code-coordinator 0.5.46__py3-none-any.whl

coord/health/checks/unit_enablement.py

@@ -0,0 +1,171 @@
+"""Systemd unit *enablement* vs. the per-role manifest (#2098).
+
+`unit_drift` (:mod:`coord.health.checks.unit_drift`) answers "does the
+installed unit's *content* match the release" and deliberately treats
+absence as OK — most hosts don't run every deploy-lane unit, and which
+units a host runs is a topology decision neither that check nor
+`coord.deploy_units` will infer (#1831). Neither of those probes answers
+the question that actually cost a day: **an installed unit whose content
+is byte-perfect can still be sitting there disabled.**
+
+`coord-release-propagate.timer` was `cp`'d onto dellserver correctly and
+then never `systemctl --user enable --now`'d, because the runbook that
+would have said so never mentioned it (fixed in
+`docs/AGENT_OPERATIONS.md`, but the doc fix alone leaves the *next* missed
+enable step just as invisible). A disabled timer and a deferring timer
+both produce zero log lines — the fleet ran 11 releases behind for a day
+with every readout looking normal, because nothing distinguished "not
+enabled" from "enabled and just hasn't fired yet".
+
+`coord.deploy_manifest` is this check's reference: a role -> unit-name
+table that used to exist only as prose (or, before that, only as
+`~/.config/systemd/user/timers.target.wants/` symlinks on dellserver, which
+is machine state that dies with the machine). This probe reads
+:func:`~coord.deploy_manifest.all_manifest_units` but only asks about units
+this host has *already chosen to install* — same "don't guess topology"
+boundary as `unit_drift`: an uninstalled manifest unit is not a fault,
+because most hosts are workers and are not supposed to run the daemon
+lanes. An *installed* one that the manifest expects and that
+`systemctl --user is-enabled` reports as anything other than enabled is
+exactly the state that hid the propagate timer.
+"""
+
+from __future__ import annotations
+
+import subprocess
+
+from coord.deploy_manifest import all_manifest_units
+from coord.health.checks.unit_drift import resolve_systemd_user_dir
+from coord.health.models import CheckResult, HealthContext, Severity
+from coord.health.registry import check
+
+# `systemctl --user is-enabled` states that mean "this will run". `static`
+# is deliberately excluded: every manifest unit ships an `[Install]`
+# section (it is either a timer or a persistent service), so an installed
+# manifest unit reporting `static` has lost that section relative to
+# `deploy/` — a real fault, not a unit that can't be enabled by design.
+#
+# `alias` means systemctl resolved the queried name to a *different* unit
+# via an `Alias=` in that other unit's `[Install]` section and reports the
+# alias target's own state — i.e. "enabled, under another name" — not a
+# distinct half-enabled state. None of the units this check reads out of
+# `deploy/` declare an `Alias=`, so this should not currently be reachable
+# in practice; it is kept in the accepted set (rather than omitted or
+# treated as WARN) because, per `systemd.unit`(5), it never means "this
+# will NOT run" — the one thing this check exists to catch — the opposite
+# would risk a false WARN on a legitimately-running unit.
+#
+# systemctl/`coord --version` are both fast, but a wedged one must not eat
+# the ~2s registry budget for the whole tick (mirrors
+# `coord.health.checks.spawned_coord`'s `_SYSTEMCTL_TIMEOUT`).
+_SYSTEMCTL_TIMEOUT = 5.0
+
+_ENABLED_STATES = {"enabled", "enabled-runtime", "alias"}
+
+
+def _is_enabled(unit: str, *, runner=None) -> tuple[str | None, str | None]:
+    """`(state, error)` from `systemctl --user is-enabled <unit>`.
+
+    systemctl exits non-zero for every state that isn't `enabled` —
+    `disabled`, `static`, `masked`, `not-found`... — so returncode is
+    never the signal here, only stdout is. Split out with an injectable
+    *runner* (mirrors `coord.deploy_units.daemon_reload`) so the probe is
+    testable without a real systemd.
+    """
+    run = runner or subprocess.run
+    try:
+        proc = run(
+            ["systemctl", "--user", "is-enabled", unit],
+            capture_output=True,
+            text=True,
+            timeout=_SYSTEMCTL_TIMEOUT,
+        )
+    except FileNotFoundError:
+        return None, "systemctl not found (no systemd on this host)"
+    except (OSError, subprocess.SubprocessError) as exc:
+        return None, f"{type(exc).__name__}: {exc}"
+    state = (proc.stdout or "").strip() or (proc.stderr or "").strip() or "unknown"
+    return state, None
+
+
+@check(
+    id="unit_enablement",
+    scope="machine",
+    title="unit enablement",
+    order=48,
+    description=(
+        "Installed deploy-lane units the per-role manifest "
+        "(coord/deploy_manifest.py) says should run are actually "
+        "`systemctl --user enable`d, not just present (#2098)."
+    ),
+)
+def probe_unit_enablement(ctx: HealthContext) -> list[CheckResult]:
+    installed_dir = resolve_systemd_user_dir(ctx)
+    results: list[CheckResult] = []
+
+    for name in all_manifest_units():
+        installed_path = installed_dir / name
+        if not installed_path.exists():
+            # Not this host's topology. `unit_drift` reports the same
+            # absence as OK for the same reason (#1831/#1927) — this probe
+            # only judges units a host has already chosen to install.
+            continue
+
+        state, error = _is_enabled(name)
+        values: dict = {"installed_path": str(installed_path), "state": state}
+
+        if error:
+            results.append(
+                CheckResult(
+                    check_id="unit_enablement",
+                    scope="machine",
+                    subject=name,
+                    severity=Severity.UNKNOWN,
+                    headroom=f"could not check enablement: {error}",
+                    error=error,
+                    values=values,
+                )
+            )
+            continue
+
+        if state in _ENABLED_STATES:
+            results.append(
+                CheckResult(
+                    check_id="unit_enablement",
+                    scope="machine",
+                    subject=name,
+                    severity=Severity.OK,
+                    headroom=state,
+                    values=values,
+                )
+            )
+            continue
+
+        results.append(
+            CheckResult(
+                check_id="unit_enablement",
+                scope="machine",
+                subject=name,
+                severity=Severity.WARN,
+                headroom=(
+                    f"installed but {state} — a disabled unit and a working "
+                    "one produce identical evidence until something needed "
+                    "it (#2098)"
+                ),
+                detail=f"systemctl --user daemon-reload && systemctl --user enable --now {name}",
+                threshold="warn when a manifest-listed, installed unit is not enabled",
+                values=values,
+            )
+        )
+
+    if not results:
+        return [
+            CheckResult(
+                check_id="unit_enablement",
+                scope="machine",
+                severity=Severity.OK,
+                headroom="no manifest-listed unit installed on this host",
+                values={"installed_dir": str(installed_dir)},
+            )
+        ]
+    return results

coord/health/checks/worktrees.py

@@ -0,0 +1,96 @@
+"""Stale coordinator worktrees under ``~/.coord/worktrees`` (#1628).
+
+Every dispatched worker gets an ephemeral checkout here and
+``AgentServer._cleanup_worktree`` is supposed to remove it.  When cleanup
+doesn't run — a killed daemon, a worker that died mid-task, a dirty tree the
+pruner correctly refuses to delete — the directory survives, and each one
+carries a full checkout (and, on Rust repos, potentially a ``target/``).
+They accumulate silently.
+
+**"Stale" here is deliberately mtime-based, not liveness-based.**
+``coord diagnose --orphan-worktrees`` already does the precise thing —
+cross-reference each worktree's assignment id against live tmux sessions and
+running DB rows — but that needs board state, and board state is H-3's job,
+not this child's.  Reading a directory's mtime is one ``stat`` per entry, no
+DB, no network, and a worktree nothing has touched in two days is a
+perfectly good proxy for one nobody is using.  When the fleet-scope probes
+land they can sharpen this; until then it is honest and cheap.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from coord.health.models import CheckResult, HealthContext, Severity
+from coord.health.registry import check
+from coord.health.units import human_hours
+
+
+@check(
+    id="worktrees",
+    scope="machine",
+    title="worktrees",
+    order=30,
+    description="Coordinator worktrees nothing has touched recently.",
+)
+def probe_worktrees(ctx: HealthContext) -> CheckResult | None:
+    th = ctx.thresholds
+    root: Path = ctx.coord_dir / "worktrees"
+    try:
+        entries = [e for e in root.iterdir() if e.is_dir() and not e.is_symlink()]
+    except OSError:
+        # No worktrees dir at all is the normal state on a machine that has
+        # never been dispatched to — not a finding.
+        return None
+
+    stale_cutoff = ctx.now - th.worktree_stale_hours * 3600.0
+    stale: list[tuple[str, float]] = []
+    for entry in entries:
+        try:
+            mtime = entry.stat().st_mtime
+        except OSError:
+            continue
+        if mtime < stale_cutoff:
+            stale.append((entry.name, ctx.now - mtime))
+
+    count = len(stale)
+    if count > th.worktree_crit_count:
+        severity = Severity.CRIT
+    elif count > th.worktree_warn_count:
+        severity = Severity.WARN
+    else:
+        severity = Severity.OK
+
+    stale.sort(key=lambda pair: pair[1], reverse=True)
+    if count == 0:
+        headroom = f"0 stale of {len(entries)}"
+    else:
+        oldest_name, oldest_age = stale[0]
+        headroom = (
+            f"{count} stale of {len(entries)} "
+            f"(oldest {oldest_name} {human_hours(oldest_age)})"
+        )
+
+    return CheckResult(
+        check_id="worktrees",
+        scope="machine",
+        severity=severity,
+        headroom=headroom,
+        threshold=f"crit above {th.worktree_crit_count}",
+        detail=(
+            "prune with `coord diagnose --orphan-worktrees`"
+            if severity is not Severity.OK
+            else ""
+        ),
+        values={
+            "root": str(root),
+            "total": len(entries),
+            "stale_count": count,
+            "stale_hours_threshold": th.worktree_stale_hours,
+            "stale": [
+                {"name": name, "age_hours": round(age / 3600.0, 2)} for name, age in stale
+            ],
+            "warn_count": th.worktree_warn_count,
+            "crit_count": th.worktree_crit_count,
+        },
+    )

coord/health/cli.py

@@ -0,0 +1,121 @@
+"""``coord health`` — run the check registry against this machine (#1628).
+
+Deliberately the *only* consumer of the engine in this child.  No transport,
+no board write, no daemon: H-1 ends at a CLI you can run by hand.  H-3 adds
+the fleet-scope probes and the board projection; H-4 adds renderers beyond
+this one.  Both consume ``--json``, whose shape is
+:meth:`coord.health.registry.HealthReport.to_dict`.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import click
+
+from coord.commands._common import _CONFIG_OPTION
+from coord.health.context import build_context
+from coord.health.models import Severity
+from coord.health.registry import run_all
+from coord.health.render import render_report
+
+# Exit codes under --exit-code.  Chosen so a timer can distinguish "needs a
+# human now" from "worth a look" without parsing anything.
+EXIT_OK = 0
+EXIT_WARN = 1
+EXIT_CRIT = 2
+
+
+@click.command("health")
+@_CONFIG_OPTION
+@click.option(
+    "--local",
+    is_flag=True,
+    default=False,
+    help=(
+        "Restrict to machine- and checkout-scope checks (this machine and its "
+        "checkouts). Fleet-scope probes do not exist yet, so today this is "
+        "equivalent to the default — pass it to keep the intent explicit."
+    ),
+)
+@click.option("--json", "output_json", is_flag=True, default=False,
+              help="Emit the machine-readable report instead of the text one.")
+@click.option(
+    "--no-network",
+    is_flag=True,
+    default=False,
+    help=(
+        "Skip probes that leave the machine (the PyPI simple-index fetch and "
+        "the Max-plan usage probe). Skipped checks are listed, never silently "
+        "reported as healthy."
+    ),
+)
+@click.option("--check", "only", multiple=True,
+              help="Run only these check ids (repeatable).")
+@click.option("-v", "--verbose", is_flag=True, default=False,
+              help="Show each check's detail/fix line, including for healthy ones.")
+@click.option(
+    "--exit-code",
+    is_flag=True,
+    default=False,
+    help=f"Exit {EXIT_CRIT} on any CRIT, {EXIT_WARN} on any WARN (default: always 0).",
+)
+def health(
+    config_path: Path,
+    local: bool,
+    output_json: bool,
+    no_network: bool,
+    only: tuple[str, ...],
+    verbose: bool,
+    exit_code: bool,
+) -> None:
+    """Report how much headroom is left on this machine and its checkouts.
+
+    Read-only and non-destructive by construction: every probe stats, reads,
+    or shells out to a read-only command. Nothing here writes, prunes, or
+    remediates — deciding what may self-heal is a separate call.
+    """
+    config = _load_config_or_none(config_path)
+    ctx = build_context(config, allow_network=not no_network)
+
+    if not getattr(ctx.thresholds, "enabled", True):
+        # A disabled engine still reports *that* it is disabled — a health
+        # check silently switched off looks identical to a healthy fleet.
+        if output_json:
+            click.echo(json.dumps({"schema": 1, "enabled": False, "results": []}))
+        else:
+            click.echo("health checks disabled in coordinator.yml (health.enabled: false)")
+        return
+
+    scopes = ("machine", "checkout") if local else None
+    report = run_all(ctx, scopes=scopes, only=list(only) or None)
+
+    if output_json:
+        click.echo(json.dumps(report.to_dict(), indent=2, sort_keys=True))
+    else:
+        click.echo(render_report(report, verbose=verbose))
+
+    if exit_code:
+        if report.severity is Severity.CRIT:
+            raise SystemExit(EXIT_CRIT)
+        if report.severity is Severity.WARN:
+            raise SystemExit(EXIT_WARN)
+
+
+def _load_config_or_none(config_path: Path):
+    """Load ``coordinator.yml``, or carry on without one.
+
+    A machine with no config still has disks, worktrees, a venv and a
+    ``claude`` binary — refusing to report any of that because a YAML file is
+    missing would make the check useless on exactly the machine most likely
+    to be misconfigured.
+    """
+    from coord.config import ConfigError, load  # noqa: PLC0415
+
+    try:
+        return load(config_path)
+    except (ConfigError, OSError) as exc:
+        click.echo(f"note: no usable coordinator.yml ({exc}); "
+                   f"running machine-scope checks only", err=True)
+        return None

coord/health/context.py

@@ -0,0 +1,106 @@
+"""Build the :class:`~coord.health.models.HealthContext` a run needs (#1628).
+
+Kept out of the registry so probes never reach for ``coord.config.load()``
+or ``Path.home()`` themselves — a probe that only reads its context is one
+that a unit test can drive with a fake filesystem and no ``coordinator.yml``.
+"""
+
+from __future__ import annotations
+
+import socket
+import time
+from pathlib import Path
+from typing import Any
+
+from coord.health.models import Checkout, HealthContext
+
+
+def local_checkouts(config: Any) -> tuple[Checkout, ...]:
+    """The checkouts named in ``coordinator.yml`` that actually exist here.
+
+    Same scope rule as ``coord diagnose --graph`` / ``--orphan-worktrees``:
+    local machine only.  Machines whose name/host matches this hostname are
+    considered first so a repo configured on several machines resolves to
+    *this* box's path; any other machine's path is a fallback for the common
+    single-user case where every machine shares a layout.
+    """
+    if config is None:
+        return ()
+    try:
+        local_hostname = socket.gethostname().split(".")[0]
+    except OSError:  # pragma: no cover — gethostname basically cannot fail
+        local_hostname = ""
+
+    seen: set[Path] = set()
+    # Also deduped by repo NAME, not just path: a repo listed on several
+    # machines with different paths, more than one of which happens to exist
+    # here, must still be one row in the report. Without this the fallback
+    # pass re-adds another machine's path for a repo the hostname-matched
+    # pass already resolved, and every checkout-scope check reports it twice
+    # — with the *other* machine's answer.
+    seen_names: set[str] = set()
+    out: list[Checkout] = []
+    for pass_no in range(2):
+        for machine in getattr(config, "machines", ()) or ():
+            on_this_machine = (
+                machine.name == local_hostname
+                or machine.host.split(".")[0] == local_hostname
+            )
+            if pass_no == 0 and not on_this_machine:
+                continue
+            if pass_no == 1 and on_this_machine:
+                continue
+            for repo in getattr(config, "repos", ()) or ():
+                raw = machine.repo_path(repo.name)
+                if not raw:
+                    continue
+                if repo.name in seen_names:
+                    continue
+                path = Path(raw).expanduser()
+                if path in seen or not (path / ".git").exists():
+                    continue
+                seen.add(path)
+                seen_names.add(repo.name)
+                out.append(
+                    Checkout(
+                        name=repo.name,
+                        path=path,
+                        default_branch=getattr(repo, "default_branch", "main") or "main",
+                        develop_branch=getattr(repo, "develop_branch", None),
+                    )
+                )
+    return tuple(out)
+
+
+def build_context(
+    config: Any = None,
+    *,
+    thresholds: Any = None,
+    home: Path | None = None,
+    coord_dir: Path | None = None,
+    allow_network: bool = True,
+    now: float | None = None,
+) -> HealthContext:
+    """Assemble a context from a loaded config (or from nothing at all).
+
+    ``config=None`` is a supported mode: ``coord health`` on a machine with
+    no ``coordinator.yml`` still reports every ``machine``-scope check, it
+    just has no checkouts to sweep.
+    """
+    if thresholds is None:
+        if config is not None and getattr(config, "health", None) is not None:
+            thresholds = config.health
+        else:
+            from coord.config import HealthConfig  # noqa: PLC0415 — avoid import cycle
+
+            thresholds = HealthConfig()
+    resolved_home = home or Path.home()
+    return HealthContext(
+        thresholds=thresholds,
+        home=resolved_home,
+        coord_dir=coord_dir or (resolved_home / ".coord"),
+        now=time.time() if now is None else now,
+        checkouts=local_checkouts(config),
+        config=config,
+        allow_network=allow_network,
+    )