code-coordinator 0.5.331__py3-none-any.whl → 0.5.332__py3-none-any.whl

{code_coordinator-0.5.331.dist-info → code_coordinator-0.5.332.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.331
+Version: 0.5.332
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.331.dist-info → code_coordinator-0.5.332.dist-info}/RECORD

@@ -1,4 +1,4 @@
-code_coordinator-0.5.331.dist-info/licenses/LICENSE,sha256=851BVZZwf_pEv2ydSlMmcZrwnwSS5YJ98wdLze7lyL8,4212
+code_coordinator-0.5.332.dist-info/licenses/LICENSE,sha256=851BVZZwf_pEv2ydSlMmcZrwnwSS5YJ98wdLze7lyL8,4212
 coord/__init__.py,sha256=3FN2qrrxtW87xSsSD557YcMwHLDahnFK8LvqQI7Wlf4,9783
 coord/_board_mapping.py,sha256=SGq8hC-gsyXHEZe2L58qy6pyOVQvswTpQ5FzPa_vZ8w,11012
 coord/acceptance.py,sha256=v7k1e_shUIjWKolKDQ9oJv67o0O8L659El3grvcF8ns,75452
@@ -63,6 +63,7 @@ coord/interactive.py,sha256=AEK-5S2aMknpP6k3Xx-aBuTEulw0_c-gbYPEKTwUjRE,219083
 coord/issue_store.py,sha256=abywea0vpXE_BZ5F3QHLDxLKYKDYF7iQYabZhppVYto,69451
 coord/issues_sync_status.py,sha256=_6ucx7oW3TUGZZvs93wRbzR_SUQcNlz-7RptOzr4alo,9262
 coord/liveness_auditor.py,sha256=sf_xBklt37DUnrIEeD4fVp4_gWPDaXbydZgfuWAPFRE,12478
+coord/machine_metrics.py,sha256=mBEciin6gdYJaPw73gY7mrDnalm01WhX7kpzt8t0NEk,9396
 coord/machine_onboard.py,sha256=BTUAC7bdERcDyaIvO2yKkQ8xPwsdR063yHUrOznmHWk,58555
 coord/machine_pause.py,sha256=6s7Dtr5zLs1qB2NWI2-cfbMdPyigK62FTtLjzmY6slE,47521
 coord/merge_queue.py,sha256=UdwyoyfE9DplS6Mia4_aY6TK5a-f933M04Ns5mVHnsw,392677
@@ -107,7 +108,7 @@ coord/revalidate.py,sha256=fQaYCvObqUawSZWMEOkVWFtYswz9PQQwSMApAyP5zeA,66266
 coord/review.py,sha256=hsY6HUif44HL3pY52H7SoqNuMMZzA_9gfHfJKnC-fWE,183282
 coord/scorecard.py,sha256=axe_doBJBBFWtN21JZDRSijwDI0xaZrt2oVYbwJPW7U,20211
 coord/self_health.py,sha256=9PLVdaEReFLUY6y2QXMGVd6U1CdHAh9_8MzduwBOzAY,10651
-coord/serve_app.py,sha256=SXHTyxt7PdAH_zCUE6srqUvkx6Ps1PbEbW2MjLOpsGE,507632
+coord/serve_app.py,sha256=_vyNSXqAUWVfsQfYCjJRnqefczof7qKObrMnTysyUkw,509398
 coord/smoke.py,sha256=INLh9Wu57WOcCoOY4CMOcbzArIhMxMolsV2D8hAW7zw,70481
 coord/spend_ceiling.py,sha256=hfuiXlnD4or7Wqoqf6BHcOMT2CXd92FBZuVdnQVlI2M,13573
 coord/split_work.py,sha256=Gp_cNtYBe7nDxBN7cDRs8CJG3V_Y0C1QUi35fHGcvnY,8502
@@ -248,8 +249,8 @@ coord/skills/release-now/SKILL.md,sha256=u5JY0Jm0IiZYs2JGjRSqNj265CUWcIA6pMaeulb
 coord/skills/review-verdict-recovery/SKILL.md,sha256=Grg8GWkJLK20ct2YiigbTFfNOkit7tNrdRIhdocKGJk,3941
 coord/skills/tui-quadraui-workflow/SKILL.md,sha256=4hUZza9wSSnPhf0eRAOh2r_MQZxK2pXYO5g-kxmJq-U,3832
 coord/skills/update-issue/SKILL.md,sha256=VrPinEmLZrCSNDrnz96hQomyO403o5c4i0RdsnUnWA0,3642
-code_coordinator-0.5.331.dist-info/METADATA,sha256=isRG7tzhdNWK37Ho940K1tFu_LWHd8Sg3LrzAur5FYU,27178
-code_coordinator-0.5.331.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-code_coordinator-0.5.331.dist-info/entry_points.txt,sha256=o-Hq8pvG7ReGePvN_4iOKjLNqiq-MJ43HbPurqjNPZo,41
-code_coordinator-0.5.331.dist-info/top_level.txt,sha256=w5sTrXesTCebhXyYv2qufOZGo0PoD1Lxgn0Q3MAXjh4,6
-code_coordinator-0.5.331.dist-info/RECORD,,
+code_coordinator-0.5.332.dist-info/METADATA,sha256=1zZHh_75sV2go8WtKq1cX-YI4TfozgKZnV3-wkhwgDk,27178
+code_coordinator-0.5.332.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+code_coordinator-0.5.332.dist-info/entry_points.txt,sha256=o-Hq8pvG7ReGePvN_4iOKjLNqiq-MJ43HbPurqjNPZo,41
+code_coordinator-0.5.332.dist-info/top_level.txt,sha256=w5sTrXesTCebhXyYv2qufOZGo0PoD1Lxgn0Q3MAXjh4,6
+code_coordinator-0.5.332.dist-info/RECORD,,

coord/machine_metrics.py

@@ -0,0 +1,217 @@
+"""Coordinator-side machine metrics sampler + bounded ring buffer (#3020).
+
+Part of milestone #71 (Fleet metrics + machines API) — the backend half of
+the coord-web Machines panel.
+
+``coord-tui`` renders CPU/mem sparklines by polling **each agent's own**
+``/metrics`` endpoint directly on port 7433 (``coord.agent_app.metrics``, a
+``psutil`` cpu/mem snapshot) and buffering samples in-process for as long as
+the panel stays open (see ``coord-tui/src/app/data.rs``
+``spawn_machine_metrics``). A browser can't do that — ``coord web`` is a
+single-origin HTTP client of the dashboard API, not something that can fan
+out to N agent hosts — and nothing anywhere persists a series; the TUI's
+``VecDeque`` dies with the panel.
+
+This module is the one missing thing: a coordinator-side sampler, owned by
+the board daemon (``coord serve``) and driven off its existing periodic-loop
+machinery (mirrors ``coord.health.fleet_snapshot.FleetHealthRefresher`` and
+``coord.gate_snapshot.GateSnapshotRefresher`` — see ``coord.serve_app``'s
+``_machine_metrics_loop``, isolated from ``_tick_loop`` for the identical
+reason those two are: a fan-out HTTP GET per agent must never block, or be
+blocked by, the reconcile/enqueue/drain steps).
+
+**Bounded, not incidental (#3020).** Each machine gets its own
+``collections.deque(maxlen=...)`` ring buffer sized for a ~6h window at the
+default ~15s cadence (:data:`DEFAULT_MAX_SAMPLES` = 1440) — old samples are
+evicted automatically as new ones arrive; the process can never accumulate
+unbounded history no matter how long the daemon stays up.
+
+**``unknown``, never a silent gap (#1485).** A failed, timed-out, or 503
+(psutil not installed) poll still appends a sample — with
+``status="unknown"`` — never nothing. A renderer building a sparkline from
+this series must be able to tell "no data point" (a gap it filled in itself)
+from "the daemon tried and the agent didn't answer" (an explicit unknown
+sample); silently skipping the append would collapse that distinction back
+into "looks the same as healthy", the exact failure mode
+``coord.health.fleet_snapshot`` already guards against for machine health.
+
+**No persistence.** This is purely in-memory; history is lost on daemon
+restart by design (a sqlite time-series was considered and deliberately
+rejected for this milestone — see the issue).
+
+**Out of scope here.** The read endpoint for this series (#A2) and putting
+it on ``/board`` (explicitly rejected — every TUI client polls ``/board`` on
+a short interval and ``FleetHealthSnapshot`` already carries a 256 KiB cap
+for exactly this reason) are not this module's job.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+
+import httpx
+
+from coord.network import AGENT_PORT, DEFAULT_TIMEOUT, classify_error
+
+log = logging.getLogger("coord.serve")
+
+# ~6h of history at the default ~15s cadence. Explicit and enforced (each
+# per-machine buffer is a `deque(maxlen=...)`, not a list someone forgot to
+# trim) rather than incidental.
+DEFAULT_MAX_SAMPLES = 1440
+DEFAULT_CADENCE_SECONDS = 15.0
+
+STATUS_OK = "ok"
+STATUS_UNKNOWN = "unknown"
+
+
+@dataclass(frozen=True)
+class MetricsSample:
+    """One machine, one point in time. ``status="unknown"`` carries no
+    cpu/mem values — never fabricated zeros — only a human-readable
+    ``reason`` for why the poll didn't produce a real reading.
+    """
+
+    timestamp: float
+    status: str  # STATUS_OK | STATUS_UNKNOWN
+    cpu_percent: float | None = None
+    mem_percent: float | None = None
+    mem_used_mb: float | None = None
+    mem_total_mb: float | None = None
+    reason: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "timestamp": self.timestamp,
+            "status": self.status,
+            "cpu_percent": self.cpu_percent,
+            "mem_percent": self.mem_percent,
+            "mem_used_mb": self.mem_used_mb,
+            "mem_total_mb": self.mem_total_mb,
+            "reason": self.reason,
+        }
+
+
+def _unknown(reason: str, *, now: float | None = None) -> MetricsSample:
+    return MetricsSample(
+        timestamp=now if now is not None else time.time(),
+        status=STATUS_UNKNOWN,
+        reason=reason,
+    )
+
+
+def fetch_metrics(machine, timeout: float = DEFAULT_TIMEOUT) -> MetricsSample:  # noqa: ANN001 — coord.models.Machine
+    """GET one machine's agent ``/metrics`` and classify the result.
+
+    Mirrors ``coord.network.check_machine``'s error classification but
+    targets ``/metrics`` instead of ``/health``. Never raises: every failure
+    mode — connect/read timeout, DNS failure, connection refused, a non-200
+    status (including the documented 503 an agent returns when ``psutil``
+    isn't installed), and malformed/missing-field JSON — becomes an explicit
+    :data:`STATUS_UNKNOWN` sample rather than an exception or a dropped
+    point. The bounded ``timeout`` (default :data:`coord.network.
+    DEFAULT_TIMEOUT`, 3s) is what keeps one hanging agent from stalling this
+    call past that ceiling; :class:`MachineMetricsSampler.refresh` is what
+    keeps that bounded stall from serializing behind every other machine.
+    """
+    url = f"http://{machine.host}:{AGENT_PORT}/metrics"
+    try:
+        resp = httpx.get(url, timeout=timeout)
+    except Exception as e:  # noqa: BLE001 — classify every network error, never raise
+        _, reason = classify_error(e)
+        return _unknown(reason)
+
+    if resp.status_code == 503:
+        # #207's documented shape: psutil not installed on that agent.
+        return _unknown("psutil not installed on agent")
+    if resp.status_code != 200:
+        return _unknown(f"HTTP {resp.status_code}")
+
+    try:
+        data = resp.json()
+    except ValueError:
+        return _unknown("invalid JSON from /metrics")
+
+    try:
+        return MetricsSample(
+            timestamp=time.time(),
+            status=STATUS_OK,
+            cpu_percent=float(data["cpu_percent"]),
+            mem_percent=float(data["mem_percent"]),
+            mem_used_mb=float(data["mem_used_mb"]),
+            mem_total_mb=float(data["mem_total_mb"]),
+        )
+    except (KeyError, TypeError, ValueError):
+        return _unknown("malformed /metrics payload")
+
+
+class MachineMetricsSampler:
+    """Owns the bounded per-machine ring buffers; refreshed by the daemon tick.
+
+    ``series()``/``all_series()`` are the read path — bare in-memory reads,
+    no I/O, safe to call from a request handler. ``refresh(config)`` is the
+    only method that talks to agents and must only ever run from the
+    daemon's tick machinery (or a test driving it explicitly), exactly the
+    same split as ``coord.health.fleet_snapshot.FleetHealthRefresher``.
+    """
+
+    def __init__(
+        self,
+        *,
+        max_samples: int = DEFAULT_MAX_SAMPLES,
+        timeout: float = DEFAULT_TIMEOUT,
+    ) -> None:
+        if max_samples <= 0:
+            raise ValueError("max_samples must be positive")
+        self._max_samples = max_samples
+        self._timeout = timeout
+        self._buffers: dict[str, deque] = {}
+
+    def series(self, machine_name: str) -> list[dict]:
+        """This machine's samples, oldest first. ``[]`` if never polled."""
+        buf = self._buffers.get(machine_name)
+        return [s.to_dict() for s in buf] if buf else []
+
+    def all_series(self) -> dict[str, list[dict]]:
+        """Every known machine's series, oldest-first, keyed by name."""
+        return {name: [s.to_dict() for s in buf] for name, buf in self._buffers.items()}
+
+    def refresh(self, config) -> None:  # noqa: ANN001 — coord.config.Config
+        """Poll every machine in ``config.machines`` once and append a sample
+        each — :data:`STATUS_OK` or :data:`STATUS_UNKNOWN`, never neither.
+
+        Polls concurrently (mirrors ``coord.network.check_all``): each call
+        is individually bounded by ``timeout``, and running them in a thread
+        pool rather than sequentially means one slow/hanging agent's bounded
+        stall does not serialize in front of the rest of the fleet's polls —
+        the whole ``refresh()`` call takes roughly one ``timeout``, not
+        ``len(machines) * timeout``, no matter which machine is the slow
+        one. Called off the event loop (``run_in_threadpool`` from
+        ``coord.serve_app``'s own loop, isolated from ``_tick_loop``) so
+        even that bound never touches board-read latency.
+        """
+        machines = list(getattr(config, "machines", ()) or ())
+        if not machines:
+            return
+
+        workers = min(8, len(machines))
+        with ThreadPoolExecutor(max_workers=workers) as pool:
+            samples = list(pool.map(self._poll_one, machines))
+
+        for machine, sample in zip(machines, samples):
+            buf = self._buffers.get(machine.name)
+            if buf is None:
+                buf = deque(maxlen=self._max_samples)
+                self._buffers[machine.name] = buf
+            buf.append(sample)
+
+    def _poll_one(self, machine) -> MetricsSample:  # noqa: ANN001
+        try:
+            return fetch_metrics(machine, timeout=self._timeout)
+        except Exception as exc:  # noqa: BLE001 — one bad machine must never abort the refresh
+            log.warning("machine-metrics poll: %s raised %s", machine.name, exc)
+            return _unknown(f"poll raised: {exc}")

coord/serve_app.py

@@ -5340,6 +5340,15 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
 
     _fleet_health_refresher = FleetHealthRefresher()
 
+    # #3020: same shape again — polling every agent's own /metrics is real
+    # per-machine I/O, so it runs on the tick loop's cadence
+    # (`_machine_metrics_loop` below), never inline off a request handler.
+    # Series live only in this sampler's bounded in-memory ring buffers; no
+    # read endpoint exists yet (that's #A2) and nothing here touches /board.
+    from coord.machine_metrics import MachineMetricsSampler  # noqa: PLC0415
+
+    _machine_metrics_sampler = MachineMetricsSampler()
+
     # Short-TTL cache for the computed /board projection so burst polls from the
     # TUI don't each pay the full board_projection + merge-plan + stage-projection
     # recomputation (~465-issue load measured in the issue). Keyed to nothing
@@ -9452,6 +9461,26 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
                 except Exception:  # noqa: BLE001 — keep serving the old snapshot
                     log.warning("fleet-health refresh failed", exc_info=True)
 
+        # #3020: CPU/mem sampler for the coord-web Machines panel — default
+        # 15s (env COORD_METRICS_POLL_INTERVAL; 0 disables). Own loop, not a
+        # `_tick_loop` step, for the identical reason as `_health_refresh_loop`
+        # just above: it fans out one HTTP GET per agent and must never be
+        # held up behind, or hold up, the reconcile/enqueue/drain steps.
+        try:
+            metrics_poll_interval = float(
+                os.environ.get("COORD_METRICS_POLL_INTERVAL", "15")
+            )
+        except ValueError:
+            metrics_poll_interval = 15.0
+
+        async def _machine_metrics_loop() -> None:
+            while True:
+                await asyncio.sleep(metrics_poll_interval)
+                try:
+                    await run_in_threadpool(_machine_metrics_sampler.refresh, config)
+                except Exception:  # noqa: BLE001 — a tick must never crash the daemon
+                    log.warning("machine-metrics refresh failed", exc_info=True)
+
         # #2570: the #2536 phantom-row auto-heal, run a SECOND time from
         # inside this long-lived process rather than relying solely on
         # `coord-notify.timer`'s oneshot `~/.coord-venv/bin/coord notify`
@@ -10218,12 +10247,18 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
                 else None,
                 "auto-revalidate",
             )
+            machine_metrics_task = _watch(
+                asyncio.create_task(_machine_metrics_loop())
+                if interval > 0 and metrics_poll_interval > 0
+                else None,
+                "machine-metrics",
+            )
             try:
                 yield
             finally:
                 for t in (
                     task, gate_task, health_task, phantom_heal_task,
-                    auto_revalidate_task,
+                    auto_revalidate_task, machine_metrics_task,
                 ):
                     if t is not None:
                         t.cancel()