cloudagent-daemon 0.2.1__py3-none-any.whl

cloudagent_daemon/metrics.py

@@ -0,0 +1,315 @@
+"""Wave C #3 — in-process metric registry + Prometheus text renderer.
+
+Lightweight intentionally: we avoid pulling in ``prometheus_client`` (a
+~30 KB dep) for a daemon that ships in user homedirs. The registry is
+process-local — single asyncio loop, no threading — so a plain dict
+backed by a per-name lock isn't necessary; we still wrap mutating
+operations in an :class:`asyncio.Lock` for safety against test
+fixtures that exercise the same registry concurrently.
+
+Type model:
+  * :class:`_Counter` — monotonic int, ``inc(n=1)`` is the only mutation.
+    Optional ``labels: dict[str, str]`` (rendered as Prometheus label set).
+  * :class:`_Gauge` — float, ``set(v)`` and ``inc(n=...)`` allowed.
+
+Rendering: :func:`render_prometheus_text` produces the
+``text/plain; version=0.0.4`` format scraped by Prometheus 2.x —
+``# HELP``, ``# TYPE``, then one sample per (name, labels) pair.
+
+This module is deliberately not exposed via ``__all__`` from the
+package root — callers should import from ``cloudagent_daemon.metrics``
+directly so the dependency graph stays explicit.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Callable
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Internal sample shapes
+# ---------------------------------------------------------------------------
+
+
+def _format_labels(labels: dict[str, str]) -> str:
+    """Render ``{k="v",k2="v2"}`` with Prometheus-safe escaping.
+
+    Per Prometheus exposition format: ``\\`` → ``\\\\``, ``"`` → ``\\"``,
+    newline → ``\\n``. Empty labels render as empty string (no braces).
+    """
+    if not labels:
+        return ""
+    parts = []
+    for k in sorted(labels):
+        v = str(labels[k])
+        v = v.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
+        parts.append(f'{k}="{v}"')
+    return "{" + ",".join(parts) + "}"
+
+
+@dataclass
+class _Counter:
+    """Monotonically-increasing 64-bit-ish int. Labels are optional."""
+
+    name: str
+    help: str
+    # samples[label_tuple] -> int value. label_tuple is a frozenset of
+    # (k, v) so identical labels with different insertion order collapse.
+    samples: dict[frozenset, int] = field(default_factory=dict)
+
+    def inc(self, n: int = 1, labels: dict[str, str] | None = None) -> None:
+        key = frozenset((labels or {}).items())
+        self.samples[key] = self.samples.get(key, 0) + n
+
+    def get(self, labels: dict[str, str] | None = None) -> int:
+        key = frozenset((labels or {}).items())
+        return self.samples.get(key, 0)
+
+    def render(self) -> list[str]:
+        out = [
+            f"# HELP {self.name} {self.help}",
+            f"# TYPE {self.name} counter",
+        ]
+        for key, val in sorted(
+            self.samples.items(), key=lambda kv: sorted(kv[0])
+        ):
+            labels = dict(key)
+            out.append(f"{self.name}{_format_labels(labels)} {val}")
+        if not self.samples:
+            # Emit a 0 line so scrapers see the series exists even when
+            # nothing has incremented yet.
+            out.append(f"{self.name} 0")
+        return out
+
+
+@dataclass
+class _Gauge:
+    """Float-valued sample; ``set`` overwrites, ``inc`` deltas.
+
+    Gauges are not labelled in this registry (we don't currently have a
+    use case that requires them; adding labels later is a strict
+    superset and would not break callers)."""
+
+    name: str
+    help: str
+    value: float = 0.0
+    # Optional resolver: if set, ``render()`` calls it to produce a
+    # fresh value at scrape time. Used for gauges that derive from
+    # external state we don't want to mirror in-memory (e.g.
+    # ``last_event_age_seconds`` on :class:`SSEClient`).
+    resolver: Callable[[], float] | None = None
+
+    def set(self, v: float) -> None:
+        self.value = float(v)
+
+    def inc(self, n: float = 1.0) -> None:
+        self.value += float(n)
+
+    def get(self) -> float:
+        if self.resolver is not None:
+            try:
+                return float(self.resolver())
+            except Exception:  # noqa: BLE001
+                logger.debug(
+                    "metrics: gauge %s resolver raised; falling back to %.3f",
+                    self.name, self.value,
+                )
+        return self.value
+
+    def render(self) -> list[str]:
+        return [
+            f"# HELP {self.name} {self.help}",
+            f"# TYPE {self.name} gauge",
+            f"{self.name} {self.get()}",
+        ]
+
+
+# ---------------------------------------------------------------------------
+# Registry
+# ---------------------------------------------------------------------------
+
+
+class MetricsRegistry:
+    """Process-local metric store.
+
+    The daemon constructs a single instance at boot (see
+    :func:`get_default`) and threads it into the components that need
+    to report (frame dispatcher, spawn queue, claude runner, SSE
+    client, main_loop). Tests can spin up a fresh registry without
+    polluting the default.
+    """
+
+    def __init__(self) -> None:
+        self._counters: dict[str, _Counter] = {}
+        self._gauges: dict[str, _Gauge] = {}
+        self._lock = asyncio.Lock()
+        self._boot_at = time.monotonic()
+
+    # --- registration --------------------------------------------------
+
+    def counter(self, name: str, help: str) -> _Counter:
+        c = self._counters.get(name)
+        if c is None:
+            c = _Counter(name=name, help=help)
+            self._counters[name] = c
+        return c
+
+    def gauge(
+        self,
+        name: str,
+        help: str,
+        *,
+        resolver: Callable[[], float] | None = None,
+    ) -> _Gauge:
+        g = self._gauges.get(name)
+        if g is None:
+            g = _Gauge(name=name, help=help, resolver=resolver)
+            self._gauges[name] = g
+        elif resolver is not None and g.resolver is None:
+            # Allow late binding of a resolver — the SSE client may be
+            # constructed after the registry on cold boot.
+            g.resolver = resolver
+        return g
+
+    # --- mutation helpers (concurrency-safe) ---------------------------
+
+    async def inc_counter(
+        self,
+        name: str,
+        n: int = 1,
+        labels: dict[str, str] | None = None,
+    ) -> None:
+        async with self._lock:
+            c = self._counters.get(name)
+            if c is None:
+                # Lazy register with a placeholder help string so a typo
+                # at call-site doesn't silently drop metrics.
+                c = self.counter(name, f"(auto-registered) {name}")
+            c.inc(n, labels)
+
+    async def set_gauge(self, name: str, value: float) -> None:
+        async with self._lock:
+            g = self._gauges.get(name)
+            if g is None:
+                g = self.gauge(name, f"(auto-registered) {name}")
+            g.set(value)
+
+    # --- introspection --------------------------------------------------
+
+    @property
+    def uptime_seconds(self) -> float:
+        return max(0.0, time.monotonic() - self._boot_at)
+
+    def snapshot(self) -> dict:
+        """Return a JSON-able snapshot for ``doctor`` and tests."""
+        out: dict = {
+            "uptime_seconds": self.uptime_seconds,
+            "counters": {},
+            "gauges": {},
+        }
+        for name, c in self._counters.items():
+            entries: list[dict] = []
+            for key, val in c.samples.items():
+                entries.append({"labels": dict(key), "value": val})
+            out["counters"][name] = entries
+        for name, g in self._gauges.items():
+            out["gauges"][name] = g.get()
+        return out
+
+    def render_prometheus_text(self) -> str:
+        """Render the full registry in Prometheus 0.0.4 text format."""
+        lines: list[str] = []
+        # Always emit uptime as a baseline series so scrapers can detect
+        # daemon restarts even before anything else increments.
+        uptime = _Gauge(
+            name="cloudagent_daemon_uptime_seconds",
+            help="Seconds since daemon process boot.",
+            value=self.uptime_seconds,
+        )
+        lines.extend(uptime.render())
+        for name in sorted(self._counters):
+            lines.extend(self._counters[name].render())
+        for name in sorted(self._gauges):
+            lines.extend(self._gauges[name].render())
+        return "\n".join(lines) + "\n"
+
+
+# ---------------------------------------------------------------------------
+# Module-level default registry
+# ---------------------------------------------------------------------------
+
+_DEFAULT_REGISTRY: MetricsRegistry | None = None
+
+
+def get_default() -> MetricsRegistry:
+    """Return the process-wide default registry.
+
+    Created lazily on first access. Tests should pass a fresh
+    :class:`MetricsRegistry` into components instead of mutating the
+    default (which would leak between tests).
+    """
+    global _DEFAULT_REGISTRY
+    if _DEFAULT_REGISTRY is None:
+        _DEFAULT_REGISTRY = MetricsRegistry()
+        _register_standard_metrics(_DEFAULT_REGISTRY)
+    return _DEFAULT_REGISTRY
+
+
+def reset_default_for_tests() -> None:
+    """Wipe the default registry — fixture cleanup hook only."""
+    global _DEFAULT_REGISTRY
+    _DEFAULT_REGISTRY = None
+
+
+def _register_standard_metrics(reg: MetricsRegistry) -> None:
+    """Pre-declare the well-known names so the first scrape produces
+    a complete set of series (avoid Prometheus' "missing series at
+    scrape N+1" gotcha)."""
+    reg.counter(
+        "cloudagent_daemon_deliver_frames_total",
+        "Total deliver SSE frames accepted by the dispatcher.",
+    )
+    reg.counter(
+        "cloudagent_daemon_spawn_completed_total",
+        "Total claude wakes completed, labelled by terminal status.",
+    )
+    reg.counter(
+        "cloudagent_daemon_spawn_fallback_send_message_total",
+        "Times the daemon POSTed send_message on claude's behalf (1k #63 Layer B).",
+    )
+    reg.counter(
+        "cloudagent_daemon_sse_reconnects_total",
+        "Total SSE reconnects, including active-watchdog forced ones.",
+    )
+    reg.counter(
+        "cloudagent_daemon_sse_active_watchdog_fires_total",
+        "Times the active liveness watchdog forced a reconnect (Wave C #2).",
+    )
+    reg.gauge(
+        "cloudagent_daemon_outbox_pending",
+        "Current outbox entries pending a spawn (received-without-done).",
+    )
+    reg.gauge(
+        "cloudagent_daemon_outbox_stuck",
+        "Outbox entries past their per-priority threshold at last scan.",
+    )
+    reg.gauge(
+        "cloudagent_daemon_agents_bound",
+        "Agents bound to this host as of the last ready/assign event.",
+    )
+    reg.gauge(
+        "cloudagent_daemon_sse_last_event_age_seconds",
+        "Seconds since the last parsed SSE event arrived.",
+    )
+
+
+__all__ = [
+    "MetricsRegistry",
+    "get_default",
+    "reset_default_for_tests",
+]

cloudagent_daemon/metrics_server.py

@@ -0,0 +1,149 @@
+"""Wave C #3 — minimal unix-socket HTTP server exposing ``/metrics``.
+
+Why unix socket instead of TCP HTTP:
+  * No port allocation surprises on user machines (laptops behind
+    pf/firewall rules can't reliably open arbitrary listening ports).
+  * Permissioned by filesystem (``0o600``) — only the daemon-owning
+    user can scrape. No need for a separate auth scheme.
+  * Prometheus 2.x supports unix socket scraping natively
+    (``-target=unix:///path/to/sock``) and ``curl --unix-socket`` is
+    a one-liner for ad-hoc operator inspection.
+
+Why we don't reuse ``bridge.sock``:
+  * That socket speaks line-delimited JSON to mcp_bridge subprocesses.
+    Mixing protocols invites bugs; a second socket is cheap.
+  * Decoupling lets us shut down ``bridge.sock`` independently if a
+    future refactor moves the mcp bridge into its own process tree.
+
+Wire format (server response):
+
+::
+
+  HTTP/1.1 200 OK\r\n
+  Content-Type: text/plain; version=0.0.4; charset=utf-8\r\n
+  Content-Length: <n>\r\n
+  Connection: close\r\n
+  \r\n
+  <prometheus text>
+
+Request parsing is intentionally permissive: we ignore the path and
+respond to ANY connection with the metrics body. This keeps the server
+3 dozen lines and avoids re-implementing HTTP. A caller that opens the
+socket and immediately reads (e.g. ``curl --unix-socket sock http://x/``)
+gets a valid HTTP response either way.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from pathlib import Path
+
+from cloudagent_daemon.metrics import MetricsRegistry
+
+logger = logging.getLogger(__name__)
+
+# Default location. The daemon's pid + state live in the same dir so
+# operators inspecting one usually want the other.
+DEFAULT_METRICS_SOCKET_PATH = (
+    Path.home() / ".cloudagent" / "daemon" / "metrics.sock"
+)
+
+
+class MetricsServer:
+    """Unix-socket HTTP-ish responder serving Prometheus text format."""
+
+    def __init__(
+        self,
+        registry: MetricsRegistry,
+        *,
+        socket_path: Path | None = None,
+    ) -> None:
+        self._registry = registry
+        self._socket_path = Path(socket_path or DEFAULT_METRICS_SOCKET_PATH)
+        self._server: asyncio.base_events.Server | None = None
+
+    @property
+    def socket_path(self) -> Path:
+        return self._socket_path
+
+    async def start(self) -> None:
+        if self._socket_path.exists():
+            # Stale socket from a crashed process.
+            try:
+                self._socket_path.unlink()
+            except OSError:  # pragma: no cover — race with another daemon
+                logger.warning(
+                    "metrics_server: could not unlink stale socket %s",
+                    self._socket_path,
+                )
+        self._socket_path.parent.mkdir(parents=True, exist_ok=True)
+        self._server = await asyncio.start_unix_server(
+            self._handle_connection, path=str(self._socket_path)
+        )
+        try:
+            os.chmod(self._socket_path, 0o600)
+        except OSError:  # pragma: no cover — exotic filesystems
+            logger.warning(
+                "metrics_server: chmod 0o600 failed on %s",
+                self._socket_path,
+            )
+
+    async def stop(self) -> None:
+        if self._server is not None:
+            self._server.close()
+            await self._server.wait_closed()
+            self._server = None
+        try:
+            if self._socket_path.exists():
+                self._socket_path.unlink()
+        except OSError:  # pragma: no cover
+            pass
+
+    async def _handle_connection(
+        self,
+        reader: asyncio.StreamReader,
+        writer: asyncio.StreamWriter,
+    ) -> None:
+        try:
+            # Drain the request lines until \r\n\r\n. We don't actually
+            # parse anything; we just need to consume the request before
+            # writing the response so the client doesn't see a half-open
+            # write.
+            try:
+                while True:
+                    line = await asyncio.wait_for(
+                        reader.readuntil(b"\r\n"),
+                        timeout=2.0,
+                    )
+                    if line in (b"\r\n", b""):
+                        break
+            except asyncio.TimeoutError:
+                logger.debug("metrics_server: request read timeout — serving anyway")
+            except asyncio.IncompleteReadError:
+                pass  # client hung up; that's fine, write response anyway
+
+            body = self._registry.render_prometheus_text().encode("utf-8")
+            response = (
+                b"HTTP/1.1 200 OK\r\n"
+                b"Content-Type: text/plain; version=0.0.4; charset=utf-8\r\n"
+                b"Content-Length: " + str(len(body)).encode("ascii") + b"\r\n"
+                b"Connection: close\r\n"
+                b"\r\n"
+            ) + body
+            writer.write(response)
+            try:
+                await writer.drain()
+            except (ConnectionResetError, BrokenPipeError):
+                # Scraper hung up before we finished — common with
+                # ``curl --unix-socket`` short-lived connections.
+                pass
+        finally:
+            try:
+                writer.close()
+                await writer.wait_closed()
+            except Exception:  # noqa: BLE001
+                pass
+
+
+__all__ = ["MetricsServer", "DEFAULT_METRICS_SOCKET_PATH"]