cloudagent-daemon 0.2.1__py3-none-any.whl

cloudagent_daemon/spawn_outbox.py

@@ -0,0 +1,443 @@
+"""Durable spawn outbox — append-only journal for accepted deliver frames.
+
+Bug 4 in ``docs/debug/2026-05-12-task-multi-agent-debug.md``:
+``frame_dispatch._on_deliver`` ack'd inbound deliver frames before the
+claude spawn actually ran. If the daemon crashed (SIGKILL, OOM, panic)
+or :class:`SpawnQueue` lost the in-memory queue entry, the server saw
+the ack and never redelivered → the spawn was silently lost (the
+chairman's "msg_209 / msg_210 sat unread for 24h" prod symptom).
+
+This module makes spawn-acceptance durable:
+
+1. :meth:`SpawnOutbox.record_received` is invoked from the dispatcher
+   **before** the server-side ack. The payload + msg_id + wake_id are
+   appended to
+   ``<workspace>/<agent_id>/.meta/pending-spawn.jsonl``.
+2. :meth:`SpawnOutbox.record_done` is invoked by :class:`SpawnQueue`
+   after the runner returns (success or error). ``ok=False`` is the
+   "runner raised" path — we still mark done so the wake isn't
+   replayed forever on each boot.
+3. :meth:`SpawnOutbox.find_pending` is called from the daemon main
+   loop on startup and yields the original payloads of any received
+   entry without a matching done entry. The main loop re-enqueues
+   those onto :class:`SpawnQueue` so the spawn finally runs.
+4. :meth:`SpawnOutbox.compact_agent` rewrites the jsonl with only the
+   unresolved received entries. Triggered automatically after each
+   done write once the file exceeds ``compact_threshold`` lines.
+
+File format — newline-delimited JSON, one entry per line. Two shapes:
+
+  ``{"event": "received", "msg_id": str, "wake_id": str | null,
+       "ts": iso, "payload": {...}}``
+  ``{"event": "done", "msg_id": str, "ts": iso, "ok": bool}``
+
+Concurrency: a per-agent :class:`asyncio.Lock` serialises writes /
+compaction. ``find_pending`` runs on boot before any writer starts so
+needs no lock. The boot path is single-threaded by design.
+
+Durability tradeoff: writes go through ``open("a") + flush()`` — no
+``fsync``. A full OS crash / power cut can still lose the most recent
+line. For the prod use case (daemon crash / SIGKILL only) the page
+cache survives. ``fsync`` per write would halve daemon throughput at
+load; revisit only if real-world data loss surfaces.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from cloudagent_daemon.workspace import Workspace
+
+logger = logging.getLogger(__name__)
+
+# Default compaction trigger — when a single agent's jsonl exceeds this
+# many lines, rewrite as only the pending entries. Picked so steady-state
+# busy daemon doesn't grow unbounded. Tunable per-instance.
+DEFAULT_COMPACT_THRESHOLD = 200
+
+
+# ---------------------------------------------------------------------------
+# Wave C #2 — outbox-stuck self-kill thresholds
+# ---------------------------------------------------------------------------
+#
+# Maps deliver-frame ``priority`` to "received-without-done is stuck"
+# seconds. Anchored to contract D4 (``docs/debug/2026-05-12-comm-contract-v1.md``
+# §D4): P0=5min, P1=15min, P2=60min. The ``None`` (default) bucket is
+# 30 min per the Wave C brief — half-way between P1 (15 min) and P2
+# (60 min) — to bound stuck windows for backends that don't yet stamp
+# the priority field.
+#
+# Keys are the lowercase stringified priority values; missing /
+# unrecognised priorities use the ``None`` bucket. main_loop's
+# outbox-watchdog uses these to decide when to log + SystemExit(1) so
+# launchd respawns and the boot-replay path re-enqueues the stuck spawn.
+STUCK_THRESHOLDS_SECONDS: dict[str | None, float] = {
+    "p0": 5 * 60.0,
+    "p1": 15 * 60.0,
+    "p2": 60 * 60.0,
+    None: 30 * 60.0,
+}
+
+
+@dataclass(frozen=True)
+class StuckSpawn:
+    """One ``received``-without-``done`` entry past its priority threshold.
+
+    Returned by :meth:`SpawnOutbox.find_stuck`. Surfaced into the
+    daemon's structured-error log and the ``/metrics`` payload before
+    the outbox-watchdog escalates to ``SystemExit(1)``.
+    """
+
+    agent_id: str
+    msg_id: str
+    received_at: str
+    age_seconds: float
+    priority: str | None
+    threshold_seconds: float
+    payload: dict
+
+
+class SpawnOutbox:
+    """Per-host durable journal of inbound deliver frames.
+
+    One instance lives on the daemon. The jsonl files are sharded
+    per-agent so an active agent's append churn doesn't bottleneck
+    on another agent's compaction.
+    """
+
+    def __init__(
+        self,
+        workspace: Workspace,
+        *,
+        compact_threshold: int = DEFAULT_COMPACT_THRESHOLD,
+    ) -> None:
+        self.workspace = workspace
+        self.compact_threshold = compact_threshold
+        self._locks: dict[str, asyncio.Lock] = {}
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _path_for(self, agent_id: str) -> Path:
+        return (
+            self.workspace.for_agent(agent_id).path
+            / ".meta"
+            / "pending-spawn.jsonl"
+        )
+
+    def _lock_for(self, agent_id: str) -> asyncio.Lock:
+        lock = self._locks.get(agent_id)
+        if lock is None:
+            lock = asyncio.Lock()
+            self._locks[agent_id] = lock
+        return lock
+
+    @staticmethod
+    def _now_iso() -> str:
+        return datetime.now(UTC).isoformat()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def record_received(self, payload: dict) -> None:
+        """Persist a freshly accepted deliver payload.
+
+        Must run BEFORE the server-side ack, otherwise the durability
+        guarantee is lost. Errors propagate so the caller can refuse
+        to ack (server will redeliver next reconnect).
+        """
+        agent_id = str(payload.get("agent_id") or "")
+        msg_id = str(payload.get("message_id") or "")
+        if not agent_id or not msg_id:
+            raise ValueError(
+                "spawn_outbox: payload missing agent_id / message_id"
+            )
+        wake_id = payload.get("wake_id")
+        path = self._path_for(agent_id)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        line = json.dumps(
+            {
+                "event": "received",
+                "msg_id": msg_id,
+                "wake_id": wake_id,
+                "ts": self._now_iso(),
+                "payload": payload,
+            },
+            ensure_ascii=False,
+        )
+        async with self._lock_for(agent_id):
+            with path.open("a", encoding="utf-8") as f:
+                f.write(line + "\n")
+                f.flush()
+        logger.debug(
+            "spawn_outbox: recorded received agent=%s msg=%s",
+            agent_id,
+            msg_id,
+        )
+
+    async def record_done(
+        self, agent_id: str, msg_id: str, *, ok: bool = True
+    ) -> None:
+        """Mark a spawn as completed (success or error path).
+
+        Called after ``runner.run_one`` returns. ``ok=False`` is the
+        runner-raised-exception path; we still mark done so the wake
+        isn't replayed every boot. Idempotent: repeated calls just
+        append another done line, which the compact path collapses.
+        """
+        agent_id = str(agent_id or "")
+        msg_id = str(msg_id or "")
+        if not agent_id or not msg_id:
+            return
+        path = self._path_for(agent_id)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        line = json.dumps(
+            {
+                "event": "done",
+                "msg_id": msg_id,
+                "ts": self._now_iso(),
+                "ok": ok,
+            },
+            ensure_ascii=False,
+        )
+        async with self._lock_for(agent_id):
+            with path.open("a", encoding="utf-8") as f:
+                f.write(line + "\n")
+                f.flush()
+            # Lazy compaction inside the same critical section so a
+            # concurrent writer doesn't race with the rewrite.
+            try:
+                lines = path.read_text(encoding="utf-8").splitlines()
+            except OSError:
+                lines = []
+            if len(lines) > self.compact_threshold:
+                self._compact_locked(agent_id, lines)
+        logger.debug(
+            "spawn_outbox: recorded done agent=%s msg=%s ok=%s",
+            agent_id,
+            msg_id,
+            ok,
+        )
+
+    async def find_pending(self) -> list[dict[str, Any]]:
+        """Scan every agent's jsonl for received-without-done entries.
+
+        Returns the original payload dicts (same shape as the deliver
+        frame's ``payload`` key) ready to feed back into
+        :meth:`SpawnQueue.enqueue`. Order is stable: per-agent
+        directory sort + per-file insertion order (oldest first).
+        """
+        out: list[dict[str, Any]] = []
+        if not self.workspace.root.exists():
+            return out
+        for agent_dir in sorted(self.workspace.root.iterdir()):
+            if not agent_dir.is_dir():
+                continue
+            jsonl = agent_dir / ".meta" / "pending-spawn.jsonl"
+            if not jsonl.exists():
+                continue
+            received: dict[str, dict[str, Any]] = {}
+            done_ids: set[str] = set()
+            try:
+                lines = jsonl.read_text(encoding="utf-8").splitlines()
+            except OSError:
+                continue
+            for raw in lines:
+                raw = raw.strip()
+                if not raw:
+                    continue
+                try:
+                    entry = json.loads(raw)
+                except (TypeError, ValueError):
+                    # Skip corrupt / partial lines (mid-write crash).
+                    continue
+                event = entry.get("event")
+                msg_id = str(entry.get("msg_id") or "")
+                if not msg_id:
+                    continue
+                if event == "received":
+                    received[msg_id] = entry
+                elif event == "done":
+                    done_ids.add(msg_id)
+            for msg_id, entry in received.items():
+                if msg_id in done_ids:
+                    continue
+                payload = entry.get("payload")
+                if isinstance(payload, dict):
+                    out.append(payload)
+        if out:
+            logger.info(
+                "spawn_outbox: boot-replay found %d pending spawn(s)",
+                len(out),
+            )
+        return out
+
+    async def find_stuck(
+        self,
+        *,
+        now: datetime | None = None,
+        thresholds: dict[str | None, float] | None = None,
+    ) -> list[StuckSpawn]:
+        """Return all ``received``-without-``done`` entries past their
+        per-priority age threshold (Wave C #2).
+
+        ``thresholds`` falls back to :data:`STUCK_THRESHOLDS_SECONDS` —
+        callers should generally pass ``None`` and let the module-level
+        contract decide. ``now`` defaults to ``datetime.now(UTC)``; tests
+        inject a fixed clock.
+
+        Resilient to corrupt JSONL lines (skipped) and unparseable
+        timestamps (treated as not-stuck, since we can't meaningfully
+        compute age — emits a debug log so an operator can audit).
+        Priority lookup is case-insensitive and falls through to the
+        ``None`` (default) bucket for any unrecognised value.
+        """
+        out: list[StuckSpawn] = []
+        if not self.workspace.root.exists():
+            return out
+        thresholds = thresholds or STUCK_THRESHOLDS_SECONDS
+        default_threshold = thresholds.get(None, 30 * 60.0)
+        now = now or datetime.now(UTC)
+
+        for agent_dir in sorted(self.workspace.root.iterdir()):
+            if not agent_dir.is_dir():
+                continue
+            jsonl = agent_dir / ".meta" / "pending-spawn.jsonl"
+            if not jsonl.exists():
+                continue
+            received: dict[str, dict[str, Any]] = {}
+            done_ids: set[str] = set()
+            try:
+                lines = jsonl.read_text(encoding="utf-8").splitlines()
+            except OSError:
+                continue
+            for raw in lines:
+                raw = raw.strip()
+                if not raw:
+                    continue
+                try:
+                    entry = json.loads(raw)
+                except (TypeError, ValueError):
+                    continue
+                event = entry.get("event")
+                msg_id = str(entry.get("msg_id") or "")
+                if not msg_id:
+                    continue
+                if event == "received":
+                    received[msg_id] = entry
+                elif event == "done":
+                    done_ids.add(msg_id)
+            for msg_id, entry in received.items():
+                if msg_id in done_ids:
+                    continue
+                received_at = str(entry.get("ts") or "")
+                try:
+                    received_dt = datetime.fromisoformat(received_at)
+                except (TypeError, ValueError):
+                    logger.debug(
+                        "spawn_outbox.find_stuck: unparseable ts on "
+                        "agent=%s msg=%s — skipping", agent_dir.name, msg_id,
+                    )
+                    continue
+                # Naive timestamps default to UTC for backwards-compat;
+                # all writes since the outbox shipped have used tz-aware
+                # ISO strings but tests may construct fixtures differently.
+                if received_dt.tzinfo is None:
+                    received_dt = received_dt.replace(tzinfo=UTC)
+                age = (now - received_dt).total_seconds()
+                payload = entry.get("payload") or {}
+                priority_raw = (
+                    payload.get("priority") if isinstance(payload, dict) else None
+                )
+                priority_key: str | None = (
+                    str(priority_raw).lower() if priority_raw else None
+                )
+                threshold = thresholds.get(priority_key, default_threshold)
+                if age > threshold:
+                    out.append(
+                        StuckSpawn(
+                            agent_id=agent_dir.name,
+                            msg_id=msg_id,
+                            received_at=received_at,
+                            age_seconds=age,
+                            priority=priority_key,
+                            threshold_seconds=threshold,
+                            payload=payload if isinstance(payload, dict) else {},
+                        )
+                    )
+        return out
+
+    async def compact_agent(self, agent_id: str) -> int:
+        """Manually compact one agent's jsonl. Returns kept line count.
+
+        Public entry point for cron-style maintenance; normal flow
+        triggers compaction automatically inside :meth:`record_done`.
+        """
+        path = self._path_for(agent_id)
+        if not path.exists():
+            return 0
+        async with self._lock_for(agent_id):
+            try:
+                lines = path.read_text(encoding="utf-8").splitlines()
+            except OSError:
+                return 0
+            return self._compact_locked(agent_id, lines)
+
+    # ------------------------------------------------------------------
+    # Locked helpers
+    # ------------------------------------------------------------------
+
+    def _compact_locked(self, agent_id: str, lines: list[str]) -> int:
+        """Rewrite the jsonl with only received-without-done entries.
+
+        Caller must hold ``_lock_for(agent_id)``. Returns the number
+        of pending entries kept after rewrite.
+        """
+        path = self._path_for(agent_id)
+        received: dict[str, str] = {}  # msg_id → raw line (preserves ts)
+        done_ids: set[str] = set()
+        for raw in lines:
+            raw = raw.strip()
+            if not raw:
+                continue
+            try:
+                entry = json.loads(raw)
+            except (TypeError, ValueError):
+                continue
+            event = entry.get("event")
+            msg_id = str(entry.get("msg_id") or "")
+            if not msg_id:
+                continue
+            if event == "received":
+                received[msg_id] = raw
+            elif event == "done":
+                done_ids.add(msg_id)
+
+        keep = [raw for msg, raw in received.items() if msg not in done_ids]
+        tmp = path.with_suffix(path.suffix + ".tmp")
+        with tmp.open("w", encoding="utf-8") as f:
+            for raw in keep:
+                f.write(raw + "\n")
+        tmp.replace(path)
+        logger.info(
+            "spawn_outbox: compacted agent=%s lines=%d→%d",
+            agent_id,
+            len(lines),
+            len(keep),
+        )
+        return len(keep)
+
+
+__all__ = [
+    "SpawnOutbox",
+    "StuckSpawn",
+    "STUCK_THRESHOLDS_SECONDS",
+    "DEFAULT_COMPACT_THRESHOLD",
+]

cloudagent_daemon/spawn_queue.py

@@ -0,0 +1,200 @@
+"""SpawnQueue: per-agent FIFO + global concurrency cap.
+
+Implements spec §5 L4 (per-agent cap) and L5 (global concurrency cap).
+
+Design
+------
+* One ``asyncio.Queue`` per agent_id, bounded by ``max_per_agent``.
+* One drain task per agent — created on the first enqueue; exits when the
+  agent queue empties; re-created if a new message arrives later.
+* A single ``asyncio.Semaphore(max_concurrent)`` gates all concurrent
+  ``runner.run_one`` calls across all agents.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+import os
+from collections.abc import Awaitable, Callable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from cloudagent_daemon.spawn_outbox import SpawnOutbox
+
+logger = logging.getLogger(__name__)
+
+
+def _default_max_concurrent() -> int:
+    return min(8, math.ceil((os.cpu_count() or 4) / 2))
+
+
+class SpawnQueue:
+    """Per-agent FIFO queue with a global concurrency cap.
+
+    Parameters
+    ----------
+    runner:
+        Any object with ``async def run_one(agent_id: str, payload: dict)``.
+        The real implementation is ``ClaudeRunner`` (Task 9); tests use a fake.
+    max_per_agent:
+        Spec §5 L4 — maximum queued items per agent (default 100).
+    max_concurrent:
+        Spec §5 L5 — global concurrency cap. ``None`` defaults to
+        ``min(8, ceil(cpu_count / 2))``.
+    queue_full_callback:
+        Called with ``(agent_id, message_id)`` when a per-agent queue is full
+        so the daemon can ack the deliver as ``queue_full``.
+    spawn_outbox:
+        Optional; Bug 4 durable journal. When provided, every drained
+        payload triggers :meth:`SpawnOutbox.record_done` after the
+        runner returns (success or error) so the boot-replay path can
+        tell the spawn finished. See ``spawn_outbox.py``.
+    """
+
+    def __init__(
+        self,
+        runner,
+        *,
+        max_per_agent: int = 100,
+        max_concurrent: int | None = None,
+        queue_full_callback: Callable[[str, str], Awaitable[None]] | None = None,
+        spawn_outbox: "SpawnOutbox | None" = None,
+    ) -> None:
+        self.runner = runner
+        self.max_per_agent = max_per_agent
+        self.max_concurrent: int = max_concurrent or _default_max_concurrent()
+        self.queue_full_callback = queue_full_callback
+        self.spawn_outbox = spawn_outbox
+
+        self._semaphore = asyncio.Semaphore(self.max_concurrent)
+        # agent_id → asyncio.Queue
+        self._queues: dict[str, asyncio.Queue] = {}
+        # agent_id → running drain Task
+        self._tasks: dict[str, asyncio.Task] = {}
+        self._shutdown = False
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def enqueue(self, deliver_payload: dict) -> bool:
+        """Queue a deliver payload for the agent.
+
+        Returns
+        -------
+        True
+            Item was accepted into the queue.
+        False
+            Per-agent queue is full; item was dropped (callback fired if set).
+        """
+        if self._shutdown:
+            return False
+
+        agent_id: str = deliver_payload["agent_id"]
+        message_id: str = deliver_payload["message_id"]
+
+        # Lazily create a per-agent queue.
+        if agent_id not in self._queues:
+            self._queues[agent_id] = asyncio.Queue(maxsize=self.max_per_agent)
+
+        queue = self._queues[agent_id]
+
+        try:
+            queue.put_nowait(deliver_payload)
+        except asyncio.QueueFull:
+            logger.warning(
+                "SpawnQueue: agent=%s queue full (cap=%d), dropping message=%s",
+                agent_id,
+                self.max_per_agent,
+                message_id,
+            )
+            if self.queue_full_callback is not None:
+                await self.queue_full_callback(agent_id, message_id)
+            return False
+
+        # 1i Bug C — Phase 1 demo visibility.
+        logger.info(
+            "spawn_queue: enqueued agent=%s msg=%s queue_depth=%d",
+            agent_id,
+            message_id,
+            queue.qsize(),
+        )
+
+        # Ensure a drain task is running for this agent.
+        existing = self._tasks.get(agent_id)
+        if existing is None or existing.done():
+            task = asyncio.create_task(
+                self._drain(agent_id), name=f"drain-{agent_id}"
+            )
+            self._tasks[agent_id] = task
+
+        return True
+
+    async def shutdown(self) -> None:
+        """Signal shutdown and wait for all in-flight drain tasks to finish."""
+        self._shutdown = True
+        tasks = list(self._tasks.values())
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def cancel_all(self) -> None:
+        """Cancel all drain tasks immediately (used on SIGTERM for fast exit).
+
+        Unlike ``shutdown()``, this does not wait for in-flight ``run_one``
+        calls to complete — it cancels them. Use when a fast, clean exit is
+        required (e.g., SIGTERM handling in the daemon start command).
+        """
+        self._shutdown = True
+        tasks = list(self._tasks.values())
+        for t in tasks:
+            t.cancel()
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    async def _drain(self, agent_id: str) -> None:
+        """Drain the per-agent queue until it is empty."""
+        queue = self._queues[agent_id]
+        while True:
+            try:
+                payload = queue.get_nowait()
+            except asyncio.QueueEmpty:
+                break
+
+            msg_id = str(payload.get("message_id") or "")
+            ok = True
+            async with self._semaphore:
+                try:
+                    await self.runner.run_one(agent_id, payload)
+                except Exception:
+                    ok = False
+                    logger.exception(
+                        "SpawnQueue: runner.run_one raised for agent=%s", agent_id
+                    )
+                finally:
+                    queue.task_done()
+                    # Bug 4 — mark the spawn done in the outbox (regardless
+                    # of runner outcome) so the boot replay path stops
+                    # considering it pending. Failure to write the done
+                    # marker is logged but non-fatal — next compact + next
+                    # boot will re-enqueue, runner has its own dedupe.
+                    if self.spawn_outbox is not None and msg_id:
+                        try:
+                            await self.spawn_outbox.record_done(
+                                agent_id, msg_id, ok=ok
+                            )
+                        except Exception:
+                            logger.exception(
+                                "SpawnQueue: spawn_outbox.record_done failed "
+                                "agent=%s msg=%s",
+                                agent_id,
+                                msg_id,
+                            )
+
+        # Remove self from the task registry so the next enqueue can
+        # re-create a fresh drain task.
+        self._tasks.pop(agent_id, None)