cloudagent-daemon 0.2.4__tar.gz → 0.2.6__tar.gz

{cloudagent_daemon-0.2.4 → cloudagent_daemon-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudagent-daemon
-Version: 0.2.4
+Version: 0.2.6
 Summary: Local daemon for cloudagent — bridges Claude Code subprocesses to the cloudagent backend over WebSocket.
 Project-URL: Homepage, https://cloudagent.chat
 Project-URL: Repository, https://codeup.aliyun.com/69bd0a4d29ad98af4065ce71/cloudagent

{cloudagent_daemon-0.2.4 → cloudagent_daemon-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cloudagent-daemon"
-version = "0.2.4"
+version = "0.2.6"
 description = "Local daemon for cloudagent — bridges Claude Code subprocesses to the cloudagent backend over WebSocket."
 authors = [{name = "chaoyin", email = "tudoucyx@gmail.com"}]
 readme = "README.md"

cloudagent_daemon-0.2.6/src/cloudagent_daemon/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.6"

{cloudagent_daemon-0.2.4 → cloudagent_daemon-0.2.6}/src/cloudagent_daemon/main_loop.py

@@ -51,14 +51,14 @@ _HEARTBEAT_INTERVAL_S = HEARTBEAT_INTERVAL_SECONDS
 
 
 # Wave C #2 — outbox stuck watchdog cadence. Scans every agent's
-# ``pending-spawn.jsonl`` for ``received``-without-``done`` entries
-# older than their per-priority threshold (see
-# :data:`spawn_outbox.STUCK_THRESHOLDS_SECONDS`). The first stuck spawn
-# escalates to ``SystemExit(1)`` so launchd respawns the daemon and the
-# boot-replay path re-enqueues. 60s scan period is tight enough that a
-# pathological 30-min default threshold gets caught within ~1 min of
-# crossing the line, slack enough that the scan cost (~1 ms per agent
-# per pass) is negligible.
+# ``pending-spawn.jsonl`` for ``received`` entries with neither a
+# ``done`` nor a recent ``heartbeat``, older than their per-priority
+# threshold (see :data:`spawn_outbox.STUCK_THRESHOLDS_SECONDS`). The
+# first stuck spawn escalates to ``SystemExit(1)`` so launchd respawns
+# the daemon and the boot-replay path re-enqueues. 60s scan period
+# catches a genuinely stuck spawn within ~1 min of it crossing the
+# threshold, slack enough that the scan cost (~1 ms per agent per pass)
+# is negligible.
 OUTBOX_WATCHDOG_INTERVAL_SECONDS = 60.0
 
 

{cloudagent_daemon-0.2.4 → cloudagent_daemon-0.2.6}/src/cloudagent_daemon/service.py

@@ -17,6 +17,7 @@ import os
 import platform
 import shutil
 import subprocess
+import time
 from pathlib import Path
 
 from .paths import (
@@ -200,29 +201,69 @@ def install_macos(
     plist_path.chmod(0o600)
 
     uid = os.getuid()
-    # Reinstall-safe (2026-05-19). A plain ``bootstrap`` is a no-op when
-    # the service is already loaded — launchctl keeps the OLD in-memory
-    # plist and never re-reads the file we just wrote. Without a prior
-    # ``bootout`` a re-run of ``install-service`` silently keeps the
-    # stale definition (e.g. a changed CLOUDAGENT_DAEMON_PERMISSION_MODE
-    # never takes effect). So: bootout first (unload any loaded
-    # instance), then bootstrap genuinely loads the fresh plist.
-    # ``bootout`` exits non-zero when nothing is loaded (first install)
-    # — ``check=False`` swallows that, making install == reinstall.
-    subprocess.run(
-        ["launchctl", "bootout", f"gui/{uid}/{label}"],
-        check=False,
-    )
-    subprocess.run(
-        ["launchctl", "bootstrap", f"gui/{uid}", str(plist_path)],
-        check=False,
-    )
+    domain = f"gui/{uid}"
+    service_target = f"{domain}/{label}"
+    _launchd_reinstall(domain, service_target, plist_path)
+    return plist_path
+
+
+def _launchd_reinstall(
+    domain: str, service_target: str, plist_path: Path
+) -> None:
+    """Reinstall a launchd job: bootout → wait → bootstrap → kickstart.
+
+    Reinstall-safe + race-safe. Two problems this handles:
+
+    1. A plain ``bootstrap`` is a no-op when the service is already
+       loaded — launchd keeps the OLD in-memory plist and never re-reads
+       the file. So we ``bootout`` first to unload any loaded instance.
+
+    2. ``bootout`` is *asynchronous*: launchctl returns before launchd
+       has finished tearing the job down. A ``bootstrap`` fired
+       immediately races the still-unloading job and fails with
+       ``Input/output error`` (5). So after bootout we POLL
+       ``launchctl print`` until the job is genuinely gone, then
+       bootstrap — with a few retries for slow teardown.
+
+    Raises RuntimeError if bootstrap never succeeds (cli surfaces it as
+    a clean error instead of the old silent "✓ Installed" lie).
+    """
+    subprocess.run(["launchctl", "bootout", service_target], check=False)
+
+    # Poll until the job is no longer registered (``print`` exits
+    # non-zero). ~5s budget — teardown is normally well under 1s.
+    for _ in range(50):
+        probe = subprocess.run(
+            ["launchctl", "print", service_target],
+            capture_output=True,
+            check=False,
+        )
+        if probe.returncode != 0:
+            break
+        time.sleep(0.1)
+
+    last_err = ""
+    for _ in range(5):
+        result = subprocess.run(
+            ["launchctl", "bootstrap", domain, str(plist_path)],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.returncode == 0:
+            break
+        last_err = (result.stderr or result.stdout or "").strip()
+        time.sleep(0.3)
+    else:
+        raise RuntimeError(
+            f"launchctl bootstrap failed after retries: {last_err or 'unknown'}"
+        )
+
     # Kickstart -k forces an immediate (re)start.
     subprocess.run(
-        ["launchctl", "kickstart", "-k", f"gui/{uid}/{label}"],
+        ["launchctl", "kickstart", "-k", service_target],
         check=False,
     )
-    return plist_path
 
 
 def uninstall_macos(profile: str | None = None) -> bool:

{cloudagent_daemon-0.2.4 → cloudagent_daemon-0.2.6}/src/cloudagent_daemon/spawn_outbox.py

@@ -25,11 +25,20 @@ This module makes spawn-acceptance durable:
    unresolved received entries. Triggered automatically after each
    done write once the file exceeds ``compact_threshold`` lines.
 
-File format — newline-delimited JSON, one entry per line. Two shapes:
+File format — newline-delimited JSON, one entry per line. Three shapes:
 
   ``{"event": "received", "msg_id": str, "wake_id": str | null,
        "ts": iso, "payload": {...}}``
   ``{"event": "done", "msg_id": str, "ts": iso, "ok": bool}``
+  ``{"event": "heartbeat", "msg_id": str, "ts": iso}``
+
+The ``heartbeat`` shape is appended by :class:`SpawnQueue` every
+``HEARTBEAT_INTERVAL_SECONDS`` for as long as it is actively running a
+spawn. :meth:`SpawnOutbox.find_stuck` ages each entry from its latest
+sign of life (``received`` OR ``heartbeat``), so a legitimately long
+wake — a 30-60+ min coding spawn — is not mistaken for a hung one.
+(Bug: the outbox-watchdog self-killed the daemon mid-spawn because a
+coding wake outran the flat 30-min threshold — 2026-05-20.)
 
 Concurrency: a per-agent :class:`asyncio.Lock` serialises writes /
 compaction. ``find_pending`` runs on boot before any writer starts so
@@ -65,12 +74,27 @@ DEFAULT_COMPACT_THRESHOLD = 200
 # Wave C #2 — outbox-stuck self-kill thresholds
 # ---------------------------------------------------------------------------
 #
-# Maps deliver-frame ``priority`` to "received-without-done is stuck"
-# seconds. Anchored to contract D4 (``docs/debug/2026-05-12-comm-contract-v1.md``
-# §D4): P0=5min, P1=15min, P2=60min. The ``None`` (default) bucket is
-# 30 min per the Wave C brief — half-way between P1 (15 min) and P2
-# (60 min) — to bound stuck windows for backends that don't yet stamp
-# the priority field.
+# A spawn counts as "stuck" when its ``received`` entry has had no
+# ``done`` AND no ``heartbeat`` for longer than the threshold below.
+# :class:`SpawnQueue` appends a ``heartbeat`` every
+# :data:`HEARTBEAT_INTERVAL_SECONDS` for as long as it is actively
+# running the spawn, and :meth:`find_stuck` ages each entry from its
+# *latest sign of life* (newest of received / heartbeat) — NOT from
+# ``received`` alone.
+#
+# Why: a coding wake legitimately runs 30-60+ min. Ageing from
+# ``received`` made the flat 30-min default fire mid-spawn, the daemon
+# ``SystemExit``-ed, launchd respawned, boot-replay re-enqueued — and
+# the agent was guillotined every 30 min, never finishing (the
+# 2026-05-20 "Dev never reports milestone 2" incident). With heartbeat
+# ageing a healthy spawn keeps its age near zero however long it runs.
+#
+# So the threshold now bounds "how long with NO heartbeat before we
+# treat the spawn as lost" — it only fires when SpawnQueue is no longer
+# running the spawn at all (drain task died / spawn lost before
+# enqueue). 2h on the ``None`` (default) bucket is a deliberately
+# conservative backstop: a healthy spawn heartbeats every 60s so never
+# nears it, and a real daemon crash is caught far sooner by boot-replay.
 #
 # Keys are the lowercase stringified priority values; missing /
 # unrecognised priorities use the ``None`` bucket. main_loop's
@@ -80,9 +104,15 @@ STUCK_THRESHOLDS_SECONDS: dict[str | None, float] = {
     "p0": 5 * 60.0,
     "p1": 15 * 60.0,
     "p2": 60 * 60.0,
-    None: 30 * 60.0,
+    None: 120 * 60.0,
 }
 
+# How often :class:`SpawnQueue` appends a ``heartbeat`` entry while a
+# spawn is in flight. Must stay comfortably below the smallest STUCK
+# threshold (``p0`` = 5 min) so a healthy spawn never drifts into
+# "stuck" between two heartbeats.
+HEARTBEAT_INTERVAL_SECONDS: float = 60.0
+
 
 @dataclass(frozen=True)
 class StuckSpawn:
@@ -226,6 +256,42 @@ class SpawnOutbox:
             ok,
         )
 
+    async def record_heartbeat(self, agent_id: str, msg_id: str) -> None:
+        """Append a ``heartbeat`` event — proof the spawn is still running.
+
+        Called periodically by :class:`SpawnQueue` while it is actively
+        running a spawn (cadence :data:`HEARTBEAT_INTERVAL_SECONDS`).
+        :meth:`find_stuck` ages each pending entry from the newest of its
+        ``received`` / ``heartbeat`` timestamps, so a long-but-healthy
+        wake is not mistaken for a hung spawn. The caller treats this as
+        best-effort — a transient write error only shortens one
+        heartbeat gap — but the error still propagates so the caller can
+        log it.
+        """
+        agent_id = str(agent_id or "")
+        msg_id = str(msg_id or "")
+        if not agent_id or not msg_id:
+            return
+        path = self._path_for(agent_id)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        line = json.dumps(
+            {
+                "event": "heartbeat",
+                "msg_id": msg_id,
+                "ts": self._now_iso(),
+            },
+            ensure_ascii=False,
+        )
+        async with self._lock_for(agent_id):
+            with path.open("a", encoding="utf-8") as f:
+                f.write(line + "\n")
+                f.flush()
+        logger.debug(
+            "spawn_outbox: recorded heartbeat agent=%s msg=%s",
+            agent_id,
+            msg_id,
+        )
+
     async def find_pending(self) -> list[dict[str, Any]]:
         """Scan every agent's jsonl for received-without-done entries.
 
@@ -314,6 +380,7 @@ class SpawnOutbox:
                 continue
             received: dict[str, dict[str, Any]] = {}
             done_ids: set[str] = set()
+            heartbeats: dict[str, str] = {}  # msg_id → latest heartbeat ts
             try:
                 lines = jsonl.read_text(encoding="utf-8").splitlines()
             except OSError:
@@ -334,6 +401,11 @@ class SpawnOutbox:
                     received[msg_id] = entry
                 elif event == "done":
                     done_ids.add(msg_id)
+                elif event == "heartbeat":
+                    # jsonl is append-ordered → the last write wins.
+                    hb_ts = str(entry.get("ts") or "")
+                    if hb_ts:
+                        heartbeats[msg_id] = hb_ts
             for msg_id, entry in received.items():
                 if msg_id in done_ids:
                     continue
@@ -351,7 +423,27 @@ class SpawnOutbox:
                 # ISO strings but tests may construct fixtures differently.
                 if received_dt.tzinfo is None:
                     received_dt = received_dt.replace(tzinfo=UTC)
-                age = (now - received_dt).total_seconds()
+                # Heartbeat-aware ageing: SpawnQueue writes a heartbeat
+                # every HEARTBEAT_INTERVAL_SECONDS while it is actively
+                # running the spawn, so age from the latest sign of life
+                # (received OR heartbeat). Without this a legitimately
+                # long coding wake looks identical to a hung spawn.
+                last_alive_dt = received_dt
+                hb_ts = heartbeats.get(msg_id)
+                if hb_ts:
+                    try:
+                        hb_dt = datetime.fromisoformat(hb_ts)
+                        if hb_dt.tzinfo is None:
+                            hb_dt = hb_dt.replace(tzinfo=UTC)
+                        if hb_dt > last_alive_dt:
+                            last_alive_dt = hb_dt
+                    except (TypeError, ValueError):
+                        logger.debug(
+                            "spawn_outbox.find_stuck: unparseable "
+                            "heartbeat ts on agent=%s msg=%s — ignoring",
+                            agent_dir.name, msg_id,
+                        )
+                age = (now - last_alive_dt).total_seconds()
                 payload = entry.get("payload") or {}
                 priority_raw = (
                     payload.get("priority") if isinstance(payload, dict) else None
@@ -375,10 +467,12 @@ class SpawnOutbox:
         return out
 
     async def compact_agent(self, agent_id: str) -> int:
-        """Manually compact one agent's jsonl. Returns kept line count.
+        """Manually compact one agent's jsonl.
 
-        Public entry point for cron-style maintenance; normal flow
-        triggers compaction automatically inside :meth:`record_done`.
+        Returns the number of still-pending spawns kept (heartbeat lines
+        retained alongside them do not count). Public entry point for
+        cron-style maintenance; normal flow triggers compaction
+        automatically inside :meth:`record_done`.
         """
         path = self._path_for(agent_id)
         if not path.exists():
@@ -403,6 +497,7 @@ class SpawnOutbox:
         path = self._path_for(agent_id)
         received: dict[str, str] = {}  # msg_id → raw line (preserves ts)
         done_ids: set[str] = set()
+        heartbeats: dict[str, str] = {}  # msg_id → latest heartbeat raw line
         for raw in lines:
             raw = raw.strip()
             if not raw:
@@ -419,8 +514,25 @@ class SpawnOutbox:
                 received[msg_id] = raw
             elif event == "done":
                 done_ids.add(msg_id)
-
-        keep = [raw for msg, raw in received.items() if msg not in done_ids]
+            elif event == "heartbeat":
+                # Append-ordered → last write wins.
+                heartbeats[msg_id] = raw
+
+        # Keep each still-pending ``received`` line plus its latest
+        # ``heartbeat`` (if any) so find_stuck's heartbeat-aware ageing
+        # survives compaction — dropping the heartbeat would snap a
+        # long-running spawn back to "aged from received" and the
+        # watchdog would false-fire.
+        keep: list[str] = []
+        pending_count = 0
+        for msg, raw in received.items():
+            if msg in done_ids:
+                continue
+            keep.append(raw)
+            pending_count += 1
+            hb = heartbeats.get(msg)
+            if hb is not None:
+                keep.append(hb)
         tmp = path.with_suffix(path.suffix + ".tmp")
         with tmp.open("w", encoding="utf-8") as f:
             for raw in keep:
@@ -432,12 +544,13 @@ class SpawnOutbox:
             len(lines),
             len(keep),
         )
-        return len(keep)
+        return pending_count
 
 
 __all__ = [
     "SpawnOutbox",
     "StuckSpawn",
     "STUCK_THRESHOLDS_SECONDS",
+    "HEARTBEAT_INTERVAL_SECONDS",
     "DEFAULT_COMPACT_THRESHOLD",
 ]

{cloudagent_daemon-0.2.4 → cloudagent_daemon-0.2.6}/src/cloudagent_daemon/spawn_queue.py

@@ -13,12 +13,15 @@ Design
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import logging
 import math
 import os
 from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING
 
+from cloudagent_daemon.spawn_outbox import HEARTBEAT_INTERVAL_SECONDS
+
 if TYPE_CHECKING:
     from cloudagent_daemon.spawn_outbox import SpawnOutbox
 
@@ -168,6 +171,18 @@ class SpawnQueue:
             msg_id = str(payload.get("message_id") or "")
             ok = True
             async with self._semaphore:
+                # 2026-05-20 — keep a heartbeat ticking for as long as
+                # run_one is in flight so the outbox-stuck watchdog can
+                # tell a legitimately long spawn (a 30-60+ min coding
+                # wake) from a hung one. Without this the watchdog aged
+                # the spawn from ``received`` and self-killed the daemon
+                # mid-wake. See spawn_outbox.STUCK_THRESHOLDS_SECONDS.
+                heartbeat_task: asyncio.Task | None = None
+                if self.spawn_outbox is not None and msg_id:
+                    heartbeat_task = asyncio.create_task(
+                        self._heartbeat_loop(agent_id, msg_id),
+                        name=f"spawn-heartbeat-{agent_id}",
+                    )
                 try:
                     await self.runner.run_one(agent_id, payload)
                 except Exception:
@@ -176,6 +191,12 @@ class SpawnQueue:
                         "SpawnQueue: runner.run_one raised for agent=%s", agent_id
                     )
                 finally:
+                    if heartbeat_task is not None:
+                        heartbeat_task.cancel()
+                        with contextlib.suppress(
+                            asyncio.CancelledError, Exception
+                        ):
+                            await heartbeat_task
                     queue.task_done()
                     # Bug 4 — mark the spawn done in the outbox (regardless
                     # of runner outcome) so the boot replay path stops
@@ -198,3 +219,31 @@ class SpawnQueue:
         # Remove self from the task registry so the next enqueue can
         # re-create a fresh drain task.
         self._tasks.pop(agent_id, None)
+
+    async def _heartbeat_loop(self, agent_id: str, msg_id: str) -> None:
+        """Append a spawn heartbeat every ``HEARTBEAT_INTERVAL_SECONDS``.
+
+        Started by :meth:`_drain` immediately before ``runner.run_one``
+        and cancelled in its ``finally`` block, so a heartbeat is on
+        record for the whole lifetime of the spawn. This is what tells
+        the outbox-stuck watchdog that a long-running spawn is alive and
+        progressing rather than hung — see
+        ``spawn_outbox.STUCK_THRESHOLDS_SECONDS``.
+
+        Only runs when an outbox is wired (``self.spawn_outbox`` set).
+        A heartbeat write failure is best-effort: it is logged and the
+        loop continues, since one missed beat only shortens a single
+        gap and never warrants killing the loop.
+        """
+        if self.spawn_outbox is None:
+            return
+        while True:
+            await asyncio.sleep(HEARTBEAT_INTERVAL_SECONDS)
+            try:
+                await self.spawn_outbox.record_heartbeat(agent_id, msg_id)
+            except Exception:
+                logger.debug(
+                    "SpawnQueue: spawn heartbeat write failed agent=%s msg=%s",
+                    agent_id,
+                    msg_id,
+                )

cloudagent_daemon-0.2.4/src/cloudagent_daemon/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.4"