cloudagent-daemon 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudagent_daemon/__init__.py +1 -0
- cloudagent_daemon/__main__.py +4 -0
- cloudagent_daemon/claude_runner.py +1068 -0
- cloudagent_daemon/cli.py +833 -0
- cloudagent_daemon/config.py +197 -0
- cloudagent_daemon/doctor.py +534 -0
- cloudagent_daemon/frame_dispatch.py +216 -0
- cloudagent_daemon/http_outbound.py +80 -0
- cloudagent_daemon/jsonl_logging.py +98 -0
- cloudagent_daemon/login.py +34 -0
- cloudagent_daemon/main_loop.py +891 -0
- cloudagent_daemon/mcp_bridge.py +441 -0
- cloudagent_daemon/metrics.py +315 -0
- cloudagent_daemon/metrics_server.py +149 -0
- cloudagent_daemon/outbound_queue.py +289 -0
- cloudagent_daemon/paths.py +204 -0
- cloudagent_daemon/protocol.py +141 -0
- cloudagent_daemon/runtime_detect.py +135 -0
- cloudagent_daemon/service.py +355 -0
- cloudagent_daemon/spawn_outbox.py +443 -0
- cloudagent_daemon/spawn_queue.py +200 -0
- cloudagent_daemon/sse_client.py +439 -0
- cloudagent_daemon/transport.py +105 -0
- cloudagent_daemon/unix_socket.py +278 -0
- cloudagent_daemon/workspace.py +441 -0
- cloudagent_daemon/ws_client.py +431 -0
- cloudagent_daemon-0.2.1.dist-info/METADATA +322 -0
- cloudagent_daemon-0.2.1.dist-info/RECORD +31 -0
- cloudagent_daemon-0.2.1.dist-info/WHEEL +4 -0
- cloudagent_daemon-0.2.1.dist-info/entry_points.txt +2 -0
- cloudagent_daemon-0.2.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,891 @@
|
|
|
1
|
+
"""Daemon main loop — wires transport, dispatcher, queues, runner, and IPC.
|
|
2
|
+
|
|
3
|
+
Sub-plan 1j (v0.2.0): the daemon defaults to SSE+POST transport. The
|
|
4
|
+
WebSocket path is still selectable via ``--transport ws`` for environments
|
|
5
|
+
where SSE causes friction (it shouldn't, but flags are cheap).
|
|
6
|
+
|
|
7
|
+
Both paths share the same downstream wiring (OutboundQueue, FrameDispatcher,
|
|
8
|
+
SpawnQueue, ClaudeRunner, IPC server) — only the transport plumbing differs.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import contextlib
|
|
14
|
+
import logging
|
|
15
|
+
import platform
|
|
16
|
+
import socket
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Callable
|
|
19
|
+
|
|
20
|
+
from . import __version__
|
|
21
|
+
from .claude_runner import ClaudeRunner
|
|
22
|
+
from .config import DaemonConfig, DaemonState
|
|
23
|
+
from .frame_dispatch import FrameDispatcher
|
|
24
|
+
from .http_outbound import HTTPOutbound
|
|
25
|
+
from .metrics import MetricsRegistry, get_default as get_default_metrics
|
|
26
|
+
from .metrics_server import MetricsServer
|
|
27
|
+
from .outbound_queue import OutboundQueue
|
|
28
|
+
from .runtime_detect import DetectedRuntime, detect_runtimes
|
|
29
|
+
from .spawn_outbox import SpawnOutbox
|
|
30
|
+
from .spawn_queue import SpawnQueue
|
|
31
|
+
from .sse_client import SSEClient
|
|
32
|
+
from .transport import HTTPTransport
|
|
33
|
+
from .unix_socket import DEFAULT_SOCKET_PATH, UnixSocketServer
|
|
34
|
+
from .workspace import Workspace
|
|
35
|
+
from .ws_client import WSClient, run_forever
|
|
36
|
+
|
|
37
|
+
logger = logging.getLogger(__name__)
|
|
38
|
+
|
|
39
|
+
# Heartbeat POST cadence — server's SSE ``:hb`` already keeps the daemon
|
|
40
|
+
# fed, but a pure-receive stream gives the server no liveness signal,
|
|
41
|
+
# hence this complementary daemon→server POST.
|
|
42
|
+
#
|
|
43
|
+
# 1k #61: tightened from 30s → 10s so backend can detect a dead daemon
|
|
44
|
+
# within ≤30s (3 missed heartbeats × 10s). Backend sweep defaults moved
|
|
45
|
+
# to interval=10s, threshold=30s in lockstep — see
|
|
46
|
+
# ``backend/src/cloudagent/harness/host_presence_sweep.py``.
|
|
47
|
+
HEARTBEAT_INTERVAL_SECONDS = 10.0
|
|
48
|
+
# Backwards-compat alias — kept until any external readers (tests, ops
|
|
49
|
+
# scripts) migrate. Internal code uses HEARTBEAT_INTERVAL_SECONDS.
|
|
50
|
+
_HEARTBEAT_INTERVAL_S = HEARTBEAT_INTERVAL_SECONDS
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# Wave C #2 — outbox stuck watchdog cadence. Scans every agent's
|
|
54
|
+
# ``pending-spawn.jsonl`` for ``received``-without-``done`` entries
|
|
55
|
+
# older than their per-priority threshold (see
|
|
56
|
+
# :data:`spawn_outbox.STUCK_THRESHOLDS_SECONDS`). The first stuck spawn
|
|
57
|
+
# escalates to ``SystemExit(1)`` so launchd respawns the daemon and the
|
|
58
|
+
# boot-replay path re-enqueues. 60s scan period is tight enough that a
|
|
59
|
+
# pathological 30-min default threshold gets caught within ~1 min of
|
|
60
|
+
# crossing the line, slack enough that the scan cost (~1 ms per agent
|
|
61
|
+
# per pass) is negligible.
|
|
62
|
+
OUTBOX_WATCHDOG_INTERVAL_SECONDS = 60.0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
# Wave C #1 — assign / unassign payload → workspace materialisation
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _identity_from_assign_payload(payload: dict) -> dict | None:
|
|
71
|
+
"""Translate an ``assign`` SSE frame payload into a Workspace identity.
|
|
72
|
+
|
|
73
|
+
Returns ``None`` when the payload has no ``agent_id`` (defensive —
|
|
74
|
+
the dispatcher already drops malformed frames upstream but the
|
|
75
|
+
daemon shouldn't crash if a future contract change strips fields).
|
|
76
|
+
|
|
77
|
+
Defaults mirror :func:`claude_runner._build_lazy_identity` so the
|
|
78
|
+
eager (assign-driven) and lazy (deliver-driven) workspace creation
|
|
79
|
+
paths are byte-identical for identical inputs:
|
|
80
|
+
|
|
81
|
+
* ``agent_name`` → agent_id when missing
|
|
82
|
+
* ``is_chief`` → False (security: daemon never auto-promotes)
|
|
83
|
+
* ``role`` → "staff"
|
|
84
|
+
* ``workspace_slug`` → "default"
|
|
85
|
+
* ``reports_to`` → "(unset)"
|
|
86
|
+
"""
|
|
87
|
+
agent_id = str(payload.get("agent_id") or "")
|
|
88
|
+
if not agent_id:
|
|
89
|
+
return None
|
|
90
|
+
return {
|
|
91
|
+
"agent_id": agent_id,
|
|
92
|
+
"agent_name": str(payload.get("agent_name") or agent_id),
|
|
93
|
+
"is_chief": bool(payload.get("is_chief", False)),
|
|
94
|
+
"role": str(payload.get("role") or "staff"),
|
|
95
|
+
"workspace_slug": str(payload.get("workspace_slug") or "default"),
|
|
96
|
+
"reports_to": str(payload.get("reports_to") or "(unset)"),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
async def _handle_assign(workspace, payload: dict) -> str | None:
|
|
101
|
+
"""Materialise an agent workspace from an ``assign`` payload.
|
|
102
|
+
|
|
103
|
+
Returns the agent_id on success, ``None`` on no-op (missing
|
|
104
|
+
agent_id) or :class:`Exception` swallowed (logged with full stack).
|
|
105
|
+
The caller — :class:`FrameDispatcher` via ``workspace_assign_callback``
|
|
106
|
+
— does not care about the return value; the value is surfaced for
|
|
107
|
+
unit tests + structured logging.
|
|
108
|
+
|
|
109
|
+
Wave C #1: pre-Wave-C the daemon only learned about agents via the
|
|
110
|
+
``/v1/daemon/ready`` snapshot, so a staff created on the server
|
|
111
|
+
after the daemon started was invisible until launchd kickstart.
|
|
112
|
+
Backend now pushes ``assign`` frames; this handler is where we land
|
|
113
|
+
them.
|
|
114
|
+
|
|
115
|
+
Task #118 (2026-05-12) — payload may carry ``workspace_root`` (the
|
|
116
|
+
per-agent cwd override). When present we pass it to
|
|
117
|
+
``Workspace.ensure`` so the spawn uses the user's chosen directory.
|
|
118
|
+
"""
|
|
119
|
+
identity = _identity_from_assign_payload(payload)
|
|
120
|
+
if identity is None:
|
|
121
|
+
logger.warning("main_loop._handle_assign: payload missing agent_id; ignoring")
|
|
122
|
+
return None
|
|
123
|
+
agent_id = identity["agent_id"]
|
|
124
|
+
agent_workspace_root = payload.get("workspace_root")
|
|
125
|
+
if agent_workspace_root is not None and not isinstance(
|
|
126
|
+
agent_workspace_root, str
|
|
127
|
+
):
|
|
128
|
+
# Defensive: silently drop a malformed (non-string) value
|
|
129
|
+
# rather than break the assign. Backend validates the shape at
|
|
130
|
+
# /v1/daemon/ready + on agent create/patch.
|
|
131
|
+
agent_workspace_root = None
|
|
132
|
+
try:
|
|
133
|
+
workspace.ensure(
|
|
134
|
+
agent_id,
|
|
135
|
+
identity,
|
|
136
|
+
agent_workspace_root=agent_workspace_root,
|
|
137
|
+
)
|
|
138
|
+
except Exception:
|
|
139
|
+
logger.exception(
|
|
140
|
+
"main_loop._handle_assign: workspace.ensure failed agent=%s",
|
|
141
|
+
agent_id,
|
|
142
|
+
)
|
|
143
|
+
return None
|
|
144
|
+
logger.info(
|
|
145
|
+
"main_loop._handle_assign: workspace materialised agent=%s role=%s "
|
|
146
|
+
"is_chief=%s workspace_root=%s (assign frame)",
|
|
147
|
+
agent_id, identity["role"], identity["is_chief"],
|
|
148
|
+
agent_workspace_root,
|
|
149
|
+
)
|
|
150
|
+
return agent_id
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
async def _outbox_stuck_watchdog(
|
|
154
|
+
spawn_outbox: SpawnOutbox,
|
|
155
|
+
*,
|
|
156
|
+
interval_seconds: float = OUTBOX_WATCHDOG_INTERVAL_SECONDS,
|
|
157
|
+
) -> None:
|
|
158
|
+
"""Wave C #2 — periodically scan the spawn outbox for stuck entries.
|
|
159
|
+
|
|
160
|
+
Returns ONLY by raising :class:`SystemExit` so the supervisor in
|
|
161
|
+
:func:`_run_sse_post` treats it as a fatal task completion and
|
|
162
|
+
triggers launchd respawn. Per-priority thresholds come from
|
|
163
|
+
:data:`spawn_outbox.STUCK_THRESHOLDS_SECONDS`.
|
|
164
|
+
|
|
165
|
+
Rationale (Wave C audit §6 P0): pre-Wave-C a daemon-side bug that
|
|
166
|
+
silently dropped spawns left the outbox to grow forever with no
|
|
167
|
+
upstream signal. The supervisor only saw ``sse`` + ``heartbeat``
|
|
168
|
+
so an outbox-only failure looked like "healthy idle daemon".
|
|
169
|
+
Wiring this watchdog into the supervisor closes that observability
|
|
170
|
+
gap by promoting outbox staleness to a respawn-triggering event.
|
|
171
|
+
"""
|
|
172
|
+
while True:
|
|
173
|
+
await asyncio.sleep(interval_seconds)
|
|
174
|
+
try:
|
|
175
|
+
stuck = await spawn_outbox.find_stuck()
|
|
176
|
+
except Exception: # noqa: BLE001
|
|
177
|
+
# Defensive — a transient disk read error must not kill
|
|
178
|
+
# the watchdog (else SystemExit cascades for the wrong
|
|
179
|
+
# reason). Log and try again next tick.
|
|
180
|
+
logger.exception(
|
|
181
|
+
"main_loop._outbox_stuck_watchdog: find_stuck raised; "
|
|
182
|
+
"will retry next tick"
|
|
183
|
+
)
|
|
184
|
+
continue
|
|
185
|
+
# Wave C #3 — publish the per-scan stuck count even when zero
|
|
186
|
+
# so /metrics has a stable series. Best-effort: a missing
|
|
187
|
+
# metrics import (test fixture) must not block the watchdog.
|
|
188
|
+
try:
|
|
189
|
+
from cloudagent_daemon.metrics import get_default as _md
|
|
190
|
+
await _md().set_gauge(
|
|
191
|
+
"cloudagent_daemon_outbox_stuck", float(len(stuck))
|
|
192
|
+
)
|
|
193
|
+
except Exception: # noqa: BLE001
|
|
194
|
+
pass
|
|
195
|
+
if not stuck:
|
|
196
|
+
continue
|
|
197
|
+
for entry in stuck:
|
|
198
|
+
logger.error(
|
|
199
|
+
"main_loop.outbox_watchdog: STUCK SPAWN agent=%s msg=%s "
|
|
200
|
+
"age=%.0fs priority=%s threshold=%.0fs — exiting so "
|
|
201
|
+
"launchd respawns and boot-replay re-enqueues",
|
|
202
|
+
entry.agent_id,
|
|
203
|
+
entry.msg_id,
|
|
204
|
+
entry.age_seconds,
|
|
205
|
+
entry.priority,
|
|
206
|
+
entry.threshold_seconds,
|
|
207
|
+
)
|
|
208
|
+
raise SystemExit(1)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
async def _handle_unassign(workspace, payload: dict) -> str | None:
|
|
212
|
+
"""Log an ``unassign`` payload; preserve on-disk workspace files.
|
|
213
|
+
|
|
214
|
+
Wave C #1: we deliberately do NOT delete ``~/.cloudagent/agents/<id>/``
|
|
215
|
+
— MEMORY.md / notes / partial claude session state may be useful
|
|
216
|
+
post-mortem. Backend re-issuing ``assign`` is enough to re-arm. The
|
|
217
|
+
``workspace`` arg is unused today but kept for symmetry with
|
|
218
|
+
:func:`_handle_assign` and so a future tombstone-marker write
|
|
219
|
+
(e.g. dropping a ``.meta/unassigned-at.txt``) lives in one place.
|
|
220
|
+
"""
|
|
221
|
+
agent_id = str(payload.get("agent_id") or "")
|
|
222
|
+
reason = payload.get("reason")
|
|
223
|
+
if not agent_id:
|
|
224
|
+
return None
|
|
225
|
+
logger.info(
|
|
226
|
+
"main_loop._handle_unassign: agent=%s reason=%s "
|
|
227
|
+
"(workspace files preserved on disk)",
|
|
228
|
+
agent_id, reason,
|
|
229
|
+
)
|
|
230
|
+
return agent_id
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
async def run_daemon(
|
|
234
|
+
cfg: DaemonConfig,
|
|
235
|
+
state: DaemonState,
|
|
236
|
+
*,
|
|
237
|
+
transport: str = "auto",
|
|
238
|
+
claude_binary: str = "claude",
|
|
239
|
+
workspace_root: Path | None = None,
|
|
240
|
+
socket_path: Path | None = None,
|
|
241
|
+
state_path: Path | None = None,
|
|
242
|
+
insecure: bool = False,
|
|
243
|
+
detect_runtimes_fn: Callable[[], list[DetectedRuntime]] | None = None,
|
|
244
|
+
profile: str | None = None,
|
|
245
|
+
) -> None:
|
|
246
|
+
"""Wire all components and run forever (until cancelled).
|
|
247
|
+
|
|
248
|
+
``transport`` selects the wire:
|
|
249
|
+
* ``"sse"`` / ``"auto"`` — SSE+POST (v0.2.0+ default).
|
|
250
|
+
* ``"ws"`` — legacy WebSocket (kept for fallback / regression tests).
|
|
251
|
+
|
|
252
|
+
Path overrides (``workspace_root``, ``socket_path``, ``state_path``)
|
|
253
|
+
default to ``~/.cloudagent/...``; tests inject ``tmp_path`` paths to
|
|
254
|
+
avoid touching the real home dir. *profile* (M6 multi-profile) is
|
|
255
|
+
resolved through ``paths.resolve_profile`` so it picks up
|
|
256
|
+
``CLOUDAGENT_DAEMON_PROFILE`` from the env when the caller didn't
|
|
257
|
+
pass an explicit value.
|
|
258
|
+
"""
|
|
259
|
+
from .paths import resolve_profile
|
|
260
|
+
|
|
261
|
+
profile_name = resolve_profile(profile)
|
|
262
|
+
|
|
263
|
+
if transport == "ws":
|
|
264
|
+
await _run_legacy_ws(
|
|
265
|
+
cfg,
|
|
266
|
+
state,
|
|
267
|
+
claude_binary=claude_binary,
|
|
268
|
+
workspace_root=workspace_root,
|
|
269
|
+
socket_path=socket_path,
|
|
270
|
+
state_path=state_path,
|
|
271
|
+
insecure=insecure,
|
|
272
|
+
profile=profile_name,
|
|
273
|
+
)
|
|
274
|
+
return
|
|
275
|
+
|
|
276
|
+
# SSE+POST path (default for v0.2.0+).
|
|
277
|
+
await _run_sse_post(
|
|
278
|
+
cfg,
|
|
279
|
+
state,
|
|
280
|
+
claude_binary=claude_binary,
|
|
281
|
+
workspace_root=workspace_root,
|
|
282
|
+
socket_path=socket_path,
|
|
283
|
+
state_path=state_path,
|
|
284
|
+
insecure=insecure,
|
|
285
|
+
detect_runtimes_fn=detect_runtimes_fn,
|
|
286
|
+
profile=profile_name,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
# ---------------------------------------------------------------------------
|
|
291
|
+
# SSE+POST path (1j default)
|
|
292
|
+
# ---------------------------------------------------------------------------
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
async def _run_sse_post(
|
|
296
|
+
cfg: DaemonConfig,
|
|
297
|
+
state: DaemonState,
|
|
298
|
+
*,
|
|
299
|
+
claude_binary: str,
|
|
300
|
+
workspace_root: Path | None,
|
|
301
|
+
socket_path: Path | None,
|
|
302
|
+
state_path: Path | None,
|
|
303
|
+
insecure: bool,
|
|
304
|
+
detect_runtimes_fn: Callable[[], list[DetectedRuntime]] | None,
|
|
305
|
+
profile: str = "default",
|
|
306
|
+
) -> None:
|
|
307
|
+
# Task #118 (2026-05-12) — ``workspace_root`` is the host-level
|
|
308
|
+
# default for agent cwd, NOT the metadata root. Metadata lives at
|
|
309
|
+
# the per-profile compiled-in default (``~/.cloudagent/agents/`` for
|
|
310
|
+
# default profile, ``~/.cloudagent/agents-<name>/`` otherwise) so a
|
|
311
|
+
# user-chosen project directory never gets polluted with
|
|
312
|
+
# identity.json / mcp-config / MEMORY.md.
|
|
313
|
+
workspace = Workspace(host_workspace_root=workspace_root, profile=profile)
|
|
314
|
+
effective_state_path = state_path or DaemonState.default_path(profile)
|
|
315
|
+
|
|
316
|
+
http = HTTPOutbound(cfg.server_url, cfg.machine_api_key, insecure=insecure)
|
|
317
|
+
|
|
318
|
+
# Step 1: POST /v1/daemon/ready — replaces the WS hello+ready handshake.
|
|
319
|
+
runtimes_list = (detect_runtimes_fn or detect_runtimes)()
|
|
320
|
+
runtimes_payload = [r.to_payload() for r in runtimes_list]
|
|
321
|
+
# Task #113 (2026-05-12) — surface ``--workspace-root`` to backend
|
|
322
|
+
# so the fr UI can render "agents live at <path>" on host detail.
|
|
323
|
+
# When the user didn't pass the flag we omit the field rather
|
|
324
|
+
# than send the daemon's compiled-in default — backend treats
|
|
325
|
+
# omission as "no change" (preserves any previously-reported
|
|
326
|
+
# value, idempotent re-ready).
|
|
327
|
+
ready_payload: dict = {
|
|
328
|
+
"daemon_ver": __version__,
|
|
329
|
+
"runtimes": runtimes_payload,
|
|
330
|
+
"hostname": socket.gethostname(),
|
|
331
|
+
"os": platform.system().lower(),
|
|
332
|
+
}
|
|
333
|
+
if workspace_root is not None:
|
|
334
|
+
ready_payload["workspace_root"] = str(workspace_root)
|
|
335
|
+
try:
|
|
336
|
+
ready_resp = await http.post(
|
|
337
|
+
"/v1/daemon/ready",
|
|
338
|
+
ready_payload,
|
|
339
|
+
)
|
|
340
|
+
except Exception:
|
|
341
|
+
# Ready POST is the gate to everything else; if it fails we have
|
|
342
|
+
# nothing to do but bail out so the supervisor sees the error.
|
|
343
|
+
await http.close()
|
|
344
|
+
raise
|
|
345
|
+
|
|
346
|
+
host_id = ready_resp.get("host_id", "")
|
|
347
|
+
last_acked_seq = int(ready_resp.get("last_acked_seq", 0) or 0)
|
|
348
|
+
bound_agents = ready_resp.get("agents", []) or []
|
|
349
|
+
# Task #118 (2026-05-12) — eagerly materialize each bound agent's
|
|
350
|
+
# workspace so per-agent ``workspace_root`` overrides take effect
|
|
351
|
+
# BEFORE the first deliver lands (the deliver frame doesn't carry
|
|
352
|
+
# ``workspace_root`` — only ready / assign do). Idempotent re-
|
|
353
|
+
# materialization is safe (Workspace.ensure contract).
|
|
354
|
+
for bound in bound_agents:
|
|
355
|
+
try:
|
|
356
|
+
a_id = str(bound.get("agent_id") or "")
|
|
357
|
+
if not a_id:
|
|
358
|
+
continue
|
|
359
|
+
identity = {
|
|
360
|
+
"agent_id": a_id,
|
|
361
|
+
"agent_name": a_id,
|
|
362
|
+
"is_chief": False,
|
|
363
|
+
"role": "staff",
|
|
364
|
+
"workspace_slug": "default",
|
|
365
|
+
"reports_to": "(unset)",
|
|
366
|
+
}
|
|
367
|
+
workspace.ensure(
|
|
368
|
+
a_id,
|
|
369
|
+
identity,
|
|
370
|
+
agent_workspace_root=bound.get("workspace_root"),
|
|
371
|
+
)
|
|
372
|
+
except Exception: # noqa: BLE001
|
|
373
|
+
logger.exception(
|
|
374
|
+
"main_loop._run_sse_post: failed to materialize bound "
|
|
375
|
+
"agent=%s (will retry on next deliver)",
|
|
376
|
+
bound.get("agent_id"),
|
|
377
|
+
)
|
|
378
|
+
# 2026-05-15 — detect host change before reconciling seq. When the
|
|
379
|
+
# ready response carries a host_id different from what state.json
|
|
380
|
+
# remembers (e.g. user re-installed the daemon against a different
|
|
381
|
+
# ``--server-url``, or backend recreated the host row), the local
|
|
382
|
+
# ``last_seq`` is from a different cluster and must be dropped —
|
|
383
|
+
# otherwise the SSE GET keeps sending stale ``Last-Event-ID`` and
|
|
384
|
+
# the new cluster's fresh frames silently get deduped.
|
|
385
|
+
prev_host = state.adopt_host_id(host_id)
|
|
386
|
+
if prev_host is not None:
|
|
387
|
+
logger.warning(
|
|
388
|
+
"daemon host_id changed (was=%s now=%s) — reset local "
|
|
389
|
+
"seq/dedupe state",
|
|
390
|
+
prev_host, host_id,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
# Server is authoritative on last-acked — but never roll BACK the
|
|
394
|
+
# local state.last_seq if the daemon happens to have processed (and
|
|
395
|
+
# not yet acked) frames newer than the server has on file.
|
|
396
|
+
state.last_seq = max(state.last_seq, last_acked_seq)
|
|
397
|
+
# Persist the host fingerprint immediately. The frame_dispatch path
|
|
398
|
+
# also saves on every ack, but for a fresh start that processes no
|
|
399
|
+
# frames we still want state.json on disk so the next launch can
|
|
400
|
+
# detect mismatch instead of treating itself as first-ever.
|
|
401
|
+
state.save(effective_state_path)
|
|
402
|
+
|
|
403
|
+
# Wave C #3 — metrics registry. Initialized here (after ready POST)
|
|
404
|
+
# so the SSE client + spawn pipeline can publish into a shared
|
|
405
|
+
# default registry; the /metrics endpoint scrapes from the same.
|
|
406
|
+
metrics = get_default_metrics()
|
|
407
|
+
await metrics.set_gauge(
|
|
408
|
+
"cloudagent_daemon_agents_bound", float(len(bound_agents))
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
logger.info(
|
|
412
|
+
"daemon ready: host_id=%s last_acked_seq=%d agents_bound=%d",
|
|
413
|
+
host_id,
|
|
414
|
+
last_acked_seq,
|
|
415
|
+
len(bound_agents),
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
# Step 2: build outbound + components.
|
|
419
|
+
transport_obj = HTTPTransport(http)
|
|
420
|
+
pending_tool_calls: dict[str, asyncio.Future] = {}
|
|
421
|
+
outbound = OutboundQueue(
|
|
422
|
+
transport=transport_obj, pending_tool_calls=pending_tool_calls
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
# 1k74 — token-level streaming publisher. claude_runner calls this
|
|
426
|
+
# for every text_delta chunk; we just POST it through the existing
|
|
427
|
+
# bearer-authenticated httpx client. Backend's endpoint publishes
|
|
428
|
+
# to Redis without a DB write, so the daemon→backend cost stays
|
|
429
|
+
# bounded (1 HTTP POST per ~80 chars of model output ≈ a dozen
|
|
430
|
+
# POSTs for a typical reply). Only wired on the SSE-POST path —
|
|
431
|
+
# the legacy WS path keeps the pre-1k74 buffered-reply behaviour.
|
|
432
|
+
async def _publish_partial(
|
|
433
|
+
session_id: str, agent_id: str, body: str
|
|
434
|
+
) -> None:
|
|
435
|
+
await http.post(
|
|
436
|
+
"/v1/daemon/publish-partial",
|
|
437
|
+
{
|
|
438
|
+
"session_id": session_id,
|
|
439
|
+
"agent_id": agent_id,
|
|
440
|
+
"body": body,
|
|
441
|
+
},
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
# (c) IPC architecture upgrade — shared agent→session memory map.
|
|
445
|
+
# ClaudeRunner registers the current session_id before spawn and
|
|
446
|
+
# removes it after. mcp_bridge queries via the ``get_current_session``
|
|
447
|
+
# IPC op instead of reading a file; file fallback kept for backward
|
|
448
|
+
# compat (old daemons without this map still work via the file).
|
|
449
|
+
_agent_to_session: dict[str, str | None] = {}
|
|
450
|
+
|
|
451
|
+
runner = ClaudeRunner(
|
|
452
|
+
outbound=outbound,
|
|
453
|
+
workspace=workspace,
|
|
454
|
+
claude_binary=claude_binary,
|
|
455
|
+
partial_publisher=_publish_partial,
|
|
456
|
+
agent_to_session=_agent_to_session,
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
async def _queue_full(agent_id: str, message_id: str) -> None:
|
|
460
|
+
await outbound.send_ack(seq=None, message_id=message_id, status="queue_full")
|
|
461
|
+
|
|
462
|
+
# Bug 4 (2026-05-12) — durable spawn outbox so deliver frames acked
|
|
463
|
+
# but not yet spawned survive daemon crashes. See spawn_outbox.py.
|
|
464
|
+
spawn_outbox = SpawnOutbox(workspace=workspace)
|
|
465
|
+
spawn = SpawnQueue(
|
|
466
|
+
runner=runner,
|
|
467
|
+
queue_full_callback=_queue_full,
|
|
468
|
+
spawn_outbox=spawn_outbox,
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
# Boot-time replay: any frames recorded as ``received`` without a
|
|
472
|
+
# matching ``done`` from the previous run get re-enqueued before we
|
|
473
|
+
# start accepting new frames. Skipping ack here — server already
|
|
474
|
+
# ack'd these in the prior run (that's exactly why they're stuck).
|
|
475
|
+
pending = await spawn_outbox.find_pending()
|
|
476
|
+
for payload in pending:
|
|
477
|
+
logger.info(
|
|
478
|
+
"main_loop: replaying pending spawn agent=%s msg=%s",
|
|
479
|
+
payload.get("agent_id"),
|
|
480
|
+
payload.get("message_id"),
|
|
481
|
+
)
|
|
482
|
+
await spawn.enqueue(payload)
|
|
483
|
+
|
|
484
|
+
# Wave C #1 (2026-05-12) — dynamic agent discovery.
|
|
485
|
+
#
|
|
486
|
+
# Pre-Wave-C, ``agents_bound`` was only refreshed on daemon restart
|
|
487
|
+
# (snapshot from ``/v1/daemon/ready``). A staff created on the
|
|
488
|
+
# server after the daemon already started would be invisible until
|
|
489
|
+
# the next launchd kickstart — that's the structural root of Bug 4b
|
|
490
|
+
# in ``docs/debug/2026-05-12-task-multi-agent-debug.md`` ("StaffPro
|
|
491
|
+
# 没 runner"). Backend now pushes ``assign`` SSE frames; the
|
|
492
|
+
# handler bodies live in :func:`_handle_assign` /
|
|
493
|
+
# :func:`_handle_unassign` so they're directly unit-testable.
|
|
494
|
+
|
|
495
|
+
async def _on_assign(payload: dict) -> None:
|
|
496
|
+
await _handle_assign(workspace, payload)
|
|
497
|
+
|
|
498
|
+
async def _on_unassign(payload: dict) -> None:
|
|
499
|
+
await _handle_unassign(workspace, payload)
|
|
500
|
+
|
|
501
|
+
dispatcher = FrameDispatcher(
|
|
502
|
+
state=state,
|
|
503
|
+
state_path=effective_state_path,
|
|
504
|
+
ack_callback=outbound.send_ack,
|
|
505
|
+
spawn_callback=spawn.enqueue,
|
|
506
|
+
tool_result_pending=pending_tool_calls,
|
|
507
|
+
spawn_outbox=spawn_outbox,
|
|
508
|
+
workspace_assign_callback=_on_assign,
|
|
509
|
+
workspace_unassign_callback=_on_unassign,
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
from .unix_socket import default_socket_path as _default_socket_path
|
|
513
|
+
|
|
514
|
+
sock_path = socket_path or _default_socket_path(profile)
|
|
515
|
+
sock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
516
|
+
|
|
517
|
+
async def ipc_handler(req: dict) -> dict:
|
|
518
|
+
# 1k #55 — IPC entry/exit logging for multi-tool-wake debugging.
|
|
519
|
+
ipc_rid = req.get("request_id")
|
|
520
|
+
op = req.get("op")
|
|
521
|
+
|
|
522
|
+
# (c) IPC architecture upgrade — daemon-memory session query.
|
|
523
|
+
# mcp_bridge sends this at startup to get the current session_id
|
|
524
|
+
# without reading a file; more reliable than the file-based path.
|
|
525
|
+
if op == "get_current_session":
|
|
526
|
+
a = req.get("args") or {}
|
|
527
|
+
agent_id_q = a.get("agent_id") or ""
|
|
528
|
+
sid = _agent_to_session.get(agent_id_q)
|
|
529
|
+
logger.info(
|
|
530
|
+
"main_loop.ipc_handler: get_current_session agent=%s sid=%s",
|
|
531
|
+
agent_id_q, sid,
|
|
532
|
+
)
|
|
533
|
+
return {"result": {"session_id": sid}}
|
|
534
|
+
|
|
535
|
+
if op != "tool_call":
|
|
536
|
+
logger.warning(
|
|
537
|
+
"main_loop.ipc_handler: unknown op=%r ipc_rid=%s", op, ipc_rid,
|
|
538
|
+
)
|
|
539
|
+
return {"error": {"code": "UNKNOWN_OP", "message": str(op)}}
|
|
540
|
+
a = req.get("args") or {}
|
|
541
|
+
logger.info(
|
|
542
|
+
"main_loop.ipc_handler: ENTER ipc_rid=%s tool=%s agent=%s "
|
|
543
|
+
"session=%s",
|
|
544
|
+
ipc_rid, a.get("tool_name"), a.get("agent_id"), a.get("session_id"),
|
|
545
|
+
)
|
|
546
|
+
try:
|
|
547
|
+
result = await outbound.send_tool_call(
|
|
548
|
+
agent_id=a["agent_id"],
|
|
549
|
+
session_id=a.get("session_id"),
|
|
550
|
+
tool_name=a["tool_name"],
|
|
551
|
+
args=a.get("tool_args") or {},
|
|
552
|
+
)
|
|
553
|
+
except Exception as exc: # noqa: BLE001
|
|
554
|
+
logger.exception(
|
|
555
|
+
"main_loop.ipc_handler: dispatch raised ipc_rid=%s tool=%s",
|
|
556
|
+
ipc_rid, a.get("tool_name"),
|
|
557
|
+
)
|
|
558
|
+
return {
|
|
559
|
+
"error": {
|
|
560
|
+
"code": "BRIDGE_DISPATCH_FAILED",
|
|
561
|
+
"message": f"{type(exc).__name__}: {exc}",
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
logger.info(
|
|
565
|
+
"main_loop.ipc_handler: EXIT ipc_rid=%s tool=%s ok=%s",
|
|
566
|
+
ipc_rid, a.get("tool_name"), result.get("ok"),
|
|
567
|
+
)
|
|
568
|
+
if result.get("ok"):
|
|
569
|
+
return {"result": result.get("data", {})}
|
|
570
|
+
return {"error": result.get("error", {"code": "UNKNOWN_ERROR"})}
|
|
571
|
+
|
|
572
|
+
ipc_server = UnixSocketServer(sock_path, ipc_handler)
|
|
573
|
+
await ipc_server.start()
|
|
574
|
+
|
|
575
|
+
# Wave C #3 — Prometheus-scrape unix socket. Lives alongside
|
|
576
|
+
# ``bridge.sock`` under ``~/.cloudagent/daemon/`` with 0o600 perms.
|
|
577
|
+
# Lifecycle mirrors the IPC server: start now, stop in the
|
|
578
|
+
# outer-finally cleanup so a daemon restart cycle doesn't leak
|
|
579
|
+
# stale socket files.
|
|
580
|
+
metrics_server = MetricsServer(metrics)
|
|
581
|
+
try:
|
|
582
|
+
await metrics_server.start()
|
|
583
|
+
except Exception:
|
|
584
|
+
# Best-effort — if the metrics socket can't bind (e.g. perms
|
|
585
|
+
# / readonly home dir) we proceed without /metrics. The
|
|
586
|
+
# daemon must NOT refuse to serve agents because Prometheus
|
|
587
|
+
# is down.
|
|
588
|
+
logger.exception(
|
|
589
|
+
"main_loop: metrics_server.start failed; /metrics unavailable"
|
|
590
|
+
)
|
|
591
|
+
metrics_server = None
|
|
592
|
+
|
|
593
|
+
# Step 3: heartbeat task. Best-effort liveness POST every
|
|
594
|
+
# ``HEARTBEAT_INTERVAL_SECONDS`` (1k #61: 10s) — if the server didn't
|
|
595
|
+
# already see us via an ack/status frame, this nudges ``last_seen_at``
|
|
596
|
+
# so the dashboard's online indicator stays green and the host-presence
|
|
597
|
+
# reaper doesn't flip us to offline.
|
|
598
|
+
async def heartbeat() -> None:
|
|
599
|
+
while True:
|
|
600
|
+
await asyncio.sleep(HEARTBEAT_INTERVAL_SECONDS)
|
|
601
|
+
try:
|
|
602
|
+
await http.post("/v1/daemon/heartbeat", {})
|
|
603
|
+
except asyncio.CancelledError:
|
|
604
|
+
raise
|
|
605
|
+
except Exception: # noqa: BLE001
|
|
606
|
+
logger.debug("heartbeat POST failed", exc_info=True)
|
|
607
|
+
|
|
608
|
+
# Step 4: SSE consumer — the inbound side.
|
|
609
|
+
sse = SSEClient(
|
|
610
|
+
cfg.server_url,
|
|
611
|
+
cfg.machine_api_key,
|
|
612
|
+
insecure=insecure,
|
|
613
|
+
last_event_id=str(state.last_seq),
|
|
614
|
+
)
|
|
615
|
+
# Wave C #3 — gauges that derive from the SSE client at scrape time.
|
|
616
|
+
metrics.gauge(
|
|
617
|
+
"cloudagent_daemon_sse_last_event_age_seconds",
|
|
618
|
+
"Seconds since the last parsed SSE event arrived.",
|
|
619
|
+
resolver=lambda: sse.last_event_age_seconds,
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
async def on_frame(frame: dict) -> None:
|
|
623
|
+
# Wave C #3 — count every frame the dispatcher sees, including
|
|
624
|
+
# duplicates (which we still ack). Failure modes (raised
|
|
625
|
+
# exceptions in dispatcher) still propagate to the SSE task.
|
|
626
|
+
if frame.get("type") == "deliver":
|
|
627
|
+
await metrics.inc_counter(
|
|
628
|
+
"cloudagent_daemon_deliver_frames_total"
|
|
629
|
+
)
|
|
630
|
+
await dispatcher.dispatch(frame)
|
|
631
|
+
|
|
632
|
+
cancelled = False
|
|
633
|
+
try:
|
|
634
|
+
# 1k78 — explicit task supervisor in place of ``asyncio.gather``.
|
|
635
|
+
#
|
|
636
|
+
# Pre-1k78 we used ``await asyncio.gather(sse.run_forever,
|
|
637
|
+
# heartbeat)``: documented to propagate any task's first
|
|
638
|
+
# exception and cancel siblings. In production we observed the
|
|
639
|
+
# SSE task silently entering a "done" state (never reconnecting,
|
|
640
|
+
# never raising) while the heartbeat task kept POSTing for
|
|
641
|
+
# hours — the daemon process was technically alive but useless
|
|
642
|
+
# because every command sent from the UI piled up in
|
|
643
|
+
# ``undelivered_frames`` with no consumer. Root cause for the
|
|
644
|
+
# gather bypass was never pinned (likely an httpx-sse internal
|
|
645
|
+
# path swallowed an exception), so this defensive supervisor
|
|
646
|
+
# explicitly polls each task's completion and force-exits the
|
|
647
|
+
# daemon — letting launchd / systemd auto-respawn it — the
|
|
648
|
+
# moment either task ends unexpectedly.
|
|
649
|
+
sse_task = asyncio.create_task(
|
|
650
|
+
sse.run_forever(on_frame), name="daemon.sse"
|
|
651
|
+
)
|
|
652
|
+
heartbeat_task = asyncio.create_task(heartbeat(), name="daemon.heartbeat")
|
|
653
|
+
# Wave C #2 — outbox stuck self-kill. The watchdog only returns
|
|
654
|
+
# by raising SystemExit, so the supervisor's FIRST_COMPLETED
|
|
655
|
+
# treats a stuck spawn the same as a dead SSE: log + respawn.
|
|
656
|
+
outbox_watchdog_task = asyncio.create_task(
|
|
657
|
+
_outbox_stuck_watchdog(spawn_outbox),
|
|
658
|
+
name="daemon.outbox_watchdog",
|
|
659
|
+
)
|
|
660
|
+
watched = {sse_task, heartbeat_task, outbox_watchdog_task}
|
|
661
|
+
try:
|
|
662
|
+
done, pending = await asyncio.wait(
|
|
663
|
+
watched, return_when=asyncio.FIRST_COMPLETED
|
|
664
|
+
)
|
|
665
|
+
# First task done. If it raised, surface that and abort —
|
|
666
|
+
# the supervisor's job from here on is to make sure the
|
|
667
|
+
# daemon doesn't linger in a half-dead state.
|
|
668
|
+
for t in done:
|
|
669
|
+
exc = t.exception()
|
|
670
|
+
if exc is not None and not isinstance(
|
|
671
|
+
exc, asyncio.CancelledError
|
|
672
|
+
):
|
|
673
|
+
logger.error(
|
|
674
|
+
"daemon supervisor: task %s died with %r — exiting "
|
|
675
|
+
"so launchd respawns",
|
|
676
|
+
t.get_name(), exc,
|
|
677
|
+
)
|
|
678
|
+
else:
|
|
679
|
+
logger.error(
|
|
680
|
+
"daemon supervisor: task %s ended unexpectedly "
|
|
681
|
+
"(no exception) — exiting so launchd respawns",
|
|
682
|
+
t.get_name(),
|
|
683
|
+
)
|
|
684
|
+
for t in pending:
|
|
685
|
+
t.cancel()
|
|
686
|
+
# Drain cancellations so cleanup hooks below get a chance
|
|
687
|
+
# to run. Don't propagate the underlying error; we're
|
|
688
|
+
# exiting deliberately.
|
|
689
|
+
for t in pending:
|
|
690
|
+
with contextlib.suppress(Exception):
|
|
691
|
+
await t
|
|
692
|
+
# Force non-zero exit so launchd's KeepAlive respawns us.
|
|
693
|
+
raise SystemExit(1)
|
|
694
|
+
except asyncio.CancelledError:
|
|
695
|
+
cancelled = True
|
|
696
|
+
raise
|
|
697
|
+
finally:
|
|
698
|
+
sse.stop()
|
|
699
|
+
try:
|
|
700
|
+
await ipc_server.stop()
|
|
701
|
+
except Exception: # noqa: BLE001
|
|
702
|
+
pass
|
|
703
|
+
if metrics_server is not None:
|
|
704
|
+
try:
|
|
705
|
+
await metrics_server.stop()
|
|
706
|
+
except Exception: # noqa: BLE001
|
|
707
|
+
pass
|
|
708
|
+
if cancelled:
|
|
709
|
+
await spawn.cancel_all()
|
|
710
|
+
else:
|
|
711
|
+
await spawn.shutdown()
|
|
712
|
+
await transport_obj.close()
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
# ---------------------------------------------------------------------------
|
|
716
|
+
# Legacy WebSocket path (kept for --transport ws)
|
|
717
|
+
# ---------------------------------------------------------------------------
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
async def _run_legacy_ws(
|
|
721
|
+
cfg: DaemonConfig,
|
|
722
|
+
state: DaemonState,
|
|
723
|
+
*,
|
|
724
|
+
claude_binary: str,
|
|
725
|
+
workspace_root: Path | None,
|
|
726
|
+
socket_path: Path | None,
|
|
727
|
+
state_path: Path | None,
|
|
728
|
+
insecure: bool,
|
|
729
|
+
profile: str = "default",
|
|
730
|
+
) -> None:
|
|
731
|
+
"""Pre-1j WebSocket path. Identical to the v0.1.0 main loop body."""
|
|
732
|
+
# Task #118 (2026-05-12) — see SSE path's matching comment.
|
|
733
|
+
workspace = Workspace(host_workspace_root=workspace_root, profile=profile)
|
|
734
|
+
effective_state_path = state_path or DaemonState.default_path(profile)
|
|
735
|
+
|
|
736
|
+
pending_tool_calls: dict[str, asyncio.Future] = {}
|
|
737
|
+
outbound = OutboundQueue(transport=None, pending_tool_calls=pending_tool_calls)
|
|
738
|
+
|
|
739
|
+
runner = ClaudeRunner(
|
|
740
|
+
outbound=outbound, workspace=workspace, claude_binary=claude_binary
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
async def _queue_full(agent_id: str, message_id: str) -> None:
|
|
744
|
+
await outbound.send_ack(seq=None, message_id=message_id, status="queue_full")
|
|
745
|
+
|
|
746
|
+
# Bug 4 — same outbox + replay wiring as the SSE path. Both
|
|
747
|
+
# transports share the same crash-survival behaviour.
|
|
748
|
+
spawn_outbox = SpawnOutbox(workspace=workspace)
|
|
749
|
+
spawn = SpawnQueue(
|
|
750
|
+
runner=runner,
|
|
751
|
+
queue_full_callback=_queue_full,
|
|
752
|
+
spawn_outbox=spawn_outbox,
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
pending = await spawn_outbox.find_pending()
|
|
756
|
+
for payload in pending:
|
|
757
|
+
logger.info(
|
|
758
|
+
"main_loop: replaying pending spawn agent=%s msg=%s",
|
|
759
|
+
payload.get("agent_id"),
|
|
760
|
+
payload.get("message_id"),
|
|
761
|
+
)
|
|
762
|
+
await spawn.enqueue(payload)
|
|
763
|
+
|
|
764
|
+
# Wave C #1 — same assign/unassign wiring as the SSE path.
|
|
765
|
+
async def _on_assign_ws(payload: dict) -> None:
|
|
766
|
+
await _handle_assign(workspace, payload)
|
|
767
|
+
|
|
768
|
+
async def _on_unassign_ws(payload: dict) -> None:
|
|
769
|
+
await _handle_unassign(workspace, payload)
|
|
770
|
+
|
|
771
|
+
dispatcher = FrameDispatcher(
|
|
772
|
+
state=state,
|
|
773
|
+
state_path=effective_state_path,
|
|
774
|
+
ack_callback=outbound.send_ack,
|
|
775
|
+
spawn_callback=spawn.enqueue,
|
|
776
|
+
tool_result_pending=pending_tool_calls,
|
|
777
|
+
spawn_outbox=spawn_outbox,
|
|
778
|
+
workspace_assign_callback=_on_assign_ws,
|
|
779
|
+
workspace_unassign_callback=_on_unassign_ws,
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
from .unix_socket import default_socket_path as _default_socket_path_ws
|
|
783
|
+
|
|
784
|
+
sock_path = socket_path or _default_socket_path_ws(profile)
|
|
785
|
+
sock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
786
|
+
|
|
787
|
+
async def ipc_handler(req: dict) -> dict:
|
|
788
|
+
if req.get("op") != "tool_call":
|
|
789
|
+
return {"error": {"code": "UNKNOWN_OP", "message": str(req.get("op"))}}
|
|
790
|
+
a = req.get("args") or {}
|
|
791
|
+
try:
|
|
792
|
+
result = await outbound.send_tool_call(
|
|
793
|
+
agent_id=a["agent_id"],
|
|
794
|
+
session_id=a.get("session_id"),
|
|
795
|
+
tool_name=a["tool_name"],
|
|
796
|
+
args=a.get("tool_args") or {},
|
|
797
|
+
)
|
|
798
|
+
except Exception as exc: # noqa: BLE001
|
|
799
|
+
return {
|
|
800
|
+
"error": {
|
|
801
|
+
"code": "BRIDGE_DISPATCH_FAILED",
|
|
802
|
+
"message": f"{type(exc).__name__}: {exc}",
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
if result.get("ok"):
|
|
806
|
+
return {"result": result.get("data", {})}
|
|
807
|
+
return {"error": result.get("error", {"code": "UNKNOWN_ERROR"})}
|
|
808
|
+
|
|
809
|
+
ipc_server = UnixSocketServer(sock_path, ipc_handler)
|
|
810
|
+
await ipc_server.start()
|
|
811
|
+
|
|
812
|
+
def _client_factory() -> WSClient:
|
|
813
|
+
return WSClient(
|
|
814
|
+
server_url=cfg.server_url,
|
|
815
|
+
api_key=cfg.machine_api_key,
|
|
816
|
+
insecure=insecure,
|
|
817
|
+
# Task #113 — propagate workspace_root onto every WS
|
|
818
|
+
# reconnect's ``ready`` frame so a daemon that auto-
|
|
819
|
+
# reconnects after a network drop still surfaces the
|
|
820
|
+
# user-chosen root to backend.
|
|
821
|
+
workspace_root=(
|
|
822
|
+
str(workspace_root) if workspace_root is not None else None
|
|
823
|
+
),
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
async def _on_connect(client: WSClient) -> None:
|
|
827
|
+
# ``outbound.ws = client`` is the legacy assignment shape — the new
|
|
828
|
+
# property setter on OutboundQueue adapts a WSClient into a
|
|
829
|
+
# Transport (_LegacyWSAdapter) so existing code works unchanged.
|
|
830
|
+
outbound.ws = client
|
|
831
|
+
# Same host_id-change reset the SSE path does. Runs BEFORE
|
|
832
|
+
# ws_client.run_forever calls state_loader → send_resume so a
|
|
833
|
+
# cluster switch doesn't replay stale Last-Event-ID upstream.
|
|
834
|
+
prev_host = state.adopt_host_id(client.host_id)
|
|
835
|
+
if prev_host is not None:
|
|
836
|
+
logger.warning(
|
|
837
|
+
"daemon host_id changed (was=%s now=%s) — reset local "
|
|
838
|
+
"seq/dedupe state",
|
|
839
|
+
prev_host, client.host_id,
|
|
840
|
+
)
|
|
841
|
+
state.save(effective_state_path)
|
|
842
|
+
logger.info("daemon connected: host_id=%s", client.host_id)
|
|
843
|
+
|
|
844
|
+
async def _on_frame(frame: dict) -> None:
|
|
845
|
+
await dispatcher.dispatch(frame)
|
|
846
|
+
|
|
847
|
+
cancelled = False
|
|
848
|
+
try:
|
|
849
|
+
# Wave C #2 — same outbox watchdog wiring as the SSE path.
|
|
850
|
+
# Wrap the WS run_forever + watchdog in a wait(...) so a stuck
|
|
851
|
+
# spawn here also escalates to launchd-respawn instead of
|
|
852
|
+
# sitting silently while the WS keeps heart-beating.
|
|
853
|
+
ws_task = asyncio.create_task(
|
|
854
|
+
run_forever(
|
|
855
|
+
client_factory=_client_factory,
|
|
856
|
+
on_frame=_on_frame,
|
|
857
|
+
state_loader=lambda: state.last_seq,
|
|
858
|
+
on_connect=_on_connect,
|
|
859
|
+
),
|
|
860
|
+
name="daemon.ws",
|
|
861
|
+
)
|
|
862
|
+
outbox_task = asyncio.create_task(
|
|
863
|
+
_outbox_stuck_watchdog(spawn_outbox),
|
|
864
|
+
name="daemon.outbox_watchdog",
|
|
865
|
+
)
|
|
866
|
+
watched = {ws_task, outbox_task}
|
|
867
|
+
done, pending = await asyncio.wait(
|
|
868
|
+
watched, return_when=asyncio.FIRST_COMPLETED
|
|
869
|
+
)
|
|
870
|
+
for t in pending:
|
|
871
|
+
t.cancel()
|
|
872
|
+
for t in pending:
|
|
873
|
+
with contextlib.suppress(Exception):
|
|
874
|
+
await t
|
|
875
|
+
# Surface the completed task's exception (if any) so callers /
|
|
876
|
+
# launchd see the right exit code.
|
|
877
|
+
for t in done:
|
|
878
|
+
exc = t.exception()
|
|
879
|
+
if exc is not None and not isinstance(
|
|
880
|
+
exc, asyncio.CancelledError
|
|
881
|
+
):
|
|
882
|
+
raise exc
|
|
883
|
+
except asyncio.CancelledError:
|
|
884
|
+
cancelled = True
|
|
885
|
+
raise
|
|
886
|
+
finally:
|
|
887
|
+
await ipc_server.stop()
|
|
888
|
+
if cancelled:
|
|
889
|
+
await spawn.cancel_all()
|
|
890
|
+
else:
|
|
891
|
+
await spawn.shutdown()
|