cloudagent-daemon 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,891 @@
1
+ """Daemon main loop — wires transport, dispatcher, queues, runner, and IPC.
2
+
3
+ Sub-plan 1j (v0.2.0): the daemon defaults to SSE+POST transport. The
4
+ WebSocket path is still selectable via ``--transport ws`` for environments
5
+ where SSE causes friction (it shouldn't, but flags are cheap).
6
+
7
+ Both paths share the same downstream wiring (OutboundQueue, FrameDispatcher,
8
+ SpawnQueue, ClaudeRunner, IPC server) — only the transport plumbing differs.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ import contextlib
14
+ import logging
15
+ import platform
16
+ import socket
17
+ from pathlib import Path
18
+ from typing import Callable
19
+
20
+ from . import __version__
21
+ from .claude_runner import ClaudeRunner
22
+ from .config import DaemonConfig, DaemonState
23
+ from .frame_dispatch import FrameDispatcher
24
+ from .http_outbound import HTTPOutbound
25
+ from .metrics import MetricsRegistry, get_default as get_default_metrics
26
+ from .metrics_server import MetricsServer
27
+ from .outbound_queue import OutboundQueue
28
+ from .runtime_detect import DetectedRuntime, detect_runtimes
29
+ from .spawn_outbox import SpawnOutbox
30
+ from .spawn_queue import SpawnQueue
31
+ from .sse_client import SSEClient
32
+ from .transport import HTTPTransport
33
+ from .unix_socket import DEFAULT_SOCKET_PATH, UnixSocketServer
34
+ from .workspace import Workspace
35
+ from .ws_client import WSClient, run_forever
36
+
37
+ logger = logging.getLogger(__name__)
38
+
39
+ # Heartbeat POST cadence — server's SSE ``:hb`` already keeps the daemon
40
+ # fed, but a pure-receive stream gives the server no liveness signal,
41
+ # hence this complementary daemon→server POST.
42
+ #
43
+ # 1k #61: tightened from 30s → 10s so backend can detect a dead daemon
44
+ # within ≤30s (3 missed heartbeats × 10s). Backend sweep defaults moved
45
+ # to interval=10s, threshold=30s in lockstep — see
46
+ # ``backend/src/cloudagent/harness/host_presence_sweep.py``.
47
+ HEARTBEAT_INTERVAL_SECONDS = 10.0
48
+ # Backwards-compat alias — kept until any external readers (tests, ops
49
+ # scripts) migrate. Internal code uses HEARTBEAT_INTERVAL_SECONDS.
50
+ _HEARTBEAT_INTERVAL_S = HEARTBEAT_INTERVAL_SECONDS
51
+
52
+
53
+ # Wave C #2 — outbox stuck watchdog cadence. Scans every agent's
54
+ # ``pending-spawn.jsonl`` for ``received``-without-``done`` entries
55
+ # older than their per-priority threshold (see
56
+ # :data:`spawn_outbox.STUCK_THRESHOLDS_SECONDS`). The first stuck spawn
57
+ # escalates to ``SystemExit(1)`` so launchd respawns the daemon and the
58
+ # boot-replay path re-enqueues. 60s scan period is tight enough that a
59
+ # pathological 30-min default threshold gets caught within ~1 min of
60
+ # crossing the line, slack enough that the scan cost (~1 ms per agent
61
+ # per pass) is negligible.
62
+ OUTBOX_WATCHDOG_INTERVAL_SECONDS = 60.0
63
+
64
+
65
+ # ---------------------------------------------------------------------------
66
+ # Wave C #1 — assign / unassign payload → workspace materialisation
67
+ # ---------------------------------------------------------------------------
68
+
69
+
70
+ def _identity_from_assign_payload(payload: dict) -> dict | None:
71
+ """Translate an ``assign`` SSE frame payload into a Workspace identity.
72
+
73
+ Returns ``None`` when the payload has no ``agent_id`` (defensive —
74
+ the dispatcher already drops malformed frames upstream but the
75
+ daemon shouldn't crash if a future contract change strips fields).
76
+
77
+ Defaults mirror :func:`claude_runner._build_lazy_identity` so the
78
+ eager (assign-driven) and lazy (deliver-driven) workspace creation
79
+ paths are byte-identical for identical inputs:
80
+
81
+ * ``agent_name`` → agent_id when missing
82
+ * ``is_chief`` → False (security: daemon never auto-promotes)
83
+ * ``role`` → "staff"
84
+ * ``workspace_slug`` → "default"
85
+ * ``reports_to`` → "(unset)"
86
+ """
87
+ agent_id = str(payload.get("agent_id") or "")
88
+ if not agent_id:
89
+ return None
90
+ return {
91
+ "agent_id": agent_id,
92
+ "agent_name": str(payload.get("agent_name") or agent_id),
93
+ "is_chief": bool(payload.get("is_chief", False)),
94
+ "role": str(payload.get("role") or "staff"),
95
+ "workspace_slug": str(payload.get("workspace_slug") or "default"),
96
+ "reports_to": str(payload.get("reports_to") or "(unset)"),
97
+ }
98
+
99
+
100
+ async def _handle_assign(workspace, payload: dict) -> str | None:
101
+ """Materialise an agent workspace from an ``assign`` payload.
102
+
103
+ Returns the agent_id on success, ``None`` on no-op (missing
104
+ agent_id) or :class:`Exception` swallowed (logged with full stack).
105
+ The caller — :class:`FrameDispatcher` via ``workspace_assign_callback``
106
+ — does not care about the return value; the value is surfaced for
107
+ unit tests + structured logging.
108
+
109
+ Wave C #1: pre-Wave-C the daemon only learned about agents via the
110
+ ``/v1/daemon/ready`` snapshot, so a staff created on the server
111
+ after the daemon started was invisible until launchd kickstart.
112
+ Backend now pushes ``assign`` frames; this handler is where we land
113
+ them.
114
+
115
+ Task #118 (2026-05-12) — payload may carry ``workspace_root`` (the
116
+ per-agent cwd override). When present we pass it to
117
+ ``Workspace.ensure`` so the spawn uses the user's chosen directory.
118
+ """
119
+ identity = _identity_from_assign_payload(payload)
120
+ if identity is None:
121
+ logger.warning("main_loop._handle_assign: payload missing agent_id; ignoring")
122
+ return None
123
+ agent_id = identity["agent_id"]
124
+ agent_workspace_root = payload.get("workspace_root")
125
+ if agent_workspace_root is not None and not isinstance(
126
+ agent_workspace_root, str
127
+ ):
128
+ # Defensive: silently drop a malformed (non-string) value
129
+ # rather than break the assign. Backend validates the shape at
130
+ # /v1/daemon/ready + on agent create/patch.
131
+ agent_workspace_root = None
132
+ try:
133
+ workspace.ensure(
134
+ agent_id,
135
+ identity,
136
+ agent_workspace_root=agent_workspace_root,
137
+ )
138
+ except Exception:
139
+ logger.exception(
140
+ "main_loop._handle_assign: workspace.ensure failed agent=%s",
141
+ agent_id,
142
+ )
143
+ return None
144
+ logger.info(
145
+ "main_loop._handle_assign: workspace materialised agent=%s role=%s "
146
+ "is_chief=%s workspace_root=%s (assign frame)",
147
+ agent_id, identity["role"], identity["is_chief"],
148
+ agent_workspace_root,
149
+ )
150
+ return agent_id
151
+
152
+
153
+ async def _outbox_stuck_watchdog(
154
+ spawn_outbox: SpawnOutbox,
155
+ *,
156
+ interval_seconds: float = OUTBOX_WATCHDOG_INTERVAL_SECONDS,
157
+ ) -> None:
158
+ """Wave C #2 — periodically scan the spawn outbox for stuck entries.
159
+
160
+ Returns ONLY by raising :class:`SystemExit` so the supervisor in
161
+ :func:`_run_sse_post` treats it as a fatal task completion and
162
+ triggers launchd respawn. Per-priority thresholds come from
163
+ :data:`spawn_outbox.STUCK_THRESHOLDS_SECONDS`.
164
+
165
+ Rationale (Wave C audit §6 P0): pre-Wave-C a daemon-side bug that
166
+ silently dropped spawns left the outbox to grow forever with no
167
+ upstream signal. The supervisor only saw ``sse`` + ``heartbeat``
168
+ so an outbox-only failure looked like "healthy idle daemon".
169
+ Wiring this watchdog into the supervisor closes that observability
170
+ gap by promoting outbox staleness to a respawn-triggering event.
171
+ """
172
+ while True:
173
+ await asyncio.sleep(interval_seconds)
174
+ try:
175
+ stuck = await spawn_outbox.find_stuck()
176
+ except Exception: # noqa: BLE001
177
+ # Defensive — a transient disk read error must not kill
178
+ # the watchdog (else SystemExit cascades for the wrong
179
+ # reason). Log and try again next tick.
180
+ logger.exception(
181
+ "main_loop._outbox_stuck_watchdog: find_stuck raised; "
182
+ "will retry next tick"
183
+ )
184
+ continue
185
+ # Wave C #3 — publish the per-scan stuck count even when zero
186
+ # so /metrics has a stable series. Best-effort: a missing
187
+ # metrics import (test fixture) must not block the watchdog.
188
+ try:
189
+ from cloudagent_daemon.metrics import get_default as _md
190
+ await _md().set_gauge(
191
+ "cloudagent_daemon_outbox_stuck", float(len(stuck))
192
+ )
193
+ except Exception: # noqa: BLE001
194
+ pass
195
+ if not stuck:
196
+ continue
197
+ for entry in stuck:
198
+ logger.error(
199
+ "main_loop.outbox_watchdog: STUCK SPAWN agent=%s msg=%s "
200
+ "age=%.0fs priority=%s threshold=%.0fs — exiting so "
201
+ "launchd respawns and boot-replay re-enqueues",
202
+ entry.agent_id,
203
+ entry.msg_id,
204
+ entry.age_seconds,
205
+ entry.priority,
206
+ entry.threshold_seconds,
207
+ )
208
+ raise SystemExit(1)
209
+
210
+
211
+ async def _handle_unassign(workspace, payload: dict) -> str | None:
212
+ """Log an ``unassign`` payload; preserve on-disk workspace files.
213
+
214
+ Wave C #1: we deliberately do NOT delete ``~/.cloudagent/agents/<id>/``
215
+ — MEMORY.md / notes / partial claude session state may be useful
216
+ post-mortem. Backend re-issuing ``assign`` is enough to re-arm. The
217
+ ``workspace`` arg is unused today but kept for symmetry with
218
+ :func:`_handle_assign` and so a future tombstone-marker write
219
+ (e.g. dropping a ``.meta/unassigned-at.txt``) lives in one place.
220
+ """
221
+ agent_id = str(payload.get("agent_id") or "")
222
+ reason = payload.get("reason")
223
+ if not agent_id:
224
+ return None
225
+ logger.info(
226
+ "main_loop._handle_unassign: agent=%s reason=%s "
227
+ "(workspace files preserved on disk)",
228
+ agent_id, reason,
229
+ )
230
+ return agent_id
231
+
232
+
233
+ async def run_daemon(
234
+ cfg: DaemonConfig,
235
+ state: DaemonState,
236
+ *,
237
+ transport: str = "auto",
238
+ claude_binary: str = "claude",
239
+ workspace_root: Path | None = None,
240
+ socket_path: Path | None = None,
241
+ state_path: Path | None = None,
242
+ insecure: bool = False,
243
+ detect_runtimes_fn: Callable[[], list[DetectedRuntime]] | None = None,
244
+ profile: str | None = None,
245
+ ) -> None:
246
+ """Wire all components and run forever (until cancelled).
247
+
248
+ ``transport`` selects the wire:
249
+ * ``"sse"`` / ``"auto"`` — SSE+POST (v0.2.0+ default).
250
+ * ``"ws"`` — legacy WebSocket (kept for fallback / regression tests).
251
+
252
+ Path overrides (``workspace_root``, ``socket_path``, ``state_path``)
253
+ default to ``~/.cloudagent/...``; tests inject ``tmp_path`` paths to
254
+ avoid touching the real home dir. *profile* (M6 multi-profile) is
255
+ resolved through ``paths.resolve_profile`` so it picks up
256
+ ``CLOUDAGENT_DAEMON_PROFILE`` from the env when the caller didn't
257
+ pass an explicit value.
258
+ """
259
+ from .paths import resolve_profile
260
+
261
+ profile_name = resolve_profile(profile)
262
+
263
+ if transport == "ws":
264
+ await _run_legacy_ws(
265
+ cfg,
266
+ state,
267
+ claude_binary=claude_binary,
268
+ workspace_root=workspace_root,
269
+ socket_path=socket_path,
270
+ state_path=state_path,
271
+ insecure=insecure,
272
+ profile=profile_name,
273
+ )
274
+ return
275
+
276
+ # SSE+POST path (default for v0.2.0+).
277
+ await _run_sse_post(
278
+ cfg,
279
+ state,
280
+ claude_binary=claude_binary,
281
+ workspace_root=workspace_root,
282
+ socket_path=socket_path,
283
+ state_path=state_path,
284
+ insecure=insecure,
285
+ detect_runtimes_fn=detect_runtimes_fn,
286
+ profile=profile_name,
287
+ )
288
+
289
+
290
+ # ---------------------------------------------------------------------------
291
+ # SSE+POST path (1j default)
292
+ # ---------------------------------------------------------------------------
293
+
294
+
295
+ async def _run_sse_post(
296
+ cfg: DaemonConfig,
297
+ state: DaemonState,
298
+ *,
299
+ claude_binary: str,
300
+ workspace_root: Path | None,
301
+ socket_path: Path | None,
302
+ state_path: Path | None,
303
+ insecure: bool,
304
+ detect_runtimes_fn: Callable[[], list[DetectedRuntime]] | None,
305
+ profile: str = "default",
306
+ ) -> None:
307
+ # Task #118 (2026-05-12) — ``workspace_root`` is the host-level
308
+ # default for agent cwd, NOT the metadata root. Metadata lives at
309
+ # the per-profile compiled-in default (``~/.cloudagent/agents/`` for
310
+ # default profile, ``~/.cloudagent/agents-<name>/`` otherwise) so a
311
+ # user-chosen project directory never gets polluted with
312
+ # identity.json / mcp-config / MEMORY.md.
313
+ workspace = Workspace(host_workspace_root=workspace_root, profile=profile)
314
+ effective_state_path = state_path or DaemonState.default_path(profile)
315
+
316
+ http = HTTPOutbound(cfg.server_url, cfg.machine_api_key, insecure=insecure)
317
+
318
+ # Step 1: POST /v1/daemon/ready — replaces the WS hello+ready handshake.
319
+ runtimes_list = (detect_runtimes_fn or detect_runtimes)()
320
+ runtimes_payload = [r.to_payload() for r in runtimes_list]
321
+ # Task #113 (2026-05-12) — surface ``--workspace-root`` to backend
322
+ # so the fr UI can render "agents live at <path>" on host detail.
323
+ # When the user didn't pass the flag we omit the field rather
324
+ # than send the daemon's compiled-in default — backend treats
325
+ # omission as "no change" (preserves any previously-reported
326
+ # value, idempotent re-ready).
327
+ ready_payload: dict = {
328
+ "daemon_ver": __version__,
329
+ "runtimes": runtimes_payload,
330
+ "hostname": socket.gethostname(),
331
+ "os": platform.system().lower(),
332
+ }
333
+ if workspace_root is not None:
334
+ ready_payload["workspace_root"] = str(workspace_root)
335
+ try:
336
+ ready_resp = await http.post(
337
+ "/v1/daemon/ready",
338
+ ready_payload,
339
+ )
340
+ except Exception:
341
+ # Ready POST is the gate to everything else; if it fails we have
342
+ # nothing to do but bail out so the supervisor sees the error.
343
+ await http.close()
344
+ raise
345
+
346
+ host_id = ready_resp.get("host_id", "")
347
+ last_acked_seq = int(ready_resp.get("last_acked_seq", 0) or 0)
348
+ bound_agents = ready_resp.get("agents", []) or []
349
+ # Task #118 (2026-05-12) — eagerly materialize each bound agent's
350
+ # workspace so per-agent ``workspace_root`` overrides take effect
351
+ # BEFORE the first deliver lands (the deliver frame doesn't carry
352
+ # ``workspace_root`` — only ready / assign do). Idempotent re-
353
+ # materialization is safe (Workspace.ensure contract).
354
+ for bound in bound_agents:
355
+ try:
356
+ a_id = str(bound.get("agent_id") or "")
357
+ if not a_id:
358
+ continue
359
+ identity = {
360
+ "agent_id": a_id,
361
+ "agent_name": a_id,
362
+ "is_chief": False,
363
+ "role": "staff",
364
+ "workspace_slug": "default",
365
+ "reports_to": "(unset)",
366
+ }
367
+ workspace.ensure(
368
+ a_id,
369
+ identity,
370
+ agent_workspace_root=bound.get("workspace_root"),
371
+ )
372
+ except Exception: # noqa: BLE001
373
+ logger.exception(
374
+ "main_loop._run_sse_post: failed to materialize bound "
375
+ "agent=%s (will retry on next deliver)",
376
+ bound.get("agent_id"),
377
+ )
378
+ # 2026-05-15 — detect host change before reconciling seq. When the
379
+ # ready response carries a host_id different from what state.json
380
+ # remembers (e.g. user re-installed the daemon against a different
381
+ # ``--server-url``, or backend recreated the host row), the local
382
+ # ``last_seq`` is from a different cluster and must be dropped —
383
+ # otherwise the SSE GET keeps sending stale ``Last-Event-ID`` and
384
+ # the new cluster's fresh frames silently get deduped.
385
+ prev_host = state.adopt_host_id(host_id)
386
+ if prev_host is not None:
387
+ logger.warning(
388
+ "daemon host_id changed (was=%s now=%s) — reset local "
389
+ "seq/dedupe state",
390
+ prev_host, host_id,
391
+ )
392
+
393
+ # Server is authoritative on last-acked — but never roll BACK the
394
+ # local state.last_seq if the daemon happens to have processed (and
395
+ # not yet acked) frames newer than the server has on file.
396
+ state.last_seq = max(state.last_seq, last_acked_seq)
397
+ # Persist the host fingerprint immediately. The frame_dispatch path
398
+ # also saves on every ack, but for a fresh start that processes no
399
+ # frames we still want state.json on disk so the next launch can
400
+ # detect mismatch instead of treating itself as first-ever.
401
+ state.save(effective_state_path)
402
+
403
+ # Wave C #3 — metrics registry. Initialized here (after ready POST)
404
+ # so the SSE client + spawn pipeline can publish into a shared
405
+ # default registry; the /metrics endpoint scrapes from the same.
406
+ metrics = get_default_metrics()
407
+ await metrics.set_gauge(
408
+ "cloudagent_daemon_agents_bound", float(len(bound_agents))
409
+ )
410
+
411
+ logger.info(
412
+ "daemon ready: host_id=%s last_acked_seq=%d agents_bound=%d",
413
+ host_id,
414
+ last_acked_seq,
415
+ len(bound_agents),
416
+ )
417
+
418
+ # Step 2: build outbound + components.
419
+ transport_obj = HTTPTransport(http)
420
+ pending_tool_calls: dict[str, asyncio.Future] = {}
421
+ outbound = OutboundQueue(
422
+ transport=transport_obj, pending_tool_calls=pending_tool_calls
423
+ )
424
+
425
+ # 1k74 — token-level streaming publisher. claude_runner calls this
426
+ # for every text_delta chunk; we just POST it through the existing
427
+ # bearer-authenticated httpx client. Backend's endpoint publishes
428
+ # to Redis without a DB write, so the daemon→backend cost stays
429
+ # bounded (1 HTTP POST per ~80 chars of model output ≈ a dozen
430
+ # POSTs for a typical reply). Only wired on the SSE-POST path —
431
+ # the legacy WS path keeps the pre-1k74 buffered-reply behaviour.
432
+ async def _publish_partial(
433
+ session_id: str, agent_id: str, body: str
434
+ ) -> None:
435
+ await http.post(
436
+ "/v1/daemon/publish-partial",
437
+ {
438
+ "session_id": session_id,
439
+ "agent_id": agent_id,
440
+ "body": body,
441
+ },
442
+ )
443
+
444
+ # (c) IPC architecture upgrade — shared agent→session memory map.
445
+ # ClaudeRunner registers the current session_id before spawn and
446
+ # removes it after. mcp_bridge queries via the ``get_current_session``
447
+ # IPC op instead of reading a file; file fallback kept for backward
448
+ # compat (old daemons without this map still work via the file).
449
+ _agent_to_session: dict[str, str | None] = {}
450
+
451
+ runner = ClaudeRunner(
452
+ outbound=outbound,
453
+ workspace=workspace,
454
+ claude_binary=claude_binary,
455
+ partial_publisher=_publish_partial,
456
+ agent_to_session=_agent_to_session,
457
+ )
458
+
459
+ async def _queue_full(agent_id: str, message_id: str) -> None:
460
+ await outbound.send_ack(seq=None, message_id=message_id, status="queue_full")
461
+
462
+ # Bug 4 (2026-05-12) — durable spawn outbox so deliver frames acked
463
+ # but not yet spawned survive daemon crashes. See spawn_outbox.py.
464
+ spawn_outbox = SpawnOutbox(workspace=workspace)
465
+ spawn = SpawnQueue(
466
+ runner=runner,
467
+ queue_full_callback=_queue_full,
468
+ spawn_outbox=spawn_outbox,
469
+ )
470
+
471
+ # Boot-time replay: any frames recorded as ``received`` without a
472
+ # matching ``done`` from the previous run get re-enqueued before we
473
+ # start accepting new frames. Skipping ack here — server already
474
+ # ack'd these in the prior run (that's exactly why they're stuck).
475
+ pending = await spawn_outbox.find_pending()
476
+ for payload in pending:
477
+ logger.info(
478
+ "main_loop: replaying pending spawn agent=%s msg=%s",
479
+ payload.get("agent_id"),
480
+ payload.get("message_id"),
481
+ )
482
+ await spawn.enqueue(payload)
483
+
484
+ # Wave C #1 (2026-05-12) — dynamic agent discovery.
485
+ #
486
+ # Pre-Wave-C, ``agents_bound`` was only refreshed on daemon restart
487
+ # (snapshot from ``/v1/daemon/ready``). A staff created on the
488
+ # server after the daemon already started would be invisible until
489
+ # the next launchd kickstart — that's the structural root of Bug 4b
490
+ # in ``docs/debug/2026-05-12-task-multi-agent-debug.md`` ("StaffPro
491
+ # 没 runner"). Backend now pushes ``assign`` SSE frames; the
492
+ # handler bodies live in :func:`_handle_assign` /
493
+ # :func:`_handle_unassign` so they're directly unit-testable.
494
+
495
+ async def _on_assign(payload: dict) -> None:
496
+ await _handle_assign(workspace, payload)
497
+
498
+ async def _on_unassign(payload: dict) -> None:
499
+ await _handle_unassign(workspace, payload)
500
+
501
+ dispatcher = FrameDispatcher(
502
+ state=state,
503
+ state_path=effective_state_path,
504
+ ack_callback=outbound.send_ack,
505
+ spawn_callback=spawn.enqueue,
506
+ tool_result_pending=pending_tool_calls,
507
+ spawn_outbox=spawn_outbox,
508
+ workspace_assign_callback=_on_assign,
509
+ workspace_unassign_callback=_on_unassign,
510
+ )
511
+
512
+ from .unix_socket import default_socket_path as _default_socket_path
513
+
514
+ sock_path = socket_path or _default_socket_path(profile)
515
+ sock_path.parent.mkdir(parents=True, exist_ok=True)
516
+
517
+ async def ipc_handler(req: dict) -> dict:
518
+ # 1k #55 — IPC entry/exit logging for multi-tool-wake debugging.
519
+ ipc_rid = req.get("request_id")
520
+ op = req.get("op")
521
+
522
+ # (c) IPC architecture upgrade — daemon-memory session query.
523
+ # mcp_bridge sends this at startup to get the current session_id
524
+ # without reading a file; more reliable than the file-based path.
525
+ if op == "get_current_session":
526
+ a = req.get("args") or {}
527
+ agent_id_q = a.get("agent_id") or ""
528
+ sid = _agent_to_session.get(agent_id_q)
529
+ logger.info(
530
+ "main_loop.ipc_handler: get_current_session agent=%s sid=%s",
531
+ agent_id_q, sid,
532
+ )
533
+ return {"result": {"session_id": sid}}
534
+
535
+ if op != "tool_call":
536
+ logger.warning(
537
+ "main_loop.ipc_handler: unknown op=%r ipc_rid=%s", op, ipc_rid,
538
+ )
539
+ return {"error": {"code": "UNKNOWN_OP", "message": str(op)}}
540
+ a = req.get("args") or {}
541
+ logger.info(
542
+ "main_loop.ipc_handler: ENTER ipc_rid=%s tool=%s agent=%s "
543
+ "session=%s",
544
+ ipc_rid, a.get("tool_name"), a.get("agent_id"), a.get("session_id"),
545
+ )
546
+ try:
547
+ result = await outbound.send_tool_call(
548
+ agent_id=a["agent_id"],
549
+ session_id=a.get("session_id"),
550
+ tool_name=a["tool_name"],
551
+ args=a.get("tool_args") or {},
552
+ )
553
+ except Exception as exc: # noqa: BLE001
554
+ logger.exception(
555
+ "main_loop.ipc_handler: dispatch raised ipc_rid=%s tool=%s",
556
+ ipc_rid, a.get("tool_name"),
557
+ )
558
+ return {
559
+ "error": {
560
+ "code": "BRIDGE_DISPATCH_FAILED",
561
+ "message": f"{type(exc).__name__}: {exc}",
562
+ }
563
+ }
564
+ logger.info(
565
+ "main_loop.ipc_handler: EXIT ipc_rid=%s tool=%s ok=%s",
566
+ ipc_rid, a.get("tool_name"), result.get("ok"),
567
+ )
568
+ if result.get("ok"):
569
+ return {"result": result.get("data", {})}
570
+ return {"error": result.get("error", {"code": "UNKNOWN_ERROR"})}
571
+
572
+ ipc_server = UnixSocketServer(sock_path, ipc_handler)
573
+ await ipc_server.start()
574
+
575
+ # Wave C #3 — Prometheus-scrape unix socket. Lives alongside
576
+ # ``bridge.sock`` under ``~/.cloudagent/daemon/`` with 0o600 perms.
577
+ # Lifecycle mirrors the IPC server: start now, stop in the
578
+ # outer-finally cleanup so a daemon restart cycle doesn't leak
579
+ # stale socket files.
580
+ metrics_server = MetricsServer(metrics)
581
+ try:
582
+ await metrics_server.start()
583
+ except Exception:
584
+ # Best-effort — if the metrics socket can't bind (e.g. perms
585
+ # / readonly home dir) we proceed without /metrics. The
586
+ # daemon must NOT refuse to serve agents because Prometheus
587
+ # is down.
588
+ logger.exception(
589
+ "main_loop: metrics_server.start failed; /metrics unavailable"
590
+ )
591
+ metrics_server = None
592
+
593
+ # Step 3: heartbeat task. Best-effort liveness POST every
594
+ # ``HEARTBEAT_INTERVAL_SECONDS`` (1k #61: 10s) — if the server didn't
595
+ # already see us via an ack/status frame, this nudges ``last_seen_at``
596
+ # so the dashboard's online indicator stays green and the host-presence
597
+ # reaper doesn't flip us to offline.
598
+ async def heartbeat() -> None:
599
+ while True:
600
+ await asyncio.sleep(HEARTBEAT_INTERVAL_SECONDS)
601
+ try:
602
+ await http.post("/v1/daemon/heartbeat", {})
603
+ except asyncio.CancelledError:
604
+ raise
605
+ except Exception: # noqa: BLE001
606
+ logger.debug("heartbeat POST failed", exc_info=True)
607
+
608
+ # Step 4: SSE consumer — the inbound side.
609
+ sse = SSEClient(
610
+ cfg.server_url,
611
+ cfg.machine_api_key,
612
+ insecure=insecure,
613
+ last_event_id=str(state.last_seq),
614
+ )
615
+ # Wave C #3 — gauges that derive from the SSE client at scrape time.
616
+ metrics.gauge(
617
+ "cloudagent_daemon_sse_last_event_age_seconds",
618
+ "Seconds since the last parsed SSE event arrived.",
619
+ resolver=lambda: sse.last_event_age_seconds,
620
+ )
621
+
622
+ async def on_frame(frame: dict) -> None:
623
+ # Wave C #3 — count every frame the dispatcher sees, including
624
+ # duplicates (which we still ack). Failure modes (raised
625
+ # exceptions in dispatcher) still propagate to the SSE task.
626
+ if frame.get("type") == "deliver":
627
+ await metrics.inc_counter(
628
+ "cloudagent_daemon_deliver_frames_total"
629
+ )
630
+ await dispatcher.dispatch(frame)
631
+
632
+ cancelled = False
633
+ try:
634
+ # 1k78 — explicit task supervisor in place of ``asyncio.gather``.
635
+ #
636
+ # Pre-1k78 we used ``await asyncio.gather(sse.run_forever,
637
+ # heartbeat)``: documented to propagate any task's first
638
+ # exception and cancel siblings. In production we observed the
639
+ # SSE task silently entering a "done" state (never reconnecting,
640
+ # never raising) while the heartbeat task kept POSTing for
641
+ # hours — the daemon process was technically alive but useless
642
+ # because every command sent from the UI piled up in
643
+ # ``undelivered_frames`` with no consumer. Root cause for the
644
+ # gather bypass was never pinned (likely an httpx-sse internal
645
+ # path swallowed an exception), so this defensive supervisor
646
+ # explicitly polls each task's completion and force-exits the
647
+ # daemon — letting launchd / systemd auto-respawn it — the
648
+ # moment either task ends unexpectedly.
649
+ sse_task = asyncio.create_task(
650
+ sse.run_forever(on_frame), name="daemon.sse"
651
+ )
652
+ heartbeat_task = asyncio.create_task(heartbeat(), name="daemon.heartbeat")
653
+ # Wave C #2 — outbox stuck self-kill. The watchdog only returns
654
+ # by raising SystemExit, so the supervisor's FIRST_COMPLETED
655
+ # treats a stuck spawn the same as a dead SSE: log + respawn.
656
+ outbox_watchdog_task = asyncio.create_task(
657
+ _outbox_stuck_watchdog(spawn_outbox),
658
+ name="daemon.outbox_watchdog",
659
+ )
660
+ watched = {sse_task, heartbeat_task, outbox_watchdog_task}
661
+ try:
662
+ done, pending = await asyncio.wait(
663
+ watched, return_when=asyncio.FIRST_COMPLETED
664
+ )
665
+ # First task done. If it raised, surface that and abort —
666
+ # the supervisor's job from here on is to make sure the
667
+ # daemon doesn't linger in a half-dead state.
668
+ for t in done:
669
+ exc = t.exception()
670
+ if exc is not None and not isinstance(
671
+ exc, asyncio.CancelledError
672
+ ):
673
+ logger.error(
674
+ "daemon supervisor: task %s died with %r — exiting "
675
+ "so launchd respawns",
676
+ t.get_name(), exc,
677
+ )
678
+ else:
679
+ logger.error(
680
+ "daemon supervisor: task %s ended unexpectedly "
681
+ "(no exception) — exiting so launchd respawns",
682
+ t.get_name(),
683
+ )
684
+ for t in pending:
685
+ t.cancel()
686
+ # Drain cancellations so cleanup hooks below get a chance
687
+ # to run. Don't propagate the underlying error; we're
688
+ # exiting deliberately.
689
+ for t in pending:
690
+ with contextlib.suppress(Exception):
691
+ await t
692
+ # Force non-zero exit so launchd's KeepAlive respawns us.
693
+ raise SystemExit(1)
694
+ except asyncio.CancelledError:
695
+ cancelled = True
696
+ raise
697
+ finally:
698
+ sse.stop()
699
+ try:
700
+ await ipc_server.stop()
701
+ except Exception: # noqa: BLE001
702
+ pass
703
+ if metrics_server is not None:
704
+ try:
705
+ await metrics_server.stop()
706
+ except Exception: # noqa: BLE001
707
+ pass
708
+ if cancelled:
709
+ await spawn.cancel_all()
710
+ else:
711
+ await spawn.shutdown()
712
+ await transport_obj.close()
713
+
714
+
715
+ # ---------------------------------------------------------------------------
716
+ # Legacy WebSocket path (kept for --transport ws)
717
+ # ---------------------------------------------------------------------------
718
+
719
+
720
+ async def _run_legacy_ws(
721
+ cfg: DaemonConfig,
722
+ state: DaemonState,
723
+ *,
724
+ claude_binary: str,
725
+ workspace_root: Path | None,
726
+ socket_path: Path | None,
727
+ state_path: Path | None,
728
+ insecure: bool,
729
+ profile: str = "default",
730
+ ) -> None:
731
+ """Pre-1j WebSocket path. Identical to the v0.1.0 main loop body."""
732
+ # Task #118 (2026-05-12) — see SSE path's matching comment.
733
+ workspace = Workspace(host_workspace_root=workspace_root, profile=profile)
734
+ effective_state_path = state_path or DaemonState.default_path(profile)
735
+
736
+ pending_tool_calls: dict[str, asyncio.Future] = {}
737
+ outbound = OutboundQueue(transport=None, pending_tool_calls=pending_tool_calls)
738
+
739
+ runner = ClaudeRunner(
740
+ outbound=outbound, workspace=workspace, claude_binary=claude_binary
741
+ )
742
+
743
+ async def _queue_full(agent_id: str, message_id: str) -> None:
744
+ await outbound.send_ack(seq=None, message_id=message_id, status="queue_full")
745
+
746
+ # Bug 4 — same outbox + replay wiring as the SSE path. Both
747
+ # transports share the same crash-survival behaviour.
748
+ spawn_outbox = SpawnOutbox(workspace=workspace)
749
+ spawn = SpawnQueue(
750
+ runner=runner,
751
+ queue_full_callback=_queue_full,
752
+ spawn_outbox=spawn_outbox,
753
+ )
754
+
755
+ pending = await spawn_outbox.find_pending()
756
+ for payload in pending:
757
+ logger.info(
758
+ "main_loop: replaying pending spawn agent=%s msg=%s",
759
+ payload.get("agent_id"),
760
+ payload.get("message_id"),
761
+ )
762
+ await spawn.enqueue(payload)
763
+
764
+ # Wave C #1 — same assign/unassign wiring as the SSE path.
765
+ async def _on_assign_ws(payload: dict) -> None:
766
+ await _handle_assign(workspace, payload)
767
+
768
+ async def _on_unassign_ws(payload: dict) -> None:
769
+ await _handle_unassign(workspace, payload)
770
+
771
+ dispatcher = FrameDispatcher(
772
+ state=state,
773
+ state_path=effective_state_path,
774
+ ack_callback=outbound.send_ack,
775
+ spawn_callback=spawn.enqueue,
776
+ tool_result_pending=pending_tool_calls,
777
+ spawn_outbox=spawn_outbox,
778
+ workspace_assign_callback=_on_assign_ws,
779
+ workspace_unassign_callback=_on_unassign_ws,
780
+ )
781
+
782
+ from .unix_socket import default_socket_path as _default_socket_path_ws
783
+
784
+ sock_path = socket_path or _default_socket_path_ws(profile)
785
+ sock_path.parent.mkdir(parents=True, exist_ok=True)
786
+
787
+ async def ipc_handler(req: dict) -> dict:
788
+ if req.get("op") != "tool_call":
789
+ return {"error": {"code": "UNKNOWN_OP", "message": str(req.get("op"))}}
790
+ a = req.get("args") or {}
791
+ try:
792
+ result = await outbound.send_tool_call(
793
+ agent_id=a["agent_id"],
794
+ session_id=a.get("session_id"),
795
+ tool_name=a["tool_name"],
796
+ args=a.get("tool_args") or {},
797
+ )
798
+ except Exception as exc: # noqa: BLE001
799
+ return {
800
+ "error": {
801
+ "code": "BRIDGE_DISPATCH_FAILED",
802
+ "message": f"{type(exc).__name__}: {exc}",
803
+ }
804
+ }
805
+ if result.get("ok"):
806
+ return {"result": result.get("data", {})}
807
+ return {"error": result.get("error", {"code": "UNKNOWN_ERROR"})}
808
+
809
+ ipc_server = UnixSocketServer(sock_path, ipc_handler)
810
+ await ipc_server.start()
811
+
812
+ def _client_factory() -> WSClient:
813
+ return WSClient(
814
+ server_url=cfg.server_url,
815
+ api_key=cfg.machine_api_key,
816
+ insecure=insecure,
817
+ # Task #113 — propagate workspace_root onto every WS
818
+ # reconnect's ``ready`` frame so a daemon that auto-
819
+ # reconnects after a network drop still surfaces the
820
+ # user-chosen root to backend.
821
+ workspace_root=(
822
+ str(workspace_root) if workspace_root is not None else None
823
+ ),
824
+ )
825
+
826
+ async def _on_connect(client: WSClient) -> None:
827
+ # ``outbound.ws = client`` is the legacy assignment shape — the new
828
+ # property setter on OutboundQueue adapts a WSClient into a
829
+ # Transport (_LegacyWSAdapter) so existing code works unchanged.
830
+ outbound.ws = client
831
+ # Same host_id-change reset the SSE path does. Runs BEFORE
832
+ # ws_client.run_forever calls state_loader → send_resume so a
833
+ # cluster switch doesn't replay stale Last-Event-ID upstream.
834
+ prev_host = state.adopt_host_id(client.host_id)
835
+ if prev_host is not None:
836
+ logger.warning(
837
+ "daemon host_id changed (was=%s now=%s) — reset local "
838
+ "seq/dedupe state",
839
+ prev_host, client.host_id,
840
+ )
841
+ state.save(effective_state_path)
842
+ logger.info("daemon connected: host_id=%s", client.host_id)
843
+
844
+ async def _on_frame(frame: dict) -> None:
845
+ await dispatcher.dispatch(frame)
846
+
847
+ cancelled = False
848
+ try:
849
+ # Wave C #2 — same outbox watchdog wiring as the SSE path.
850
+ # Wrap the WS run_forever + watchdog in a wait(...) so a stuck
851
+ # spawn here also escalates to launchd-respawn instead of
852
+ # sitting silently while the WS keeps heart-beating.
853
+ ws_task = asyncio.create_task(
854
+ run_forever(
855
+ client_factory=_client_factory,
856
+ on_frame=_on_frame,
857
+ state_loader=lambda: state.last_seq,
858
+ on_connect=_on_connect,
859
+ ),
860
+ name="daemon.ws",
861
+ )
862
+ outbox_task = asyncio.create_task(
863
+ _outbox_stuck_watchdog(spawn_outbox),
864
+ name="daemon.outbox_watchdog",
865
+ )
866
+ watched = {ws_task, outbox_task}
867
+ done, pending = await asyncio.wait(
868
+ watched, return_when=asyncio.FIRST_COMPLETED
869
+ )
870
+ for t in pending:
871
+ t.cancel()
872
+ for t in pending:
873
+ with contextlib.suppress(Exception):
874
+ await t
875
+ # Surface the completed task's exception (if any) so callers /
876
+ # launchd see the right exit code.
877
+ for t in done:
878
+ exc = t.exception()
879
+ if exc is not None and not isinstance(
880
+ exc, asyncio.CancelledError
881
+ ):
882
+ raise exc
883
+ except asyncio.CancelledError:
884
+ cancelled = True
885
+ raise
886
+ finally:
887
+ await ipc_server.stop()
888
+ if cancelled:
889
+ await spawn.cancel_all()
890
+ else:
891
+ await spawn.shutdown()