cloudagent-daemon 0.2.1__py3-none-any.whl

cloudagent_daemon/sse_client.py

@@ -0,0 +1,439 @@
+"""Server-sent events consumer for the cloudagent daemon.
+
+This is the SSE side of Sub-plan 1j (SSE+POST transport replacing the
+WebSocket).  The daemon GETs ``/v1/daemon/stream`` with a Bearer token and
+``Last-Event-ID`` set to the highest ``seq`` it has durably processed.
+The server replies with ``text/event-stream``; each ``event:``/``id:``/
+``data:`` block is a server→daemon frame.
+
+Reconnect strategy
+------------------
+The HTTP+SSE world is intentionally less precise than WebSocket — proxies
+may close idle streams, NAT may cull connections, etc. We:
+
+* Reconnect on any non-auth error with exponential backoff (1→2→4→8→16→30s
+  cap), reset to 1s on a successful event.
+* Always re-send ``Last-Event-ID`` on reconnect so the server replays
+  unacked frames from its catchup buffer (per SSE spec, this is the
+  client's responsibility — the server only sees the header).
+* Treat ``401``/``403`` as terminal (``AuthFailed``) — re-trying with the
+  same key wastes server time and can trigger abuse mitigations.
+
+Out of scope:
+  * POSTing daemon→server frames — that's ``http_outbound.HTTPOutbound``.
+
+Stale-stream watchdogs (1k #64 passive + Wave C #2 active)
+----------------------------------------------------------
+Clash TUN / NAT middleware silently swallows long-idle SSE streams: the
+TCP socket stays open from our side but no bytes ever arrive again. Two
+layers of defence:
+
+* **Passive (1k #64)** — httpx's ``read`` timeout. The server sends
+  ``:hb`` comment lines every 30s when idle (see
+  ``backend/.../daemon_sse.py``), which counts as bytes at the socket
+  layer (``httpx-sse`` filters comment lines out only at the parse
+  layer). If no bytes arrive within ``READ_TIMEOUT_SECONDS`` (60s = 2×
+  server heartbeat), httpx raises ``ReadTimeout`` and we reconnect with
+  ``Last-Event-ID``. Covers "TCP socket dead, no bytes flowing".
+
+* **Active (Wave C #2)** — :data:`ACTIVE_LIVENESS_TIMEOUT_SECONDS`
+  watchdog **on the parsed-event stream**. ``httpx-sse`` filters
+  ``:hb`` comments before yielding to user code, so even on a fully
+  healthy stream we may go minutes between user-visible events. The
+  passive timeout (TCP bytes) can hide a pathological case where the
+  underlying httpx-sse buffer is starved of real events forever. If we
+  see no real frame for ``ACTIVE_LIVENESS_TIMEOUT_SECONDS`` we close
+  the stream from our side and reconnect — the next ``Last-Event-ID``
+  GET re-arms the catchup buffer and the worst-case stuck-frame window
+  is bounded by the watchdog. Default is intentionally LONGER than the
+  passive timeout so an idle backend doesn't reconnect every minute.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import ssl
+import time
+from collections.abc import Awaitable, Callable
+
+import httpx
+from httpx_sse import aconnect_sse
+
+logger = logging.getLogger(__name__)
+
+# Reconnect backoff curve: 1, 2, 4, 8, 16, 30, 30, … (capped at 30s).
+_BACKOFF_INITIAL_SECONDS = 1.0
+_BACKOFF_CAP_SECONDS = 30.0
+
+# Passive stale-stream watchdog (1k #64). Server emits a ``:hb`` comment
+# line every 30s when no real frame flows; httpx counts each ``:hb`` as
+# bytes at the socket layer (httpx-sse filters them out only at parse
+# layer). 60s = tolerate one missed heartbeat before httpx raises
+# ``ReadTimeout`` and we reconnect via ``Last-Event-ID``. Tests
+# monkeypatch this down to a few seconds.
+READ_TIMEOUT_SECONDS = 60.0
+
+# Active liveness watchdog (Wave C #2). httpx-sse swallows ``:hb``
+# comment lines before yielding to user code, so even on a healthy
+# stream we may go minutes between user-visible events. If no parsed
+# event arrives within this window, force-close the SSE response and
+# trigger a fresh ``Last-Event-ID`` GET. Default is 180s = 6× server
+# heartbeat — wide enough that a genuinely idle backend doesn't
+# thrash, tight enough to cap stuck-event windows at 3 min in
+# worst-case pathological httpx-sse buffer starvation. Tests
+# monkeypatch this to a few seconds.
+ACTIVE_LIVENESS_TIMEOUT_SECONDS = 180.0
+
+
+class AuthFailed(Exception):
+    """Raised on terminal 401/403 — caller should not retry."""
+
+    def __init__(self, status_code: int) -> None:
+        self.status_code = status_code
+        super().__init__(f"SSE auth failed: HTTP {status_code}")
+
+
+class SSEClient:
+    """Async SSE consumer with auto-reconnect via ``Last-Event-ID``.
+
+    Usage::
+
+        client = SSEClient(server_url, api_key)
+        await client.run_forever(on_frame)
+
+    ``on_frame`` is awaited for each parsed JSON event (heartbeat comment
+    lines and non-JSON event data are silently dropped).
+    """
+
+    def __init__(
+        self,
+        server_url: str,
+        api_key: str,
+        *,
+        insecure: bool = False,
+        last_event_id: str = "0",
+    ) -> None:
+        self._server_url = server_url.rstrip("/")
+        self._api_key = api_key
+        self._insecure = insecure
+        self._last_event_id = last_event_id
+        self._stop_event = asyncio.Event()
+        self._reconnects = 0
+        # Wave C #2 — active liveness watchdog timestamp. Reset on
+        # every successfully-parsed event (and on connection open).
+        # Compared against ``ACTIVE_LIVENESS_TIMEOUT_SECONDS`` inside
+        # ``_consume_sse_until_stop`` to force a reconnect when the
+        # event stream stalls below the passive read-timeout layer.
+        self._last_event_at: float = time.monotonic()
+        # Diagnostic counters surfaced by /metrics + doctor (Wave C #3).
+        self._active_watchdog_fires = 0
+        # SSL context only used for https:// URLs; httpx ignores ``verify``
+        # values that aren't bool/path/SSLContext, so we map insecure→False.
+        self._verify: ssl.SSLContext | bool = not insecure
+
+    @property
+    def last_event_id(self) -> str:
+        return self._last_event_id
+
+    @property
+    def reconnect_count(self) -> int:
+        return self._reconnects
+
+    @property
+    def last_event_age_seconds(self) -> float:
+        """Seconds since the last parsed SSE event (Wave C #2 / #3).
+
+        Used by ``cloudagent-daemon doctor`` and the ``/metrics`` endpoint
+        to surface stream liveness without having to plumb the timestamp
+        through callbacks.
+        """
+        return max(0.0, time.monotonic() - self._last_event_at)
+
+    @property
+    def active_watchdog_fires(self) -> int:
+        """How many times the active liveness watchdog has forced a
+        reconnect (cumulative since process start)."""
+        return self._active_watchdog_fires
+
+    def stop(self) -> None:
+        """Signal ``run_forever`` to exit at the next checkpoint."""
+        self._stop_event.set()
+
+    async def run_forever(
+        self,
+        on_frame: Callable[[dict], Awaitable[None]],
+    ) -> None:
+        """Connect, stream, reconnect on errors. Returns on ``stop()``.
+
+        Raises:
+            AuthFailed: 401/403 on the SSE GET — auth is dead, do not retry.
+            asyncio.CancelledError: propagated unchanged for cooperative
+                shutdown (so signal handlers / ``task.cancel()`` work).
+        """
+        backoff = _BACKOFF_INITIAL_SECONDS
+        url = f"{self._server_url}/v1/daemon/stream"
+
+        # One client survives the entire run_forever call; httpx pools the
+        # underlying TCP connection so each reconnect is cheap.
+        # Read READ_TIMEOUT_SECONDS at call-time (not import-time) so tests
+        # can monkeypatch the module constant before invoking run_forever.
+        import cloudagent_daemon.sse_client as _self_module  # local re-import
+        read_timeout = _self_module.READ_TIMEOUT_SECONDS
+        async with httpx.AsyncClient(
+            verify=self._verify,
+            # ``read=READ_TIMEOUT_SECONDS`` is the stale-stream watchdog —
+            # server sends ``:hb`` comments every 30s, so 60s of no inbound
+            # bytes means Clash TUN / NAT silently dropped the stream
+            # → httpx.ReadTimeout → reconnect with Last-Event-ID (1k #64).
+            timeout=httpx.Timeout(
+                timeout=None, connect=30.0, read=read_timeout
+            ),
+            # No proxy override — daemon CLI's _bypass_system_proxy_by_default
+            # already cleared the env vars before we got here. Honour any
+            # explicit env var set by ``CLOUDAGENT_USE_SYSTEM_PROXY=1`` users.
+            trust_env=True,
+            # 2026-05-15 — Alibaba vipserver fronts the pre/daily/prod
+            # intranet ingresses (``pre-agentinfra.alibabacloud.com`` →
+            # vipserver TLS terminate → SLB → nginx-ingress). Vipserver
+            # buffers HTTP/1.1 chunked-transfer bodies, so the SSE long-
+            # poll yields zero bytes until the upstream closes — the
+            # daemon's stale-stream watchdog then fires every 60s and the
+            # stream looks broken even though the server is happily
+            # publishing frames into Redis. Negotiating HTTP/2 dodges
+            # the chunked-transfer path entirely (HTTP/2 frames the body
+            # natively). Cleartext HTTP and servers without h2 just
+            # transparently fall back to 1.1 (httpx ALPN).
+            http2=True,
+        ) as client:
+            while not self._stop_event.is_set():
+                try:
+                    headers = {
+                        "Authorization": f"Bearer {self._api_key}",
+                        "Accept": "text/event-stream",
+                        "Last-Event-ID": self._last_event_id,
+                    }
+                    async with aconnect_sse(
+                        client, "GET", url, headers=headers
+                    ) as event_source:
+                        # Surface auth failures BEFORE entering the
+                        # iteration loop — httpx-sse defers the status
+                        # check to ``response.raise_for_status()``.
+                        try:
+                            event_source.response.raise_for_status()
+                        except httpx.HTTPStatusError as exc:
+                            await _handle_status_error(exc)
+                            # _handle_status_error either raises AuthFailed
+                            # or returns; non-auth status falls through to
+                            # the backoff path below.
+                            raise
+                        # Iterate manually so we can cooperatively check
+                        # stop_event between event arrivals — if the stream
+                        # only emits heartbeat comments (which httpx-sse
+                        # filters out before yielding), a plain ``async for``
+                        # would block forever ignoring stop_event.
+                        sse_iter = event_source.aiter_sse()
+                        backoff = await _consume_sse_until_stop(
+                            sse_iter,
+                            on_frame,
+                            self,
+                            backoff,
+                        )
+                except AuthFailed:
+                    raise
+                except httpx.HTTPStatusError as exc:
+                    # Already handled above for 401/403; non-auth status
+                    # codes (5xx, 502, etc.) are transient — back off.
+                    logger.warning(
+                        "sse: HTTP %d, reconnecting in %.1fs",
+                        exc.response.status_code,
+                        min(backoff, _BACKOFF_CAP_SECONDS),
+                    )
+                except asyncio.CancelledError:
+                    raise
+                except httpx.ReadTimeout:
+                    # 1k #64 — stale-stream watchdog fired. Clash TUN / NAT
+                    # silently dropped the stream; the OS-level read sat
+                    # idle past READ_TIMEOUT_SECONDS without even a ``:hb``
+                    # comment line arriving. Reconnect via Last-Event-ID.
+                    if self._stop_event.is_set():
+                        break
+                    logger.warning(
+                        "sse: stream read timeout (%.0fs idle) — "
+                        "reconnecting via Last-Event-ID=%s in %.1fs",
+                        read_timeout,
+                        self._last_event_id,
+                        min(backoff, _BACKOFF_CAP_SECONDS),
+                    )
+                except (httpx.HTTPError, OSError) as exc:
+                    if self._stop_event.is_set():
+                        break
+                    logger.warning(
+                        "sse: stream broken (%s), reconnecting in %.1fs",
+                        type(exc).__name__,
+                        min(backoff, _BACKOFF_CAP_SECONDS),
+                    )
+                except Exception:  # noqa: BLE001
+                    # 1k78 — catch-all so silent failures stop being
+                    # silent. Previously an unexpected exception type
+                    # (e.g. RuntimeError from httpx-sse internals or
+                    # ``on_frame`` raising) would propagate out of
+                    # run_forever without hitting any logger — making
+                    # post-mortem debugging impossible. Now we log
+                    # with full traceback and re-raise so the
+                    # supervisor in main_loop sees the task die.
+                    if self._stop_event.is_set():
+                        raise
+                    logger.exception(
+                        "sse: unexpected error in run_forever (will "
+                        "propagate to supervisor for daemon restart)"
+                    )
+                    raise
+
+                if self._stop_event.is_set():
+                    break
+
+                self._reconnects += 1
+                # Wave C #3 — surface reconnects to the default metrics
+                # registry. Imported lazily so this module stays cheap
+                # to import in tests that don't exercise metrics.
+                try:
+                    from cloudagent_daemon.metrics import get_default as _md
+                    await _md().inc_counter(
+                        "cloudagent_daemon_sse_reconnects_total"
+                    )
+                except Exception:  # noqa: BLE001
+                    pass
+                await asyncio.sleep(min(backoff, _BACKOFF_CAP_SECONDS))
+                backoff = min(backoff * 2.0, _BACKOFF_CAP_SECONDS)
+
+        # 1k78 — if we exit the loop via stop_event normally, log
+        # that explicitly. Pre-1k78 a normal return looked identical
+        # to a silent failure in the main_loop supervisor.
+        logger.info(
+            "sse: run_forever exited normally (stop_event=%s reconnects=%d)",
+            self._stop_event.is_set(),
+            self._reconnects,
+        )
+
+
+async def _handle_status_error(exc: httpx.HTTPStatusError) -> None:
+    """Raise AuthFailed for terminal codes; fall through for transient."""
+    code = exc.response.status_code
+    if code in (401, 403):
+        logger.error("sse: auth failed (%d), aborting", code)
+        raise AuthFailed(code) from exc
+    # Non-auth status — let the caller handle backoff.
+
+
+async def _consume_sse_until_stop(
+    sse_iter,
+    on_frame: Callable[[dict], Awaitable[None]],
+    sse_client: "SSEClient",
+    backoff: float,
+) -> float:
+    """Drive ``sse_iter`` while honouring ``sse_client._stop_event``.
+
+    Returns the (possibly reset) backoff so the outer reconnect loop sees
+    the latest value after a successful event delivery.
+
+    Wave C #2 — also gates each ``__anext__`` call on the active
+    liveness watchdog: if no parsed event arrives in
+    :data:`ACTIVE_LIVENESS_TIMEOUT_SECONDS` (re-read from the module
+    each iteration so tests can monkeypatch), we exit so the outer
+    loop reconnects via ``Last-Event-ID``. Resetting ``_last_event_at``
+    on connection open + each successfully-parsed event keeps an idle
+    backend from thrashing.
+    """
+    # Reset the watchdog clock when we (re)enter the consume loop —
+    # otherwise a previous run's last_event_at could still be stale
+    # at the moment we connect.
+    sse_client._last_event_at = time.monotonic()
+    while True:
+        if sse_client._stop_event.is_set():
+            break
+
+        # Re-read each iter so tests can monkeypatch the module-level
+        # constant between runs without reconstructing the client.
+        import cloudagent_daemon.sse_client as _self_module
+        active_timeout = _self_module.ACTIVE_LIVENESS_TIMEOUT_SECONDS
+        elapsed = time.monotonic() - sse_client._last_event_at
+        remaining = active_timeout - elapsed
+        if remaining <= 0:
+            # Edge case: clock jumped (sleep / hibernate) past the
+            # threshold while we were between iterations. Treat
+            # exactly like a watchdog fire.
+            sse_client._active_watchdog_fires += 1
+            try:
+                from cloudagent_daemon.metrics import get_default as _md
+                await _md().inc_counter(
+                    "cloudagent_daemon_sse_active_watchdog_fires_total"
+                )
+            except Exception:  # noqa: BLE001
+                pass
+            logger.warning(
+                "sse: active watchdog — no event in %.0fs (>= %.0fs), "
+                "reconnecting via Last-Event-ID=%s",
+                elapsed, active_timeout, sse_client._last_event_id,
+            )
+            return backoff
+
+        next_task = asyncio.create_task(_anext(sse_iter))
+        stop_task = asyncio.create_task(sse_client._stop_event.wait())
+        watchdog_task = asyncio.create_task(asyncio.sleep(remaining))
+        try:
+            done, _pending = await asyncio.wait(
+                {next_task, stop_task, watchdog_task},
+                return_when=asyncio.FIRST_COMPLETED,
+            )
+        finally:
+            for t in (next_task, stop_task, watchdog_task):
+                if not t.done():
+                    t.cancel()
+        if stop_task in done:
+            return backoff
+        if watchdog_task in done and next_task not in done:
+            # Active watchdog won the race — the parsed-event stream
+            # has been silent past the threshold. Exit so the outer
+            # loop reconnects; ``Last-Event-ID`` re-arms catchup.
+            sse_client._active_watchdog_fires += 1
+            try:
+                from cloudagent_daemon.metrics import get_default as _md
+                await _md().inc_counter(
+                    "cloudagent_daemon_sse_active_watchdog_fires_total"
+                )
+            except Exception:  # noqa: BLE001
+                pass
+            elapsed = time.monotonic() - sse_client._last_event_at
+            logger.warning(
+                "sse: active watchdog — no event in %.0fs (>= %.0fs), "
+                "reconnecting via Last-Event-ID=%s",
+                elapsed, active_timeout, sse_client._last_event_id,
+            )
+            return backoff
+        # next_task done — propagate exceptions, otherwise consume.
+        try:
+            sse_event = next_task.result()
+        except StopAsyncIteration:
+            return backoff
+        sse_client._last_event_at = time.monotonic()
+        if sse_event.id:
+            sse_client._last_event_id = sse_event.id
+        if not sse_event.data:
+            continue
+        try:
+            frame = json.loads(sse_event.data)
+        except json.JSONDecodeError:
+            logger.warning(
+                "sse: non-JSON event data %r",
+                sse_event.data[:120],
+            )
+            continue
+        backoff = _BACKOFF_INITIAL_SECONDS
+        await on_frame(frame)
+    return backoff
+
+
+async def _anext(aiter):
+    """Polyfill for ``anext`` on a generic async iterator."""
+    return await aiter.__anext__()

cloudagent_daemon/transport.py

@@ -0,0 +1,105 @@
+"""Transport abstraction for daemon→server frames.
+
+Sub-plan 1j replaces the WebSocket with an SSE stream (server→daemon)
+plus per-frame-type POST endpoints (daemon→server). Outbound senders
+(``OutboundQueue``, ``ClaudeRunner``) shouldn't care which wire is used —
+they call ``transport.send_frame(frame_dict)`` and let the implementation
+route.
+
+Two concrete transports today:
+  * ``HTTPTransport`` — routes each frame ``type`` to the matching POST
+    endpoint. Used in production with the SSE stream.
+  * ``WSTransport`` — wraps a connected ``WSClient`` so the legacy
+    ``--transport ws`` path still works through the same OutboundQueue.
+"""
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from .http_outbound import HTTPOutbound
+    from .ws_client import WSClient
+
+logger = logging.getLogger(__name__)
+
+
+class Transport(Protocol):
+    """Sends daemon→server frames.
+
+    Implementations decide whether the frame goes over WS as JSON or
+    over HTTP as a POST to a per-type endpoint.
+    """
+
+    async def send_frame(self, frame: dict) -> None: ...
+
+    async def close(self) -> None: ...
+
+
+# Frame ``type`` → POST path. ``None`` means the SSE path doesn't have a
+# corresponding POST (the WS-only frame is silently dropped). We list all
+# the daemon→server frame types from spec §4.2 explicitly so future frame
+# types fail loudly (logged) rather than getting silently sent to ``None``.
+_HTTP_PATHS: dict[str, str | None] = {
+    "delivery_ack": "/v1/daemon/delivery-ack",
+    "agent_status": "/v1/daemon/agent-status",
+    "agent_session_start": "/v1/daemon/agent-session-start",
+    "tool_call": "/v1/daemon/tool-call",
+    # SSE provides server-side ``:hb`` heartbeat; the daemon doesn't need
+    # to pong (and the server has no /pong endpoint). The legacy WS still
+    # uses pong via WSTransport.
+    "pong": None,
+    # Resume happens implicitly via the ``Last-Event-ID`` header on the
+    # SSE GET; no POST equivalent.
+    "resume": None,
+    # Ready is sent ONCE during /v1/daemon/ready (handled by main_loop
+    # before the OutboundQueue is wired up); no need to re-send.
+    "ready": None,
+}
+
+
+class HTTPTransport:
+    """Transport implementation that routes by frame type to POST endpoints."""
+
+    def __init__(self, http: HTTPOutbound) -> None:
+        self._http = http
+
+    async def send_frame(self, frame: dict) -> None:
+        ftype = frame.get("type")
+        if ftype not in _HTTP_PATHS:
+            logger.warning(
+                "http_transport: unknown frame type %r — dropping", ftype
+            )
+            return
+        path = _HTTP_PATHS[ftype]
+        if path is None:
+            # Intentionally not POSTed — see _HTTP_PATHS comments above.
+            logger.debug(
+                "http_transport: frame type %r has no POST path — skipping",
+                ftype,
+            )
+            return
+        payload = frame.get("payload") or {}
+        # tool_call and delivery_ack include the request_id / message_id
+        # inside payload already; the backend POST schemas mirror those
+        # fields, so passing the inner payload as the JSON body is enough.
+        await self._http.post(path, payload)
+
+    async def close(self) -> None:
+        await self._http.close()
+
+
+class WSTransport:
+    """Transport wrapping a connected ``WSClient`` for the legacy path."""
+
+    def __init__(self, ws: "WSClient") -> None:
+        self._ws = ws
+
+    async def send_frame(self, frame: dict) -> None:
+        await self._ws.send(frame)
+
+    async def close(self) -> None:
+        # WSClient is owned (entered/exited) by ``run_forever`` in the
+        # legacy path; closing here would race with the supervisor. Keep
+        # this a no-op so the supervisor stays in charge of the lifecycle.
+        return