smol-symphony 0.2.0 → 0.3.0

package/scripts/vm-agent.mjs

@@ -1,38 +1,65 @@
 #!/usr/bin/env node
 // symphony-vm-agent — in-guest proxy that runs the ACP adapter and bridges its stdio
-// to a TCP socket dialed back to the host orchestrator.
+// to a WebSocket dialed back to the host orchestrator's unified HTTP server.
 //
-// Why TCP and not stdio: the previous version of this proxy bridged the in-VM adapter's
-// stdio to the VM-exec stdio channel directly. That channel has a stdin-pump bug
-// that wedges the adapter after the SDK's first `available_commands_update` notification
-// (the in-guest reader never wakes for subsequent kernel events unless host stdin keeps
-// writing). We worked around it for a while with a 1.5 s `\n` keepalive on the host
-// side, but the proper fix is to stop piping ACP through the VM-exec stdio at all. This proxy
-// dials back to a TCP listener on the host (`SYMPHONY_ACP_URL`), authenticates with a
-// per-dispatch bearer token (`SYMPHONY_ACP_TOKEN`), and from then on talks ACP over a
-// plain socket the host owns. the VM exec is reduced to a launcher: its stdio just
-// carries this proxy's own diagnostic stderr and the adapter's stderr (`inherit`'d).
+// Why a WebSocket and not stdio or raw TCP: the original proxy piped the in-VM adapter's
+// stdio through the VM-exec stdio channel, which has a stdin-pump bug that wedges the
+// adapter after the SDK's first notification. That was replaced by a raw-TCP bridge on a
+// dedicated host port; this version collapses that bridge onto the SAME host listener
+// that already serves MCP + the dashboard — ACP now rides a `/acp` WebSocket `Upgrade`
+// reached through the SAME `tcp.hosts` tunnel MCP uses (the guest dials
+// `ws://symphony-mcp:7001/acp`). One mapping, one synthetic host, one host port; the
+// VM-exec stdio is reduced to a launcher whose stderr carries only diagnostics.
 //
-// The same proxy works under any sandbox tech that can launch a process with env vars
-// and reach the host loopback — that's the portability win.
+// The proxy authenticates with a per-dispatch bearer sent as the FIRST WebSocket
+// message (the WHATWG `WebSocket` client cannot set custom upgrade headers, so a header
+// bearer is not an option for the guest dialer). The auth frame is
+// `{"type":"auth","token":"<token>","proto":1}`; the host validates the bearer
+// constant-time and then replies `{"type":"ready","proto":1}` BEFORE any ACP byte
+// flows. Only after that ready frame does the proxy spawn the adapter and start pumping
+// ACP JSON-RPC bytes as WebSocket frames: inbound frame payloads are written to the
+// adapter's stdin; the adapter's stdout is sent as binary frames. Each chunk is
+// forwarded straight through as it arrives — NEVER buffered — preserving streaming
+// (issue 135). Backpressure pauses the SOURCE stream (child.stdout or the WS receive
+// side), it never accumulates a frame/turn before forwarding.
+//
+// Liveness: the proxy sends a WS PING every ~15 s and tracks the last pong; if no pong
+// arrives for ~3 intervals the half-open socket is torn down so the dispatch fails fast
+// instead of hanging until the host's prompt timeout. Both ends ping independently, so a
+// long silent tool call (no ACP frames) does NOT trip the watchdog.
 //
 // Configuration (all required unless noted):
-//   SYMPHONY_ACP_URL          — `tcp://host:port` to dial. `127.0.0.1:8788` under the Gondolin bridge mapping
-//                               since the guest loopback is rewritten to the host
-//                               loopback.
-//   SYMPHONY_ACP_TOKEN        — opaque per-dispatch bearer; sent as `Bearer <token>\n`
-//                               first line.
+//   SYMPHONY_ACP_URL          — `ws://host:port/acp` (or `wss://…`) to dial. Under the
+//                               Gondolin tunnel this is `ws://symphony-mcp:7001/acp`.
+//   SYMPHONY_ACP_TOKEN        — opaque per-dispatch bearer; sent as the first WS message
+//                               `{"type":"auth","token":"<token>","proto":1}`.
 //   SYMPHONY_ADAPTER_BIN      — adapter executable to spawn (PATH-resolved).
 //   SYMPHONY_ADAPTER_ARGS     — JSON array of extra argv. Optional; defaults to `[]`.
+//   SYMPHONY_HEARTBEAT_INTERVAL_MS — optional; WS ping interval. Defaults to 15000.
+//   SYMPHONY_HEARTBEAT_TIMEOUT_MS  — optional; dead-peer threshold. Defaults to 45000.
 //   SYMPHONY_VM_AGENT_DEBUG   — optional; truthy → log lifecycle to stderr.
 //
 // Exit code: mirrors the adapter's exit when it exits, or 1 if the connection fails
 // before the adapter starts.
 
-import { connect as netConnect } from 'node:net';
 import { spawn } from 'node:child_process';
 import { constants as osConstants } from 'node:os';
 import { URL } from 'node:url';
+import { Buffer } from 'node:buffer';
+
+// Wire protocol version exchanged in the auth/ready handshake; bumped lockstep with the
+// host (src/acp-ws.ts). The launcher and host now move together (PR-A injects this file
+// from the host's own disk at dispatch), so a single integer suffices.
+const PROTO = 1;
+
+// Outbound backpressure (child.stdout → ws.send): when the WebSocket's send queue grows
+// past the high-water mark we PAUSE child.stdout (stops the source — never accumulates a
+// frame on our side) and resume once the queue drains below the low-water mark. WHATWG
+// WebSocket exposes no 'drain' event, only `bufferedAmount`, so the drain is detected by a
+// short poll. The marks are generous so a normal streaming turn never pauses.
+const WS_HIGH_WATER_BYTES = 1 << 20; // 1 MiB queued in the WS send buffer → pause source
+const WS_LOW_WATER_BYTES = 256 * 1024; // resume once it drains below this
+const WS_DRAIN_POLL_MS = 10;
 
 const requiredEnv = (name) => {
   const v = process.env[name];
@@ -43,9 +70,18 @@ const requiredEnv = (name) => {
   return v;
 };
 
+const intEnv = (name, dflt) => {
+  const raw = process.env[name];
+  if (!raw) return dflt;
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : dflt;
+};
+
 const acpUrl = requiredEnv('SYMPHONY_ACP_URL');
 const acpToken = requiredEnv('SYMPHONY_ACP_TOKEN');
 const adapterBin = requiredEnv('SYMPHONY_ADAPTER_BIN');
+const heartbeatIntervalMs = intEnv('SYMPHONY_HEARTBEAT_INTERVAL_MS', 15_000);
+const heartbeatTimeoutMs = intEnv('SYMPHONY_HEARTBEAT_TIMEOUT_MS', 45_000);
 
 let adapterArgs = [];
 const rawArgs = process.env.SYMPHONY_ADAPTER_ARGS;
@@ -74,83 +110,278 @@ try {
   process.stderr.write(`symphony-vm-agent: SYMPHONY_ACP_URL invalid: ${err.message}\n`);
   process.exit(2);
 }
-if (parsed.protocol !== 'tcp:') {
+if (parsed.protocol !== 'ws:' && parsed.protocol !== 'wss:') {
   process.stderr.write(
-    `symphony-vm-agent: SYMPHONY_ACP_URL must use tcp:// scheme (got ${parsed.protocol})\n`,
+    `symphony-vm-agent: SYMPHONY_ACP_URL must use ws:// or wss:// scheme (got ${parsed.protocol})\n`,
   );
   process.exit(2);
 }
-const acpHost = parsed.hostname;
-const acpPort = parseInt(parsed.port, 10);
-if (!Number.isFinite(acpPort) || acpPort <= 0) {
-  process.stderr.write(`symphony-vm-agent: SYMPHONY_ACP_URL has no port\n`);
+
+if (typeof WebSocket !== 'function') {
+  process.stderr.write('symphony-vm-agent: global WebSocket is unavailable (need Node >= 21)\n');
   process.exit(2);
 }
 
-log(`dialing ${acpHost}:${acpPort}`);
+log(`dialing ${acpUrl}`);
 
-const socket = netConnect({ host: acpHost, port: acpPort, allowHalfOpen: false });
-socket.setNoDelay(true);
+let child = null;
+// The adapter's exit code/signal, captured on `child` close so we can flush the WebSocket
+// before exiting. Null until the adapter exits.
+let pendingExit = null;
+let finalized = false;
+let opened = false;
+// The host must send `{type:'ready',proto:1}` after validating the bearer; only then do we
+// spawn the adapter and start the ACP byte pump. Guards against starting the adapter before
+// the host has accepted us (and lets the host gate the spawn until it is ready to relay).
+let ready = false;
+let bridgeStarted = false;
 
-// Pre-connect error handler: a TCP-level failure here means we couldn't reach the host
-// bridge at all. Exit hard so symphony's host-side bridge-connect-timeout fires its own
-// error path. We REMOVE this listener once `connect` fires; otherwise it would also kick
-// in for post-auth errors and bypass startBridge()'s graceful adapter teardown.
-const onPreConnectError = (err) => {
-  process.stderr.write(`symphony-vm-agent: socket error: ${err.message}\n`);
-  process.exit(1);
-};
-socket.once('error', onPreConnectError);
-
-socket.once('connect', () => {
-  socket.off('error', onPreConnectError);
-  log('connected; sending bearer line');
-  // The bearer line MUST end with a bare \n; the host expects exactly that delimiter.
-  socket.write(`Bearer ${acpToken}\n`);
-  startBridge();
+// --- Heartbeat (WS ping / pong watchdog) ----------------------------------------------
+// Both ends ping independently. We send an application PING frame every interval and record
+// the last pong arrival; if the gap exceeds the dead-peer threshold we tear the socket down
+// so the dispatch fails fast (host classifies the drop as a transport_error). The watchdog
+// is INDEPENDENT of ACP frames, so a long silent tool call does not trip it. Every timer is
+// cleared on close (no dangling interval after teardown).
+let pingTimer = null;
+let lastPongAt = 0;
+function startHeartbeat() {
+  lastPongAt = Date.now();
+  pingTimer = setInterval(() => {
+    if (ws.readyState !== WebSocket.OPEN) return;
+    if (Date.now() - lastPongAt > heartbeatTimeoutMs) {
+      log(`no pong within ${heartbeatTimeoutMs}ms; tearing down (dead host)`);
+      stopHeartbeat();
+      try {
+        ws.close(1001, 'heartbeat timeout');
+      } catch {
+        /* ignore */
+      }
+      // Belt-and-braces: a half-open socket may never deliver `close`; kill the adapter
+      // and finalize so the dispatch unwinds rather than hanging.
+      if (child) {
+        try {
+          child.kill('SIGTERM');
+        } catch {
+          /* ignore */
+        }
+      }
+      const t = setTimeout(finalize, 2_000);
+      if (typeof t.unref === 'function') t.unref();
+      return;
+    }
+    try {
+      ws.send(JSON.stringify({ type: 'ping', t: Date.now() }));
+    } catch {
+      /* socket closing — the close handler clears the timer */
+    }
+  }, heartbeatIntervalMs);
+  if (typeof pingTimer.unref === 'function') pingTimer.unref();
+}
+function stopHeartbeat() {
+  if (pingTimer) {
+    clearInterval(pingTimer);
+    pingTimer = null;
+  }
+}
+
+const ws = new WebSocket(acpUrl);
+ws.binaryType = 'arraybuffer';
+
+function finalize() {
+  if (finalized) return;
+  finalized = true;
+  stopHeartbeat();
+  stopOutboundDrainPoll();
+  if (pendingExit && pendingExit.signal) {
+    const sig = osConstants.signals[pendingExit.signal] ?? 0;
+    process.exit(128 + sig);
+  }
+  process.exit(pendingExit ? (pendingExit.code ?? 0) : 0);
+}
+
+ws.addEventListener('open', () => {
+  opened = true;
+  log('connected; sending auth message');
+  // The bearer is the FIRST message; the host validates it before resolving the session,
+  // then replies `{type:'ready',proto:1}`. We spawn the adapter only after that ready frame.
+  ws.send(JSON.stringify({ type: 'auth', token: acpToken, proto: PROTO }));
+  startHeartbeat();
+});
+
+ws.addEventListener('error', (ev) => {
+  const msg = ev?.message || (ev?.error && ev.error.message) || 'websocket error';
+  if (!opened) {
+    // Couldn't reach the host at all → exit hard so symphony's connect-timeout error path
+    // fires (mirrors the raw bridge's pre-connect error handling).
+    process.stderr.write(`symphony-vm-agent: websocket error before open: ${msg}\n`);
+    process.exit(1);
+  }
+  log(`websocket error after open: ${msg}`);
+  if (child) {
+    try {
+      child.kill('SIGTERM');
+    } catch {
+      /* ignore */
+    }
+  }
+});
+
+ws.addEventListener('close', (ev) => {
+  log(`websocket closed code=${ev?.code}`);
+  stopHeartbeat();
+  stopOutboundDrainPoll();
+  // Adapter already exited: the close completes our flush → exit with its code.
+  if (pendingExit) return finalize();
+  // Host closed first (cancel / attempt teardown): end the adapter's stdin so it unwinds
+  // (parity with the raw socket half-close), and kill it as a belt-and-braces.
+  if (child) {
+    try {
+      child.stdin.end();
+    } catch {
+      /* ignore */
+    }
+    try {
+      child.kill('SIGTERM');
+    } catch {
+      /* ignore */
+    }
+  }
+});
+
+// The control + handshake frame handler. Installed before `startBridge()` so the ready
+// frame (and pongs) are observed even before the adapter exists. `startBridge()` adds a
+// SECOND `message` listener for ACP bytes once ready; this one only consumes JSON control
+// frames (auth-ready handshake + pong) and ignores everything else, so it never competes
+// with the ACP byte pump for opaque adapter bytes.
+ws.addEventListener('message', (ev) => {
+  const ctrl = parseControlFrame(ev.data);
+  if (!ctrl) return; // opaque ACP bytes — handled by startBridge's pump (once ready)
+  if (ctrl.type === 'ready') {
+    if (ready) return;
+    ready = true;
+    log('host ready; spawning adapter');
+    startBridge();
+    return;
+  }
+  if (ctrl.type === 'pong' || ctrl.type === 'ping') {
+    // Any control traffic proves the peer is alive; refresh the watchdog. We reply to a
+    // host ping with a pong so the host's own watchdog stays satisfied symmetrically.
+    lastPongAt = Date.now();
+    if (ctrl.type === 'ping') {
+      try {
+        ws.send(JSON.stringify({ type: 'pong', t: Date.now() }));
+      } catch {
+        /* socket closing */
+      }
+    }
+  }
 });
 
+// Parse a host→guest control frame. Returns `{type}` for our small JSON control vocabulary
+// (`ready`/`ping`/`pong`), or null when the payload is opaque ACP bytes (binary frame, or
+// text that is not one of our control messages). Kept deliberately strict so adapter JSON
+// that happens to be text can never be mistaken for a control frame: ACP bytes are sent by
+// the host as BINARY frames (ArrayBuffer here), so a string payload that parses to a known
+// control `type` is the only thing treated as control.
+function parseControlFrame(data) {
+  if (typeof data !== 'string') return null; // binary → ACP bytes
+  let msg;
+  try {
+    msg = JSON.parse(data);
+  } catch {
+    return null;
+  }
+  if (msg && (msg.type === 'ready' || msg.type === 'ping' || msg.type === 'pong')) {
+    return { type: msg.type };
+  }
+  return null;
+}
+
+// --- Outbound backpressure poll (child.stdout → ws.send) -------------------------------
+let outboundDrainTimer = null;
+function stopOutboundDrainPoll() {
+  if (outboundDrainTimer) {
+    clearInterval(outboundDrainTimer);
+    outboundDrainTimer = null;
+  }
+}
+
 function startBridge() {
+  if (bridgeStarted) return;
+  bridgeStarted = true;
   log(`spawn ${adapterBin} ${JSON.stringify(adapterArgs)}`);
-  const child = spawn(adapterBin, adapterArgs, {
+  child = spawn(adapterBin, adapterArgs, {
     // Adapter stdio: kernel pipes we fully own. Adapter stderr is inherited so any
     // crashes / warnings show up on this proxy's stderr → VM-exec stderr → host
-    // orchestrator stderr capture. ACP frames flow ONLY over the TCP socket.
+    // orchestrator stderr capture. ACP frames flow ONLY over the WebSocket.
     stdio: ['pipe', 'pipe', 'inherit'],
   });
-  // Bridge socket ↔ adapter stdio with explicit `data` handlers + `write()` (NOT
-  // stream.pipe()): we observed during the bisect that pipe() can interact badly with
-  // certain transport quirks, so we forward bytes synchronously and let the kernel pipe
-  // buffer absorb any short-term mismatch. With a clean TCP socket this is moot but
-  // costs nothing and keeps the pattern uniform.
-  socket.on('data', (chunk) => {
+
+  // Inbound (host → adapter): each WS binary frame's payload is written to the adapter's
+  // stdin AS IT ARRIVES — never buffered. We deliberately do NOT track `write()`'s backpressure
+  // return here: the WHATWG WebSocket receive side has no pause/pull API, so there is nothing to
+  // throttle locally. The real backpressure is end-to-end instead — a slow adapter fills its
+  // stdin pipe, then the guest's TCP receive window, which pauses the HOST's socket reads
+  // (src/acp-ws.ts) where the source actually lives. We never accumulate a frame here; Node's
+  // own bounded stdin buffer absorbs the in-flight bytes during the brief drain window.
+  ws.addEventListener('message', (ev) => {
+    const data = ev.data;
+    if (typeof data === 'string') {
+      // A string frame post-ready is one of our control messages (handled by the control
+      // listener above), never ACP bytes. Skip it here so control traffic is not written
+      // into the adapter's stdin.
+      if (parseControlFrame(data)) return;
+    }
+    const buf = typeof data === 'string' ? Buffer.from(data, 'utf8') : Buffer.from(data);
     try {
-      child.stdin.write(chunk);
+      // A `false` return (kernel pipe momentarily full) is intentionally not acted on — see
+      // above; Node buffers the write and the host-side socket pause is what throttles the source.
+      child.stdin.write(buf);
     } catch {
       /* adapter stdin closed while we were writing — see exit handler */
     }
   });
+
+  // Outbound (adapter → host): adapter stdout is sent as binary frames. Each chunk is
+  // forwarded as it arrives — never buffered (issue 135). Backpressure GATES on the WS send
+  // queue: when `ws.bufferedAmount` exceeds the high-water mark we PAUSE child.stdout (the
+  // SOURCE) and resume once the queue drains below the low-water mark. Pausing the source —
+  // not accumulating — is what keeps this from re-introducing the issue-135 full-buffering
+  // stall while still bounding memory if the host socket stalls.
+  const maybeResumeStdout = () => {
+    if (outboundDrainTimer && ws.bufferedAmount <= WS_LOW_WATER_BYTES) {
+      stopOutboundDrainPoll();
+      if (!child.stdout.destroyed) child.stdout.resume();
+    }
+  };
   child.stdout.on('data', (chunk) => {
+    if (ws.readyState !== WebSocket.OPEN) return;
     try {
-      socket.write(chunk);
+      // Send a tight ArrayBuffer copy of exactly the chunk's bytes as one binary frame.
+      // `chunk` is a Buffer that may be a view into Node's shared allocation pool (non-zero
+      // byteOffset); slicing its backing ArrayBuffer yields a fresh buffer of exactly the
+      // chunk's data, so no surrounding pool bytes can ever leak into the frame regardless
+      // of how the WebSocket impl treats ArrayBufferView offsets.
+      ws.send(chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength));
     } catch {
-      /* host socket closed while we were writing — see exit handlers */
+      /* host socket closing while we were writing — see exit handlers */
     }
-  });
-  // Half-closes: when the host closes its write side, the adapter should see stdin EOF;
-  // when the adapter closes stdout, the host should see socket-half-close.
-  socket.on('end', () => {
-    log('host closed socket; ending adapter stdin');
-    try {
-      child.stdin.end();
-    } catch {
-      /* ignore */
+    // Gate AFTER forwarding this chunk: the chunk is already on the wire, so we never hold
+    // it back; we only pause the NEXT read when the send queue is over the high-water mark.
+    if (ws.bufferedAmount > WS_HIGH_WATER_BYTES && !outboundDrainTimer) {
+      child.stdout.pause();
+      // WHATWG WebSocket has no 'drain'; poll `bufferedAmount` until it falls to low-water,
+      // then resume. The timer is cleared on resume AND on close/finalize (no dangling poll).
+      outboundDrainTimer = setInterval(maybeResumeStdout, WS_DRAIN_POLL_MS);
+      if (typeof outboundDrainTimer.unref === 'function') outboundDrainTimer.unref();
     }
   });
+  // Half-close: when the adapter closes stdout, close the WebSocket write side so the host
+  // sees end-of-stream.
   child.stdout.on('end', () => {
-    log('adapter closed stdout; ending socket write side');
+    log('adapter closed stdout; closing websocket');
     try {
-      socket.end();
+      ws.close(1000, 'adapter stdout end');
     } catch {
       /* ignore */
     }
@@ -158,44 +389,35 @@ function startBridge() {
   child.stdin.on('error', (err) => {
     if (err.code !== 'EPIPE') log(`adapter stdin error: ${err.message}`);
   });
-  socket.on('error', (err) => {
-    log(`socket error after auth: ${err.message}`);
-    try {
-      child.kill('SIGTERM');
-    } catch {
-      /* ignore */
-    }
-  });
   child.on('error', (err) => {
     process.stderr.write(`symphony-vm-agent: failed to spawn adapter: ${err.message}\n`);
     try {
-      socket.end();
+      ws.close(1011, 'spawn failed');
     } catch {
       /* ignore */
     }
     process.exit(127);
   });
-  // Use `close` not `exit`: `exit` fires when the adapter process terminates but its
-  // stdio pipes may still have buffered bytes our `data` handlers haven't drained yet.
-  // `close` fires only after all stdio streams have been closed AND drained, so by then
-  // every byte the adapter wrote has been forwarded to our socket.write() above. We
-  // still need to wait for the socket itself to flush those forwarded bytes to the
-  // kernel before exiting, hence the `socket.end(callback)` pattern: end() signals
-  // shutdown and the callback fires after the write buffer has drained.
+  // Use `close` not `exit`: `exit` fires when the adapter process terminates but its stdio
+  // pipes may still have buffered bytes our `data` handlers haven't drained yet. `close`
+  // fires only after all stdio streams have closed AND drained, so by then every byte the
+  // adapter wrote has been forwarded to ws.send() above. We then close the WebSocket and
+  // let its `close` event (or the grace timer) finalize the exit so the forwarded frames
+  // flush to the host before we terminate.
   child.on('close', (code, signal) => {
     log(`adapter close code=${code} signal=${signal}`);
-    const finalize = () => {
-      if (signal) {
-        const sig = osConstants.signals[signal] ?? 0;
-        process.exit(128 + sig);
-      }
-      process.exit(code ?? 0);
-    };
+    pendingExit = { code, signal };
+    stopHeartbeat();
+    stopOutboundDrainPoll();
     try {
-      socket.end(finalize);
+      ws.close(1000, 'adapter exit');
     } catch {
       finalize();
+      return;
     }
+    // Safety net: if the close handshake stalls, exit anyway rather than hang the dispatch.
+    const t = setTimeout(finalize, 2_000);
+    if (typeof t.unref === 'function') t.unref();
   });
   // Forward host-side termination signals to the adapter.
   for (const sig of ['SIGTERM', 'SIGINT', 'SIGHUP']) {