pi-browser-harness 0.5.0 → 0.8.0

package/src/daemon/server.ts

@@ -0,0 +1,245 @@
+/**
+ * IpcServer — Unix socket server for daemon ↔ pi client communication.
+ *
+ * Accepts multiple concurrent pi client connections over a Unix domain
+ * stream socket. Uses newline-delimited JSON framing (one WireMessage per
+ * line). Clients are anonymous until they send a control.register message.
+ *
+ * References:
+ *   - net.createServer: https://nodejs.org/docs/latest-v22.x/api/net.html#netcreateserveroptions-connectionlistener
+ *   - server.listen(path): https://nodejs.org/docs/latest-v22.x/api/net.html#serverlistenpath-backlog-callback
+ *   - readline.createInterface: https://nodejs.org/docs/latest-v22.x/api/readline.html#readlinecreateinterfaceoptions
+ */
+
+import { createServer, type Server, type Socket } from "node:net";
+import { createInterface, type Interface } from "node:readline";
+import { unlinkSync } from "node:fs";
+import {
+  type WireMessage,
+  type WireControl,
+  isWireMessage,
+  DAEMON_SOCKET_PATH,
+  DAEMON_MAX_CLIENTS,
+  DAEMON_STALE_SOCKET_CLEANUP,
+} from "./protocol";
+
+// ── Types ──────────────────────────────────────────────────────────────────────
+
+/** A connected pi client with its socket and readline interface. */
+export type ClientSocket = {
+  /** Client namespace (set after registration). */
+  id: string;
+  readonly socket: Socket;
+  readonly rl: Interface;
+  /** True once a valid control.register message has been received. */
+  registered: boolean;
+};
+
+/** Handler called for every valid WireMessage from any client. */
+export type MessageHandler = (msg: WireMessage, client: ClientSocket) => void;
+
+/** Handler called when a client connects or disconnects. */
+export type ConnectionHandler = (client: ClientSocket) => void;
+
+export type IpcServer = {
+  /** Start listening on the Unix socket. Resolves when the server is ready. */
+  start(): Promise<void>;
+  /** Close the server and all client connections. Resolves when cleaned up. */
+  stop(): Promise<void>;
+  /** Register a handler for incoming WireMessages from any client. */
+  onMessage(handler: MessageHandler): void;
+  /** Register a handler for client connect events. */
+  onConnect(handler: ConnectionHandler): void;
+  /** Register a handler for client disconnect events. */
+  onDisconnect(handler: ConnectionHandler): void;
+  /** Send a WireMessage to a specific client by clientId. No-op if client not found. */
+  send(clientId: string, msg: WireMessage): void;
+  /** Send a WireMessage to all connected (registered) clients. */
+  broadcast(msg: WireMessage): void;
+  /** Force-disconnect a client by clientId. */
+  disconnectClient(clientId: string): void;
+  /** All currently connected clients (by clientId). */
+  clients(): ReadonlyMap<string, ClientSocket>;
+  /** Number of connected clients. */
+  clientCount(): number;
+};
+
+// ── Implementation ─────────────────────────────────────────────────────────────
+
+export const createIpcServer = (): IpcServer => {
+  let server: Server | null = null;
+  // Anonymous temp-id → ClientSocket. Temp ids are assigned before registration.
+  const sockets = new Map<string, ClientSocket>();
+  // clientId → ClientSocket (only registered clients).
+  const clients = new Map<string, ClientSocket>();
+  let messageHandler: MessageHandler | null = null;
+  let connectHandler: ConnectionHandler | null = null;
+  let disconnectHandler: ConnectionHandler | null = null;
+  let nextTempId = 1;
+
+  const start = (): Promise<void> =>
+    new Promise((resolve, reject) => {
+      // Clean up stale socket file from a previous crashed daemon.
+      if (DAEMON_STALE_SOCKET_CLEANUP) {
+        try {
+          unlinkSync(DAEMON_SOCKET_PATH);
+        } catch {
+          // ENOENT is expected — no stale socket to clean.
+        }
+      }
+
+      server = createServer({ pauseOnConnect: false }, (socket: Socket) => {
+        // Reject if at capacity (check registered clients only).
+        if (clients.size >= DAEMON_MAX_CLIENTS) {
+          socket.destroy();
+          return;
+        }
+
+        const tempId = `anon-${nextTempId++}`;
+        const rl = createInterface({ input: socket, crlfDelay: Infinity });
+        let currentClient: ClientSocket = { id: tempId, socket, rl, registered: false };
+
+        sockets.set(tempId, currentClient);
+
+        // Buffer for partial lines. readline handles line splitting, but
+        // we store a reference in case we need it for diagnostics.
+        let lineCount = 0;
+
+        rl.on("line", (line: string) => {
+          lineCount++;
+          let msg: WireMessage | null;
+          try {
+            const parsed: unknown = JSON.parse(line);
+            if (!isWireMessage(parsed)) return;
+            msg = parsed;
+          } catch {
+            // Malformed JSON — ignore silently, don't crash the daemon.
+            return;
+          }
+
+          // Handle registration before forwarding to the general handler.
+          if (msg.type === "control" && msg.action === "register" && !currentClient.registered) {
+            const clientId = msg.clientId;
+            if (!clientId) return;
+
+            // If a client with this id is already connected, reject.
+            if (clients.has(clientId)) {
+              const reply: WireControl = {
+                type: "control",
+                action: "shutdown",
+                reason: `clientId ${clientId} is already connected`,
+              };
+              socket.write(JSON.stringify(reply) + "\n");
+              socket.destroy();
+              return;
+            }
+
+            // Promote from anonymous to registered — mutate in place so the
+            // closure captures the updated client reference.
+            sockets.delete(tempId);
+            currentClient.id = clientId;
+            currentClient.registered = true;
+            clients.set(clientId, currentClient);
+
+            // Acknowledge registration.
+            const ack: WireControl = { type: "control", action: "registered", clientId };
+            socket.write(JSON.stringify(ack) + "\n");
+
+            connectHandler?.(currentClient);
+            return;
+          }
+
+          // Require registration for non-control messages.
+          if (!currentClient.registered) return;
+
+          // Forward to the general handler.
+          messageHandler?.(msg, currentClient);
+        });
+
+        socket.on("error", () => {
+          // Socket errors (ECONNRESET, etc.) are handled by the 'close' event.
+        });
+
+        socket.on("close", () => {
+          // Clean up whichever map the client is in.
+          const found = clients.get(currentClient.id) ?? sockets.get(tempId);
+          if (found) {
+            clients.delete(found.id);
+            sockets.delete(tempId);
+            // Notify if this was a registered client.
+            if (found.registered) {
+              disconnectHandler?.(found);
+            }
+          }
+          // Clean up readline to prevent memory leaks.
+          rl.close();
+        });
+      });
+
+      server.on("error", (err: NodeJS.ErrnoException) => {
+        reject(err);
+      });
+
+      server.listen(DAEMON_SOCKET_PATH, () => {
+        resolve();
+      });
+    });
+
+  const stop = (): Promise<void> =>
+    new Promise((resolve) => {
+      // Close all client sockets.
+      for (const [, client] of clients) {
+        try { client.socket.destroy(); } catch { /* best-effort */ }
+      }
+      for (const [, client] of sockets) {
+        try { client.socket.destroy(); } catch { /* best-effort */ }
+      }
+      clients.clear();
+      sockets.clear();
+
+      if (server) {
+        server.close(() => {
+          // Clean up the socket file.
+          try { unlinkSync(DAEMON_SOCKET_PATH); } catch { /* ENOENT ok */ }
+          resolve();
+        });
+      } else {
+        resolve();
+      }
+    });
+
+  return {
+    start,
+    stop,
+    onMessage(handler) {
+      messageHandler = handler;
+    },
+    onConnect(handler) {
+      connectHandler = handler;
+    },
+    onDisconnect(handler) {
+      disconnectHandler = handler;
+    },
+    send(clientId, msg) {
+      const client = clients.get(clientId);
+      if (!client) return;
+      try {
+        client.socket.write(JSON.stringify(msg) + "\n");
+      } catch {
+        // Socket write error — the close handler will clean up.
+      }
+    },
+    broadcast(msg) {
+      for (const [clientId] of clients) {
+        this.send(clientId, msg);
+      }
+    },
+    disconnectClient(clientId) {
+      const client = clients.get(clientId);
+      if (!client) return;
+      client.socket.destroy();
+    },
+    clients: (): ReadonlyMap<string, ClientSocket> => clients,
+    clientCount: (): number => clients.size,
+  };
+};

package/src/daemon/spawn.ts

@@ -0,0 +1,85 @@
+/**
+ * Daemon lifecycle helpers — spawn detection and lazy launch.
+ *
+ * The daemon is a standalone long-lived process that owns the single WebSocket
+ * to Chrome. pi instances connect to it via a Unix domain socket. This module
+ * provides the plumbing to detect a running daemon and spawn one if needed.
+ *
+ * Reference:
+ *   - child_process.spawn detached: https://nodejs.org/docs/latest-v22.x/api/child_process.html#optionsdetached
+ */
+
+import { spawn, type ChildProcess } from "node:child_process";
+import { access } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+import { DAEMON_SOCKET_PATH } from "./protocol";
+
+const moduleDir = dirname(fileURLToPath(import.meta.url));
+
+/**
+ * Check whether the daemon is running by testing if its Unix socket exists.
+ * The socket file is created by the daemon on startup and removed on shutdown.
+ */
+export const isDaemonRunning = async (): Promise<boolean> => {
+  try {
+    await access(DAEMON_SOCKET_PATH);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * Spawn the daemon as a detached child process. The daemon will continue
+ * running after the parent pi process exits.
+ *
+ * The daemon entry point is resolved relative to this module. When running
+ * from source (tsx), the entry is src/daemon/index.ts. When running compiled,
+ * it's dist/daemon/index.js.
+ *
+ * Returns the child process handle (caller should call child.unref()).
+ */
+export const spawnDaemon = (): ChildProcess | null => {
+  // Resolve the daemon entry point and tsx binary.
+  const daemonScript = join(moduleDir, "index.ts");
+  const tsxBin = join(moduleDir, "..", "..", "node_modules", ".bin", "tsx");
+
+  // Use tsx to run the daemon TypeScript source directly.
+  // The spawed process inherits the parent's env so module resolution works.
+  // Fall back to `npx tsx` if the local binary isn't available.
+  const cmd = existsSync(tsxBin) ? tsxBin : "npx";
+  const args = existsSync(tsxBin) ? [daemonScript] : ["tsx", daemonScript];
+
+  const child = spawn(cmd, args, {
+    detached: true,
+    stdio: "ignore",
+    env: { ...process.env },
+  });
+
+  child.unref();
+  return child;
+};
+
+/**
+ * Ensure the daemon is running. If not, spawn it and wait (up to timeoutMs)
+ * for the socket file to appear.
+ *
+ * Returns true if the daemon is running (or was successfully spawned).
+ */
+export const ensureDaemon = async (timeoutMs = 10_000): Promise<boolean> => {
+  if (await isDaemonRunning()) return true;
+
+  const child = spawnDaemon();
+  if (!child) return false;
+
+  // Poll for the socket file to appear
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await isDaemonRunning()) return true;
+    await new Promise((r) => setTimeout(r, 200));
+  }
+
+  return false;
+};

package/src/domains/click.ts

@@ -1,13 +1,21 @@
-import { Type, type Static } from "typebox";
+import { Type } from "typebox";
 import { Text } from "@mariozechner/pi-tui";
 import type { BrowserClient } from "../client";
 import { Coords, MouseButton } from "../schemas/common";
 import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr } from "../util/tool";
 import { captureWithCrosshair } from "./screenshot";
+import { interactiveDiff, resolveRefToBox } from "./ref-resolve";
 
 const ClickArgs = Type.Object({
-  ...Coords.properties,
+  ref: Type.Optional(
+    Type.String({
+      description:
+        "Stable element ref from browser_snapshot (e.g. 'e12'). PREFERRED over x/y — survives re-renders. When set, x/y are ignored.",
+    }),
+  ),
+  x: Type.Optional(Coords.properties.x),
+  y: Type.Optional(Coords.properties.y),
   button: Type.Optional(MouseButton),
   count: Type.Optional(
     Type.Integer({
@@ -18,20 +26,27 @@ const ClickArgs = Type.Object({
     }),
   ),
 });
-type ClickArgsT = Static<typeof ClickArgs>;
 
 const dispatchClick = async (
   client: BrowserClient,
-  args: ClickArgsT,
+  x: number,
+  y: number,
+  button: "left" | "right" | "middle",
+  count: number,
 ): Promise<Result<void, ToolErr>> => {
-  const button = args.button ?? "left";
-  const count = args.count ?? 1;
+  // Move the pointer to the target first. Some focus/hover handlers (and React
+  // synthetic events) only fire when a mousemove precedes the press, so without
+  // it a click can land without focusing the element.
+  const moved = await client.session().call("Input.dispatchMouseEvent", {
+    type: "mouseMoved", x, y,
+  });
+  if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message });
   const pressed = await client.session().call("Input.dispatchMouseEvent", {
-    type: "mousePressed", x: args.x, y: args.y, button, clickCount: count,
+    type: "mousePressed", x, y, button, clickCount: count,
   });
   if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
   const released = await client.session().call("Input.dispatchMouseEvent", {
-    type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count,
+    type: "mouseReleased", x, y, button, clickCount: count,
   });
   if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
   return ok(undefined);
@@ -41,34 +56,55 @@ export const clickTool = defineBrowserTool({
   name: "browser_click",
   label: "Browser Click",
   description:
-    "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.",
-  promptSnippet: "Click at pixel coordinates (x, y) on the page",
+    "Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.",
+  promptSnippet: "Click an element by ref (preferred) or pixel coordinates",
   promptGuidelines: [
-    "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.",
-    "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.",
-    "Coordinates are viewport CSS pixels (not device pixels).",
-    "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.",
-    "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
-    "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).",
-    "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
+    "PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.",
+    "Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.",
+    "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
+    "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
+    "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
+    "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.",
   ],
   parameters: ClickArgs,
-  renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0),
+  renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0),
   async handler(args, { client }) {
-    const clicked = await dispatchClick(client, args);
+    const button = args.button ?? "left";
+    const count = args.count ?? 1;
+
+    // Resolve the click point: ref re-resolves to the element's CURRENT box
+    // (so re-renders don't matter); otherwise use the literal x/y.
+    let x: number;
+    let y: number;
+    if (args.ref !== undefined) {
+      const box = await resolveRefToBox(client, args.ref);
+      if (!box.success) return box;
+      x = box.data.cx;
+      y = box.data.cy;
+    } else if (args.x !== undefined && args.y !== undefined) {
+      x = args.x;
+      y = args.y;
+    } else {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." });
+    }
+
+    const clicked = await dispatchClick(client, x, y, button, count);
     if (!clicked.success) return clicked;
+
+    const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`;
+    const diff = await interactiveDiff(client);
     if (process.env["BH_DEBUG_CLICKS"]) {
-      const debug = await captureWithCrosshair(client, { x: args.x, y: args.y });
+      const debug = await captureWithCrosshair(client, { x, y });
       if (debug.success) {
         return ok({
-          text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`,
-          details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y },
+          text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`,
+          details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref },
         });
       }
     }
     return ok({
-      text: `Clicked at (${args.x}, ${args.y})`,
-      details: { x: args.x, y: args.y },
+      text: `Clicked at ${target}${diff}`,
+      details: { x, y, ref: args.ref },
     });
   },
 });

package/src/domains/files.ts

@@ -6,12 +6,28 @@ import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
 import { safeJs } from "../util/js-template";
 import { pdfPath } from "../util/paths";
+import { resolveRefToBackendId } from "./ref-resolve";
 
 const UploadArgs = Type.Object({
-  selector: Type.String({ description: "CSS selector of the file <input>" }),
+  ref: Type.Optional(
+    Type.String({ description: "Stable ref of the file <input> from browser_snapshot. PREFERRED over selector." }),
+  ),
+  selector: Type.Optional(Type.String({ description: "CSS selector of the file <input> (fallback when no ref)" })),
   filePath: Type.String({ description: "Absolute path to the file to upload" }),
 });
 
+// Ref path: set files directly on the resolved backendNodeId via CDP — simplest
+// and most reliable, no selector needed.
+const tryRefUpload = async (
+  client: BrowserClient,
+  backendId: number,
+  filePath: string,
+): Promise<Result<void, ToolErr>> => {
+  const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId });
+  if (!set.success) return err({ kind: "cdp_error", message: set.error.message });
+  return ok(undefined);
+};
+
 const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => {
   try {
     await access(filePath, constants.R_OK);
@@ -89,16 +105,29 @@ const jsFallbackUpload = async (
 export const uploadFileTool = defineBrowserTool({
   name: "browser_upload_file",
   label: "Browser Upload File",
-  description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages.",
-  promptSnippet: "Upload a file to a file input",
+  description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.",
+  promptSnippet: "Upload a file to a file input by ref (preferred) or selector",
   promptGuidelines: [
+    "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)",
     "File path must be absolute and readable.",
-    "Selector must match a file input.",
   ],
   parameters: UploadArgs,
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
     const readable = await verifyReadable(args.filePath);
     if (!readable.success) return readable;
+
+    // Ref path: resolve to backendNodeId and set files directly via CDP.
+    if (args.ref !== undefined) {
+      const backendId = resolveRefToBackendId(client, args.ref);
+      if (!backendId.success) return backendId;
+      const up = await tryRefUpload(client, backendId.data, args.filePath);
+      if (!up.success) return up;
+      return ok({ text: `Uploaded ${args.filePath} via CDP (ref ${args.ref})`, details: { mode: "cdp", ref: args.ref, filePath: args.filePath } });
+    }
+
+    if (args.selector === undefined) {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+    }
     const cdp = await tryCdpUpload(client, args.selector, args.filePath);
     if (cdp.success) {
       return ok({