pi-browser-harness 0.6.0 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-browser-harness",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "Browser control extension for pi — navigate, click, type, screenshot, and extract data from real Chrome via CDP",
   "keywords": [
     "pi-package",

package/src/cdp/daemon-transport.ts

@@ -17,6 +17,7 @@ import { type Result, err, ok } from "../util/result";
 import { type CdpError, cdpError } from "../cdp/errors";
 import type { CdpTransport } from "../cdp/transport";
 import type { CdpEvent } from "../cdp/types";
+import { type Pending, makeEventQueue, makeOnClose, rejectAllPending, sendWithTimeout } from "./event-queue";
 import {
   DAEMON_SOCKET_PATH,
   type WireRequest,
@@ -25,47 +26,6 @@ import {
   serialize,
 } from "../daemon/protocol";
 
-// ── Types ──────────────────────────────────────────────────────────────────────
-
-type EventQueue = {
-  readonly push: (e: CdpEvent) => void;
-  readonly end: () => void;
-  readonly iter: AsyncIterable<CdpEvent>;
-};
-
-// ── Helpers ────────────────────────────────────────────────────────────────────
-
-const makeEventQueue = (): EventQueue => {
-  const buf: CdpEvent[] = [];
-  const waiters: Array<(v: IteratorResult<CdpEvent>) => void> = [];
-  let ended = false;
-  return {
-    push(e) {
-      if (ended) return;
-      const w = waiters.shift();
-      if (w) w({ value: e, done: false });
-      else buf.push(e);
-    },
-    end() {
-      ended = true;
-      for (const w of waiters.splice(0)) w({ value: undefined as unknown as CdpEvent, done: true });
-    },
-    iter: {
-      [Symbol.asyncIterator]() {
-        return {
-          next: (): Promise<IteratorResult<CdpEvent>> =>
-            new Promise((resolve) => {
-              const next = buf.shift();
-              if (next) resolve({ value: next, done: false });
-              else if (ended) resolve({ value: undefined as unknown as CdpEvent, done: true });
-              else waiters.push(resolve);
-            }),
-        };
-      },
-    },
-  };
-};
-
 // ── Implementation ─────────────────────────────────────────────────────────────
 
 export const createDaemonTransport = (clientId: string): CdpTransport => {
@@ -73,20 +33,12 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
   let rl: ReturnType<typeof createInterface> | null = null;
   let queue = makeEventQueue();
   const closeListeners = new Set<() => void>();
-  const pending = new Map<number, {
-    resolve: (v: Result<unknown, CdpError>) => void;
-    timer: ReturnType<typeof setTimeout>;
-    method: string;
-  }>();
+  const pending = new Map<number, Pending>();
   let registered = false;
   let nextRequestId = 1;
 
   const cleanup = (reason: string): void => {
-    for (const [, p] of pending) {
-      clearTimeout(p.timer);
-      p.resolve(err(cdpError("transport_closed", reason, p.method)));
-    }
-    pending.clear();
+    rejectAllPending(pending, reason);
     queue.end();
     queue = makeEventQueue();
     registered = false;
@@ -237,22 +189,7 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
       ...(sessionId !== null && sessionId !== undefined ? { sessionId } : {}),
     };
 
-    return new Promise((resolve) => {
-      const timer = setTimeout(() => {
-        pending.delete(id);
-        resolve(err(cdpError("timeout", `Daemon timeout after ${timeoutMs}ms: ${method}`, method)));
-      }, timeoutMs);
-
-      pending.set(id, { resolve, timer, method });
-
-      try {
-        socket!.write(serialize(req) + "\n");
-      } catch (e) {
-        clearTimeout(timer);
-        pending.delete(id);
-        resolve(err(cdpError("transport_closed", e instanceof Error ? e.message : String(e), method)));
-      }
-    });
+    return sendWithTimeout(pending, id, method, timeoutMs, "Daemon", () => socket!.write(serialize(req) + "\n"));
   };
 
   // ── Events ─────────────────────────────────────────────────────────────
@@ -268,10 +205,7 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
     return "open";
   };
 
-  const onClose = (cb: () => void): (() => void) => {
-    closeListeners.add(cb);
-    return () => closeListeners.delete(cb);
-  };
+  const onClose = makeOnClose(closeListeners);
 
   return { connect, close, request, events, state, onClose };
 };

package/src/cdp/event-queue.ts

@@ -0,0 +1,128 @@
+/**
+ * Shared transport primitives for CdpTransport implementations.
+ *
+ * Both the WebSocket transport (transport.ts, talking to Chrome directly) and
+ * the daemon transport (daemon-transport.ts, talking to the daemon over a Unix
+ * socket) need an identical single-consumer async event queue plus the same
+ * pending-request bookkeeping. This module is the single source of truth for
+ * that shared machinery.
+ */
+
+import { type Result, err } from "../util/result";
+import { type CdpError, cdpError } from "./errors";
+import type { CdpEvent } from "./types";
+
+export type CdpResult = Result<unknown, CdpError>;
+
+/**
+ * A single-consumer async queue of CDP events.
+ *
+ * `push` enqueues an event (dropped silently after `end`). `end` terminates the
+ * stream — any pending and future iterator `next()` calls resolve `done:true`.
+ * `iter` is the AsyncIterable consumed by one for-await loop; a second consumer
+ * would silently steal events from the first.
+ */
+export type EventQueue = {
+  readonly push: (e: CdpEvent) => void;
+  readonly end: () => void;
+  readonly iter: AsyncIterable<CdpEvent>;
+};
+
+export const makeEventQueue = (): EventQueue => {
+  const buf: CdpEvent[] = [];
+  const waiters: Array<(v: IteratorResult<CdpEvent>) => void> = [];
+  let ended = false;
+  return {
+    push(e) {
+      if (ended) return;
+      const w = waiters.shift();
+      if (w) w({ value: e, done: false });
+      else buf.push(e);
+    },
+    end() {
+      ended = true;
+      // value: undefined as unknown as CdpEvent is the correct iterator-protocol
+      // pattern: when done=true the value field is conventionally undefined but
+      // the TS type still requires CdpEvent.
+      for (const w of waiters.splice(0)) w({ value: undefined as unknown as CdpEvent, done: true });
+    },
+    iter: {
+      [Symbol.asyncIterator]() {
+        return {
+          next: (): Promise<IteratorResult<CdpEvent>> =>
+            new Promise((resolve) => {
+              const next = buf.shift();
+              if (next) resolve({ value: next, done: false });
+              else if (ended) resolve({ value: undefined as unknown as CdpEvent, done: true });
+              else waiters.push(resolve);
+            }),
+        };
+      },
+    },
+  };
+};
+
+/** An in-flight CDP request awaiting its response or timeout. */
+export type Pending = {
+  readonly resolve: (v: Result<unknown, CdpError>) => void;
+  readonly timer: ReturnType<typeof setTimeout>;
+  readonly method: string;
+};
+
+/**
+ * Reject every in-flight request with a transport_closed error and clear the
+ * map. Called from each transport's cleanup path when the underlying connection
+ * goes away.
+ */
+export const rejectAllPending = (pending: Map<number, Pending>, reason: string): void => {
+  for (const [, p] of pending) {
+    clearTimeout(p.timer);
+    p.resolve(err(cdpError("transport_closed", reason, p.method)));
+  }
+  pending.clear();
+};
+
+/**
+ * Send a CDP request and return a promise that settles when the matching
+ * response arrives, the request times out, or the send throws.
+ *
+ * The transport supplies a `send` callback that puts the already-serialized
+ * payload onto its wire (WebSocket frame or Unix-socket line); everything else
+ * — registering the pending entry, the timeout timer, and synchronous send
+ * failure handling — is identical across transports and lives here.
+ *
+ * `timeoutLabel` is the transport's prefix for the timeout error message
+ * (e.g. "CDP" or "Daemon").
+ */
+export const sendWithTimeout = (
+  pending: Map<number, Pending>,
+  id: number,
+  method: string,
+  timeoutMs: number,
+  timeoutLabel: string,
+  send: () => void,
+): Promise<CdpResult> =>
+  new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      pending.delete(id);
+      resolve(err(cdpError("timeout", `${timeoutLabel} timeout after ${timeoutMs}ms: ${method}`, method)));
+    }, timeoutMs);
+    pending.set(id, { resolve, timer, method });
+    try {
+      send();
+    } catch (e) {
+      clearTimeout(timer);
+      pending.delete(id);
+      resolve(err(cdpError("transport_closed", e instanceof Error ? e.message : String(e), method)));
+    }
+  });
+
+/**
+ * Register a close listener, returning an unsubscribe function. Identical
+ * across transports, so it lives here.
+ */
+export const makeOnClose = (closeListeners: Set<() => void>) =>
+  (cb: () => void): (() => void) => {
+    closeListeners.add(cb);
+    return () => closeListeners.delete(cb);
+  };

package/src/cdp/session.ts

@@ -17,12 +17,25 @@ type TabSession = {
   pageInfoDirty: boolean;
   networkBuffer: ReturnType<typeof createNetworkBuffer>;
   consoleBuffer: ReturnType<typeof createConsoleBuffer>;
+  // Stable element refs (e1, e2, …) → CDP backendNodeId, from the latest
+  // snapshot of this tab. Replaced wholesale on every snapshot — old refs are
+  // stale anyway, so this stays bounded (~50–60 KB worst case).
+  refMap: Map<string, number>;
+  // Per-ref signature ("role|name|value") from the latest snapshot, used as the
+  // baseline for the post-mutation auto-diff. Replaced alongside refMap.
+  refSig: Map<string, string>;
 };
 
 export type CdpSession = {
   attachFirstPage(): Promise<Result<{ readonly targetId: string; readonly sessionId: string }, CdpError>>;
   switchTo(targetId: string): Promise<Result<void, CdpError>>;
   current(): { readonly sessionId: string; readonly targetId: string } | null;
+  /** Replace the active tab's ref map and ref signatures (from a fresh snapshot). */
+  setRefMap(refMap: Map<string, number>, refSig: Map<string, string>): void;
+  /** Resolve a ref (e.g. "e12") to its backendNodeId on the active tab, or undefined if unknown/stale. */
+  resolveRef(ref: string): number | undefined;
+  /** The active tab's prior ref signatures — baseline for the post-mutation diff. */
+  refSignatures(): ReadonlyMap<string, string>;
   call(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
   callOnTarget(method: string, params: Record<string, unknown>, sessionId: string, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
   callBrowser(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
@@ -211,6 +224,8 @@ export const createCdpSession = (
         pageInfoDirty: false,
         networkBuffer: createNetworkBuffer(),
         consoleBuffer: createConsoleBuffer(),
+        refMap: new Map(),
+        refSig: new Map(),
       });
       sessionIdToTargetId.set(a.sessionId, pickTargetId);
       return ok({ targetId: pickTargetId, sessionId: a.sessionId });
@@ -230,6 +245,8 @@ export const createCdpSession = (
         pageInfoDirty: true,
         networkBuffer: createNetworkBuffer(),
         consoleBuffer: createConsoleBuffer(),
+        refMap: new Map(),
+        refSig: new Map(),
       };
       if (existing) {
         // Each Target.attachToTarget produces a new sessionId — update it.
@@ -249,6 +266,20 @@ export const createCdpSession = (
     current() {
       return sessionId && targetId ? { sessionId, targetId } : null;
     },
+    setRefMap(refMap, refSig) {
+      const tab = targetId ? tabs.get(targetId) : undefined;
+      if (!tab) return;
+      tab.refMap = refMap;
+      tab.refSig = refSig;
+    },
+    resolveRef(ref) {
+      const tab = targetId ? tabs.get(targetId) : undefined;
+      return tab?.refMap.get(ref);
+    },
+    refSignatures() {
+      const tab = targetId ? tabs.get(targetId) : undefined;
+      return tab?.refSig ?? new Map();
+    },
     call(method, params = {}, opts = {}) {
       return transport.request(method, params, { sessionId, ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}) });
     },

package/src/cdp/transport.ts

@@ -3,15 +3,10 @@ import { type Result, err, ok } from "../util/result";
 import { type CdpError, cdpError } from "./errors";
 import { isCdpRawMessage } from "./types";
 import type { CdpEvent, CdpRawMessage } from "./types";
+import { type Pending, makeEventQueue, makeOnClose, rejectAllPending, sendWithTimeout } from "./event-queue";
 
 const DEFAULT_TIMEOUT_MS = 15_000;
 
-type Pending = {
-  readonly resolve: (v: Result<unknown, CdpError>) => void;
-  readonly timer: ReturnType<typeof setTimeout>;
-  readonly method: string;
-};
-
 export type CdpTransport = {
   connect(url: string, opts?: { timeoutMs?: number }): Promise<Result<void, CdpError>>;
   close(): Promise<void>;
@@ -33,46 +28,6 @@ export const createCdpTransport = (): CdpTransport => {
   const pending = new Map<number, Pending>();
   const closeListeners = new Set<() => void>();
 
-  type EventQueue = {
-    readonly push: (e: CdpEvent) => void;
-    readonly end: () => void;
-    readonly iter: AsyncIterable<CdpEvent>;
-  };
-
-  const makeEventQueue = (): EventQueue => {
-    const buf: CdpEvent[] = [];
-    const waiters: Array<(v: IteratorResult<CdpEvent>) => void> = [];
-    let ended = false;
-    return {
-      push(e) {
-        if (ended) return;
-        const w = waiters.shift();
-        if (w) w({ value: e, done: false });
-        else buf.push(e);
-      },
-      end() {
-        ended = true;
-        // value: undefined as unknown as CdpEvent is the correct iterator-protocol
-        // pattern: when done=true the value field is conventionally undefined but
-        // the TS type still requires CdpEvent.
-        for (const w of waiters.splice(0)) w({ value: undefined as unknown as CdpEvent, done: true });
-      },
-      iter: {
-        [Symbol.asyncIterator]() {
-          return {
-            next: (): Promise<IteratorResult<CdpEvent>> =>
-              new Promise((resolve) => {
-                const next = buf.shift();
-                if (next) resolve({ value: next, done: false });
-                else if (ended) resolve({ value: undefined as unknown as CdpEvent, done: true });
-                else waiters.push(resolve);
-              }),
-          };
-        },
-      },
-    };
-  };
-
   let queue = makeEventQueue();
 
   const handleMessage = (raw: string): void => {
@@ -109,11 +64,7 @@ export const createCdpTransport = (): CdpTransport => {
   };
 
   const cleanup = (reason: string): void => {
-    for (const [, p] of pending) {
-      clearTimeout(p.timer);
-      p.resolve(err(cdpError("transport_closed", reason, p.method)));
-    }
-    pending.clear();
+    rejectAllPending(pending, reason);
     queue.end();
     queue = makeEventQueue();
     for (const cb of closeListeners) cb();
@@ -184,20 +135,7 @@ export const createCdpTransport = (): CdpTransport => {
       const payload: Record<string, unknown> = { id, method, params };
       if (sessionId) payload["sessionId"] = sessionId;
       const json = JSON.stringify(payload);
-      return new Promise((resolve) => {
-        const timer = setTimeout(() => {
-          pending.delete(id);
-          resolve(err(cdpError("timeout", `CDP timeout after ${timeoutMs}ms: ${method}`, method)));
-        }, timeoutMs);
-        pending.set(id, { resolve, timer, method });
-        try {
-          sock.send(json);
-        } catch (e) {
-          clearTimeout(timer);
-          pending.delete(id);
-          resolve(err(cdpError("transport_closed", e instanceof Error ? e.message : String(e), method)));
-        }
-      });
+      return sendWithTimeout(pending, id, method, timeoutMs, "CDP", () => sock.send(json));
     },
     events(): AsyncIterable<CdpEvent> {
       // Single-consumer: each connection's event stream may be iterated by
@@ -213,9 +151,6 @@ export const createCdpTransport = (): CdpTransport => {
       if (ws.readyState === WebSocket.OPEN) return "open";
       return "closed";
     },
-    onClose(cb): () => void {
-      closeListeners.add(cb);
-      return () => closeListeners.delete(cb);
-    },
+    onClose: makeOnClose(closeListeners),
   };
 };

package/src/domains/click.ts

@@ -1,13 +1,21 @@
-import { Type, type Static } from "typebox";
+import { Type } from "typebox";
 import { Text } from "@mariozechner/pi-tui";
 import type { BrowserClient } from "../client";
 import { Coords, MouseButton } from "../schemas/common";
 import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr } from "../util/tool";
 import { captureWithCrosshair } from "./screenshot";
+import { interactiveDiff, resolveRefToBox } from "./ref-resolve";
 
 const ClickArgs = Type.Object({
-  ...Coords.properties,
+  ref: Type.Optional(
+    Type.String({
+      description:
+        "Stable element ref from browser_snapshot (e.g. 'e12'). PREFERRED over x/y — survives re-renders. When set, x/y are ignored.",
+    }),
+  ),
+  x: Type.Optional(Coords.properties.x),
+  y: Type.Optional(Coords.properties.y),
   button: Type.Optional(MouseButton),
   count: Type.Optional(
     Type.Integer({
@@ -18,20 +26,27 @@ const ClickArgs = Type.Object({
     }),
   ),
 });
-type ClickArgsT = Static<typeof ClickArgs>;
 
 const dispatchClick = async (
   client: BrowserClient,
-  args: ClickArgsT,
+  x: number,
+  y: number,
+  button: "left" | "right" | "middle",
+  count: number,
 ): Promise<Result<void, ToolErr>> => {
-  const button = args.button ?? "left";
-  const count = args.count ?? 1;
+  // Move the pointer to the target first. Some focus/hover handlers (and React
+  // synthetic events) only fire when a mousemove precedes the press, so without
+  // it a click can land without focusing the element.
+  const moved = await client.session().call("Input.dispatchMouseEvent", {
+    type: "mouseMoved", x, y,
+  });
+  if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message });
   const pressed = await client.session().call("Input.dispatchMouseEvent", {
-    type: "mousePressed", x: args.x, y: args.y, button, clickCount: count,
+    type: "mousePressed", x, y, button, clickCount: count,
   });
   if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
   const released = await client.session().call("Input.dispatchMouseEvent", {
-    type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count,
+    type: "mouseReleased", x, y, button, clickCount: count,
   });
   if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
   return ok(undefined);
@@ -41,34 +56,55 @@ export const clickTool = defineBrowserTool({
   name: "browser_click",
   label: "Browser Click",
   description:
-    "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.",
-  promptSnippet: "Click at pixel coordinates (x, y) on the page",
+    "Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.",
+  promptSnippet: "Click an element by ref (preferred) or pixel coordinates",
   promptGuidelines: [
-    "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.",
-    "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.",
-    "Coordinates are viewport CSS pixels (not device pixels).",
-    "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.",
-    "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
-    "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).",
-    "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
+    "PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.",
+    "Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.",
+    "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
+    "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
+    "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
+    "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.",
   ],
   parameters: ClickArgs,
-  renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0),
+  renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0),
   async handler(args, { client }) {
-    const clicked = await dispatchClick(client, args);
+    const button = args.button ?? "left";
+    const count = args.count ?? 1;
+
+    // Resolve the click point: ref re-resolves to the element's CURRENT box
+    // (so re-renders don't matter); otherwise use the literal x/y.
+    let x: number;
+    let y: number;
+    if (args.ref !== undefined) {
+      const box = await resolveRefToBox(client, args.ref);
+      if (!box.success) return box;
+      x = box.data.cx;
+      y = box.data.cy;
+    } else if (args.x !== undefined && args.y !== undefined) {
+      x = args.x;
+      y = args.y;
+    } else {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." });
+    }
+
+    const clicked = await dispatchClick(client, x, y, button, count);
     if (!clicked.success) return clicked;
+
+    const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`;
+    const diff = await interactiveDiff(client);
     if (process.env["BH_DEBUG_CLICKS"]) {
-      const debug = await captureWithCrosshair(client, { x: args.x, y: args.y });
+      const debug = await captureWithCrosshair(client, { x, y });
       if (debug.success) {
         return ok({
-          text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`,
-          details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y },
+          text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`,
+          details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref },
         });
       }
     }
     return ok({
-      text: `Clicked at (${args.x}, ${args.y})`,
-      details: { x: args.x, y: args.y },
+      text: `Clicked at ${target}${diff}`,
+      details: { x, y, ref: args.ref },
     });
   },
 });

package/src/domains/files.ts

@@ -6,12 +6,28 @@ import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
 import { safeJs } from "../util/js-template";
 import { pdfPath } from "../util/paths";
+import { resolveRefToBackendId } from "./ref-resolve";
 
 const UploadArgs = Type.Object({
-  selector: Type.String({ description: "CSS selector of the file <input>" }),
+  ref: Type.Optional(
+    Type.String({ description: "Stable ref of the file <input> from browser_snapshot. PREFERRED over selector." }),
+  ),
+  selector: Type.Optional(Type.String({ description: "CSS selector of the file <input> (fallback when no ref)" })),
   filePath: Type.String({ description: "Absolute path to the file to upload" }),
 });
 
+// Ref path: set files directly on the resolved backendNodeId via CDP — simplest
+// and most reliable, no selector needed.
+const tryRefUpload = async (
+  client: BrowserClient,
+  backendId: number,
+  filePath: string,
+): Promise<Result<void, ToolErr>> => {
+  const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId });
+  if (!set.success) return err({ kind: "cdp_error", message: set.error.message });
+  return ok(undefined);
+};
+
 const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => {
   try {
     await access(filePath, constants.R_OK);
@@ -89,16 +105,29 @@ const jsFallbackUpload = async (
 export const uploadFileTool = defineBrowserTool({
   name: "browser_upload_file",
   label: "Browser Upload File",
-  description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages.",
-  promptSnippet: "Upload a file to a file input",
+  description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.",
+  promptSnippet: "Upload a file to a file input by ref (preferred) or selector",
   promptGuidelines: [
+    "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)",
     "File path must be absolute and readable.",
-    "Selector must match a file input.",
   ],
   parameters: UploadArgs,
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
     const readable = await verifyReadable(args.filePath);
     if (!readable.success) return readable;
+
+    // Ref path: resolve to backendNodeId and set files directly via CDP.
+    if (args.ref !== undefined) {
+      const backendId = resolveRefToBackendId(client, args.ref);
+      if (!backendId.success) return backendId;
+      const up = await tryRefUpload(client, backendId.data, args.filePath);
+      if (!up.success) return up;
+      return ok({ text: `Uploaded ${args.filePath} via CDP (ref ${args.ref})`, details: { mode: "cdp", ref: args.ref, filePath: args.filePath } });
+    }
+
+    if (args.selector === undefined) {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+    }
     const cdp = await tryCdpUpload(client, args.selector, args.filePath);
     if (cdp.success) {
       return ok({