pi-browser-harness 0.3.1 → 0.4.0

package/src/cdp/session.ts

@@ -1,8 +1,9 @@
-import { type Result, err, ok } from "../util/result";
-import { type CdpError, cdpError } from "./errors";
+import { type Result, ok } from "../util/result";
+import type { CdpError } from "./errors";
 import type { DialogInfo } from "./types";
-import { isInternalUrl } from "./types";
+import type { OwnershipRegistry } from "./ownership";
 import type { CdpTransport } from "./transport";
+import { createNetworkBuffer, type DrainResult, type NetworkFilter } from "./network-buffer";
 
 export type CdpSession = {
   attachFirstPage(): Promise<Result<{ readonly targetId: string; readonly sessionId: string }, CdpError>>;
@@ -13,13 +14,18 @@ export type CdpSession = {
   callBrowser(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
   takeDialog(): DialogInfo | null;
   drainPageInfoInvalidations(): boolean;
+  drainNetworkBuffer(filter: NetworkFilter): DrainResult;
 };
 
-export const createCdpSession = (transport: CdpTransport): CdpSession => {
+export const createCdpSession = (
+  transport: CdpTransport,
+  ownership?: OwnershipRegistry,
+): CdpSession => {
   let sessionId: string | null = null;
   let targetId: string | null = null;
   let dialog: DialogInfo | null = null;
   let pageInfoDirty = false;
+  const networkBuffer = createNetworkBuffer();
 
   let activeConsumer: Promise<void> = Promise.resolve();
 
@@ -41,6 +47,14 @@ export const createCdpSession = (transport: CdpTransport): CdpSession => {
       if (ev.method === "Page.frameNavigated" || ev.method === "Page.loadEventFired") {
         pageInfoDirty = true;
       }
+      if (ev.method === "Target.targetDestroyed" && ownership) {
+        const params = ev.params as { targetId?: string } | undefined;
+        if (params?.targetId) ownership.remove(params.targetId);
+      }
+      if (ev.method === "Network.requestWillBeSent") networkBuffer.ingestRequestWillBeSent(ev.params);
+      else if (ev.method === "Network.responseReceived") networkBuffer.ingestResponseReceived(ev.params);
+      else if (ev.method === "Network.loadingFinished") networkBuffer.ingestLoadingFinished(ev.params);
+      else if (ev.method === "Network.loadingFailed") networkBuffer.ingestLoadingFailed(ev.params);
     }
   };
 
@@ -68,7 +82,7 @@ export const createCdpSession = (transport: CdpTransport): CdpSession => {
   // Switching to Promise.all over a single WS pipelines the round-trips and
   // saves ~3× on tab-switch latency. Defer until session.ts has tests.
   const enableDomains = async (sid: string): Promise<void> => {
-    for (const d of ["Page", "DOM", "Runtime", "Network"]) {
+    for (const d of ["Page", "DOM", "Runtime", "Network", "Accessibility"]) {
       await transport.request(`${d}.enable`, {}, { sessionId: sid });
     }
   };
@@ -81,25 +95,51 @@ export const createCdpSession = (transport: CdpTransport): CdpSession => {
   // as a clear Error in normal use.
   return {
     async attachFirstPage() {
+      // Subscribe to Target.* events so we can react to targetDestroyed.
+      // Best-effort: failing to enable discovery is not fatal for attach.
+      await transport.request("Target.setDiscoverTargets", { discover: true }, { sessionId: null });
+
       const targets = await transport.request("Target.getTargets", {}, { sessionId: null });
       if (!targets.success) return targets;
       const data = targets.data as { targetInfos: ReadonlyArray<{ targetId: string; type: string; url: string }> };
-      let pages = data.targetInfos.filter((t) => t.type === "page" && !isInternalUrl(t.url));
-      if (pages.length === 0) {
-        const created = await transport.request("Target.createTarget", { url: "about:blank" }, { sessionId: null });
+      const allPages = data.targetInfos.filter((t) => t.type === "page");
+      // Reconcile the persisted ownership set against live targets — drop dead IDs.
+      if (ownership) {
+        const live = new Set(allPages.map((p) => p.targetId));
+        const survivors = ownership.list().filter((id) => live.has(id));
+        if (survivors.length !== ownership.list().length) ownership.replaceAll(survivors);
+        const hw = ownership.harnessWindow();
+        if (hw && !live.has(hw)) ownership.setHarnessWindow(undefined);
+      }
+
+      // Prefer attaching to a tab this session already owns. Falls back to
+      // creating a fresh harness-owned tab in a dedicated window — never
+      // grabs the user's foreground tab.
+      let pickTargetId: string | undefined;
+      if (ownership) {
+        const ownedLive = ownership.list().filter((id) => allPages.some((p) => p.targetId === id));
+        pickTargetId = ownedLive[0];
+      }
+      if (!pickTargetId) {
+        const createParams: Record<string, unknown> = { url: "about:blank" };
+        if (ownership) createParams["newWindow"] = true;
+        const created = await transport.request("Target.createTarget", createParams, { sessionId: null });
         if (!created.success) return created;
         const c = created.data as { targetId: string };
-        pages = [{ targetId: c.targetId, type: "page", url: "about:blank" }];
+        pickTargetId = c.targetId;
+        if (ownership) {
+          ownership.setHarnessWindow(c.targetId);
+          ownership.add(c.targetId);
+        }
       }
-      const first = pages[0];
-      if (!first) return err(cdpError("invalid_response", "no page targets after creation"));
-      const attached = await transport.request("Target.attachToTarget", { targetId: first.targetId, flatten: true }, { sessionId: null });
+
+      const attached = await transport.request("Target.attachToTarget", { targetId: pickTargetId, flatten: true }, { sessionId: null });
       if (!attached.success) return attached;
       const a = attached.data as { sessionId: string };
       sessionId = a.sessionId;
-      targetId = first.targetId;
+      targetId = pickTargetId;
       await enableDomains(a.sessionId);
-      return ok({ targetId: first.targetId, sessionId: a.sessionId });
+      return ok({ targetId: pickTargetId, sessionId: a.sessionId });
     },
     async switchTo(tid) {
       const activated = await transport.request("Target.activateTarget", { targetId: tid }, { sessionId: null });
@@ -110,6 +150,9 @@ export const createCdpSession = (transport: CdpTransport): CdpSession => {
       sessionId = a.sessionId;
       targetId = tid;
       pageInfoDirty = true;
+      // Buffer is page-scoped: on switch, the network history of the previous
+      // tab is no longer relevant to what the agent is now looking at.
+      networkBuffer.clear();
       await enableDomains(a.sessionId);
       return ok(undefined);
     },
@@ -135,5 +178,8 @@ export const createCdpSession = (transport: CdpTransport): CdpSession => {
       pageInfoDirty = false;
       return dirty;
     },
+    drainNetworkBuffer(filter) {
+      return networkBuffer.drain(filter);
+    },
   };
 };

package/src/cdp/types.ts

@@ -23,6 +23,7 @@ export type TabInfo = {
   readonly targetId: string;
   readonly title: string;
   readonly url: string;
+  readonly owned?: boolean;
 };
 
 export type PageInfo = {

package/src/client.ts

@@ -1,15 +1,25 @@
 import { type Result, err, ok } from "./util/result";
 import { safeJs } from "./util/js-template";
+import { type Mutex, createMutex } from "./util/mutex";
 import { discoverWsUrl } from "./cdp/discovery";
 import { type CdpError, cdpError } from "./cdp/errors";
 import type { CdpTransport } from "./cdp/transport";
 import { createCdpTransport } from "./cdp/transport";
 import { type CdpSession, createCdpSession } from "./cdp/session";
+import { type OwnershipRegistry, createOwnershipRegistry } from "./cdp/ownership";
 import type { DaemonStatus, DialogInfo, PageInfo, TabInfo } from "./cdp/types";
 
 export type BrowserClientOptions = {
   readonly namespace: string;
   readonly remote?: { readonly cdpUrl: string; readonly browserId: string };
+  readonly initialOwnership?: {
+    readonly ownedTargetIds?: ReadonlyArray<string>;
+    readonly harnessWindowTargetId?: string;
+  };
+  readonly onOwnershipChange?: (snapshot: {
+    readonly ownedTargetIds: ReadonlyArray<string>;
+    readonly harnessWindowTargetId: string | undefined;
+  }) => void;
 };
 
 export type BrowserClient = {
@@ -24,9 +34,15 @@ export type BrowserClient = {
   listTabs(includeInternal?: boolean): Promise<Result<ReadonlyArray<TabInfo>, CdpError>>;
   switchTab(targetId: string): Promise<Result<void, CdpError>>;
   newTab(url?: string): Promise<Result<string, CdpError>>;
+  closeTab(targetId: string): Promise<Result<void, CdpError>>;
+  owns(targetId: string): boolean;
+  ownership(): OwnershipRegistry;
   current(): { readonly sessionId: string; readonly targetId: string } | null;
   session(): CdpSession;
   transport(): CdpTransport;
+  /** Returns the shared async mutex that serialized browser tools must acquire
+   *  before performing mutations. Observation tools should not use this. */
+  mutationMutex(): Mutex;
 };
 
 const HEALTH_TTL_MS = 30_000;
@@ -62,7 +78,25 @@ const parsePageInfoPayload = (v: unknown): Result<PageInfo, CdpError> => {
 
 export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient => {
   const transport = createCdpTransport();
-  const session = createCdpSession(transport);
+  const ownershipInit: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {};
+  if (opts.initialOwnership?.ownedTargetIds !== undefined) {
+    ownershipInit.ownedTargetIds = opts.initialOwnership.ownedTargetIds;
+  }
+  if (opts.initialOwnership?.harnessWindowTargetId !== undefined) {
+    ownershipInit.harnessWindowTargetId = opts.initialOwnership.harnessWindowTargetId;
+  }
+  const ownership = createOwnershipRegistry(ownershipInit);
+  if (opts.onOwnershipChange) {
+    const cb = opts.onOwnershipChange;
+    ownership.onChange(() => {
+      cb({
+        ownedTargetIds: ownership.list(),
+        harnessWindowTargetId: ownership.harnessWindow(),
+      });
+    });
+  }
+  const session = createCdpSession(transport, ownership);
+  const mutationMutex = createMutex();
   let lastHealth = 0;
   let pageCache: { readonly info: PageInfo; readonly at: number } | null = null;
   let remote: BrowserClientOptions["remote"] | null = opts.remote ?? null;
@@ -181,7 +215,19 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
     const tabs = data.targetInfos
       .filter((t) => t.type === "page")
       .filter((t) => includeInternal || !t.url.startsWith("chrome://"))
-      .map((t): TabInfo => ({ targetId: t.targetId, title: t.title, url: t.url }));
+      .map((t): TabInfo => ({
+        targetId: t.targetId,
+        title: t.title,
+        url: t.url,
+        owned: ownership.has(t.targetId),
+      }));
+    // Reconcile any persisted owned ids that no longer exist as live page targets.
+    const live = new Set(tabs.map((t) => t.targetId));
+    const owned = ownership.list();
+    const survivors = owned.filter((id) => live.has(id));
+    if (survivors.length !== owned.length) ownership.replaceAll(survivors);
+    const hw = ownership.harnessWindow();
+    if (hw && !live.has(hw)) ownership.setHarnessWindow(undefined);
     return ok(tabs);
   };
 
@@ -199,9 +245,30 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
   };
 
   const newTab = async (url?: string): Promise<Result<string, CdpError>> => {
-    const created = await session.callBrowser("Target.createTarget", { url: "about:blank" });
+    // Verify the recorded harness window still exists. Querying the live
+    // target list also gives listTabs's reconciliation a chance to run.
+    const tabsResult = await listTabs(true);
+    if (!tabsResult.success) return tabsResult;
+    const live = new Set(tabsResult.data.map((t) => t.targetId));
+    const hw = ownership.harnessWindow();
+
+    const params: Record<string, unknown> = { url: "about:blank" };
+    if (hw && live.has(hw)) {
+      // Open as a child of the harness window's seed tab — Chrome places
+      // it in the same window, giving us visual grouping for free.
+      params["openerId"] = hw;
+    } else {
+      // Either fresh session or the harness window was closed by the user.
+      // Spawn a new dedicated window.
+      params["newWindow"] = true;
+    }
+    const created = await session.callBrowser("Target.createTarget", params);
     if (!created.success) return created;
     const c = created.data as { targetId: string };
+
+    if (params["newWindow"]) ownership.setHarnessWindow(c.targetId);
+    ownership.add(c.targetId);
+
     const switched = await switchTab(c.targetId);
     if (!switched.success) return switched;
     if (url && url !== "about:blank") {
@@ -212,6 +279,13 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
     return ok(c.targetId);
   };
 
+  const closeTab = async (targetId: string): Promise<Result<void, CdpError>> => {
+    const r = await session.callBrowser("Target.closeTarget", { targetId });
+    if (!r.success) return r;
+    ownership.remove(targetId);
+    return ok(undefined);
+  };
+
   const status = (): DaemonStatus => ({
     alive: transport.state() === "open" && session.current() !== null,
     sessionId: session.current()?.sessionId ?? null,
@@ -224,9 +298,12 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
     ensureAlive, status, start, stop,
     evaluateJs, pageInfo,
     takeDialog: () => session.takeDialog(),
-    listTabs, switchTab, newTab,
+    listTabs, switchTab, newTab, closeTab,
+    owns: (id: string) => ownership.has(id),
+    ownership: () => ownership,
     current: () => session.current(),
     session: () => session,
     transport: () => transport,
+    mutationMutex: () => mutationMutex,
   };
 };

package/src/domains/click.ts

@@ -41,14 +41,15 @@ export const clickTool = defineBrowserTool({
   name: "browser_click",
   label: "Browser Click",
   description:
-    "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.",
+    "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.",
   promptSnippet: "Click at pixel coordinates (x, y) on the page",
   promptGuidelines: [
-    "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).",
-    "Capture a browser_screenshot BEFORE clicking to find the right coordinates.",
-    "Capture another browser_screenshot AFTER clicking to verify the action worked.",
+    "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.",
+    "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.",
+    "Coordinates are viewport CSS pixels (not device pixels).",
+    "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.",
     "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
-    "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.",
+    "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).",
     "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
   ],
   parameters: ClickArgs,

package/src/domains/js.ts

@@ -3,10 +3,20 @@ import { resolve, isAbsolute } from "node:path";
 import { tmpdir } from "node:os";
 import { createRequire } from "node:module";
 import { Type } from "typebox";
+import { Markdown, Text } from "@mariozechner/pi-tui";
+import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
 import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
 import { applyTruncation } from "../util/truncate";
 
+const COMPACT_PREVIEW_BYTES = 120;
+
+const formatBytes = (n: number): string => {
+  if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`;
+  if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${n} B`;
+};
+
 // AsyncFunction constructor — the documented Node.js mechanism for compiling
 // arbitrary user-supplied source. Equivalent to `new Function` but produces an
 // async function we can `await`. The full RCE risk is contained at the source
@@ -29,11 +39,15 @@ export const executeJsTool = defineBrowserTool({
   name: "browser_execute_js",
   label: "Browser Execute JS",
   description:
-    "Run JavaScript in the page (or a specific iframe target). `return X` gets auto-wrapped in an IIFE for convenience. Always use safeJs / JSON.stringify when interpolating untrusted strings into source.",
-  promptSnippet: "Execute JS in the page (or iframe)",
+    "Surgical reads from the DOM. The cheapest, most precise way to extract a specific element's text, attribute, value, or geometry. Prefer over browser_screenshot for ANY data extraction or coordinate lookup. `return X` is auto-wrapped in an IIFE. Always use safeJs / JSON.stringify when interpolating untrusted strings.",
+  promptSnippet: "Execute JS in the page — surgical DOM reads (preferred over screenshot)",
   promptGuidelines: [
+    "DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.",
+    "For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.",
+    "For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.",
+    "DO NOT call browser_screenshot to read a value or check element state. Use this tool.",
     "`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.",
-    "For iframes, get a targetId via browser_execute_js with `Object.values(document.querySelectorAll('iframe'))` then call browser_execute_js with targetId.",
+    "For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.",
     "Result must be JSON-serializable (Runtime.evaluate returnByValue=true).",
   ],
   parameters: ExecuteJsArgs,
@@ -42,13 +56,58 @@ export const executeJsTool = defineBrowserTool({
     if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
     const valueStr = r.data === undefined ? "undefined" : JSON.stringify(r.data);
     const truncated = await applyTruncation(valueStr, "js");
+    // If the value is JSON, pretty-print for the expanded TUI view. Falls back
+    // to the raw string if parse fails (e.g. value is "undefined" sentinel).
+    let pretty: string | undefined;
+    try {
+      const parsed = JSON.parse(valueStr);
+      pretty = JSON.stringify(parsed, null, 2);
+    } catch {
+      pretty = undefined;
+    }
     return ok({
       text: truncated.text,
-      details: truncated.fullOutputPath !== undefined
-        ? { valueLength: valueStr.length, fullOutputPath: truncated.fullOutputPath }
-        : { valueLength: valueStr.length },
+      details: {
+        valueLength: valueStr.length,
+        // Full value for the expanded renderer. Capped at 200 KB so a giant
+        // dump doesn't bloat the conversation transcript on disk.
+        full: valueStr.length > 200_000 ? valueStr.slice(0, 200_000) : valueStr,
+        ...(pretty !== undefined ? { pretty: pretty.length > 200_000 ? pretty.slice(0, 200_000) : pretty } : {}),
+        ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
+      },
     });
   },
+
+  renderResult(result, expanded, theme) {
+    const details = result.details as
+      | { valueLength?: number; full?: string; pretty?: string; fullOutputPath?: string }
+      | undefined;
+    if (!details) return new Text(theme.fg("error", "execute_js: no details"), 0, 0);
+
+    const len = details.valueLength ?? 0;
+    const full = details.full ?? "";
+    const isJson = details.pretty !== undefined;
+
+    if (!expanded) {
+      const preview = full.length > COMPACT_PREVIEW_BYTES ? full.slice(0, COMPACT_PREVIEW_BYTES) + "…" : full;
+      const md = [
+        `**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`,
+        "```",
+        preview,
+        "```",
+        keyHint("app.tools.expand", "to expand"),
+      ].join("\n");
+      return new Markdown(md, 0, 0, getMarkdownTheme());
+    }
+
+    const body = details.pretty ?? full;
+    const fence = isJson ? "```json" : "```";
+    const tail = details.fullOutputPath
+      ? `\n\nFull value at \`${details.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
+      : `\n\n${keyHint("app.tools.expand", "to collapse")}`;
+    const md = [`**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`, fence, body, "```", tail].join("\n");
+    return new Markdown(md, 0, 0, getMarkdownTheme());
+  },
 });
 
 const RunScriptArgs = Type.Object({

package/src/domains/navigate.ts

@@ -75,7 +75,7 @@ export const openUrlsTool = defineBrowserTool({
   promptSnippet: "Open multiple URLs in new tabs (parallel)",
   promptGuidelines: [
     "Use after web_search to open citations in parallel.",
-    "After opening, use browser_list_tabs / browser_switch_tab / browser_screenshot to interact.",
+    "After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.",
     "Use browser_wait_for_load on a tab before extracting data from SPAs.",
   ],
   parameters: OpenUrlsArgs,