pi-browser-harness 0.2.0 → 0.3.1

package/src/domains/page.ts

@@ -0,0 +1,88 @@
+import { Type } from "typebox";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { sleep } from "../util/time";
+import { safeJs } from "../util/js-template";
+
+export const pageInfoTool = defineBrowserTool({
+  name: "browser_page_info",
+  label: "Browser Page Info",
+  description:
+    "Get current page state: URL, title, viewport size, scroll position, page dimensions. If a JS dialog is open, returns dialog info instead.",
+  promptSnippet: "Get current page URL, title, viewport, and scroll position",
+  promptGuidelines: [
+    "Use browser_page_info to quickly check what page you're on and whether a JS dialog is blocking interaction.",
+    "If browser_page_info returns a dialog, use browser_handle_dialog before any other browser actions.",
+    "JS dialogs freeze the page's JS thread, so no other interaction works until the dialog is handled.",
+    "browser_page_info auto-detects alert, confirm, prompt, and beforeunload dialogs.",
+  ],
+  parameters: Type.Object({}),
+  async handler(_args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const r = await client.pageInfo();
+    if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+    if ("dialog" in r.data) {
+      const d = r.data.dialog;
+      return ok({
+        text: `⚠️  DIALOG OPEN: ${d.type}\nMessage: ${d.message}${d.defaultPrompt ? `\nDefault: ${d.defaultPrompt}` : ""}\n\nUse browser_handle_dialog to accept or dismiss.`,
+        details: { dialog: d },
+      });
+    }
+    const i = r.data;
+    return ok({
+      text: `URL: ${i.url}\nTitle: ${i.title}\nViewport: ${i.width}x${i.height}\nScroll: (${i.scrollX}, ${i.scrollY})\nPage size: ${i.pageWidth}x${i.pageHeight}`,
+      details: { ...i },
+    });
+  },
+});
+
+const WaitArgs = Type.Object({
+  seconds: Type.Number({ minimum: 0, maximum: 60, description: "Seconds to wait" }),
+});
+
+export const waitTool = defineBrowserTool({
+  name: "browser_wait",
+  label: "Browser Wait",
+  description: "Wait N seconds before continuing.",
+  promptSnippet: "Wait N seconds",
+  promptGuidelines: ["Use sparingly — prefer browser_wait_for_load when waiting for a page load."],
+  parameters: WaitArgs,
+  async handler(args, { signal }): Promise<Result<ToolOk, ToolErr>> {
+    try {
+      await sleep(Math.round(args.seconds * 1000), signal);
+      return ok({ text: `Waited ${args.seconds}s` });
+    } catch (e) {
+      return err({ kind: "internal", message: e instanceof Error ? e.message : String(e) });
+    }
+  },
+});
+
+const WaitForLoadArgs = Type.Object({
+  timeout: Type.Optional(Type.Number({ default: 15, minimum: 1, maximum: 120, description: "Max seconds to wait. Default: 15." })),
+});
+
+export const waitForLoadTool = defineBrowserTool({
+  name: "browser_wait_for_load",
+  label: "Browser Wait For Load",
+  description:
+    "Poll until the current page reports document.readyState === 'complete'. Returns a typed timeout error if the deadline elapses.",
+  promptSnippet: "Wait for the page to finish loading",
+  promptGuidelines: [
+    "Call after browser_navigate / browser_open_urls before extracting data.",
+    "Returns when readyState becomes 'complete' OR the timeout elapses.",
+  ],
+  parameters: WaitForLoadArgs,
+  async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
+    const timeoutMs = (args.timeout ?? 15) * 1000;
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+      if (signal?.aborted) return err({ kind: "internal", message: "aborted" });
+      const r = await client.evaluateJs(safeJs`document.readyState`);
+      if (r.success && r.data === "complete") {
+        const ms = Date.now() - start;
+        return ok({ text: `Page loaded in ${Math.round(ms / 100) / 10}s`, details: { ms } });
+      }
+      await new Promise((res) => setTimeout(res, 50));
+    }
+    return err({ kind: "timeout", message: `Page did not finish loading in ${args.timeout ?? 15}s` });
+  },
+});

package/src/domains/screenshot.ts

@@ -0,0 +1,114 @@
+import { writeFile, rename } from "node:fs/promises";
+import { Type } from "typebox";
+import type { BrowserClient } from "../client";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { screenshotPath } from "../util/paths";
+import { loadSharp } from "../util/sharp-shim";
+import { safeJs } from "../util/js-template";
+
+const ScreenshotArgs = Type.Object({
+  fullPage: Type.Optional(Type.Boolean({ default: false, description: "Capture beyond viewport" })),
+  format: Type.Optional(Type.Union([Type.Literal("png"), Type.Literal("jpeg")], { default: "png" })),
+  quality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100, default: 80, description: "JPEG quality 1-100" })),
+  maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })),
+});
+
+type CaptureArgs = {
+  readonly fullPage?: boolean;
+  readonly format?: "png" | "jpeg";
+  readonly quality?: number;
+};
+
+const captureBase = async (
+  client: BrowserClient,
+  args: CaptureArgs,
+): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => {
+  const format = args.format ?? "png";
+  const quality = args.quality ?? 80;
+  const path = screenshotPath(client.namespace, format);
+  const params: Record<string, unknown> = { format, captureBeyondViewport: args.fullPage ?? false };
+  if (format === "jpeg") params["quality"] = quality;
+  const r = await client.session().call("Page.captureScreenshot", params);
+  if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+  // Page.captureScreenshot returns { data: base64 }
+  const data = (r.data as { data: string }).data;
+  await writeFile(path, Buffer.from(data, "base64"));
+  return ok({ path, format });
+};
+
+const resizeIfNeeded = async (
+  path: string,
+  maxDim: number,
+): Promise<{ readonly note: string }> => {
+  const load = await loadSharp();
+  if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" };
+  if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` };
+  try {
+    const meta = await load.sharp(path).metadata();
+    const w = meta.width ?? 0;
+    const h = meta.height ?? 0;
+    if (Math.max(w, h) <= maxDim) return { note: "" };
+    const tmp = `${path}.resized`;
+    await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp);
+    await rename(tmp, path);
+    return { note: ` (resized to fit ${maxDim}px)` };
+  } catch (e) {
+    return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` };
+  }
+};
+
+export const screenshotTool = defineBrowserTool({
+  name: "browser_screenshot",
+  label: "Browser Screenshot",
+  description: "Capture the current page as PNG or JPEG. JPEG is 2-5x smaller — prefer it for photo-heavy pages.",
+  promptSnippet: "Capture a screenshot of the current page",
+  promptGuidelines: [
+    "Use BEFORE clicks/scrolls to find coordinates; AFTER to verify the action.",
+    "Pass format='jpeg' with a quality (60-90) for smaller files on photo-heavy pages.",
+    "Set maxDim if the page is huge and you want to fit under LLM image-size limits.",
+  ],
+  parameters: ScreenshotArgs,
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const cap = await captureBase(client, args);
+    if (!cap.success) return cap;
+    let note = "";
+    if (args.maxDim !== undefined) {
+      const resized = await resizeIfNeeded(cap.data.path, args.maxDim);
+      note = resized.note;
+    }
+    return ok({
+      text: `Screenshot saved: ${cap.data.path}${note}`,
+      details: { path: cap.data.path, format: cap.data.format, attached: false },
+    });
+  },
+});
+
+export const captureWithCrosshair = async (
+  client: BrowserClient,
+  args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number },
+): Promise<Result<{ readonly path: string }, ToolErr>> => {
+  const cap = await captureBase(client, args);
+  if (!cap.success) return cap;
+  const load = await loadSharp();
+  if (load.kind !== "ok") return ok({ path: cap.data.path });
+  try {
+    const dprR = await client.evaluateJs(safeJs`window.devicePixelRatio`);
+    const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1;
+    const meta = await load.sharp(cap.data.path).metadata();
+    const w = meta.width ?? 0;
+    const h = meta.height ?? 0;
+    const px = Math.round(args.x * dpr);
+    const py = Math.round(args.y * dpr);
+    const r = Math.round(15 * dpr);
+    const stroke = Math.max(2, Math.round(3 * dpr));
+    const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`;
+    const tmp = `${cap.data.path}.debug`;
+    await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp);
+    await rename(tmp, cap.data.path);
+    return ok({ path: cap.data.path });
+  } catch {
+    // Crosshair overlay is best-effort; on failure return the plain screenshot.
+    return ok({ path: cap.data.path });
+  }
+};

package/src/domains/scroll.ts

@@ -0,0 +1,53 @@
+import { Type } from "typebox";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+
+const ScrollArgs = Type.Object({
+  x: Type.Optional(Type.Number({ description: "X coordinate where to scroll. Default: viewport center" })),
+  y: Type.Optional(Type.Number({ description: "Y coordinate where to scroll. Default: viewport center" })),
+  deltaX: Type.Optional(Type.Number({ default: 0, description: "Horizontal scroll delta (CSS pixels). Positive = right." })),
+  deltaY: Type.Optional(Type.Number({
+    default: -300,
+    description: "Vertical scroll delta (CSS pixels). Positive = up (matches W3C wheel events). Default: -300 (scroll down).",
+  })),
+});
+
+export const scrollTool = defineBrowserTool({
+  name: "browser_scroll",
+  label: "Browser Scroll",
+  description:
+    "Scroll the page at given coordinates. deltaY follows W3C wheel-event convention: positive = scroll up, negative = scroll down. Default scrolls down (deltaY = -300).",
+  promptSnippet: "Scroll the page (deltaY positive=up, negative=down)",
+  promptGuidelines: [
+    "Default behavior scrolls down 300px (deltaY=-300). Pass a positive deltaY to scroll up.",
+    "Pass x/y to target a specific scrollable region (e.g., a div with overflow); otherwise scrolls the page at viewport center.",
+  ],
+  parameters: ScrollArgs,
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const info = await client.pageInfo();
+    if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
+    if ("dialog" in info.data) {
+      return err({
+        kind: "invalid_state",
+        message: `Dialog open: ${info.data.dialog.type} — ${info.data.dialog.message}`,
+      });
+    }
+    const cx = args.x ?? Math.round(info.data.width / 2);
+    const cy = args.y ?? Math.round(info.data.height / 2);
+    const dx = args.deltaX ?? 0;
+    const dy = args.deltaY ?? -300;
+    // Establish compositor mouse position before mouseWheel; without this,
+    // CDP sometimes drops the wheel event with a timeout. Fix preserved
+    // from v0.2.0.
+    const moved = await client.session().call("Input.dispatchMouseEvent", { type: "mouseMoved", x: cx, y: cy });
+    if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message });
+    const wheel = await client.session().call("Input.dispatchMouseEvent", {
+      type: "mouseWheel", x: cx, y: cy, deltaX: dx, deltaY: dy,
+    });
+    if (!wheel.success) return err({ kind: "cdp_error", message: wheel.error.message });
+    return ok({
+      text: `Scrolled at (${cx}, ${cy}) by (${dx}, ${dy})`,
+      details: { x: cx, y: cy, deltaX: dx, deltaY: dy },
+    });
+  },
+});

package/src/domains/tabs.ts

@@ -0,0 +1,121 @@
+import { Type } from "typebox";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+
+const ListTabsArgs = Type.Object({
+  includeInternal: Type.Optional(Type.Boolean({ default: true, description: "Include chrome:// pages" })),
+});
+
+export const listTabsTool = defineBrowserTool({
+  name: "browser_list_tabs",
+  label: "Browser List Tabs",
+  description: "List all open browser tabs (page targets).",
+  promptSnippet: "List browser tabs",
+  promptGuidelines: [
+    "Use to find a full targetId for browser_switch_tab.",
+    "Internal tabs (chrome://) included by default; pass includeInternal=false to exclude them.",
+    "Each tab shows its full 32-char hex targetId — use the exact value with browser_switch_tab.",
+  ],
+  parameters: ListTabsArgs,
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const r = await client.listTabs(args.includeInternal ?? true);
+    if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+    const lines = r.data.map((t, i) => `  [${i}] ${t.targetId} ${t.url}\n      ${t.title}`);
+    return ok({
+      text: `Tabs (${r.data.length}):\n${lines.join("\n")}`,
+      details: { tabs: r.data },
+    });
+  },
+});
+
+export const currentTabTool = defineBrowserTool({
+  name: "browser_current_tab",
+  label: "Browser Current Tab",
+  description: "Get info about the currently attached tab.",
+  promptSnippet: "Get current tab info",
+  promptGuidelines: ["Returns targetId, url, title."],
+  parameters: Type.Object({}),
+  async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const cur = client.current();
+    if (!cur) return err({ kind: "invalid_state", message: "No tab attached." });
+    const ti = await client.session().callBrowser("Target.getTargetInfo", { targetId: cur.targetId });
+    if (!ti.success) return err({ kind: "cdp_error", message: ti.error.message });
+    const info = (ti.data as { targetInfo: { targetId: string; url: string; title: string } }).targetInfo; // CDP boundary cast: Target.getTargetInfo returns { targetInfo: { targetId, url, title } }
+    return ok({
+      text: `Current tab:\n  ${info.targetId}\n  ${info.url}\n  ${info.title}`,
+      details: { targetId: info.targetId, url: info.url, title: info.title },
+    });
+  },
+});
+
+const SwitchTabArgs = Type.Object({
+  targetId: Type.String({ description: "Target ID from browser_list_tabs" }),
+});
+
+export const switchTabTool = defineBrowserTool({
+  name: "browser_switch_tab",
+  label: "Browser Switch Tab",
+  description: "Switch to and attach to a different tab by targetId.",
+  promptSnippet: "Switch tabs",
+  promptGuidelines: [
+    "Get a targetId via browser_list_tabs first.",
+    "Accepts exact targetId or a unique prefix of at least 8 hex characters.",
+  ],
+  parameters: SwitchTabArgs,
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    // Try exact match first
+    let r = await client.switchTab(args.targetId);
+    if (r.success) {
+      return ok({ text: `Switched to ${args.targetId}`, details: { targetId: args.targetId } });
+    }
+    // If exact match failed and input looks like a short prefix (≥8 hex chars), try prefix resolution
+    const isHexPrefix = /^[0-9A-Fa-f]{8,}$/.test(args.targetId);
+    if (!isHexPrefix) {
+      return err({ kind: "cdp_error", message: r.error.message });
+    }
+    const tabs = await client.listTabs(true);
+    if (!tabs.success) return err({ kind: "cdp_error", message: r.error.message });
+    const matches = tabs.data.filter((t) => t.targetId.startsWith(args.targetId));
+    if (matches.length === 0) {
+      return err({ kind: "cdp_error", message: `No tab found with prefix "${args.targetId}"` });
+    }
+    if (matches.length > 1) {
+      const ids = matches.map((t) => t.targetId).join(", ");
+      return err({
+        kind: "invalid_state",
+        message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} tabs: ${ids}`,
+        details: { matches: matches.map((t) => ({ targetId: t.targetId, url: t.url })) },
+      });
+    }
+    const resolved = matches[0]!.targetId;
+    r = await client.switchTab(resolved);
+    if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+    return ok({
+      text: `Switched to ${resolved} (resolved from prefix "${args.targetId}")`,
+      details: { targetId: resolved, resolvedPrefix: args.targetId },
+    });
+  },
+});
+
+const NewTabArgs = Type.Object({
+  url: Type.Optional(Type.String({ description: "Optional URL to navigate to" })),
+});
+
+export const newTabTool = defineBrowserTool({
+  name: "browser_new_tab",
+  label: "Browser New Tab",
+  description: "Open a new tab and switch to it. Optionally navigate to a URL.",
+  promptSnippet: "Open a new tab",
+  promptGuidelines: ["Pass url to navigate immediately."],
+  parameters: NewTabArgs,
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const r = await client.newTab(args.url);
+    if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+    return ok({
+      text: `New tab: ${r.data}${args.url ? ` (${args.url})` : ""}`,
+      details: args.url !== undefined
+        ? { targetId: r.data, url: args.url }
+        : { targetId: r.data },
+    });
+  },
+});

package/src/domains/viewport.ts

@@ -0,0 +1,40 @@
+import { Type } from "typebox";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+
+const ViewportArgs = Type.Object({
+  width: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel width" }),
+  height: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel height" }),
+  deviceScaleFactor: Type.Optional(
+    Type.Number({ minimum: 0.5, maximum: 4, default: 1, description: "Device pixel ratio" }),
+  ),
+});
+
+export const viewportResizeTool = defineBrowserTool({
+  name: "browser_viewport_resize",
+  label: "Browser Viewport Resize",
+  description: "Override the viewport size and device pixel ratio for responsive testing.",
+  promptSnippet: "Resize the viewport (responsive testing)",
+  promptGuidelines: [
+    "Width/height in CSS pixels (e.g., 375x667 for iPhone SE).",
+    "Pass deviceScaleFactor=2 to simulate retina displays.",
+  ],
+  parameters: ViewportArgs,
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const r = await client.session().call("Emulation.setDeviceMetricsOverride", {
+      width: args.width,
+      height: args.height,
+      deviceScaleFactor: args.deviceScaleFactor ?? 1,
+      mobile: false,
+    });
+    if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+    return ok({
+      text: `Viewport set to ${args.width}x${args.height} @${args.deviceScaleFactor ?? 1}x`,
+      details: {
+        width: args.width,
+        height: args.height,
+        deviceScaleFactor: args.deviceScaleFactor ?? 1,
+      },
+    });
+  },
+});