pi-browser-harness 0.10.2 → 0.11.0

package/src/domains/drag.ts

@@ -1,6 +1,7 @@
 import { Type } from "typebox";
-import { type Result, err, ok } from "../util/result";
+import { type Result, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { cdpCall } from "./cdp-call";
 
 const DragArgs = Type.Object({
   startX: Type.Number(),
@@ -24,6 +25,7 @@ export const dragAndDropTool = defineBrowserTool({
     "dataTransfer is an optional MIME→data map (e.g., { 'text/plain': 'hello' }).",
   ],
   parameters: DragArgs,
+  concurrency: "serialized",
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
     const data = args.dataTransfer
       ? {
@@ -34,12 +36,11 @@ export const dragAndDropTool = defineBrowserTool({
           dragOperationsMask: 1,
         }
       : { items: [], dragOperationsMask: 1 };
-    const cdp = client.session();
-    const calls: ReadonlyArray<readonly [string, Record<string, unknown>]> = [
+    type InputCall = readonly ["Input.dispatchMouseEvent" | "Input.dispatchDragEvent", Record<string, unknown>];
+    const calls: ReadonlyArray<InputCall> = [
       ["Input.dispatchMouseEvent", { type: "mousePressed", x: args.startX, y: args.startY, button: "left", clickCount: 1 }],
       ["Input.dispatchDragEvent", { type: "dragEnter", x: args.startX, y: args.startY, data, modifiers: 0 }],
-      // 5 interpolated dragOver steps from start → end so the page sees a smooth drag.
-      ...Array.from({ length: 5 }, (_, i): readonly [string, Record<string, unknown>] => {
+      ...Array.from({ length: 5 }, (_, i): InputCall => {
         const t = (i + 1) / 5;
         return [
           "Input.dispatchDragEvent",
@@ -56,8 +57,8 @@ export const dragAndDropTool = defineBrowserTool({
       ["Input.dispatchMouseEvent", { type: "mouseReleased", x: args.endX, y: args.endY, button: "left", clickCount: 1 }],
     ];
     for (const [method, params] of calls) {
-      const r = await cdp.call(method, params);
-      if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+      const r = await cdpCall(client, method, params);
+      if (!r.success) return r;
     }
     return ok({
       text: `Dragged from (${args.startX},${args.startY}) to (${args.endX},${args.endY})`,

package/src/domains/element-call.ts

@@ -0,0 +1,117 @@
+import type { BrowserClient } from "../client";
+import { safeJs } from "../util/js-template";
+import { type Result, err, ok } from "../util/result";
+import type { ToolErr } from "../util/tool";
+import { cdpCall, evalJs } from "./cdp-call";
+import { resolveRefToObjectId } from "./ref-resolve";
+
+export type PageResult = {
+  readonly ok: boolean;
+  readonly reason?: string;
+  readonly kind?: string;
+  readonly value?: unknown;
+  readonly text?: string;
+  readonly checked?: boolean;
+  readonly changed?: boolean;
+  readonly tag?: string;
+  readonly options?: ReadonlyArray<{ value: string; text: string }>;
+};
+
+const isOption = (v: unknown): v is { readonly value: string; readonly text: string } =>
+  typeof v === "object" && v !== null
+  && "value" in v && typeof v.value === "string"
+  && "text" in v && typeof v.text === "string";
+
+export const isPageResult = (v: unknown): v is PageResult => {
+  if (typeof v !== "object" || v === null) return false;
+  if (!("ok" in v) || typeof v.ok !== "boolean") return false;
+  if ("reason" in v && typeof v.reason !== "string") return false;
+  if ("kind" in v && typeof v.kind !== "string") return false;
+  if ("text" in v && typeof v.text !== "string") return false;
+  if ("checked" in v && typeof v.checked !== "boolean") return false;
+  if ("changed" in v && typeof v.changed !== "boolean") return false;
+  if ("tag" in v && typeof v.tag !== "string") return false;
+  if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false;
+  return true;
+};
+
+export const runOnElement = async (
+  client: BrowserClient,
+  opts: {
+    ref?: string | undefined;
+    selector?: string | undefined;
+    fnBody: string;
+    arg: unknown;
+    notFound?: unknown;
+  },
+): Promise<Result<unknown, ToolErr>> => {
+  if (opts.ref !== undefined) {
+    const objectId = await resolveRefToObjectId(client, opts.ref);
+    if (!objectId.success) return objectId;
+    const r = await cdpCall(client, "Runtime.callFunctionOn", {
+      objectId: objectId.data,
+      functionDeclaration: `function (arg) { ${opts.fnBody} }`,
+      arguments: [{ value: opts.arg }],
+      returnByValue: true,
+    });
+    if (!r.success) return r;
+    if (r.data.exceptionDetails !== undefined) {
+      return err({
+        kind: "cdp_error",
+        message: `page function threw: ${r.data.exceptionDetails.text}`,
+        details: { ref: opts.ref },
+      });
+    }
+    return ok(r.data.result.value);
+  }
+  if (opts.selector === undefined) {
+    return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+  }
+  // fnBody is harness-authored, not user input; only the selector and value are interpolated, and those go through safeJs.
+  const prelude = safeJs`
+    (() => {
+      const el = document.querySelector(${opts.selector});
+      if (!el) return ${opts.notFound ?? { status: "not_found" }};
+      const __arg = ${opts.arg};
+      return (function (arg) {`;
+  const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`;
+  const r = await evalJs(client, expr);
+  if (!r.success) return r;
+  return ok(r.data);
+};
+
+export const resolveAndCall = async (
+  client: BrowserClient,
+  ref: string,
+  functionDeclaration: string,
+  args: ReadonlyArray<unknown>,
+): Promise<Result<PageResult, ToolErr>> => {
+  const objectId = await resolveRefToObjectId(client, ref);
+  if (!objectId.success) return objectId;
+  const handle = objectId.data;
+  try {
+    const called = await cdpCall(client, "Runtime.callFunctionOn", {
+      objectId: handle,
+      functionDeclaration,
+      arguments: args.map((v) => ({ value: v })),
+      returnByValue: true,
+      awaitPromise: true,
+    });
+    if (!called.success) return err({ ...called.error, details: { ref } });
+    if (called.data.exceptionDetails !== undefined) {
+      return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } });
+    }
+    const value = called.data.result.value;
+    if (!isPageResult(value)) {
+      return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
+    }
+    return ok(value);
+  } finally {
+    await cdpCall(client, "Runtime.releaseObject", { objectId: handle });
+  }
+};
+
+export const detailsOf = (ref: string, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
+  const { ok: _ok, ...rest } = res;
+  return { ref, ...rest, ...(extra ?? {}) };
+};

package/src/domains/files.ts

@@ -6,6 +6,7 @@ import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
 import { safeJs } from "../util/js-template";
 import { pdfPath } from "../util/paths";
+import { cdpCall, cdpCallBrowser, evalJs } from "./cdp-call";
 import { resolveRefToBackendId } from "./ref-resolve";
 
 const UploadArgs = Type.Object({
@@ -16,15 +17,13 @@ const UploadArgs = Type.Object({
   filePath: Type.String({ description: "Absolute path to the file to upload" }),
 });
 
-// Ref path: set files directly on the resolved backendNodeId via CDP — simplest
-// and most reliable, no selector needed.
 const tryRefUpload = async (
   client: BrowserClient,
   backendId: number,
   filePath: string,
 ): Promise<Result<void, ToolErr>> => {
-  const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId });
-  if (!set.success) return err({ kind: "cdp_error", message: set.error.message });
+  const set = await cdpCall(client, "DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId });
+  if (!set.success) return set;
   return ok(undefined);
 };
 
@@ -45,19 +44,16 @@ const tryCdpUpload = async (
   selector: string,
   filePath: string,
 ): Promise<Result<void, ToolErr>> => {
-  const doc = await client.session().call("DOM.getDocument", { depth: -1 });
-  if (!doc.success) return err({ kind: "cdp_error", message: doc.error.message });
-  // CDP boundary cast: DOM.getDocument returns { root: { nodeId: number } }
-  const root = (doc.data as { root: { nodeId: number } }).root;
-  const q = await client.session().call("DOM.querySelector", { nodeId: root.nodeId, selector });
-  if (!q.success) return err({ kind: "cdp_error", message: q.error.message });
-  // CDP boundary cast: DOM.querySelector returns { nodeId: number }
-  const nodeId = (q.data as { nodeId: number }).nodeId;
+  const doc = await cdpCall(client, "DOM.getDocument", { depth: -1 });
+  if (!doc.success) return doc;
+  const q = await cdpCall(client, "DOM.querySelector", { nodeId: doc.data.root.nodeId, selector });
+  if (!q.success) return q;
+  const nodeId = q.data.nodeId;
   if (!nodeId) return err({ kind: "invalid_state", message: `Selector matched 0 file inputs: ${selector}` });
-  const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], nodeId });
-  if (!set.success) return err({ kind: "cdp_error", message: set.error.message });
-  const verify = await client.evaluateJs(safeJs`document.querySelector(${selector})?.files?.length || 0`);
-  if (!verify.success) return err({ kind: "cdp_error", message: verify.error.message });
+  const set = await cdpCall(client, "DOM.setFileInputFiles", { files: [filePath], nodeId });
+  if (!set.success) return set;
+  const verify = await evalJs(client, safeJs`document.querySelector(${selector})?.files?.length || 0`);
+  if (!verify.success) return verify;
   if (Number(verify.data ?? 0) === 0) {
     return err({ kind: "invalid_state", message: "CDP upload reported success but file count is 0" });
   }
@@ -96,8 +92,8 @@ const jsFallbackUpload = async (
       return input.files.length;
     })()
   `;
-  const r = await client.evaluateJs(expr);
-  if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+  const r = await evalJs(client, expr);
+  if (!r.success) return r;
   if (!Number(r.data ?? 0)) return err({ kind: "invalid_state", message: "JS fallback set 0 files" });
   return ok(undefined);
 };
@@ -112,11 +108,11 @@ export const uploadFileTool = defineBrowserTool({
     "File path must be absolute and readable.",
   ],
   parameters: UploadArgs,
+  concurrency: "serialized",
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
     const readable = await verifyReadable(args.filePath);
     if (!readable.success) return readable;
 
-    // Ref path: resolve to backendNodeId and set files directly via CDP.
     if (args.ref !== undefined) {
       const backendId = resolveRefToBackendId(client, args.ref);
       if (!backendId.success) return backendId;
@@ -163,6 +159,7 @@ export const downloadTool = defineBrowserTool({
     "Affects all subsequent downloads on this browser.",
   ],
   parameters: DownloadArgs,
+  concurrency: "serialized",
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
     try {
       await mkdir(args.downloadPath, { recursive: true });
@@ -184,12 +181,12 @@ export const downloadTool = defineBrowserTool({
         message: `Download path unusable: ${e instanceof Error ? e.message : String(e)}`,
       });
     }
-    const r = await client.session().callBrowser("Browser.setDownloadBehavior", {
+    const r = await cdpCallBrowser(client, "Browser.setDownloadBehavior", {
       behavior: "allow",
       downloadPath: args.downloadPath,
       eventsEnabled: true,
     });
-    if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+    if (!r.success) return r;
     return ok({
       text: `Downloads will save to: ${args.downloadPath}`,
       details: { downloadPath: args.downloadPath },
@@ -208,16 +205,15 @@ export const printToPdfTool = defineBrowserTool({
   promptSnippet: "Print the current page to PDF",
   promptGuidelines: ["Default output path is in tmpdir; pass outputPath to control."],
   parameters: PrintPdfArgs,
+  concurrency: "serialized",
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
-    const r = await client.session().call("Page.printToPDF", {
+    const r = await cdpCall(client, "Page.printToPDF", {
       printBackground: true,
       preferCSSPageSize: true,
     });
-    if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
-    // CDP boundary cast: Page.printToPDF returns { data: string } (base64-encoded PDF)
-    const data = (r.data as { data: string }).data;
+    if (!r.success) return r;
     const path = args.outputPath ?? pdfPath(client.namespace);
-    await writeFile(path, Buffer.from(data, "base64"));
+    await writeFile(path, Buffer.from(r.data.data, "base64"));
     return ok({ text: `PDF saved: ${path}`, details: { path } });
   },
 });

package/src/domains/fill-engine.ts

@@ -0,0 +1,80 @@
+export type FillOptions = {
+  readonly rejectSelect: boolean;
+  readonly focusFirst: boolean;
+};
+
+const ENGINE = `
+  const el = this;
+  if (!el || el.nodeType !== 1) return { ok: false, reason: "ref does not point to an element" };
+  const tagName = el.tagName || "";
+  const tag = tagName.toLowerCase();
+  const type = (el.type || "").toLowerCase();
+
+  if (__rejectSelect && tag === "select") {
+    return { ok: false, reason: "element is a <select>", kind: "select", tag: tagName };
+  }
+  if (el.disabled) return { ok: false, reason: "element is disabled", tag: tagName };
+
+  const fire = function (t) { el.dispatchEvent(new Event(t, { bubbles: true })); };
+  if (__focusFirst) { try { el.focus(); } catch (e) {} }
+
+  if (tag === "select") {
+    const want = String(value);
+    let matched = null;
+    for (let i = 0; i < el.options.length; i++) {
+      const o = el.options[i];
+      if (o.value === want || o.label === want || o.text === want) { matched = o; break; }
+    }
+    if (!matched) {
+      const opts = [];
+      for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
+      return { ok: false, reason: "no matching option", kind: "select", options: opts, tag: tagName };
+    }
+    el.value = matched.value;
+    fire("input"); fire("change");
+    return { ok: true, kind: "select", value: el.value, text: matched.text, tag: tagName };
+  }
+
+  if (tag === "input" && (type === "checkbox" || type === "radio")) {
+    const want = value === true || value === "true" || value === "on" || value === 1;
+    let changed = false;
+    if (el.checked !== want) {
+      try { el.click(); } catch (e) {}
+      if (el.checked !== want) { el.checked = want; fire("input"); fire("change"); }
+      changed = true;
+    }
+    return { ok: true, kind: type, checked: el.checked, changed: changed, tag: tagName };
+  }
+
+  if (el.isContentEditable) {
+    try { el.focus(); } catch (e) {}
+    let done = false;
+    try {
+      const sel = window.getSelection();
+      if (sel) { sel.removeAllRanges(); const rng = document.createRange(); rng.selectNodeContents(el); sel.addRange(rng); }
+      done = document.execCommand("insertText", false, String(value));
+    } catch (e) { done = false; }
+    if (!done) {
+      el.textContent = String(value);
+      el.dispatchEvent(new InputEvent("input", { bubbles: true }));
+    }
+    return { ok: true, kind: "contenteditable", value: el.textContent || "", tag: tagName };
+  }
+
+  if (tag === "input" || tag === "textarea") {
+    const proto = tag === "textarea" ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype;
+    const desc = Object.getOwnPropertyDescriptor(proto, "value");
+    if (desc && desc.set) desc.set.call(el, String(value));
+    else el.value = String(value);
+    fire("input"); fire("change");
+    return { ok: true, kind: el.type || tag, value: el.value, tag: tagName };
+  }
+
+  return { ok: false, reason: "element is not a fillable field (tag=" + tag + ")", tag: tagName };
+`;
+
+export const fillBody = (opts: FillOptions): string =>
+  `const __rejectSelect = ${opts.rejectSelect}; const __focusFirst = ${opts.focusFirst};\n${ENGINE}`;
+
+export const fillDeclaration = (opts: FillOptions): string =>
+  `function (value) { ${fillBody(opts)} }`;

package/src/domains/form-batch.ts

@@ -0,0 +1,65 @@
+import { Type } from "typebox";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { fillDeclaration } from "./fill-engine";
+import { detailsOf, resolveAndCall } from "./element-call";
+
+const FILL_FN = fillDeclaration({ rejectSelect: false, focusFirst: true });
+
+const FieldValue = Type.Union([Type.String(), Type.Boolean()], {
+  description: "Value to set. String for text inputs / textareas / selects / contenteditable; boolean for checkboxes & radios.",
+});
+
+const FillFormArgs = Type.Object({
+  fields: Type.Array(
+    Type.Object({
+      ref: Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7')." }),
+      value: FieldValue,
+    }),
+    { minItems: 1, description: "Fields to fill in one batch, each by ref." },
+  ),
+});
+
+export const fillFormTool = defineBrowserTool({
+  name: "browser_fill_form",
+  label: "Browser Fill Form",
+  description:
+    "Fill multiple form fields in one call — the efficient way to complete a form. Each field is { ref, value }; refs come from browser_snapshot's [eN]. Handles text inputs, textareas, selects, checkboxes, radios, and contenteditable, drives React/Vue controlled inputs correctly, and reports the resulting value of every field.",
+  promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming",
+  promptGuidelines: [
+    "Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.",
+    "Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.",
+    "The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.",
+    "Re-run browser_snapshot afterwards to verify the form state if needed.",
+  ],
+  parameters: FillFormArgs,
+  concurrency: "serialized",
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const results: Array<Readonly<Record<string, unknown>>> = [];
+    let okCount = 0;
+    const issues: string[] = [];
+    for (const f of args.fields) {
+      const r = await resolveAndCall(client, f.ref, FILL_FN, [f.value]);
+      if (!r.success) {
+        results.push({ ref: f.ref, ok: false, error: r.error.message });
+        issues.push(`ref ${f.ref} (${r.error.message})`);
+        continue;
+      }
+      const res = r.data;
+      const wanted = typeof f.value === "boolean" ? undefined : String(f.value);
+      const got = res.value !== undefined ? String(res.value) : undefined;
+      const mismatch = wanted !== undefined && got !== undefined && got !== wanted && res.text !== wanted;
+      const fieldOk = res.ok === true && !mismatch;
+      if (fieldOk) okCount++;
+      else issues.push(`ref ${f.ref} (${res.ok === false ? (res.reason ?? "failed") : "value mismatch"})`);
+      results.push({ ref: f.ref, ok: fieldOk, mismatch, ...detailsOf(f.ref, res) });
+    }
+    const text =
+      `Filled ${okCount}/${args.fields.length} fields` +
+      (issues.length > 0 ? `; issues: ${issues.join(", ")}` : "");
+    if (okCount === 0) {
+      return err({ kind: "invalid_state", message: text, details: { results } });
+    }
+    return ok({ text, details: { results } });
+  },
+});