pi-feats 0.1.2 → 0.1.4

package/extensions/browser-harness/domains/drag.ts

@@ -0,0 +1,67 @@
+import { Type } from "typebox";
+import { type Result, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { cdpCall } from "./cdp-call";
+
+const DragArgs = Type.Object({
+  startX: Type.Number(),
+  startY: Type.Number(),
+  endX: Type.Number(),
+  endY: Type.Number(),
+  dataTransfer: Type.Optional(
+    Type.Record(Type.String(), Type.String(), {
+      description: "Optional MIME → data map for DataTransfer (e.g., { 'text/plain': 'hello' })",
+    }),
+  ),
+});
+
+export const dragAndDropTool = defineBrowserTool({
+  name: "browser_drag_and_drop",
+  label: "Browser Drag & Drop",
+  description: "Drag from (startX, startY) to (endX, endY) using CDP Input.dispatchDragEvent.",
+  promptSnippet: "Drag and drop between two coordinates",
+  promptGuidelines: [
+    "Coordinates in CSS pixels.",
+    "dataTransfer is an optional MIME→data map (e.g., { 'text/plain': 'hello' }).",
+  ],
+  parameters: DragArgs,
+  concurrency: "serialized",
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const data = args.dataTransfer
+      ? {
+          items: Object.entries(args.dataTransfer).map(([mimeType, payload]) => ({
+            mimeType,
+            data: Buffer.from(payload).toString("base64"),
+          })),
+          dragOperationsMask: 1,
+        }
+      : { items: [], dragOperationsMask: 1 };
+    type InputCall = readonly ["Input.dispatchMouseEvent" | "Input.dispatchDragEvent", Record<string, unknown>];
+    const calls: ReadonlyArray<InputCall> = [
+      ["Input.dispatchMouseEvent", { type: "mousePressed", x: args.startX, y: args.startY, button: "left", clickCount: 1 }],
+      ["Input.dispatchDragEvent", { type: "dragEnter", x: args.startX, y: args.startY, data, modifiers: 0 }],
+      ...Array.from({ length: 5 }, (_, i): InputCall => {
+        const t = (i + 1) / 5;
+        return [
+          "Input.dispatchDragEvent",
+          {
+            type: "dragOver",
+            x: Math.round(args.startX + (args.endX - args.startX) * t),
+            y: Math.round(args.startY + (args.endY - args.startY) * t),
+            data,
+            modifiers: 0,
+          },
+        ];
+      }),
+      ["Input.dispatchDragEvent", { type: "drop", x: args.endX, y: args.endY, data, modifiers: 0 }],
+      ["Input.dispatchMouseEvent", { type: "mouseReleased", x: args.endX, y: args.endY, button: "left", clickCount: 1 }],
+    ];
+    for (const [method, params] of calls) {
+      const r = await cdpCall(client, method, params);
+      if (!r.success) return r;
+    }
+    return ok({
+      text: `Dragged from (${args.startX},${args.startY}) to (${args.endX},${args.endY})`,
+    });
+  },
+});

package/extensions/browser-harness/domains/element-call.ts

@@ -0,0 +1,117 @@
+import type { BrowserClient } from "../client";
+import { safeJs } from "../util/js-template";
+import { type Result, err, ok } from "../util/result";
+import type { ToolErr } from "../util/tool";
+import { cdpCall, evalJs } from "./cdp-call";
+import { resolveRefToObjectId } from "./ref-resolve";
+
+export type PageResult = {
+  readonly ok: boolean;
+  readonly reason?: string;
+  readonly kind?: string;
+  readonly value?: unknown;
+  readonly text?: string;
+  readonly checked?: boolean;
+  readonly changed?: boolean;
+  readonly tag?: string;
+  readonly options?: ReadonlyArray<{ value: string; text: string }>;
+};
+
+const isOption = (v: unknown): v is { readonly value: string; readonly text: string } =>
+  typeof v === "object" && v !== null
+  && "value" in v && typeof v.value === "string"
+  && "text" in v && typeof v.text === "string";
+
+export const isPageResult = (v: unknown): v is PageResult => {
+  if (typeof v !== "object" || v === null) return false;
+  if (!("ok" in v) || typeof v.ok !== "boolean") return false;
+  if ("reason" in v && typeof v.reason !== "string") return false;
+  if ("kind" in v && typeof v.kind !== "string") return false;
+  if ("text" in v && typeof v.text !== "string") return false;
+  if ("checked" in v && typeof v.checked !== "boolean") return false;
+  if ("changed" in v && typeof v.changed !== "boolean") return false;
+  if ("tag" in v && typeof v.tag !== "string") return false;
+  if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false;
+  return true;
+};
+
+export const runOnElement = async (
+  client: BrowserClient,
+  opts: {
+    ref?: string | undefined;
+    selector?: string | undefined;
+    fnBody: string;
+    arg: unknown;
+    notFound?: unknown;
+  },
+): Promise<Result<unknown, ToolErr>> => {
+  if (opts.ref !== undefined) {
+    const objectId = await resolveRefToObjectId(client, opts.ref);
+    if (!objectId.success) return objectId;
+    const r = await cdpCall(client, "Runtime.callFunctionOn", {
+      objectId: objectId.data,
+      functionDeclaration: `function (arg) { ${opts.fnBody} }`,
+      arguments: [{ value: opts.arg }],
+      returnByValue: true,
+    });
+    if (!r.success) return r;
+    if (r.data.exceptionDetails !== undefined) {
+      return err({
+        kind: "cdp_error",
+        message: `page function threw: ${r.data.exceptionDetails.text}`,
+        details: { ref: opts.ref },
+      });
+    }
+    return ok(r.data.result.value);
+  }
+  if (opts.selector === undefined) {
+    return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+  }
+  // fnBody is harness-authored, not user input; only the selector and value are interpolated, and those go through safeJs.
+  const prelude = safeJs`
+    (() => {
+      const el = document.querySelector(${opts.selector});
+      if (!el) return ${opts.notFound ?? { status: "not_found" }};
+      const __arg = ${opts.arg};
+      return (function (arg) {`;
+  const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`;
+  const r = await evalJs(client, expr);
+  if (!r.success) return r;
+  return ok(r.data);
+};
+
+export const resolveAndCall = async (
+  client: BrowserClient,
+  ref: string,
+  functionDeclaration: string,
+  args: ReadonlyArray<unknown>,
+): Promise<Result<PageResult, ToolErr>> => {
+  const objectId = await resolveRefToObjectId(client, ref);
+  if (!objectId.success) return objectId;
+  const handle = objectId.data;
+  try {
+    const called = await cdpCall(client, "Runtime.callFunctionOn", {
+      objectId: handle,
+      functionDeclaration,
+      arguments: args.map((v) => ({ value: v })),
+      returnByValue: true,
+      awaitPromise: true,
+    });
+    if (!called.success) return err({ ...called.error, details: { ref } });
+    if (called.data.exceptionDetails !== undefined) {
+      return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } });
+    }
+    const value = called.data.result.value;
+    if (!isPageResult(value)) {
+      return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
+    }
+    return ok(value);
+  } finally {
+    await cdpCall(client, "Runtime.releaseObject", { objectId: handle });
+  }
+};
+
+export const detailsOf = (ref: string, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
+  const { ok: _ok, ...rest } = res;
+  return { ref, ...rest, ...(extra ?? {}) };
+};

package/extensions/browser-harness/domains/files.ts

@@ -0,0 +1,219 @@
+import { mkdir, access, constants, stat, readFile, writeFile } from "node:fs/promises";
+import { basename } from "node:path";
+import { Type } from "typebox";
+import type { BrowserClient } from "../client";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { safeJs } from "../util/js-template";
+import { pdfPath } from "../util/paths";
+import { cdpCall, cdpCallBrowser, evalJs } from "./cdp-call";
+import { resolveRefToBackendId } from "./ref-resolve";
+
+const UploadArgs = Type.Object({
+  ref: Type.Optional(
+    Type.String({ description: "Stable ref of the file <input> from browser_snapshot. PREFERRED over selector." }),
+  ),
+  selector: Type.Optional(Type.String({ description: "CSS selector of the file <input> (fallback when no ref)" })),
+  filePath: Type.String({ description: "Absolute path to the file to upload" }),
+});
+
+const tryRefUpload = async (
+  client: BrowserClient,
+  backendId: number,
+  filePath: string,
+): Promise<Result<void, ToolErr>> => {
+  const set = await cdpCall(client, "DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId });
+  if (!set.success) return set;
+  return ok(undefined);
+};
+
+const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => {
+  try {
+    await access(filePath, constants.R_OK);
+    return ok(undefined);
+  } catch (e) {
+    return err({
+      kind: "io_error",
+      message: `Cannot read file: ${filePath} (${e instanceof Error ? e.message : String(e)})`,
+    });
+  }
+};
+
+const tryCdpUpload = async (
+  client: BrowserClient,
+  selector: string,
+  filePath: string,
+): Promise<Result<void, ToolErr>> => {
+  const doc = await cdpCall(client, "DOM.getDocument", { depth: -1 });
+  if (!doc.success) return doc;
+  const q = await cdpCall(client, "DOM.querySelector", { nodeId: doc.data.root.nodeId, selector });
+  if (!q.success) return q;
+  const nodeId = q.data.nodeId;
+  if (!nodeId) return err({ kind: "invalid_state", message: `Selector matched 0 file inputs: ${selector}` });
+  const set = await cdpCall(client, "DOM.setFileInputFiles", { files: [filePath], nodeId });
+  if (!set.success) return set;
+  const verify = await evalJs(client, safeJs`document.querySelector(${selector})?.files?.length || 0`);
+  if (!verify.success) return verify;
+  if (Number(verify.data ?? 0) === 0) {
+    return err({ kind: "invalid_state", message: "CDP upload reported success but file count is 0" });
+  }
+  return ok(undefined);
+};
+
+const inferMime = (name: string): string => {
+  if (name.endsWith(".png")) return "image/png";
+  if (name.endsWith(".jpg") || name.endsWith(".jpeg")) return "image/jpeg";
+  if (name.endsWith(".pdf")) return "application/pdf";
+  if (name.endsWith(".json")) return "application/json";
+  return "text/plain";
+};
+
+const jsFallbackUpload = async (
+  client: BrowserClient,
+  selector: string,
+  filePath: string,
+): Promise<Result<void, ToolErr>> => {
+  const buf = await readFile(filePath);
+  const st = await stat(filePath);
+  const name = basename(filePath);
+  const mime = inferMime(name);
+  const expr = safeJs`
+    (() => {
+      const input = document.querySelector(${selector});
+      if (!input || input.type !== 'file') throw new Error('File input not found');
+      const bin = atob(${buf.toString("base64")});
+      const bytes = new Uint8Array(bin.length);
+      for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+      const file = new File([bytes], ${name}, { type: ${mime}, lastModified: ${st.mtimeMs} });
+      const dt = new DataTransfer();
+      dt.items.add(file);
+      input.files = dt.files;
+      input.dispatchEvent(new Event('change', { bubbles: true }));
+      return input.files.length;
+    })()
+  `;
+  const r = await evalJs(client, expr);
+  if (!r.success) return r;
+  if (!Number(r.data ?? 0)) return err({ kind: "invalid_state", message: "JS fallback set 0 files" });
+  return ok(undefined);
+};
+
+export const uploadFileTool = defineBrowserTool({
+  name: "browser_upload_file",
+  label: "Browser Upload File",
+  description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.",
+  promptSnippet: "Upload a file to a file input by ref (preferred) or selector",
+  promptGuidelines: [
+    "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)",
+    "File path must be absolute and readable.",
+  ],
+  parameters: UploadArgs,
+  concurrency: "serialized",
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const readable = await verifyReadable(args.filePath);
+    if (!readable.success) return readable;
+
+    if (args.ref !== undefined) {
+      const backendId = resolveRefToBackendId(client, args.ref);
+      if (!backendId.success) return backendId;
+      const up = await tryRefUpload(client, backendId.data, args.filePath);
+      if (!up.success) return up;
+      return ok({ text: `Uploaded ${args.filePath} via CDP (ref ${args.ref})`, details: { mode: "cdp", ref: args.ref, filePath: args.filePath } });
+    }
+
+    if (args.selector === undefined) {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+    }
+    const cdp = await tryCdpUpload(client, args.selector, args.filePath);
+    if (cdp.success) {
+      return ok({
+        text: `Uploaded ${args.filePath} via CDP`,
+        details: { mode: "cdp", filePath: args.filePath },
+      });
+    }
+    const js = await jsFallbackUpload(client, args.selector, args.filePath);
+    if (js.success) {
+      return ok({
+        text: `Uploaded ${args.filePath} via JS DataTransfer`,
+        details: { mode: "js", filePath: args.filePath },
+      });
+    }
+    return err({
+      kind: "cdp_error",
+      message: `Both CDP and JS fallback failed. CDP: ${cdp.error.message}; JS: ${js.error.message}`,
+    });
+  },
+});
+
+const DownloadArgs = Type.Object({
+  downloadPath: Type.String({ description: "Absolute path to a writable directory where downloads should be saved" }),
+});
+
+export const downloadTool = defineBrowserTool({
+  name: "browser_download",
+  label: "Browser Download",
+  description: "Configure Chrome's download behavior: set the save directory and disable the save-as prompt.",
+  promptSnippet: "Configure download directory",
+  promptGuidelines: [
+    "Pass an absolute path to an existing writable directory.",
+    "Affects all subsequent downloads on this browser.",
+  ],
+  parameters: DownloadArgs,
+  concurrency: "serialized",
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    try {
+      await mkdir(args.downloadPath, { recursive: true });
+    } catch (e) {
+      return err({
+        kind: "io_error",
+        message: `Cannot create download directory: ${args.downloadPath} (${e instanceof Error ? e.message : String(e)})`,
+      });
+    }
+    try {
+      const s = await stat(args.downloadPath);
+      if (!s.isDirectory()) {
+        return err({ kind: "io_error", message: `Not a directory: ${args.downloadPath}` });
+      }
+      await access(args.downloadPath, constants.W_OK);
+    } catch (e) {
+      return err({
+        kind: "io_error",
+        message: `Download path unusable: ${e instanceof Error ? e.message : String(e)}`,
+      });
+    }
+    const r = await cdpCallBrowser(client, "Browser.setDownloadBehavior", {
+      behavior: "allow",
+      downloadPath: args.downloadPath,
+      eventsEnabled: true,
+    });
+    if (!r.success) return r;
+    return ok({
+      text: `Downloads will save to: ${args.downloadPath}`,
+      details: { downloadPath: args.downloadPath },
+    });
+  },
+});
+
+const PrintPdfArgs = Type.Object({
+  outputPath: Type.Optional(Type.String({ description: "Where to save the PDF. Default: tmpdir + uuid." })),
+});
+
+export const printToPdfTool = defineBrowserTool({
+  name: "browser_print_to_pdf",
+  label: "Browser Print to PDF",
+  description: "Print the current page to a PDF file using Chrome's Page.printToPDF.",
+  promptSnippet: "Print the current page to PDF",
+  promptGuidelines: ["Default output path is in tmpdir; pass outputPath to control."],
+  parameters: PrintPdfArgs,
+  concurrency: "serialized",
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const r = await cdpCall(client, "Page.printToPDF", {
+      printBackground: true,
+      preferCSSPageSize: true,
+    });
+    if (!r.success) return r;
+    const path = args.outputPath ?? pdfPath(client.namespace);
+    await writeFile(path, Buffer.from(r.data.data, "base64"));
+    return ok({ text: `PDF saved: ${path}`, details: { path } });
+  },
+});

package/extensions/browser-harness/domains/fill-engine.ts

@@ -0,0 +1,80 @@
+export type FillOptions = {
+  readonly rejectSelect: boolean;
+  readonly focusFirst: boolean;
+};
+
+const ENGINE = `
+  const el = this;
+  if (!el || el.nodeType !== 1) return { ok: false, reason: "ref does not point to an element" };
+  const tagName = el.tagName || "";
+  const tag = tagName.toLowerCase();
+  const type = (el.type || "").toLowerCase();
+
+  if (__rejectSelect && tag === "select") {
+    return { ok: false, reason: "element is a <select>", kind: "select", tag: tagName };
+  }
+  if (el.disabled) return { ok: false, reason: "element is disabled", tag: tagName };
+
+  const fire = function (t) { el.dispatchEvent(new Event(t, { bubbles: true })); };
+  if (__focusFirst) { try { el.focus(); } catch (e) {} }
+
+  if (tag === "select") {
+    const want = String(value);
+    let matched = null;
+    for (let i = 0; i < el.options.length; i++) {
+      const o = el.options[i];
+      if (o.value === want || o.label === want || o.text === want) { matched = o; break; }
+    }
+    if (!matched) {
+      const opts = [];
+      for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
+      return { ok: false, reason: "no matching option", kind: "select", options: opts, tag: tagName };
+    }
+    el.value = matched.value;
+    fire("input"); fire("change");
+    return { ok: true, kind: "select", value: el.value, text: matched.text, tag: tagName };
+  }
+
+  if (tag === "input" && (type === "checkbox" || type === "radio")) {
+    const want = value === true || value === "true" || value === "on" || value === 1;
+    let changed = false;
+    if (el.checked !== want) {
+      try { el.click(); } catch (e) {}
+      if (el.checked !== want) { el.checked = want; fire("input"); fire("change"); }
+      changed = true;
+    }
+    return { ok: true, kind: type, checked: el.checked, changed: changed, tag: tagName };
+  }
+
+  if (el.isContentEditable) {
+    try { el.focus(); } catch (e) {}
+    let done = false;
+    try {
+      const sel = window.getSelection();
+      if (sel) { sel.removeAllRanges(); const rng = document.createRange(); rng.selectNodeContents(el); sel.addRange(rng); }
+      done = document.execCommand("insertText", false, String(value));
+    } catch (e) { done = false; }
+    if (!done) {
+      el.textContent = String(value);
+      el.dispatchEvent(new InputEvent("input", { bubbles: true }));
+    }
+    return { ok: true, kind: "contenteditable", value: el.textContent || "", tag: tagName };
+  }
+
+  if (tag === "input" || tag === "textarea") {
+    const proto = tag === "textarea" ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype;
+    const desc = Object.getOwnPropertyDescriptor(proto, "value");
+    if (desc && desc.set) desc.set.call(el, String(value));
+    else el.value = String(value);
+    fire("input"); fire("change");
+    return { ok: true, kind: el.type || tag, value: el.value, tag: tagName };
+  }
+
+  return { ok: false, reason: "element is not a fillable field (tag=" + tag + ")", tag: tagName };
+`;
+
+export const fillBody = (opts: FillOptions): string =>
+  `const __rejectSelect = ${opts.rejectSelect}; const __focusFirst = ${opts.focusFirst};\n${ENGINE}`;
+
+export const fillDeclaration = (opts: FillOptions): string =>
+  `function (value) { ${fillBody(opts)} }`;

package/extensions/browser-harness/domains/form-batch.ts

@@ -0,0 +1,65 @@
+import { Type } from "typebox";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { fillDeclaration } from "./fill-engine";
+import { detailsOf, resolveAndCall } from "./element-call";
+
+const FILL_FN = fillDeclaration({ rejectSelect: false, focusFirst: true });
+
+const FieldValue = Type.Union([Type.String(), Type.Boolean()], {
+  description: "Value to set. String for text inputs / textareas / selects / contenteditable; boolean for checkboxes & radios.",
+});
+
+const FillFormArgs = Type.Object({
+  fields: Type.Array(
+    Type.Object({
+      ref: Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7')." }),
+      value: FieldValue,
+    }),
+    { minItems: 1, description: "Fields to fill in one batch, each by ref." },
+  ),
+});
+
+export const fillFormTool = defineBrowserTool({
+  name: "browser_fill_form",
+  label: "Browser Fill Form",
+  description:
+    "Fill multiple form fields in one call — the efficient way to complete a form. Each field is { ref, value }; refs come from browser_snapshot's [eN]. Handles text inputs, textareas, selects, checkboxes, radios, and contenteditable, drives React/Vue controlled inputs correctly, and reports the resulting value of every field.",
+  promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming",
+  promptGuidelines: [
+    "Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.",
+    "Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.",
+    "The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.",
+    "Re-run browser_snapshot afterwards to verify the form state if needed.",
+  ],
+  parameters: FillFormArgs,
+  concurrency: "serialized",
+  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    const results: Array<Readonly<Record<string, unknown>>> = [];
+    let okCount = 0;
+    const issues: string[] = [];
+    for (const f of args.fields) {
+      const r = await resolveAndCall(client, f.ref, FILL_FN, [f.value]);
+      if (!r.success) {
+        results.push({ ref: f.ref, ok: false, error: r.error.message });
+        issues.push(`ref ${f.ref} (${r.error.message})`);
+        continue;
+      }
+      const res = r.data;
+      const wanted = typeof f.value === "boolean" ? undefined : String(f.value);
+      const got = res.value !== undefined ? String(res.value) : undefined;
+      const mismatch = wanted !== undefined && got !== undefined && got !== wanted && res.text !== wanted;
+      const fieldOk = res.ok === true && !mismatch;
+      if (fieldOk) okCount++;
+      else issues.push(`ref ${f.ref} (${res.ok === false ? (res.reason ?? "failed") : "value mismatch"})`);
+      results.push({ ref: f.ref, ok: fieldOk, mismatch, ...detailsOf(f.ref, res) });
+    }
+    const text =
+      `Filled ${okCount}/${args.fields.length} fields` +
+      (issues.length > 0 ? `; issues: ${issues.join(", ")}` : "");
+    if (okCount === 0) {
+      return err({ kind: "invalid_state", message: text, details: { results } });
+    }
+    return ok({ text, details: { results } });
+  },
+});