pi-browser-harness 0.14.1 → 0.16.0

package/src/domains/search/web-search.ts

@@ -4,46 +4,58 @@ import { parseJson } from "../../schemas/parse";
 import { type Result, err, ok } from "../../util/result";
 import { applyTruncation } from "../../util/truncate";
 import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
+import type { BrowserClient } from "../../client";
 import {
   closeIsolatedTab,
   evalInIsolatedTab,
+  type IsolatedTab,
   isJsonText,
   navigateIsolatedTab,
   openIsolatedTab,
   waitForIsolatedLoad,
 } from "../isolated-tab";
 import { pageCallExpression } from "../page/inject";
+import { pollUntil } from "../poll";
+import { safeJs } from "../../util/js-template";
 import { extractSerp } from "../page/serp.page";
 import type * as extracted from "../page/serp.page";
 import { ExpandableTextSchema, renderExpandableText } from "../render";
 import { deadlineSignal, runLimited } from "../concurrency";
+import { defaultEngines, type EngineRegistry } from "./engines";
 import {
-  classifySerp,
-  isGotoRedirect,
-  parseGoogleSerp,
+  classifySerpWith,
+  parseSerp,
   type ResolvedTargets,
+  type SearchEngine,
   type SearchResult,
   type SerpExtraction,
-} from "./google-serp";
+  type SerpSelectors,
+} from "./serp";
 import { fetchRedirectTarget, type RedirectResolver } from "./redirect-resolver";
 
 const LOAD_TIMEOUT_MS = 15_000;
 const RESOLVE_BUDGET_MS = 5_000;
 const RESOLVE_CONCURRENCY = 5;
+const RENDER_WAIT_MS = 4_000;
 
 export type WebSearchDeps = {
+  // The mounted engines; mount order is the fallback ladder.
+  readonly engines: EngineRegistry;
   readonly resolveRedirect: RedirectResolver;
   readonly resolveBudgetMs?: number;
 };
 
-// Resolves every /goto anchor in parallel under one time budget; a slot that fails or runs out of time is simply
-// absent from the map, and the parser falls back to that anchor's cite.
-const resolveGotoTargets = async (
+// Resolves every server-side redirect anchor (Google /goto) in parallel under one time budget; a slot that fails or
+// runs out of time is simply absent from the map, and the engine falls back to that anchor's cite.
+const resolveRedirectTargets = async (
+  engine: SearchEngine,
   anchors: ReadonlyArray<SerpExtraction["anchors"][number]>,
   deps: WebSearchDeps,
   signal: AbortSignal | undefined,
 ): Promise<ResolvedTargets> => {
-  const hrefs = anchors.map((a) => a.href).filter(isGotoRedirect);
+  const needsResolve = engine.needsRedirectResolve;
+  if (needsResolve === undefined) return new Map();
+  const hrefs = anchors.map((a) => a.href).filter(needsResolve);
   if (hrefs.length === 0) return new Map();
   const deadline = deadlineSignal(deps.resolveBudgetMs ?? RESOLVE_BUDGET_MS, signal);
   const targets = await runLimited(hrefs, { limit: RESOLVE_CONCURRENCY, signal: deadline.signal }, (href, s) =>
@@ -58,25 +70,17 @@ const resolveGotoTargets = async (
   return resolved;
 };
 
-const WebSearchArgs = Type.Object({
-  query: Type.String({ description: "The search query." }),
-  limit: Type.Optional(
-    Type.Integer({ default: 10, minimum: 1, maximum: 30, description: "Max results to return. Default: 10." }),
-  ),
-});
-
 const PAGE_TEXT_LIMIT = 4000;
 
 // The page function declares the extraction shape it produces; this pins it to the shape the SERP parser consumes,
 // so the two cannot drift apart silently (the runtime validator below still guards the actual wire value).
 type Pinned<T extends SerpExtraction> = T;
 export type ExtractionFromPage = Pinned<extracted.SerpExtraction>;
+type PinnedSelectors<T extends extracted.SerpSelectors> = T;
+export type SelectorsForPage = PinnedSelectors<SerpSelectors>;
 
-// Built once: stringifying the page function per call was the only cost of the thunk.
-const SERP_EXTRACTION_EXPR = pageCallExpression(extractSerp, PAGE_TEXT_LIMIT);
-
-const buildSerpUrl = (query: string, limit: number): string =>
-  `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
+const extractionExpression = (engine: SearchEngine): string =>
+  pageCallExpression(extractSerp, engine.selectors, PAGE_TEXT_LIMIT);
 
 const SerpExtractionSchema = Type.Object({
   anchors: Type.Array(
@@ -102,105 +106,231 @@ const SearchResultSchema = Type.Object({
   rank: Type.Number(),
 });
 
+const AttemptSchema = Type.Object({
+  engine: Type.String(),
+  kind: Type.String(),
+  reason: Type.Optional(Type.String()),
+  message: Type.String(),
+});
+
 const WebSearchDetailsSchema = Type.Object({
   results: Type.Array(SearchResultSchema),
-  engine: Type.Literal("google"),
+  engine: Type.String(),
   query: Type.String(),
+  // Engines tried and failed before `engine` answered; absent when the first engine answered.
+  fallbacks: Type.Optional(Type.Array(AttemptSchema)),
   render: ExpandableTextSchema,
   fullOutputPath: Type.Optional(Type.String()),
 });
 
 export type WebSearchDetails = DetailsOf<typeof WebSearchDetailsSchema>;
+type Attempt = DetailsOf<typeof AttemptSchema>;
 
 const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
   const lines = results.map((r) => `${r.rank}. ${r.title}\n   ${r.url}\n   ${r.snippet}`);
   return `${results.length} result(s) for "${query}":\n\n${lines.join("\n\n")}`;
 };
 
-export const createWebSearchTool = (deps: WebSearchDeps) => defineBrowserTool({
-  name: "browser_web_search",
-  label: "Browser Web Search",
-  description:
-    "Search the web and return ranked results (title, url, snippet, rank). Scrapes a Google results page in the real Chrome — no API key. Returns links only; use browser_read_page to read a result's content.",
-  promptSnippet: "Search the web for a query",
-  promptGuidelines: [
-    "On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.",
-    "On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
-    "If results were on the page but none could be parsed it fails with details.reason:'extraction_mismatch' (plus anchorCount) — Google's markup changed; report it rather than retrying.",
-    "Runs in its own isolated tab and never disturbs the user's current tab.",
-  ],
-  parameters: WebSearchArgs,
-  details: WebSearchDetailsSchema,
-  concurrency: "serialized",
-  async handler(args, { client, signal }): Promise<Result<ToolOk<WebSearchDetails>, ToolErr>> {
-    const limit = args.limit ?? 10;
-    const serpUrl = buildSerpUrl(args.query, limit);
-
-    const opened = await openIsolatedTab(client);
-    if (!opened.success) return err(opened.error);
-    const tab = opened.data;
-
-    try {
-      const navigated = await navigateIsolatedTab(client, tab, serpUrl);
-      if (!navigated.success) return err(navigated.error);
-
-      const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
-      if (!loaded.success) return err(loaded.error);
-
-      const extracted = await evalInIsolatedTab(client, tab, SERP_EXTRACTION_EXPR, isJsonText);
-      if (!extracted.success) return err(extracted.error);
-
-      const parsedJson = parseSerpExtraction(extracted.data);
-      if (!parsedJson.success) {
-        return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
-      }
-      const extraction: SerpExtraction = parsedJson.data;
-
-      const resolved = await resolveGotoTargets(extraction.anchors, deps, signal);
-      const results = parseGoogleSerp(extraction.anchors, limit, resolved);
-      const anchorCount = extraction.anchors.length;
-      const verdict = classifySerp(extraction.pageText, results.length, anchorCount);
-      if (verdict === "captcha") {
-        return err({
-          kind: "invalid_state",
-          message: "Google served a CAPTCHA / bot wall",
-          details: { reason: verdict, serpUrl },
-        });
-      }
-      if (verdict === "extraction_mismatch") {
-        return err({
-          kind: "invalid_state",
-          message: `found ${anchorCount} result anchor(s) but none parsed to an external url (Google markup may have changed)`,
-          details: { reason: verdict, anchorCount, serpUrl },
-        });
-      }
-      if (verdict === "no_results") {
-        return err({ kind: "invalid_state", message: "no results found", details: { reason: verdict, serpUrl } });
-      }
-
-      const body = formatResults(args.query, results);
-      const truncated = await applyTruncation(body, "search");
-      return ok({
-        text: truncated.text,
-        details: {
-          results,
-          engine: "google",
-          query: args.query,
-          render: {
-            summary: `${results.length} result(s) · google · "${args.query}"`,
-            body,
-            ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
-          },
-          ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
-        },
-      });
-    } finally {
-      await closeIsolatedTab(client, tab);
-    }
-  },
-  renderResult(result, expanded) {
-    return renderExpandableText(result.details.render, expanded);
+// One engine, one SERP, in the already-open isolated tab.
+const searchEngine = async (
+  engine: SearchEngine,
+  query: string,
+  limit: number,
+  ctx: {
+    readonly client: BrowserClient;
+    readonly tab: IsolatedTab;
+    readonly signal: AbortSignal | undefined;
+    readonly expression: string;
   },
+  deps: WebSearchDeps,
+): Promise<Result<ReadonlyArray<SearchResult>, ToolErr>> => {
+  const { client, tab, signal, expression } = ctx;
+  const serpUrl = engine.buildUrl(query, limit);
+
+  const navigated = await navigateIsolatedTab(client, tab, serpUrl);
+  if (!navigated.success) return err(navigated.error);
+
+  const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
+  if (!loaded.success) return err(loaded.error);
+
+  if (engine.readySelector !== undefined) {
+    // A miss is not a failure: a bot wall or an empty SERP never renders results, and extraction classifies it.
+    await pollUntil(client, safeJs`document.querySelector(${engine.readySelector}) !== null`, {
+      timeoutMs: RENDER_WAIT_MS,
+      signal,
+      sessionId: tab.sessionId,
+      what: `${engine.name} results`,
+    });
+    if (signal?.aborted === true) return err({ kind: "timeout", message: `Aborted while waiting for ${engine.name} results` });
+  }
+
+  const evaluated = await evalInIsolatedTab(client, tab, expression, isJsonText);
+  if (!evaluated.success) return err(evaluated.error);
+
+  const parsedJson = parseSerpExtraction(evaluated.data);
+  if (!parsedJson.success) {
+    return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
+  }
+  const extraction: SerpExtraction = parsedJson.data;
+
+  const resolved = await resolveRedirectTargets(engine, extraction.anchors, deps, signal);
+  const results = parseSerp(engine, extraction.anchors, limit, resolved);
+  const anchorCount = extraction.anchors.length;
+  const verdict = classifySerpWith(engine.captchaMarkers, extraction.pageText, results.length, anchorCount);
+  if (verdict === "captcha") {
+    return err({
+      kind: "invalid_state",
+      message: `${engine.name} served a CAPTCHA / bot wall`,
+      details: { reason: verdict, serpUrl },
+    });
+  }
+  if (verdict === "extraction_mismatch") {
+    return err({
+      kind: "invalid_state",
+      message: `found ${anchorCount} result anchor(s) but none parsed to an external url (${engine.name} markup may have changed)`,
+      details: { reason: verdict, anchorCount, serpUrl },
+    });
+  }
+  if (verdict === "no_results") {
+    return err({ kind: "invalid_state", message: "no results found", details: { reason: verdict, serpUrl } });
+  }
+  return ok(results);
+};
+
+const withEngine = (engine: string, error: ToolErr): ToolErr => ({
+  ...error,
+  details: { ...error.details, engine },
 });
 
-export const webSearchTool = createWebSearchTool({ resolveRedirect: fetchRedirectTarget });
+const toAttempt = (engine: string, error: ToolErr): Attempt => {
+  const reason = error.details?.["reason"];
+  return {
+    engine,
+    kind: error.kind,
+    ...(typeof reason === "string" ? { reason } : {}),
+    message: error.message,
+  };
+};
+
+// A dead browser or a cancelled call fails every engine the same way, so the ladder stops instead of trying the next.
+const stopsLadder = (error: ToolErr, signal: AbortSignal | undefined): boolean =>
+  signal?.aborted === true || error.kind === "not_connected";
+
+const exhausted = (attempts: ReadonlyArray<Attempt>): ToolErr => {
+  const allEmpty = attempts.every((a) => a.reason === "no_results");
+  const summary = attempts.map((a) => `${a.engine}: ${a.message}`).join("; ");
+  return {
+    kind: "invalid_state",
+    message: allEmpty ? `no results found on any engine (${summary})` : `every search engine failed (${summary})`,
+    details: { reason: allEmpty ? "no_results" : "all_engines_failed", attempts },
+  };
+};
+
+// A named engine is tried alone; no name means the whole mounted ladder.
+const pickLadder = (
+  engines: EngineRegistry,
+  name: string | undefined,
+): Result<ReadonlyArray<SearchEngine>, ToolErr> => {
+  if (name === undefined) return ok(engines.ladder());
+  const named = engines.get(name);
+  if (named !== undefined) return ok([named]);
+  return err({
+    kind: "invalid_args",
+    message: `unknown search engine "${name}"; mounted engines: ${engines.ids().join(", ")}`,
+    details: { engine: name, engines: engines.ids() },
+  });
+};
+
+const webSearchArgs = (engines: EngineRegistry) =>
+  Type.Object({
+    query: Type.String({ description: "The search query." }),
+    limit: Type.Optional(
+      Type.Integer({ default: 10, minimum: 1, maximum: 30, description: "Max results to return. Default: 10." }),
+    ),
+    engine: Type.Optional(
+      Type.String({
+        enum: [...engines.ids()],
+        description: `Search only this engine (${engines.ids().join(", ")}), with no fallback. Omit to walk the fallback ladder ${engines.ids().join(" → ")} until one returns results.`,
+      }),
+    ),
+  });
+
+export const createWebSearchTool = (deps: WebSearchDeps) => {
+  const { engines } = deps;
+  const ids = engines.ids();
+  if (ids.length === 0) throw new Error("browser_web_search needs at least one mounted search engine");
+  const ladderText = ids.join(" → ");
+  // Built once per engine: stringifying the page function per call was the only cost of the thunk.
+  const expressions = new Map(engines.ladder().map((e) => [e.id, extractionExpression(e)] as const));
+
+  return defineBrowserTool({
+    name: "browser_web_search",
+    label: "Browser Web Search",
+    description:
+      `Search the web and return ranked results (title, url, snippet, rank). Scrapes a search engine results page in the real Chrome — no API key. Engines: ${ids.join(", ")}. With no engine given it walks the fallback ladder ${ladderText} until one returns results. Returns links only; use browser_read_page to read a result's content.`,
+    promptSnippet: "Search the web for a query",
+    promptGuidelines: [
+      `Omit engine to get the fallback ladder (${ladderText}); details.engine says which engine answered and details.fallbacks lists the engines that failed first.`,
+      "Pass engine only when one engine is specifically needed; a named engine is tried alone, with no fallback.",
+      "On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl and engine) — surface it to the user; do not retry in a tight loop.",
+      "On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
+      "If results were on the page but none could be parsed it fails with details.reason:'extraction_mismatch' (plus anchorCount) — the engine's markup changed; report it rather than retrying.",
+      "When every engine on the ladder fails it is details.reason:'all_engines_failed' with details.attempts naming each engine's failure.",
+      "Runs in its own isolated tab and never disturbs the user's current tab.",
+    ],
+    parameters: webSearchArgs(engines),
+    details: WebSearchDetailsSchema,
+    concurrency: "serialized",
+    async handler(args, { client, signal }): Promise<Result<ToolOk<WebSearchDetails>, ToolErr>> {
+      const limit = args.limit ?? 10;
+      const ladder = pickLadder(engines, args.engine);
+      if (!ladder.success) return err(ladder.error);
+
+      const opened = await openIsolatedTab(client);
+      if (!opened.success) return err(opened.error);
+      const tab = opened.data;
+
+      try {
+        const attempts: Attempt[] = [];
+        for (const engine of ladder.data) {
+          const id = engine.id;
+          const expression = expressions.get(id) ?? extractionExpression(engine);
+          const searched = await searchEngine(engine, args.query, limit, { client, tab, signal, expression }, deps);
+          if (!searched.success) {
+            // A single named engine reports its own failure exactly, as if there were no ladder.
+            if (ladder.data.length === 1 || stopsLadder(searched.error, signal)) return err(withEngine(id, searched.error));
+            attempts.push(toAttempt(id, searched.error));
+            continue;
+          }
+          const results = searched.data;
+          const body = formatResults(args.query, results);
+          const truncated = await applyTruncation(body, "search");
+          const via = attempts.length > 0 ? ` (after ${attempts.map((a) => a.engine).join(", ")} failed)` : "";
+          return ok({
+            text: truncated.text,
+            details: {
+              results,
+              engine: id,
+              query: args.query,
+              ...(attempts.length > 0 ? { fallbacks: attempts } : {}),
+              render: {
+                summary: `${results.length} result(s) · ${id}${via} · "${args.query}"`,
+                body,
+                ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
+              },
+              ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
+            },
+          });
+        }
+        return err(exhausted(attempts));
+      } finally {
+        await closeIsolatedTab(client, tab);
+      }
+    },
+    renderResult(result, expanded) {
+      return renderExpandableText(result.details.render, expanded);
+    },
+  });
+};
+
+export const webSearchTool = createWebSearchTool({ engines: defaultEngines(), resolveRedirect: fetchRedirectTarget });

package/src/domains/snapshot.ts

@@ -115,7 +115,7 @@ export const snapshotTool = defineBrowserTool({
   name: "browser_snapshot",
   label: "Browser Snapshot",
   description:
-    "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy). Every interactive element gets a stable ref shown as [eN] plus click coordinates @(x,y). Pass the ref to browser_click/browser_fill/etc. — refs survive re-renders, coordinates don't. Use this BEFORE deciding whether you need a screenshot. Pair with browser_execute_js for surgical reads of specific element values.",
+    "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy). Every interactive element gets a stable ref shown as [eN] plus click coordinates @(x,y). Pass the ref to browser_click/browser_type/etc. — refs survive re-renders, coordinates don't. Use this BEFORE deciding whether you need a screenshot. Pair with browser_execute_js for surgical reads of specific element values.",
   promptSnippet: "Get accessibility-tree snapshot with stable element refs (default for page inspection)",
   promptGuidelines: [
     "Re-run browser_snapshot after a navigation or a major re-render to get fresh refs.",

package/src/domains/target.ts

@@ -1,7 +1,12 @@
 import type { BrowserClient } from "../client";
+import { isRecord } from "../util/guards";
 import { type Result, err, ok } from "../util/result";
 import type { ToolErr } from "../util/tool";
-import { resolveRefToBox } from "./ref-resolve";
+import { boxOf } from "./box";
+import { cdpCall } from "./cdp-call";
+import { hitInfo } from "./page/click.page";
+import { elementFunctionSource } from "./page/inject";
+import { resolveRefToBackendId } from "./ref-resolve";
 
 export type PointArgs = {
   readonly ref?: string | undefined;
@@ -13,23 +18,94 @@ export type ResolvedTarget = {
   readonly x: number;
   readonly y: number;
   readonly label: string;
+  // Set when hit-testing found something other than the ref's element under the point (an overlay, a sticky header).
+  readonly covered?: string;
+  // For an x/y click: what is under the point, so the agent can tell whether it hit what it meant to.
+  readonly hit?: string;
 };
 
+type Viewport = { readonly width: number; readonly height: number };
+
+// The layout viewport in CSS pixels, or undefined when Chrome would not say (then no bounds check is made).
+export const viewportOf = async (client: BrowserClient): Promise<Viewport | undefined> => {
+  const r = await cdpCall(client, "Page.getLayoutMetrics", {});
+  if (!r.success) return undefined;
+  const { clientWidth, clientHeight } = r.data.cssLayoutViewport;
+  return clientWidth > 0 && clientHeight > 0 ? { width: clientWidth, height: clientHeight } : undefined;
+};
+
+const inside = (vp: Viewport, x: number, y: number): boolean => x >= 0 && y >= 0 && x < vp.width && y < vp.height;
+
+const outsideErr = (x: number, y: number, vp: Viewport, ref?: string): ToolErr => ({
+  kind: "invalid_state",
+  message: ref !== undefined
+    ? `[${ref}] is at (${x}, ${y}), outside the ${vp.width}x${vp.height} viewport even after scrolling it into view — it may be clipped by a scroll container or positioned off-screen. Scroll its container (browser_scroll) and re-snapshot.`
+    : `Point (${x}, ${y}) is outside the ${vp.width}x${vp.height} viewport, so the click would hit nothing. Use the element's ref from browser_snapshot instead — browser_click scrolls it into view.`,
+  details: { x, y, viewportWidth: vp.width, viewportHeight: vp.height, ...(ref !== undefined ? { ref } : {}) },
+});
+
+const HIT_FN = elementFunctionSource(hitInfo);
+
+const isHitInfo = (v: unknown): v is { readonly onTarget: boolean; readonly hit: string } =>
+  isRecord(v) && typeof v["onTarget"] === "boolean" && typeof v["hit"] === "string";
+
+// What hit-testing finds at the point, relative to the target element (or just what is there, with no target);
+// undefined when it cannot tell.
+const hitTest = async (client: BrowserClient, backendId: number | undefined, x: number, y: number): Promise<{ readonly onTarget: boolean; readonly hit: string } | undefined> => {
+  const loc = await cdpCall(client, "DOM.getNodeForLocation", { x: Math.round(x), y: Math.round(y), ignorePointerEventsNone: true });
+  if (!loc.success) return undefined;
+  const target = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId ?? loc.data.backendNodeId });
+  const hit = await cdpCall(client, "DOM.resolveNode", { backendNodeId: loc.data.backendNodeId });
+  const targetId = target.success ? target.data.object.objectId : undefined;
+  const hitId = hit.success ? hit.data.object.objectId : undefined;
+  try {
+    if (targetId === undefined || hitId === undefined) return undefined;
+    const called = await cdpCall(client, "Runtime.callFunctionOn", {
+      objectId: targetId,
+      functionDeclaration: HIT_FN,
+      arguments: [{ objectId: hitId }],
+      returnByValue: true,
+    });
+    if (!called.success || called.data.exceptionDetails !== undefined) return undefined;
+    const v: unknown = called.data.result.value;
+    return isHitInfo(v) ? v : undefined;
+  } finally {
+    for (const id of [targetId, hitId]) if (id !== undefined) await cdpCall(client, "Runtime.releaseObject", { objectId: id });
+  }
+};
+
+// Turns a ref (or x/y) into the point a trusted mouse event should land on. For a ref: scroll the element into view
+// (nested scroll containers included), measure it now, refuse a point still outside the viewport, and hit-test it so
+// a covered element is reported instead of silently clicking whatever is on top.
 export const resolveTarget = async (
   client: BrowserClient,
   args: PointArgs,
 ): Promise<Result<ResolvedTarget, ToolErr>> => {
   if (args.ref !== undefined) {
-    const box = await resolveRefToBox(client, args.ref);
-    if (!box.success) return box;
+    const backendId = resolveRefToBackendId(client, args.ref);
+    if (!backendId.success) return backendId;
+    // A failed scroll is not fatal: the bounds check below catches an element that is still out of view.
+    await cdpCall(client, "DOM.scrollIntoViewIfNeeded", { backendNodeId: backendId.data });
+    const box = await boxOf(client, backendId.data);
+    if (!box.success) {
+      return err({ kind: "invalid_state", message: `Ref ${args.ref} is unknown or stale — re-run browser_snapshot to get fresh refs.`, details: { ref: args.ref } });
+    }
+    const { cx, cy } = box.data;
+    const vp = await viewportOf(client);
+    if (vp !== undefined && !inside(vp, cx, cy)) return err(outsideErr(cx, cy, vp, args.ref));
+    const hit = await hitTest(client, backendId.data, cx, cy);
     return ok({
-      x: box.data.cx,
-      y: box.data.cy,
-      label: `[${args.ref}] (${box.data.cx}, ${box.data.cy})`,
+      x: cx,
+      y: cy,
+      label: `[${args.ref}] (${cx}, ${cy})`,
+      ...(hit !== undefined && !hit.onTarget ? { covered: hit.hit } : {}),
     });
   }
   if (args.x !== undefined && args.y !== undefined) {
-    return ok({ x: args.x, y: args.y, label: `(${args.x}, ${args.y})` });
+    const vp = await viewportOf(client);
+    if (vp !== undefined && !inside(vp, args.x, args.y)) return err(outsideErr(args.x, args.y, vp));
+    const hit = await hitTest(client, undefined, args.x, args.y);
+    return ok({ x: args.x, y: args.y, label: `(${args.x}, ${args.y})`, ...(hit !== undefined ? { hit: hit.hit } : {}) });
   }
   return err({ kind: "invalid_args", message: "Provide either `ref` or both `x` and `y`." });
 };

package/src/recipes/recordable.ts

@@ -20,7 +20,7 @@ export const REPLAYABLE: ReadonlyMap<string, "ref" | "fields" | "none"> = new Ma
   ["browser_dispatch_key", "ref"],
   ["browser_upload_file", "ref"],
   ["browser_fill_form", "fields"],
-  ["browser_type", "none"],
+  ["browser_type", "ref"],
   ["browser_press_key", "none"],
   ["browser_navigate", "none"],
   ["browser_go_back", "none"],
@@ -303,8 +303,9 @@ export const observeCall: CallObserver = async ({ name, concurrency, args, clien
     }
     const fields = mode === "fields" ? await fieldsOf(client, args) : undefined;
     let password = fields?.some((f) => f.password) ?? false;
-    if (name === "browser_fill") password = await isPasswordTarget(client, ref, selector);
-    if (name === "browser_type") {
+    if (name === "browser_fill" || (name === "browser_type" && (ref !== undefined || selector !== undefined))) {
+      password = await isPasswordTarget(client, ref, selector);
+    } else if (name === "browser_type") {
       const r = await evalJs(client, FOCUSED_IS_PASSWORD_EXPR);
       password = r.success && r.data === true;
     }

package/src/registry.ts

@@ -4,7 +4,7 @@ import type { BrowserTool } from "./util/tool";
 import { type Result, err, ok } from "./util/result";
 import { clickTool } from "./domains/click";
 import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard";
-import { fillTool, selectOptionTool, focusTool, setCheckedTool } from "./domains/form";
+import { selectOptionTool, focusTool, setCheckedTool } from "./domains/form";
 import { fillFormTool } from "./domains/form-batch";
 import { pageInfoTool, waitTool, waitForLoadTool, waitForTool } from "./domains/page";
 import { scrollTool } from "./domains/scroll";
@@ -31,7 +31,6 @@ export const ALL_TOOLS: ReadonlyArray<BrowserTool> = [
   setupTool,
   clickTool,
   typeTool,
-  fillTool,
   fillFormTool,
   selectOptionTool,
   setCheckedTool,

package/src/util/image-size.ts

@@ -0,0 +1,16 @@
+// Reads an image's pixel width from its header, without decoding it (sharp is optional). PNG keeps it in IHDR; JPEG
+// in the first start-of-frame marker. Returns undefined for anything else or a truncated file.
+export const imageWidth = (buf: Buffer): number | undefined => {
+  if (buf.length >= 24 && buf.readUInt32BE(0) === 0x89504e47) return buf.readUInt32BE(16);
+  if (buf.length < 4 || buf[0] !== 0xff || buf[1] !== 0xd8) return undefined;
+  let i = 2;
+  while (i + 9 < buf.length) {
+    if (buf[i] !== 0xff) return undefined;
+    const marker = buf[i + 1] ?? 0;
+    const len = buf.readUInt16BE(i + 2);
+    // SOF0..SOF15, minus DHT (C4), JPG (C8) and DAC (CC), carry the frame size: height then width.
+    if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) return buf.readUInt16BE(i + 7);
+    i += 2 + len;
+  }
+  return undefined;
+};