pi-browser-harness 0.13.0 → 0.14.0

package/src/domains/search/google-serp.ts

@@ -2,6 +2,8 @@ export type SerpAnchor = {
   readonly href: string;
   readonly heading: string;
   readonly snippet: string;
+  // Visible breadcrumb url ("https://host › path › seg"); the only place the target survives once Google wraps hrefs in /goto.
+  readonly cite?: string;
 };
 
 export type SerpExtraction = {
@@ -16,7 +18,7 @@ export type SearchResult = {
   readonly rank: number;
 };
 
-export type SerpVerdict = "ok" | "captcha" | "no_results";
+export type SerpVerdict = "ok" | "captcha" | "no_results" | "extraction_mismatch";
 
 const CAPTCHA_MARKERS: ReadonlyArray<string> = [
   "unusual traffic",
@@ -36,12 +38,74 @@ const unwrapRedirect = (href: string): string => {
   return target ?? href;
 };
 
+const CITE_SEPARATOR = /\s*›\s*/;
+const TRUNCATION = /(?:\.\.\.|…)/;
+
+// Rebuilds a url from Google's visible breadcrumb cite. Segments after a truncation marker are unknowable, so the
+// path stops at the last complete segment rather than guessing.
+export const urlFromCite = (cite: string): string | undefined => {
+  const segments = cite
+    .trim()
+    .split(CITE_SEPARATOR)
+    .map((segment) => segment.trim())
+    .filter((segment) => segment.length > 0);
+  const origin = segments[0];
+  if (origin === undefined || TRUNCATION.test(origin)) return undefined;
+  // A cite naming any other scheme (ftp:, mailto:) is not a web result; prefixing it would forge a bogus https url.
+  if (/^[a-z][a-z0-9+.-]*:/i.test(origin) && !/^https?:\/\//i.test(origin)) return undefined;
+  const withScheme = /^https?:\/\//i.test(origin) ? origin : `https://${origin}`;
+  const path: string[] = [];
+  for (const segment of segments.slice(1)) {
+    if (TRUNCATION.test(segment)) break;
+    path.push(segment);
+  }
+  const url = [withScheme.replace(/\/$/, ""), ...path].join("/");
+  return isExternalResult(url) ? url : undefined;
+};
+
+// Only a real Google /goto url counts: the daemon fetches these, so a look-alike host, userinfo, a port or http
+// must never qualify. Hosts are google.com or www.google.<tld>, nothing else.
+const GOOGLE_GOTO_HOST = /^(?:www\.)?google\.[a-z]{2,3}(?:\.[a-z]{2})?$/;
+export const isGotoRedirect = (href: string): boolean => {
+  try {
+    const u = new URL(href);
+    return (
+      u.protocol === "https:" &&
+      u.username === "" &&
+      u.password === "" &&
+      u.port === "" &&
+      GOOGLE_GOTO_HOST.test(u.hostname.toLowerCase()) &&
+      u.pathname === "/goto"
+    );
+  } catch {
+    return false;
+  }
+};
+
+export type ResolvedTargets = ReadonlyMap<string, string>;
+
+// Direct external hrefs win. A /goto wrapper is opaque: the server-resolved redirect target is exact, and the cite is
+// a display breadcrumb that only approximates the path, so it is the fallback when no target was resolved.
+const resolveResultUrl = (anchor: SerpAnchor, resolved: ResolvedTargets): string | undefined => {
+  if (!isGotoRedirect(anchor.href)) {
+    const unwrapped = unwrapRedirect(anchor.href);
+    return isExternalResult(unwrapped) ? unwrapped : undefined;
+  }
+  const target = resolved.get(anchor.href);
+  if (target !== undefined && isExternalResult(target)) return target;
+  if (anchor.cite !== undefined && anchor.cite.trim() !== "") return urlFromCite(anchor.cite);
+  return undefined;
+};
+
+// Any Google property, on any TLD (google.com, google.co.uk, consent.google.de, googleusercontent.com), is never a
+// result: a sorry or consent page is where a rate-limited /goto redirect lands.
+const GOOGLE_PROPERTY = /(?:^|\.)google(?:usercontent)?\.(?:com?\.[a-z]{2}|[a-z]{2,3})$/;
+
 const isExternalResult = (url: string): boolean => {
   if (!/^https?:\/\//i.test(url)) return false;
   try {
     const host = new URL(url).hostname.toLowerCase();
-    if (host === "google.com" || host.endsWith(".google.com")) return false;
-    if (host === "googleusercontent.com" || host.endsWith(".googleusercontent.com")) return false;
+    if (GOOGLE_PROPERTY.test(host)) return false;
     return true;
   } catch {
     return false;
@@ -57,13 +121,17 @@ const dedupeKey = (url: string): string => {
   }
 };
 
-export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: number): ReadonlyArray<SearchResult> => {
+export const parseGoogleSerp = (
+  anchors: ReadonlyArray<SerpAnchor>,
+  limit: number,
+  resolved: ResolvedTargets = new Map(),
+): ReadonlyArray<SearchResult> => {
   const seen = new Set<string>();
   const results: SearchResult[] = [];
   for (const anchor of anchors) {
     if (results.length >= limit) break;
-    const url = unwrapRedirect(anchor.href);
-    if (!isExternalResult(url)) continue;
+    const url = resolveResultUrl(anchor, resolved);
+    if (url === undefined) continue;
     const key = dedupeKey(url);
     if (seen.has(key)) continue;
     seen.add(key);
@@ -77,9 +145,11 @@ export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: numbe
   return results;
 };
 
-export const classifySerp = (pageText: string, resultCount: number): SerpVerdict => {
+export const classifySerp = (pageText: string, resultCount: number, anchorCount: number): SerpVerdict => {
   if (resultCount > 0) return "ok";
   const lower = pageText.toLowerCase();
   if (CAPTCHA_MARKERS.some((marker) => lower.includes(marker))) return "captcha";
+  // Anchors were found but every one was filtered out: the markup changed, not the result set.
+  if (anchorCount > 0) return "extraction_mismatch";
   return "no_results";
 };

package/src/domains/search/redirect-resolver.ts

@@ -0,0 +1,41 @@
+// Resolves one Google /goto?url=<opaque blob> link to its target by reading the server's redirect, without
+// following it. The blob cannot be decoded client-side and the page's <cite> is only a display breadcrumb, so this
+// is the one source of the exact url. Google answers the redirect to a cookie-less GET; HEAD returns 200 instead.
+
+import { isGotoRedirect } from "./google-serp";
+
+const REDIRECT_STATUSES: ReadonlySet<number> = new Set([301, 302, 303, 307, 308]);
+
+const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36";
+
+export type RedirectResolver = (href: string, signal: AbortSignal) => Promise<string | undefined>;
+
+export type ResolveOptions = {
+  // Defence in depth: the parser already filters to /goto hrefs, and the resolver refuses anything else again so a
+  // caller bug cannot turn it into a generic url fetcher. Tests against a local server opt out explicitly.
+  readonly requireGoto?: boolean;
+};
+
+export const fetchRedirectTarget = async (
+  href: string,
+  signal: AbortSignal,
+  opts: ResolveOptions = {},
+): Promise<string | undefined> => {
+  if ((opts.requireGoto ?? true) && !isGotoRedirect(href)) return undefined;
+  try {
+    const response = await fetch(href, {
+      method: "GET",
+      redirect: "manual",
+      signal,
+      headers: { "user-agent": USER_AGENT, accept: "text/html" },
+    });
+    // The body is never read; cancelling it releases the socket without waiting for Google to finish sending.
+    await response.body?.cancel().catch(() => undefined);
+    if (!REDIRECT_STATUSES.has(response.status)) return undefined;
+    const location = response.headers.get("location");
+    if (location === null || location === "") return undefined;
+    return new URL(location, href).toString();
+  } catch {
+    return undefined;
+  }
+};

package/src/domains/search/web-search.ts

@@ -16,9 +16,47 @@ import { pageCallExpression } from "../page/inject";
 import { extractSerp } from "../page/serp.page";
 import type * as extracted from "../page/serp.page";
 import { ExpandableTextSchema, renderExpandableText } from "../render";
-import { classifySerp, parseGoogleSerp, type SearchResult, type SerpExtraction } from "./google-serp";
+import { deadlineSignal, runLimited } from "../concurrency";
+import {
+  classifySerp,
+  isGotoRedirect,
+  parseGoogleSerp,
+  type ResolvedTargets,
+  type SearchResult,
+  type SerpExtraction,
+} from "./google-serp";
+import { fetchRedirectTarget, type RedirectResolver } from "./redirect-resolver";
 
 const LOAD_TIMEOUT_MS = 15_000;
+const RESOLVE_BUDGET_MS = 5_000;
+const RESOLVE_CONCURRENCY = 5;
+
+export type WebSearchDeps = {
+  readonly resolveRedirect: RedirectResolver;
+  readonly resolveBudgetMs?: number;
+};
+
+// Resolves every /goto anchor in parallel under one time budget; a slot that fails or runs out of time is simply
+// absent from the map, and the parser falls back to that anchor's cite.
+const resolveGotoTargets = async (
+  anchors: ReadonlyArray<SerpExtraction["anchors"][number]>,
+  deps: WebSearchDeps,
+  signal: AbortSignal | undefined,
+): Promise<ResolvedTargets> => {
+  const hrefs = anchors.map((a) => a.href).filter(isGotoRedirect);
+  if (hrefs.length === 0) return new Map();
+  const deadline = deadlineSignal(deps.resolveBudgetMs ?? RESOLVE_BUDGET_MS, signal);
+  const targets = await runLimited(hrefs, { limit: RESOLVE_CONCURRENCY, signal: deadline.signal }, (href, s) =>
+    deps.resolveRedirect(href, s),
+  );
+  deadline.clear();
+  const resolved = new Map<string, string>();
+  hrefs.forEach((href, i) => {
+    const target = targets[i];
+    if (target !== undefined) resolved.set(href, target);
+  });
+  return resolved;
+};
 
 const WebSearchArgs = Type.Object({
   query: Type.String({ description: "The search query." }),
@@ -42,7 +80,12 @@ const buildSerpUrl = (query: string, limit: number): string =>
 
 const SerpExtractionSchema = Type.Object({
   anchors: Type.Array(
-    Type.Object({ href: Type.String(), heading: Type.String(), snippet: Type.String() }),
+    Type.Object({
+      href: Type.String(),
+      heading: Type.String(),
+      snippet: Type.String(),
+      cite: Type.Optional(Type.String()),
+    }),
   ),
   pageText: Type.String(),
 });
@@ -74,7 +117,7 @@ const formatResults = (query: string, results: ReadonlyArray<SearchResult>): str
   return `${results.length} result(s) for "${query}":\n\n${lines.join("\n\n")}`;
 };
 
-export const webSearchTool = defineBrowserTool({
+export const createWebSearchTool = (deps: WebSearchDeps) => defineBrowserTool({
   name: "browser_web_search",
   label: "Browser Web Search",
   description:
@@ -83,6 +126,7 @@ export const webSearchTool = defineBrowserTool({
   promptGuidelines: [
     "On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.",
     "On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
+    "If results were on the page but none could be parsed it fails with details.reason:'extraction_mismatch' (plus anchorCount) — Google's markup changed; report it rather than retrying.",
     "Runs in its own isolated tab and never disturbs the user's current tab.",
   ],
   parameters: WebSearchArgs,
@@ -112,15 +156,27 @@ export const webSearchTool = defineBrowserTool({
       }
       const extraction: SerpExtraction = parsedJson.data;
 
-      const results = parseGoogleSerp(extraction.anchors, limit);
-      const verdict = classifySerp(extraction.pageText, results.length);
-      if (verdict !== "ok") {
+      const resolved = await resolveGotoTargets(extraction.anchors, deps, signal);
+      const results = parseGoogleSerp(extraction.anchors, limit, resolved);
+      const anchorCount = extraction.anchors.length;
+      const verdict = classifySerp(extraction.pageText, results.length, anchorCount);
+      if (verdict === "captcha") {
         return err({
           kind: "invalid_state",
-          message: verdict === "captcha" ? "Google served a CAPTCHA / bot wall" : "no results found",
+          message: "Google served a CAPTCHA / bot wall",
           details: { reason: verdict, serpUrl },
         });
       }
+      if (verdict === "extraction_mismatch") {
+        return err({
+          kind: "invalid_state",
+          message: `found ${anchorCount} result anchor(s) but none parsed to an external url (Google markup may have changed)`,
+          details: { reason: verdict, anchorCount, serpUrl },
+        });
+      }
+      if (verdict === "no_results") {
+        return err({ kind: "invalid_state", message: "no results found", details: { reason: verdict, serpUrl } });
+      }
 
       const body = formatResults(args.query, results);
       const truncated = await applyTruncation(body, "search");
@@ -146,3 +202,5 @@ export const webSearchTool = defineBrowserTool({
     return renderExpandableText(result.details.render, expanded);
   },
 });
+
+export const webSearchTool = createWebSearchTool({ resolveRedirect: fetchRedirectTarget });

package/src/domains/snapshot.ts

@@ -12,7 +12,9 @@ import {
   buildTree,
   collectInteractiveTargets,
   countNodes,
+  firstNodes,
   INTERACTIVE_ROLES,
+  keepOnly,
   liveValue,
   type SlimNode,
   stripInternals,
@@ -136,12 +138,12 @@ export const snapshotTool = defineBrowserTool({
     const pageUrl = piRes.data.kind === "dialog" ? "" : piRes.data.url;
     const pageTitle = piRes.data.kind === "dialog" ? "" : piRes.data.title;
 
-    const slim = buildTree(rawNodes, {
-      interestingOnly: args.interestingOnly ?? true,
-      maxNodes: args.maxNodes ?? 1000,
-    });
+    // Every element gets a ref, so a tool or a recipe can reach one the shown outline cut off; maxNodes limits
+    // only what is shown.
+    const full = buildTree(rawNodes, { interestingOnly: args.interestingOnly ?? true, maxNodes: Number.POSITIVE_INFINITY });
+    const shown = firstNodes(full, args.maxNodes ?? 1000);
 
-    const targets = collectInteractiveTargets(slim);
+    const targets = collectInteractiveTargets(full);
 
     const refMap = new Map<string, number>();
     targets.forEach(({ node, backendId }, i) => {
@@ -153,7 +155,8 @@ export const snapshotTool = defineBrowserTool({
     if (targets.length > 0) {
       // Geometry and live values are best-effort decoration: whatever lands inside the budget is kept, the rest is omitted.
       const deadline = deadlineSignal(BOX_BUDGET_MS, signal);
-      await runLimited(targets, { limit: BOX_CONCURRENCY, signal: deadline.signal }, async ({ node, backendId }, sig) => {
+      const visible = targets.filter(({ node }) => shown.has(node));
+      await runLimited(visible, { limit: BOX_CONCURRENCY, signal: deadline.signal }, async ({ node, backendId }, sig) => {
         const box = await boxOf(client, backendId);
         if (!box.success || sig.aborted) return;
         if (box.data.x < 0 || box.data.y < 0) return;
@@ -172,6 +175,8 @@ export const snapshotTool = defineBrowserTool({
     }
     session.setRefMap(refMap, refSig);
 
+    const slim = keepOnly(full, shown);
+
     const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripInternals(slim), null, 2);
     const trunc = await applyTruncation(text, "snapshot");
 

package/src/index.ts

@@ -12,8 +12,11 @@ import { createDaemonTransport } from "./daemon/transport";
 import type { CdpTransport } from "./cdp/types";
 import { setDebugClicks } from "./util/debug";
 import { asBoolean, asString } from "./util/guards";
-import type { BrowserTool } from "./util/tool";
+import { type BrowserTool, setCallObserver } from "./util/tool";
 import type { Result } from "./util/result";
+import { activeJournal, createJournal, journalDir, setActiveJournal } from "./recipes/journal";
+import { observeCall } from "./recipes/recordable";
+import { registerRecipeCommand } from "./recipes/command";
 
 export { defineBrowserTool } from "./util/tool";
 export type {
@@ -178,9 +181,17 @@ export const createExtension = (deps: ExtensionDeps = {}): ((pi: ExtensionAPI) =
       client.setProfilePin(profilePin);
     }
 
+    setActiveJournal(createJournal(journalDir(), ctx.sessionManager.getSessionId()));
+    setCallObserver(observeCall);
+
     harnessRegistry.register(pi, client);
     registerSetupCommand(pi, client);
     registerProfileCommand(pi, client);
+    registerRecipeCommand(pi, client);
+  });
+
+  pi.on("agent_start", async () => {
+    await activeJournal()?.beginRun();
   });
 
   pi.on("session_shutdown", async () => {

package/src/prompt.ts

@@ -27,7 +27,8 @@ const joinShort = (names: ReadonlyArray<string>): string => names.map(shortToolN
 export function getBrowserSystemPrompt(connected = true, tools: ReadonlyArray<PromptTool> = ALL_TOOLS): string {
   const connectionNotice = connected
     ? ""
-    : "Browser is not connected. Run browser_setup or /browser-setup to connect.\n\n";
+    : // "Not connected" alone read to some models as "no browser tools", and they fell back to curl.
+      "The browser_* tools are available in this session; the browser is just not connected yet. Call browser_setup first to connect (the user can also run /browser-setup), then use the browser_* tools. When the user asks for the browser, do not swap in curl, another fetch tool or another browser tool.\n\n";
   const lanes = laneNames(tools);
   return `
 ## Browser Control

package/src/recipes/command.ts

@@ -0,0 +1,96 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import type { BrowserClient } from "../client";
+import { missingParams, runRecipe, unknownParams, unknownParamsMessage } from "./replay";
+import { formatFailureHandoff, formatOutputs, listText } from "./run-tool";
+import { findRecipe, listRecipes, recordRun } from "./store";
+import { approveProjectRecipe, trustWritten } from "./trust";
+
+const TOKEN = /([^\s=]+)="([^"]*)"|(\S+)/g;
+
+// `NAME key=value key="two words"`: the first bare word is the name, a double-quoted value stays whole, and the
+// bare words after an unquoted value join it, so `topic=Grace Hopper` is "Grace Hopper", not "Grace".
+export const parseCommandArgs = (raw: string): { readonly name: string | undefined; readonly params: Readonly<Record<string, string>> } => {
+  let name: string | undefined;
+  let open: string | undefined;
+  const params: Record<string, string> = {};
+  for (const m of raw.matchAll(TOKEN)) {
+    const [, quotedKey, quotedValue, bare] = m;
+    if (quotedKey !== undefined && quotedValue !== undefined) {
+      params[quotedKey] = quotedValue;
+      open = undefined;
+    } else if (bare?.includes("=") === true) {
+      const at = bare.indexOf("=");
+      open = bare.slice(0, at);
+      params[open] = bare.slice(at + 1);
+    } else if (bare !== undefined && name === undefined) {
+      name = bare;
+    } else if (bare !== undefined && open !== undefined) {
+      params[open] = `${params[open] ?? ""} ${bare}`;
+    }
+  }
+  return { name, params };
+};
+
+// Runs the replay engine directly, so a command run never starts a model turn.
+export const registerRecipeCommand = (pi: Pick<ExtensionAPI, "registerCommand" | "sendUserMessage" | "sendMessage">, client: BrowserClient): void => {
+  pi.registerCommand("browser-recipe", {
+    description: "Replay a saved browser recipe with no model calls: /browser-recipe NAME key=value. No name lists recipes.",
+    getArgumentCompletions: async (prefix) => {
+      const all = await listRecipes(process.cwd());
+      return all
+        .map((s) => s.recipe.name)
+        .filter((n) => n.startsWith(prefix))
+        .map((n) => ({ value: n, label: n }));
+    },
+    handler: async (raw, ctx) => {
+      const { name, params } = parseCommandArgs(raw);
+      if (name === undefined) {
+        ctx.ui.notify(await listText(ctx.cwd), "info");
+        return;
+      }
+      const alive = await client.ensureAlive();
+      if (!alive.success) {
+        ctx.ui.notify(`Browser not connected: ${alive.error.message}. Run /browser-setup first.`, "error");
+        return;
+      }
+      const stored = await findRecipe(ctx.cwd, name);
+      if (!stored.success) {
+        ctx.ui.notify(stored.error, "error");
+        return;
+      }
+      const unknown = unknownParams(stored.data.recipe, params);
+      if (unknown.length > 0) {
+        ctx.ui.notify(unknownParamsMessage(stored.data.recipe, unknown), "error");
+        return;
+      }
+      const missing = missingParams(stored.data.recipe, params);
+      if (missing.length > 0) {
+        ctx.ui.notify(`Recipe ${name} needs: ${missing.map((p) => `${p}=… (e.g. ${stored.data.recipe.params[p]?.example ?? ""})`).join(" ")}`, "error");
+        return;
+      }
+      const approved = await approveProjectRecipe(ctx, stored.data, params);
+      if (!approved.success) {
+        ctx.ui.notify(approved.error, "error");
+        return;
+      }
+      ctx.ui.notify(`Running recipe ${name}…`, "info");
+      const outcome = await runRecipe({ client, extensionCtx: ctx, recipe: stored.data.recipe, params });
+      const at = new Date().toISOString();
+      if (outcome.ok) {
+        await recordRun(stored.data, { at, result: "ok" });
+        await trustWritten(stored.data, ctx.cwd);
+        ctx.ui.notify(`Recipe ${name} done: ${outcome.steps} steps ✓`, "info");
+        // Shown in the session without starting a model turn; the agent sees it with the user's next message.
+        const data = formatOutputs(outcome.outputs);
+        if (data !== "") pi.sendMessage({ customType: "browser-recipe-output", content: `/browser-recipe ${name}\n${data}`, display: true }, { triggerTurn: false });
+        return;
+      }
+      await recordRun(stored.data, { at, result: "failed", step: outcome.failure.step });
+      await trustWritten(stored.data, ctx.cwd);
+      ctx.ui.notify(`Recipe ${name} failed at step ${outcome.failure.step}; handing it to the agent to finish the task and repair the recipe.`, "warning");
+      const report = `/browser-recipe ${name} failed. ${formatFailureHandoff(stored.data.recipe, params, outcome.failure)}`;
+      if (ctx.isIdle()) pi.sendUserMessage(report);
+      else pi.sendUserMessage(report, { deliverAs: "followUp" });
+    },
+  });
+};