pi-agent-browser-native 0.2.34 → 0.2.35

package/extensions/agent-browser/lib/playbook.ts

@@ -24,8 +24,8 @@ export const QUICK_START_GUIDELINES = [
 	"Locator-first clicks/fills and native select changes without hand-building argv: { semanticAction: { action: \"click\", locator: \"text\", value: \"Close\" } }, { semanticAction: { action: \"fill\", locator: \"label\", value: \"Email\", text: \"user@example.com\" } }, or { semanticAction: { action: \"select\", selector: \"#flavor\", value: \"chocolate\" } }; add semanticAction.session when targeting a named upstream browser session; details.compiledSemanticAction shows the semantic target, while details.effectiveArgs may show a resolved current @ref for active-session role/name click/check/uncheck actions to avoid hidden duplicate matches; selector-not-found failures may append bounded click try-*-candidate next actions or, for fill misses with current editable refs, details.richInputRecovery with focus/click actions that do not copy fill text; stale-ref failures can return retry-semantic-action-after-stale-ref for compiled find actions when retry safety is provable.",
 	`Common advanced calls: { args: ["batch"], stdin: "[[\"open\",\"https://example.com\"],[\"snapshot\",\"-i\"]]" }, { job: { steps: [{ action: "open", url: "https://example.com" }, { action: "assertText", text: "Example Domain" }, { action: "screenshot", path: ".dogfood/example.png" }] } }, { qa: { url: "https://example.com", expectedText: "Example Domain", screenshotPath: ".dogfood/qa-example.png" } } (example.com smoke only; elsewhere match exact visible text from snapshot -i), { electron: { action: "list", query: "code" } }, { electron: { action: "launch", appName: "Visual Studio Code", handoff: "snapshot" } }, { electron: { action: "probe" } }, { qa: { attached: true, expectedText: "Explorer" } }, { args: ["eval", "--stdin"], stdin: "document.title" }, { args: ["auth", "save", "name", "--password-stdin"], stdin: "<password from user-approved secret source>" }, { args: ["--profile", "Default", "open", "https://example.com/account"], sessionMode: "fresh" }, and { args: ["open", "--enable", "react-devtools", "https://example.com"], sessionMode: "fresh" }. For app pages with a native dropdown, job steps can include { action: "select", selector: "#flavor", value: "chocolate" } before the dependent assertion.`, 
 	"Constrained job navigation is explicit only: click (and select/submit flows that may navigate) does not prove the next page loaded; add assertUrl and/or assertText after navigation-prone steps before screenshot or later interactions. Example: { job: { steps: [{ action: \"open\", url: \"https://shop.example/checkout\" }, { action: \"fill\", selector: \"#email\", text: \"user@example.com\" }, { action: \"click\", selector: \"#continue\" }, { action: \"assertUrl\", url: \"**/shipping\" }, { action: \"assertText\", text: \"Shipping address\" }, { action: \"screenshot\", path: \".dogfood/shipping.png\" }] } }. Top-level click may add navigationSummary hints, but job never auto-inserts post-click asserts.",
-	"High-value command reference: select <selector> <value...> changes native dropdown values; download <selector> <path> saves a file triggered by a click; get title/url/text/html/value/attr/count reads page state; screenshot [path] captures an image; pdf <path> saves a PDF; tab list and tab <tab-id-or-label> inspect or recover the active tab; react tree/inspect/renders/suspense introspect React after --enable react-devtools; vitals [url] measures Core Web Vitals; pushstate <url> performs SPA navigation.",
-	"For artifact-producing commands, read the visible artifact block and details.artifactVerification before using files: check requested path, absolute path, existence, size bytes, artifact kind, optional mediaType, status, optional limitation, and verified/missing/pending/unverified counts. details.artifacts contains per-file metadata. Browser close does not delete explicit saved files; if close reports details.artifactCleanup, use host file tools to remove paths listed in explicitArtifactPaths (when non-empty) after inspection. For annotated screenshots inside batch, put --annotate in top-level args (for example { args: [\"--annotate\", \"batch\"], stdin: \"[[\\\"screenshot\\\",\\\"/tmp/page.png\\\"]]\" }) rather than inside the screenshot step.",
+	"High-value command reference: click <selector> --new-tab opens link-like targets in a new tab; select <selector> <value...> changes native dropdown values; scroll <dir> [px] --selector <sel> targets nested scrollers; download <selector> <path> saves a file triggered by a click; get title/url/text/html/value/attr/count reads page state; screenshot [selector] [path] captures a page or element image; pdf <path> saves a PDF; tab list and tab <tab-id-or-label> inspect or recover the active tab; react tree/inspect/renders/suspense introspect React after --enable react-devtools; vitals [url] measures Core Web Vitals; pushstate <url> performs SPA navigation; tap <selector> and swipe <direction> [distance] support iOS/provider touch flows.",
+	"For artifact-producing commands, read the visible artifact block and details.artifactVerification before using files: check requested path, absolute path, existence, size bytes, artifact kind, optional mediaType, status, optional limitation, and verified/missing/pending/unverified counts. details.artifacts contains per-file metadata. Browser close does not delete explicit saved files; if close reports details.artifactCleanup, use host file tools to remove paths listed in explicitArtifactPaths (when non-empty) after inspection. If close fails with details.promptGuard.reason=requested-artifacts-missing-before-close, save the exact required artifact path before closing. For annotated screenshots inside batch, put --annotate in top-level args (for example { args: [\"--annotate\", \"batch\"], stdin: \"[[\\\"screenshot\\\",\\\"/tmp/page.png\\\"]]\" }) rather than inside the screenshot step.",
 	"When details.nextActions is present, prefer those exact native agent_browser follow-up payloads over prose guidance; they may include args, stdin, sessionMode, networkSourceLookup, safety notes, or artifactPath for saved files.",
 ] as const;
 
@@ -45,13 +45,13 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
 	"If you already used the implicit session and now need launch-scoped flags like --profile, --session-name, --cdp, --state, --auto-connect, --init-script, --enable, -p/--provider, or iOS --device, retry with sessionMode set to fresh or pass an explicit --session for the new launch. After a successful unnamed fresh launch, later auto calls follow that new session.",
 	"For React introspection, launch the page with --enable react-devtools before first navigation, then use react tree, react inspect <fiberId>, sourceLookup candidates for local UI source hints, react renders start/stop, or react suspense; sourceLookup is experimental and reports confidence/evidence instead of guaranteed DOM-to-file mappings. For failed fetches and APIs, networkSourceLookup (experimental) correlates failed network requests with initiator metadata and bounded workspace URL literals—candidates only, not definitive blame. Use vitals [url] for Core Web Vitals and hydration timing, and pushstate <url> for client-side SPA navigation.",
 	"For first-navigation setup, use open without a URL plus network route --resource-type <csv>, cookies set --curl <file>, or --init-script/--enable before navigate/opening the target page.",
-	"For stateful browser context work, prefer purpose-specific page actions before dumping browser data: use auth save --password-stdin with the tool stdin field for credentials, state save/load for portable test state, cookies get/set/clear and storage local|session only when the task needs those values, and expect cookie/storage/auth/state summaries to redact credential-like fields.",
+	"For stateful browser context work, prefer purpose-specific page actions before dumping browser data: use auth save --password-stdin with the tool stdin field for credentials, auth list/show/delete/remove for local auth-profile maintenance, auth login when you need the browser to fill a saved profile, state save/load for portable test state, state list/show/rename/clear/clear -a/clean for saved-state lifecycle cleanup, cookies get/set/clear and storage local|session only when the task needs those values, and expect cookie/storage/auth/state summaries to redact credential-like fields.",
 	"For batch chains that touch cookies, storage, auth, or other secret-bearing commands, use details.batchSteps for per-step artifacts, categories, spill paths, and full structured errors; top-level details.data on batch is only a compact redacted step matrix (success, argv-redacted command, redacted result or scrubbed error text) built from the same presentation rules as standalone calls.",
-	"For non-core families, pass current upstream commands through the native tool directly: network route/requests/har, diff snapshot/screenshot/url, trace/profiler/record, console/errors/highlight/inspect/clipboard, stream enable/disable/status, dashboard start/stop, and chat. For compact network requests output, prefer details.nextActions for request detail, actionable failed-request networkSourceLookup, filtering, or HAR capture follow-ups instead of guessing request-id syntax. Artifact-producing commands report details.artifacts and verification state; long-running starts such as stream, dashboard, trace/profiler, and record should be paired with the matching stop/disable command when the task is done.",
-	"For Electron desktop apps, prefer top-level electron for wrapper-owned discovery, isolated launch, status, compact probe, and cleanup: list first, treat likely-sensitive annotations as hints rather than enforcement, launch with the default snapshot handoff unless handoff: \"tabs\" is the safer diagnostic starting point, use electron.probe or snapshot -i/qa.attached for current-session state, and always cleanup the returned launchId when done. electron.launch uses an isolated temporary profile; it does not reuse the app's normal signed-in profile or attach to an already-running authenticated app. For signed-in local app state, host-launch the normal app with --remote-debugging-port when appropriate, then use raw args connect <port|url>; after connect, inspect tab list, select the stable tab id such as tab t2, then run a condition wait or snapshot -i before using refs. close only closes the browser/CDP session; leave manually launched app shutdown, profile cleanup, and explicit artifacts to the host owner.",
-	"For provider or specialized app workflows, load version-matched upstream guidance with skills get agentcore|electron|slack|dogfood|vercel-sandbox through the native tool. Provider launches such as -p ios, --provider browserbase/kernel/browseruse/browserless/agentcore, and iOS --device are upstream-owned setup paths; use sessionMode fresh when switching providers and expect external credentials or local Appium/Xcode setup to be required.",
+	"For non-core families, pass current upstream commands through the native tool directly: network route/requests/har (including request filters like --type/--method/--status), diff snapshot/screenshot/url with scoped/baseline options, trace/profiler/record, console/errors/highlight/inspect/clipboard, stream enable/disable/status, dashboard start/stop, device list for iOS simulator inventory, and chat. For compact network requests output, prefer details.nextActions for request detail, actionable failed-request networkSourceLookup, filtering, or HAR capture follow-ups instead of guessing request-id syntax. Artifact-producing commands report details.artifacts and verification state; long-running starts such as stream, dashboard, trace/profiler, and record should be paired with the matching stop/disable command when the task is done.",
+	"For Electron desktop apps, prefer top-level electron for wrapper-owned discovery, isolated launch, status, compact probe, and cleanup: list first, treat likely-sensitive annotations as hints rather than enforcement, launch with the default snapshot handoff unless handoff: \"tabs\" is the safer diagnostic starting point, use electron.probe or snapshot -i/qa.attached for current-session state, and always cleanup the returned launchId when done. electron.launch uses an isolated temporary profile; it does not reuse the app's normal signed-in profile or attach to an already-running authenticated app. For signed-in local app state, host-launch the normal app with --remote-debugging-port when appropriate, then use raw args connect <port|url>; after connect, inspect tab list, select the stable tab id such as tab t2, then run a condition wait or snapshot -i before using refs. close commands (`close`, `quit`, or `exit`) only close the browser/CDP session; leave manually launched app shutdown, profile cleanup, and explicit artifacts to the host owner.",
+	"For provider or specialized app workflows, load version-matched upstream guidance with skills get agentcore|electron|slack|dogfood|vercel-sandbox through the native tool; add --full when you need references/templates, and use skills get --all only for broad skill audits. Provider launches such as -p ios, --provider browserbase/kernel/browseruse/browserless/agentcore, and iOS --device are upstream-owned setup paths; use sessionMode fresh when switching providers and expect external credentials or local Appium/Xcode setup to be required.",
 	"For dialogs and frames, use dialog status/accept/dismiss and frame <selector|main> through native args; when --confirm-actions produces a pending confirmation, use details.nextActions or exact confirm <id> / deny <id> calls instead of inventing ids.",
-	"If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; use electron.probe/status for wrapper-owned launch health or target mismatch. Fixed waits are a last resort, must stay below the wrapper IPC budget (wait 30000 is intentionally blocked), and a successful payload like \"waited\":\"timeout\" means elapsed time only—verify completion with an observed condition, fresh snapshot, or screenshot.",
+	"If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; for disappearance checks in agent-browser 0.27.0, use wait --fn predicates instead of stale upstream-help examples like wait <selector> --state hidden. Use electron.probe/status for wrapper-owned launch health or target mismatch. Fixed waits are a last resort, must stay below the wrapper IPC budget (wait 30000 is intentionally blocked), and a successful payload like \"waited\":\"timeout\" means elapsed time only—verify completion with an observed condition, fresh snapshot, or screenshot.",
 	"For feed, timeline, or inbox reading tasks, focus on the main timeline/list region and read the first item there rather than unrelated composer or sidebar content.",
 	"For read-only browsing tasks, prefer extracting the answer from the current snapshot, structured ref labels, or eval --stdin on the current page before navigating away. Only click into media viewers, detail routes, or new pages when the current view does not contain the needed information.",
 	"For downloads, prefer download <selector> <path> when an element click should save a file. Do not rely on click alone when you need the downloaded file on disk.",
@@ -61,7 +61,7 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
 	"When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.",
 	"When commands save or spill files (screenshots, downloads, PDFs, traces, recordings, HAR, large snapshot spills), use the user's exact requested paths when given and treat paths as provisional until details.artifactVerification shows every row verified: branch on missingCount, pendingCount, unverifiedCount, per-entry state, and optional limitation before downstream file use or PASS/FAIL reporting.",
 	"For evidence-only screenshots, QA captures, or other audit artifacts, save to an explicit path and branch on details.artifactVerification plus details.artifacts before reporting PASS/FAIL; do not require vision review of inline image attachments unless the user asked for visual inspection.",
-	"Respect explicit user stop boundaries: if the user says to stop before order/post/purchase/submit, do not click that final action.",
+	"Respect explicit user stop boundaries: if the user says to stop before order/post/purchase/submit, do not click that final action. If the wrapper returns details.promptGuard.reason=explicit-user-stop-boundary, gather evidence on the current page instead of retrying the blocked final action.",
 	"Successful record stop needs ffmpeg on PATH; the wrapper may warn after record start when ffmpeg is missing.",
 	"Do not call --help or other exploratory inspection commands unless the user explicitly asks for them or debugging the browser integration is necessary.",
 ] as const;
@@ -100,7 +100,7 @@ export const RUNTIME_PROMPT_GUIDELINES = [
 	"Use exactly one input mode: args (open→snapshot -i→@refs), semanticAction, job, qa, sourceLookup/networkSourceLookup (candidate hints), or electron. stdin only for batch/eval/auth or wrapper batch; electron rejects stdin. Do not pass --json in args; wrapper injects it.",
 	"Common flow: open, snapshot -i, use current @refs or semanticAction, then re-snapshot after navigation/scroll/rerender/DOM change. Batch same-snapshot fills unless they may submit/navigate/rerender. Respect explicit stop boundaries: if the user says stop before order/post/purchase/submit, do not click the final action.",
 	"Use sessionMode=fresh for launch-scoped flags on an active implicit session. For signed-in/account-specific content, start with --profile Default plus sessionMode=fresh unless asked otherwise; visible content is model-visible.",
-	"For artifacts, save the exact user path and check details.artifactVerification/details.artifacts before claiming success. record stop needs ffmpeg on PATH; close does not delete saved files; \"waited\":\"timeout\" is not proof.",
+	"For artifacts, save the exact user path and check details.artifactVerification/details.artifacts before claiming success. If close is blocked by details.promptGuard, save the required artifact first. record stop needs ffmpeg on PATH; close does not delete saved files; \"waited\":\"timeout\" is not proof.",
 	"When details.nextActions is present, prefer exact payloads over prose/guessed selectors. For dense snapshots, check Omitted high-value controls/details.data.highValueControlRefIds. For dashboards, verify scroll with screenshot/snapshot; if nothing moved, target the real scroll region.",
 	"For extraction, prefer get title/url/text/html/value/attr/count or eval --stdin with plain expression, not console.log. Batch three or more known refs/selectors (e.g. [[\"get\",\"text\",\"@e1\"],[\"get\",\"text\",\"@e2\"]]); selector visibility warnings → visible @refs/nextActions.",
 ] as const;

package/extensions/agent-browser/lib/prompt-policy.ts

@@ -0,0 +1,122 @@
+/**
+ * Purpose: Derive operator prompt constraints for browser-run preflight guards and legacy bash policy.
+ * Responsibilities: Parse the latest user message into stop boundaries, requested artifact paths, and legacy bash allowance.
+ * Scope: Pure prompt-text policy; enforcement lives in orchestration prompt-guards and the extension entrypoint.
+ */
+
+export interface PromptRequestedArtifact {
+	kind: "recording" | "screenshot";
+	path: string;
+	required: boolean;
+}
+
+export interface PromptStopBoundary {
+	reason: "avoid-final-submit-action";
+}
+
+export interface PromptPolicy {
+	allowLegacyAgentBrowserBash: boolean;
+	requestedArtifacts: PromptRequestedArtifact[];
+	stopBoundary?: PromptStopBoundary;
+}
+
+const BROWSER_PROMPT_PATTERNS = [
+	/\b(?:agent[_ -]?browser|browser automation|eval\s+--stdin|screenshot|snapshot|tab\s+list)\b/i,
+	/\b(?:react\s+(?:tree|inspect|renders|suspense)|web\s+vitals|core\s+web\s+vitals|pushstate)\b/i,
+	/\b(?:live\s+docs?|online\s+research|research\s+(?:online|the\s+web)|search\s+(?:online|the\s+web)|web\s+research)\b/i,
+	/\bbrowser\b.*\b(?:automation|click|fill|navigate|open|page|screenshot|site|snapshot|tab|url|visit|web(?:site| page)?)\b/i,
+	/\b(?:browse|click|fill|login|navigate|open|visit)\b.*\b(?:https?:\/\/\S+|page|site|tab|url|web(?:site| page)?)\b/i,
+];
+
+const LEGACY_BASH_ALLOW_PATTERNS = [
+	/\b(?:bash-oriented workflow|bash workflow)\b/i,
+	/\b(?:use|via|through|with)\s+bash\b/i,
+	/\bnpx\s+agent-browser\b/i,
+	/\bagent-browser\s+--(?:help|version)\b/i,
+	/\bdebug(?:ging)?\b.*\b(?:agent[_ -]?browser|agent_browser|browser integration)\b/i,
+];
+
+const STOP_BOUNDARY_PATTERNS = [
+	/\b(?:do\s+not|don't|dont|never)\s+(?:place|submit|complete|finish|finali[sz]e|confirm)\s+(?:the\s+)?(?:order|purchase|checkout|payment)\b/i,
+	/\b(?:do\s+not|don't|dont|never)\s+click\s+(?:the\s+)?(?:finish|submit|place\s+order|complete\s+order|confirm\s+order|buy\s+now|pay\s+now)\b/i,
+	/\bstop\s+(?:on|at|before)\b[^.\n]*(?:checkout\s+overview|finish|place\s+(?:the\s+)?order|submit\s+(?:the\s+)?order|complete\s+(?:the\s+)?order|purchase|payment)\b/i,
+	/\bwithout\s+(?:placing|submitting|completing|finishing|confirming)\s+(?:the\s+)?(?:order|purchase|payment)\b/i,
+];
+
+const PROMPT_ARTIFACT_PATH_PATTERN = /(?:^|[\s"'`(:])((?:\/[^\s"'`),;]+|[A-Za-z]:[\\/][^\s"'`),;]+|\.{1,2}[\\/][^\s"'`),;]+|[^\s"'`),;:\\/]+(?:[\\/][^\s"'`),;]+)+|[^\s"'`),;:\\/]+)\.(?:png|jpe?g|webp|gif|webm|mp4|har|pdf|trace|json))(?:[\s"'`),;.]|$)/gi;
+
+function buildPromptStopBoundary(prompt: string): PromptStopBoundary | undefined {
+	return STOP_BOUNDARY_PATTERNS.some((pattern) => pattern.test(prompt)) ? { reason: "avoid-final-submit-action" } : undefined;
+}
+
+function extractPromptRequestedArtifacts(prompt: string): PromptRequestedArtifact[] {
+	const artifacts: PromptRequestedArtifact[] = [];
+	const seen = new Set<string>();
+	for (const line of prompt.split(/\r?\n/)) {
+		const lowerLine = line.toLowerCase();
+		const kind = lowerLine.includes("screenshot")
+			? "screenshot"
+			: /\b(?:screen\s+recording|recording|webm|video)\b/.test(lowerLine)
+				? "recording"
+				: undefined;
+		if (!kind) continue;
+		PROMPT_ARTIFACT_PATH_PATTERN.lastIndex = 0;
+		for (const match of line.matchAll(PROMPT_ARTIFACT_PATH_PATTERN)) {
+			const path = match[1]?.trim();
+			if (!path) continue;
+			const key = `${kind}:${path}`;
+			if (seen.has(key)) continue;
+			seen.add(key);
+			artifacts.push({
+				kind,
+				path,
+				required: kind === "screenshot" || !/\b(?:if|when)\s+(?:recording\s+)?(?:is\s+)?available\b/i.test(line),
+			});
+		}
+	}
+	return artifacts;
+}
+
+export function buildPromptPolicy(prompt: string): PromptPolicy {
+	return {
+		allowLegacyAgentBrowserBash: LEGACY_BASH_ALLOW_PATTERNS.some((pattern) => pattern.test(prompt)),
+		requestedArtifacts: extractPromptRequestedArtifacts(prompt),
+		stopBoundary: buildPromptStopBoundary(prompt),
+	};
+}
+
+function getMessageText(content: unknown): string {
+	if (typeof content === "string") return content;
+	if (!Array.isArray(content)) return "";
+
+	return content
+		.map((item) => {
+			if (typeof item !== "object" || item === null) return "";
+			return item.type === "text" && typeof item.text === "string" ? item.text : "";
+		})
+		.filter((text) => text.length > 0)
+		.join("\n");
+}
+
+export function shouldAppendBrowserSystemPrompt(prompt: string): boolean {
+	const normalizedPrompt = prompt.trim();
+	if (normalizedPrompt.length === 0) {
+		return false;
+	}
+	return BROWSER_PROMPT_PATTERNS.some((pattern) => pattern.test(normalizedPrompt));
+}
+
+export function getLatestUserPrompt(branch: unknown[]): string {
+	for (let index = branch.length - 1; index >= 0; index -= 1) {
+		const entry = branch[index];
+		if (typeof entry !== "object" || entry === null || !("type" in entry) || entry.type !== "message") {
+			continue;
+		}
+		const message = "message" in entry ? entry.message : undefined;
+		if (typeof message !== "object" || message === null || !("role" in message) || message.role !== "user") {
+			continue;
+		}
+		return getMessageText("content" in message ? message.content : undefined);
+	}
+	return "";
+}

package/extensions/agent-browser/lib/results/action-recommendations.ts

@@ -6,6 +6,7 @@
  * Invariants/Assumptions: Action ids are public machine-readable contracts; preserve first-observed order.
  */
 
+import { isOpenNavigationCommand, isPageMutationCommand } from "../command-taxonomy.js";
 import { isPendingRecordingArtifact } from "./artifact-state.js";
 import type {
 	AgentBrowserFailureCategory,
@@ -56,27 +57,6 @@ function buildElectronToolAction(options: {
 	};
 }
 
-const MUTATING_COMMANDS = new Set([
-	"back",
-	"check",
-	"click",
-	"dblclick",
-	"dialog",
-	"fill",
-	"forward",
-	"hover",
-	"press",
-	"pushstate",
-	"reload",
-	"scroll",
-	"scrollintoview",
-	"select",
-	"swipe",
-	"tap",
-	"type",
-	"uncheck",
-]);
-
 function getDownloadRetryPath(args: string[] | undefined, fallback: string | undefined): string | undefined {
 	if (fallback) return fallback;
 	if (!args || args.length === 0) return undefined;
@@ -170,13 +150,13 @@ export function buildAgentBrowserNextActions(options: {
 		}
 	}
 	if (options.resultCategory === "success") {
-		if (options.command === "open") {
+		if (isOpenNavigationCommand(options.command)) {
 			actions.push(buildNextToolAction({
 				args: ["snapshot", "-i"],
 				id: "inspect-opened-page",
 				reason: "Inspect the opened page before choosing interactive refs.",
 			}));
-		} else if (options.command && MUTATING_COMMANDS.has(options.command)) {
+		} else if (isPageMutationCommand(options.command)) {
 			actions.push(buildNextToolAction({
 				args: ["snapshot", "-i"],
 				id: "inspect-after-mutation",

package/extensions/agent-browser/lib/results/presentation/navigation.ts

@@ -4,41 +4,13 @@
  * Scope: Navigation and get/eval extraction formatting only.
  */
 
+import { isNavigationObservableCommandName, isPageChangeSummaryCommand } from "../../command-taxonomy.js";
 import { isRecord } from "../../parsing.js";
 import type { CommandInfo } from "../../runtime.js";
 import { detectConfirmationRequired } from "../confirmation.js";
 import type { AgentBrowserPageChangeSummary, FileArtifactMetadata } from "../contracts.js";
 import { redactModelFacingText, stringifyModelFacing } from "./common.js";
 
-const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
-
-const PAGE_CHANGE_SUMMARY_COMMANDS = new Set([
-	"back",
-	"check",
-	"click",
-	"dblclick",
-	"dialog",
-	"download",
-	"fill",
-	"forward",
-	"goto",
-	"hover",
-	"navigate",
-	"open",
-	"pdf",
-	"press",
-	"pushstate",
-	"reload",
-	"screenshot",
-	"scroll",
-	"scrollintoview",
-	"select",
-	"swipe",
-	"tap",
-	"type",
-	"uncheck",
-]);
-
 const NAVIGATION_SUMMARY_FIELD = "navigationSummary";
 
 interface NavigationSummary {
@@ -108,7 +80,7 @@ export function formatExtractionText(commandInfo: CommandInfo, data: Record<stri
 }
 
 export function isNavigationObservableCommand(command: string | undefined): boolean {
-	return command !== undefined && NAVIGATION_SUMMARY_COMMANDS.has(command);
+	return isNavigationObservableCommandName(command);
 }
 
 function isNavigationSummary(value: unknown): value is NavigationSummary {
@@ -142,10 +114,6 @@ export function formatNavigationSummary(summary: NavigationSummary): string | un
 	return normalized.title ?? normalized.url;
 }
 
-function isPageChangeSummaryCommand(command: string | undefined): boolean {
-	return command !== undefined && PAGE_CHANGE_SUMMARY_COMMANDS.has(command);
-}
-
 export function buildPageChangeSummary(options: {
 	artifacts?: FileArtifactMetadata[];
 	commandInfo: CommandInfo;