pi-agent-browser-native 0.2.26 → 0.2.28

package/extensions/agent-browser/index.ts

@@ -6,8 +6,10 @@
  * Invariants/Assumptions: agent-browser is installed separately on PATH, the wrapper targets the current locally installed upstream version only, and no backward-compatibility shims are provided.
  */
 
-import { copyFile, mkdir, readFile, readdir, rm, stat } from "node:fs/promises";
-import { dirname, extname, isAbsolute, join, resolve } from "node:path";
+import { constants as fsConstants } from "node:fs";
+import { access, copyFile, mkdir, readFile, readdir, rm, stat } from "node:fs/promises";
+import { delimiter, dirname, extname, isAbsolute, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
 
 import { StringEnum } from "@earendil-works/pi-ai";
 import {
@@ -30,6 +32,7 @@ import {
 	buildAgentBrowserNextActions,
 	buildAgentBrowserResultCategoryDetails,
 	buildToolPresentation,
+	compareRefIds,
 	getAgentBrowserErrorText,
 	parseAgentBrowserEnvelope,
 	type AgentBrowserBatchResult,
@@ -84,6 +87,7 @@ const PACKAGE_NAME = "pi-agent-browser-native";
 const AGENT_BROWSER_SEMANTIC_ACTIONS = ["check", "click", "fill", "select", "uncheck"] as const;
 const AGENT_BROWSER_SEMANTIC_LOCATORS = ["alt", "label", "placeholder", "role", "testid", "text", "title"] as const;
 const AGENT_BROWSER_JOB_STEP_ACTIONS = ["open", "click", "fill", "wait", "assertText", "assertUrl", "waitForDownload", "screenshot"] as const;
+const AGENT_BROWSER_QA_LOAD_STATES = ["domcontentloaded", "load", "networkidle"] as const;
 const SOURCE_LOOKUP_WORKSPACE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx"]);
 const SOURCE_LOOKUP_IGNORED_DIRECTORIES = new Set([".git", "node_modules", "dist", "build", "coverage", ".next", "out", "tmp", "temp"]);
 const SOURCE_LOOKUP_DEFAULT_MAX_WORKSPACE_FILES = 2_000;
@@ -92,6 +96,7 @@ const SOURCE_LOOKUP_MAX_WORKSPACE_FILES = 5_000;
 type AgentBrowserSemanticActionName = (typeof AGENT_BROWSER_SEMANTIC_ACTIONS)[number];
 type AgentBrowserSemanticLocator = (typeof AGENT_BROWSER_SEMANTIC_LOCATORS)[number];
 type AgentBrowserJobStepAction = (typeof AGENT_BROWSER_JOB_STEP_ACTIONS)[number];
+type AgentBrowserQaLoadState = (typeof AGENT_BROWSER_QA_LOAD_STATES)[number];
 type AgentBrowserSourceLookupStatus = "candidates-found" | "no-candidates" | "unsupported";
 type AgentBrowserNetworkSourceLookupStatus = "failed-requests-found" | "no-failed-requests" | "no-candidates";
 
@@ -111,6 +116,48 @@ interface CompiledAgentBrowserSemanticAction {
 	args: string[];
 }
 
+interface ScrollPositionSnapshot {
+	containerCount: number;
+	containers: Array<{ id: string; scrollLeft: number; scrollTop: number }>;
+	innerHeight: number;
+	innerWidth: number;
+	scrollHeight: number;
+	scrollWidth: number;
+	scrollX: number;
+	scrollY: number;
+}
+
+interface ScrollNoopDiagnostic {
+	after: ScrollPositionSnapshot;
+	before: ScrollPositionSnapshot;
+	message: string;
+	reason: "no-observed-scroll-position-change";
+	recommendations: string[];
+}
+
+interface ComboboxFocusDiagnostic {
+	activeElement: {
+		expanded?: string;
+		hasPopup?: string;
+		name?: string;
+		role?: string;
+		tagName?: string;
+	};
+	message: string;
+	reason: "focused-combobox-without-visible-options";
+	recommendations: string[];
+	visibleListboxCount: number;
+	visibleOptionCount: number;
+}
+
+interface RecordingDependencyWarning {
+	command: "record start" | "record restart";
+	dependency: "ffmpeg";
+	message: string;
+	reason: "ffmpeg-missing-for-recording";
+	recommendations: string[];
+}
+
 interface CompiledAgentBrowserJobStep {
 	action: AgentBrowserJobStepAction;
 	args: string[];
@@ -127,6 +174,7 @@ interface CompiledAgentBrowserQaPreset extends CompiledAgentBrowserJob {
 		checkConsole: boolean;
 		checkErrors: boolean;
 		checkNetwork: boolean;
+		loadState: AgentBrowserQaLoadState;
 		expectedText: string[];
 		expectedSelector?: string;
 		screenshotPath?: string;
@@ -238,6 +286,7 @@ const AGENT_BROWSER_PARAMS = Type.Object({
 			checkConsole: Type.Optional(Type.Boolean({ description: "Whether to fail on console error messages. Defaults to true." })),
 			checkErrors: Type.Optional(Type.Boolean({ description: "Whether to fail on page errors. Defaults to true." })),
 			checkNetwork: Type.Optional(Type.Boolean({ description: "Whether to inspect network requests and fail on actionable request failures; benign icon misses warn. Defaults to true." })),
+			loadState: Type.Optional(StringEnum(AGENT_BROWSER_QA_LOAD_STATES, { description: "Page readiness state for the QA preset before assertions and diagnostics. Defaults to domcontentloaded; use networkidle only for pages without long-lived background requests." })),
 		}),
 	),
 	sourceLookup: Type.Optional(
@@ -450,16 +499,21 @@ function compileAgentBrowserQaPreset(input: unknown): { compiled?: CompiledAgent
 			return { error: `qa.${field} must be a boolean when provided.` };
 		}
 	}
+	const rawLoadState = input.loadState;
+	if (rawLoadState !== undefined && (typeof rawLoadState !== "string" || !AGENT_BROWSER_QA_LOAD_STATES.includes(rawLoadState as AgentBrowserQaLoadState))) {
+		return { error: `qa.loadState must be one of: ${AGENT_BROWSER_QA_LOAD_STATES.join(", ")}.` };
+	}
 	const checkConsole = input.checkConsole !== false;
 	const checkErrors = input.checkErrors !== false;
 	const checkNetwork = input.checkNetwork !== false;
+	const loadState = (rawLoadState as AgentBrowserQaLoadState | undefined) ?? "domcontentloaded";
 	const steps: CompiledAgentBrowserJobStep[] = [];
 	if (checkNetwork) steps.push({ action: "wait", args: ["network", "requests", "--clear"] });
 	if (checkConsole) steps.push({ action: "wait", args: ["console", "--clear"] });
 	if (checkErrors) steps.push({ action: "wait", args: ["errors", "--clear"] });
 	steps.push(
 		{ action: "open", args: ["open", url] },
-		{ action: "wait", args: ["wait", "--load", "networkidle"] },
+		{ action: "wait", args: ["wait", "--load", loadState] },
 	);
 	for (const text of expectedText) {
 		steps.push({ action: "assertText", args: ["wait", "--text", text] });
@@ -474,7 +528,7 @@ function compileAgentBrowserQaPreset(input: unknown): { compiled?: CompiledAgent
 	return {
 		compiled: {
 			args: ["batch"],
-			checks: { checkConsole, checkErrors, checkNetwork, expectedSelector, expectedText, screenshotPath, url },
+			checks: { checkConsole, checkErrors, checkNetwork, expectedSelector, expectedText, loadState, screenshotPath, url },
 			stdin: JSON.stringify(steps.map((step) => step.args)),
 			steps,
 		},
@@ -969,6 +1023,61 @@ function buildSemanticActionCandidateActions(compiled: CompiledAgentBrowserSeman
 	return [];
 }
 
+function normalizeSemanticActionAccessibleName(name: string): string {
+	return name.replace(/\s+/g, " ").trim().toLowerCase();
+}
+
+function semanticActionNameMatches(candidateName: string, targetName: string): boolean {
+	const normalizedCandidate = normalizeSemanticActionAccessibleName(candidateName);
+	const normalizedTarget = normalizeSemanticActionAccessibleName(targetName);
+	return normalizedCandidate === normalizedTarget || normalizedCandidate.startsWith(`${normalizedTarget} `);
+}
+
+function getCompiledSemanticActionRoleTarget(compiled: CompiledAgentBrowserSemanticAction): { role: string; targetName: string } | undefined {
+	if (compiled.locator !== "role" || !["check", "click", "uncheck"].includes(compiled.action)) return undefined;
+	const findIndex = compiled.args.indexOf("find");
+	if (findIndex < 0 || compiled.args[findIndex + 1] !== "role") return undefined;
+	const role = compiled.args[findIndex + 2];
+	const nameFlagIndex = compiled.args.indexOf("--name");
+	const targetName = nameFlagIndex >= 0 ? compiled.args[nameFlagIndex + 1] : undefined;
+	if (!role || !targetName) return undefined;
+	return { role, targetName };
+}
+
+function findSemanticActionRefInSnapshot(compiled: CompiledAgentBrowserSemanticAction, snapshotData: unknown): string | undefined {
+	const target = getCompiledSemanticActionRoleTarget(compiled);
+	const refs = getSnapshotRefRecord(snapshotData);
+	if (!target || !refs) return undefined;
+	const candidates = Object.entries(refs).flatMap(([ref, entry]) => {
+		if (!/^e\d+$/.test(ref) || !isRecord(entry)) return [];
+		const role = typeof entry.role === "string" ? entry.role : undefined;
+		const name = typeof entry.name === "string" ? entry.name : undefined;
+		if (!role || !name || role.toLowerCase() !== target.role.toLowerCase() || !semanticActionNameMatches(name, target.targetName)) return [];
+		return [{ exact: normalizeSemanticActionAccessibleName(name) === normalizeSemanticActionAccessibleName(target.targetName), name, ref }];
+	});
+	candidates.sort((left, right) => Number(right.exact) - Number(left.exact) || left.name.length - right.name.length || compareRefIds(left.ref, right.ref));
+	return candidates[0]?.ref;
+}
+
+interface SemanticActionVisibleRefResolution {
+	args: string[];
+	snapshot: SessionRefSnapshot;
+}
+
+async function resolveSemanticActionVisibleRefArgs(options: {
+	compiled: CompiledAgentBrowserSemanticAction | undefined;
+	cwd: string;
+	sessionName?: string;
+	signal?: AbortSignal;
+}): Promise<SemanticActionVisibleRefResolution | undefined> {
+	if (!options.compiled || !options.sessionName || !getCompiledSemanticActionRoleTarget(options.compiled)) return undefined;
+	const snapshotData = await runSessionCommandData({ args: ["snapshot", "-i"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
+	const ref = findSemanticActionRefInSnapshot(options.compiled, snapshotData);
+	const snapshot = extractRefSnapshotFromData(snapshotData);
+	if (!ref || !snapshot) return undefined;
+	return { args: [...getCompiledSemanticActionSessionPrefix(options.compiled), options.compiled.action, `@${ref}`], snapshot };
+}
+
 function compileAgentBrowserSemanticAction(input: unknown): { compiled?: CompiledAgentBrowserSemanticAction; error?: string } {
 	if (!isRecord(input)) {
 		return { error: "semanticAction must be an object." };
@@ -2615,6 +2724,140 @@ async function collectNavigationSummary(options: {
 	return { title, url };
 }
 
+function extractScrollPositionSnapshot(data: unknown): ScrollPositionSnapshot | undefined {
+	const result = isRecord(data) && isRecord(data.result) ? data.result : data;
+	if (!isRecord(result)) return undefined;
+	const scrollX = typeof result.scrollX === "number" ? result.scrollX : undefined;
+	const scrollY = typeof result.scrollY === "number" ? result.scrollY : undefined;
+	const innerHeight = typeof result.innerHeight === "number" ? result.innerHeight : undefined;
+	const innerWidth = typeof result.innerWidth === "number" ? result.innerWidth : undefined;
+	const scrollHeight = typeof result.scrollHeight === "number" ? result.scrollHeight : undefined;
+	const scrollWidth = typeof result.scrollWidth === "number" ? result.scrollWidth : undefined;
+	if (scrollX === undefined || scrollY === undefined || innerHeight === undefined || innerWidth === undefined || scrollHeight === undefined || scrollWidth === undefined) return undefined;
+	const containers = Array.isArray(result.containers)
+		? result.containers.flatMap((entry, index): ScrollPositionSnapshot["containers"] => {
+			if (!isRecord(entry)) return [];
+			const rawId = typeof entry.id === "string" ? entry.id : undefined;
+			const id = rawId && /^\d+:[a-z][a-z0-9-]*(?:\[role=[a-z-]+\])?$/i.test(rawId) ? rawId : `sample-${index}`;
+			const scrollTop = typeof entry.scrollTop === "number" ? entry.scrollTop : undefined;
+			const scrollLeft = typeof entry.scrollLeft === "number" ? entry.scrollLeft : undefined;
+			return scrollTop !== undefined && scrollLeft !== undefined ? [{ id, scrollLeft, scrollTop }] : [];
+		})
+		: [];
+	return {
+		containerCount: typeof result.containerCount === "number" ? result.containerCount : containers.length,
+		containers,
+		innerHeight,
+		innerWidth,
+		scrollHeight,
+		scrollWidth,
+		scrollX,
+		scrollY,
+	};
+}
+
+const SCROLL_POSITION_EVAL = `(() => {
+  const viewport = {
+    scrollX: window.scrollX,
+    scrollY: window.scrollY,
+    innerHeight: window.innerHeight,
+    innerWidth: window.innerWidth,
+    scrollHeight: Math.max(document.documentElement?.scrollHeight || 0, document.body?.scrollHeight || 0),
+    scrollWidth: Math.max(document.documentElement?.scrollWidth || 0, document.body?.scrollWidth || 0),
+  };
+  const describe = (element, index) => {
+    const role = element.getAttribute("role") || "";
+    const id = element.tagName.toLowerCase();
+    return {
+      id: String(index) + ":" + id + (role ? "[role=" + role + "]" : ""),
+      scrollTop: element.scrollTop,
+      scrollLeft: element.scrollLeft,
+      area: element.clientWidth * element.clientHeight,
+    };
+  };
+  const containers = Array.from(document.querySelectorAll("body *"))
+    .filter((element) => element instanceof HTMLElement && (element.scrollHeight > element.clientHeight + 1 || element.scrollWidth > element.clientWidth + 1))
+    .map(describe)
+    .sort((left, right) => right.area - left.area)
+    .slice(0, 10)
+    .map(({ area, ...entry }) => entry);
+  return { ...viewport, containerCount: containers.length, containers };
+})()`;
+
+async function collectScrollPositionSnapshot(options: {
+	cwd: string;
+	sessionName?: string;
+	signal?: AbortSignal;
+}): Promise<ScrollPositionSnapshot | undefined> {
+	return extractScrollPositionSnapshot(await runSessionCommandData({
+		args: ["eval", "--stdin"],
+		cwd: options.cwd,
+		sessionName: options.sessionName,
+		signal: options.signal,
+		stdin: SCROLL_POSITION_EVAL,
+	}));
+}
+
+function sameScrollPositionSnapshot(left: ScrollPositionSnapshot, right: ScrollPositionSnapshot): boolean {
+	if (
+		left.scrollX !== right.scrollX ||
+		left.scrollY !== right.scrollY ||
+		left.scrollHeight !== right.scrollHeight ||
+		left.scrollWidth !== right.scrollWidth ||
+		left.containers.length !== right.containers.length
+	) {
+		return false;
+	}
+	return left.containers.every((container, index) => {
+		const other = right.containers[index];
+		return other?.id === container.id && other.scrollTop === container.scrollTop && other.scrollLeft === container.scrollLeft;
+	});
+}
+
+function buildScrollNoopDiagnostic(before: ScrollPositionSnapshot | undefined, after: ScrollPositionSnapshot | undefined): ScrollNoopDiagnostic | undefined {
+	if (!before || !after || !sameScrollPositionSnapshot(before, after)) return undefined;
+	return {
+		after,
+		before,
+		message: "Scroll reported success, but the viewport and sampled scrollable containers did not change position.",
+		reason: "no-observed-scroll-position-change",
+		recommendations: [
+			"Run snapshot -i or screenshot to confirm what is visible before choosing the next action.",
+			"On dashboards and panes with nested scrolling, use scrollintoview <@ref> for a visible target or target the actual scrollable region instead of repeating page scrolls.",
+		],
+	};
+}
+
+function buildScrollNoopNextActions(sessionName: string | undefined): AgentBrowserNextAction[] {
+	const withSession = (args: string[]): string[] => sessionName ? ["--session", sessionName, ...args] : args;
+	return [
+		{
+			id: "inspect-after-noop-scroll",
+			params: { args: withSession(["snapshot", "-i"]) },
+			reason: "Refresh interactive refs and inspect whether the intended target is inside a nested scroll container.",
+			safety: "Do not assume repeated page scrolls will move dashboard panels or nested panes.",
+			tool: "agent_browser",
+		},
+		{
+			id: "verify-noop-scroll-visually",
+			params: { args: withSession(["screenshot"]) },
+			reason: "Capture the current viewport to verify whether the scroll actually changed visible content.",
+			safety: "Use screenshot evidence before concluding a dense dashboard did or did not move.",
+			tool: "agent_browser",
+		},
+	];
+}
+
+function formatScrollNoopDiagnosticText(diagnostic: ScrollNoopDiagnostic | undefined): string | undefined {
+	if (!diagnostic) return undefined;
+	return [
+		"Scroll diagnostic: no observed scroll movement.",
+		`Reason: ${diagnostic.message}`,
+		`Sampled scrollable containers: ${diagnostic.after.containers.length}/${diagnostic.after.containerCount}.`,
+		...diagnostic.recommendations.map((recommendation) => `- ${recommendation}`),
+	].join("\n");
+}
+
 function mergeNavigationSummaryIntoData(data: unknown, navigationSummary: NavigationSummary): unknown {
 	if (isRecord(data)) {
 		return { ...data, navigationSummary };
@@ -2622,12 +2865,187 @@ function mergeNavigationSummaryIntoData(data: unknown, navigationSummary: Naviga
 	return { navigationSummary, result: data };
 }
 
+const COMBOBOX_FOCUS_EVAL = `(() => {
+  const isVisible = (element) => {
+    if (!(element instanceof HTMLElement)) return false;
+    const style = window.getComputedStyle(element);
+    if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return false;
+    return element.getClientRects().length > 0;
+  };
+  const active = document.activeElement instanceof HTMLElement ? document.activeElement : null;
+  const role = active?.getAttribute("role") || undefined;
+  const hasPopup = active?.getAttribute("aria-haspopup") || undefined;
+  const expanded = active?.getAttribute("aria-expanded") || undefined;
+  const tagName = active?.tagName.toLowerCase();
+  const name = (active?.getAttribute("aria-label") || active?.getAttribute("placeholder") || active?.getAttribute("title") || active?.textContent || "").trim().slice(0, 80) || undefined;
+  const visibleListboxCount = Array.from(document.querySelectorAll('[role="listbox"], [role="menu"]')).filter(isVisible).length;
+  const visibleOptionCount = Array.from(document.querySelectorAll('[role="option"], option, [role="menuitem"]')).filter(isVisible).length;
+  const comboboxLike = role === "combobox" || hasPopup === "listbox" || hasPopup === "menu" || tagName === "select" || active?.getAttribute("aria-autocomplete") !== null;
+  return { activeElement: active ? { expanded, hasPopup, name, role, tagName } : undefined, comboboxLike, visibleListboxCount, visibleOptionCount };
+})()`;
+
+function extractComboboxFocusDiagnostic(data: unknown): ComboboxFocusDiagnostic | undefined {
+	const result = isRecord(data) && isRecord(data.result) ? data.result : data;
+	if (!isRecord(result) || result.comboboxLike !== true || !isRecord(result.activeElement)) return undefined;
+	const visibleListboxCount = typeof result.visibleListboxCount === "number" ? result.visibleListboxCount : 0;
+	const visibleOptionCount = typeof result.visibleOptionCount === "number" ? result.visibleOptionCount : 0;
+	const expanded = typeof result.activeElement.expanded === "string" ? result.activeElement.expanded : undefined;
+	if ((expanded !== "false" && expanded !== "true") || visibleListboxCount > 0 || visibleOptionCount > 0) return undefined;
+	return {
+		activeElement: {
+			expanded,
+			hasPopup: typeof result.activeElement.hasPopup === "string" ? result.activeElement.hasPopup : undefined,
+			name: typeof result.activeElement.name === "string" ? redactSensitiveText(result.activeElement.name) : undefined,
+			role: typeof result.activeElement.role === "string" ? result.activeElement.role : undefined,
+			tagName: typeof result.activeElement.tagName === "string" ? result.activeElement.tagName : undefined,
+		},
+		message: "A combobox-like control is focused, but no listbox or option elements are visibly open.",
+		reason: "focused-combobox-without-visible-options",
+		recommendations: [
+			"Run snapshot -i to inspect whether options appeared under a different role or portal.",
+			"Try ArrowDown or Enter to open the option list before selecting, or use select/visible option refs when available.",
+		],
+		visibleListboxCount,
+		visibleOptionCount,
+	};
+}
+
+function isComboboxFocusDiagnosticCommand(command: string | undefined, commandTokens: string[]): boolean {
+	const explicitlyTargetsCombobox = commandTokens.some((token) => /^(?:combobox|listbox)$/i.test(token));
+	if (!explicitlyTargetsCombobox) return false;
+	if (command === "click" || command === "fill") return true;
+	return command === "find" && commandTokens.some((token) => ["click", "fill", "select"].includes(token));
+}
+
+function getCompiledSemanticActionRoleValue(compiled: CompiledAgentBrowserSemanticAction): string | undefined {
+	if (compiled.locator !== "role") return undefined;
+	const findIndex = compiled.args.indexOf("find");
+	if (findIndex < 0 || compiled.args[findIndex + 1] !== "role") return undefined;
+	return compiled.args[findIndex + 2];
+}
+
+function isComboboxFocusDiagnosticSemanticAction(compiled: CompiledAgentBrowserSemanticAction | undefined): boolean {
+	if (!compiled || !["click", "fill", "select"].includes(compiled.action)) return false;
+	return /^(?:combobox|listbox)$/i.test(getCompiledSemanticActionRoleValue(compiled) ?? "");
+}
+
+async function collectComboboxFocusDiagnostic(options: {
+	command?: string;
+	commandTokens: string[];
+	cwd: string;
+	semanticAction?: CompiledAgentBrowserSemanticAction;
+	sessionName?: string;
+	signal?: AbortSignal;
+}): Promise<ComboboxFocusDiagnostic | undefined> {
+	if (!isComboboxFocusDiagnosticCommand(options.command, options.commandTokens) && !isComboboxFocusDiagnosticSemanticAction(options.semanticAction)) return undefined;
+	return extractComboboxFocusDiagnostic(await runSessionCommandData({
+		args: ["eval", "--stdin"],
+		cwd: options.cwd,
+		sessionName: options.sessionName,
+		signal: options.signal,
+		stdin: COMBOBOX_FOCUS_EVAL,
+	}));
+}
+
+function buildComboboxFocusNextActions(sessionName: string | undefined): AgentBrowserNextAction[] {
+	const withSession = (args: string[]): string[] => sessionName ? ["--session", sessionName, ...args] : args;
+	return [
+		{
+			id: "inspect-focused-combobox",
+			params: { args: withSession(["snapshot", "-i"]) },
+			reason: "Inspect the focused combobox and any portal/listbox refs before choosing an option.",
+			safety: "Prefer visible option refs or select when a native/selectable option list is exposed.",
+			tool: "agent_browser",
+		},
+		{
+			id: "try-open-combobox-with-arrow",
+			params: { args: withSession(["press", "ArrowDown"]) },
+			reason: "Many searchable comboboxes open their option list with ArrowDown after focus.",
+			safety: "Use only when the focused combobox is still the intended control, then re-snapshot before selecting.",
+			tool: "agent_browser",
+		},
+		{
+			id: "try-open-combobox-with-enter",
+			params: { args: withSession(["press", "Enter"]) },
+			reason: "Some comboboxes open or confirm their option list with Enter after focus.",
+			safety: "Enter may select a highlighted/default option; prefer ArrowDown first unless Enter is the app's expected opener.",
+			tool: "agent_browser",
+		},
+	];
+}
+
+function formatComboboxFocusDiagnosticText(diagnostic: ComboboxFocusDiagnostic | undefined): string | undefined {
+	if (!diagnostic) return undefined;
+	const label = diagnostic.activeElement.name ? ` (${diagnostic.activeElement.name})` : "";
+	return [
+		`Combobox diagnostic: focused combobox did not expose visible options${label}.`,
+		`Reason: ${diagnostic.message}`,
+		...diagnostic.recommendations.map((recommendation) => `- ${recommendation}`),
+	].join("\n");
+}
+
+function getRecordStartLikeCommand(command: string | undefined, commandTokens: string[]): RecordingDependencyWarning["command"] | undefined {
+	if (command !== "record") return undefined;
+	const subcommand = commandTokens[1]?.toLowerCase();
+	if (subcommand === "start") return "record start";
+	if (subcommand === "restart") return "record restart";
+	return undefined;
+}
+
+async function executableExistsOnPath(command: string): Promise<boolean> {
+	const pathValue = process.env.PATH ?? "";
+	const extensions = process.platform === "win32"
+		? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean)
+		: [""];
+	for (const directory of pathValue.split(delimiter).filter(Boolean)) {
+		for (const extension of extensions) {
+			try {
+				const candidate = join(directory, `${command}${extension}`);
+				await access(candidate, fsConstants.X_OK);
+				if ((await stat(candidate)).isFile()) return true;
+			} catch {
+				// Try the next candidate.
+			}
+		}
+	}
+	return false;
+}
+
+async function collectRecordingDependencyWarning(options: {
+	command: string | undefined;
+	commandTokens: string[];
+	succeeded: boolean;
+}): Promise<RecordingDependencyWarning | undefined> {
+	if (!options.succeeded) return undefined;
+	const recordCommand = getRecordStartLikeCommand(options.command, options.commandTokens);
+	if (!recordCommand) return undefined;
+	if (await executableExistsOnPath("ffmpeg")) return undefined;
+	return {
+		command: recordCommand,
+		dependency: "ffmpeg",
+		message: `${recordCommand} can begin recording, but record stop needs ffmpeg on PATH to encode the WebM output.`,
+		reason: "ffmpeg-missing-for-recording",
+		recommendations: [
+			"Install ffmpeg before relying on this recording workflow; on macOS with Homebrew, brew install ffmpeg or brew install ffmpeg-full.",
+			"If ffmpeg was just installed, restart pi or ensure the PATH visible to pi includes the ffmpeg binary before running record stop.",
+		],
+	};
+}
+
+function formatRecordingDependencyWarningText(warning: RecordingDependencyWarning | undefined): string | undefined {
+	if (!warning) return undefined;
+	return [
+		"Recording dependency warning: ffmpeg not found on PATH.",
+		`Reason: ${warning.message}`,
+		...warning.recommendations.map((recommendation) => `- ${recommendation}`),
+	].join("\n");
+}
+
 function getSnapshotRefRecord(data: unknown): Record<string, unknown> | undefined {
 	return isRecord(data) && isRecord(data.refs) ? data.refs : undefined;
 }
 
 const OVERLAY_CLOSE_NAME_PATTERN = /(?:\b(?:close|dismiss|no thanks|not now|maybe later|hide|skip|continue without|x)\b|^\s*×\s*$)/i;
-const OVERLAY_CONTEXT_NAME_PATTERN = /\b(?:banner|modal|dialog|popup|pop-up|overlay|donat(?:e|ion)|subscribe|sign in|login|cookie|privacy|consent)\b/i;
 const OVERLAY_CONTEXT_ROLES = new Set(["alertdialog", "dialog"]);
 const OVERLAY_ACTION_ROLES = new Set(["button", "link", "menuitem"]);
 const OVERLAY_BLOCKER_CANDIDATE_LIMIT = 3;
@@ -2638,8 +3056,7 @@ function getOverlayBlockerCandidates(snapshotData: unknown): OverlayBlockerCandi
 	const hasOverlayContext = Object.values(refs).some((entry) => {
 		if (!isRecord(entry)) return false;
 		const role = typeof entry.role === "string" ? entry.role : "";
-		const name = typeof entry.name === "string" ? entry.name : "";
-		return OVERLAY_CONTEXT_ROLES.has(role.toLowerCase()) || OVERLAY_CONTEXT_NAME_PATTERN.test(name);
+		return OVERLAY_CONTEXT_ROLES.has(role.toLowerCase());
 	});
 	if (!hasOverlayContext) return [];
 	const candidates: OverlayBlockerCandidate[] = [];
@@ -2810,16 +3227,27 @@ function formatEvalStdinHintText(hint: EvalStdinHint | undefined): string | unde
 	return hint ? `Eval stdin hint: ${hint.reason} ${hint.suggestion}` : undefined;
 }
 
-function getArtifactCleanupGuidance(options: { command?: string; manifest?: SessionArtifactManifest; succeeded: boolean }): ArtifactCleanupGuidance | undefined {
+async function getArtifactCleanupGuidance(options: { command?: string; cwd: string; manifest?: SessionArtifactManifest; succeeded: boolean }): Promise<ArtifactCleanupGuidance | undefined> {
 	if (!options.succeeded || options.command !== "close" || !options.manifest || options.manifest.entries.length === 0) return undefined;
-	const explicitArtifactPaths = options.manifest.entries
-		.filter((entry) => entry.storageScope === "explicit-path")
-		.map((entry) => entry.path)
-		.filter((path, index, paths) => paths.indexOf(path) === index)
-		.slice(0, 10);
+	const explicitEntries = options.manifest.entries.filter((entry) => entry.storageScope === "explicit-path");
+	const explicitArtifactPaths: string[] = [];
+	const seenPaths = new Set<string>();
+	for (const entry of explicitEntries) {
+		if (explicitArtifactPaths.length >= 10) break;
+		const displayPath = entry.path;
+		if (seenPaths.has(displayPath)) continue;
+		const absolutePath = entry.absolutePath ?? (isAbsolute(entry.path) ? entry.path : resolve(options.cwd, entry.path));
+		try {
+			await stat(absolutePath);
+		} catch {
+			continue;
+		}
+		seenPaths.add(displayPath);
+		explicitArtifactPaths.push(displayPath);
+	}
 	return {
 		explicitArtifactPaths,
-		note: "Closing the browser session does not delete explicit screenshots, downloads, PDFs, traces, HAR files, or recordings; clean those paths with host file tools when no longer needed.",
+		note: "Closing the browser session does not delete explicit screenshots, downloads, PDFs, traces, HAR files, or recordings; clean existing paths with host file tools when no longer needed.",
 		owner: "host-file-tools",
 		summary: formatSessionArtifactRetentionSummary(options.manifest),
 	};
@@ -3283,10 +3711,19 @@ async function closeManagedSession(options: { cwd: string; sessionName: string;
 	}
 }
 
+function getInstalledDocsPaths(): { readmePath: string; commandReferencePath: string; toolContractPath: string } {
+	const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
+	return {
+		readmePath: join(packageRoot, "README.md"),
+		commandReferencePath: join(packageRoot, "docs", "COMMAND_REFERENCE.md"),
+		toolContractPath: join(packageRoot, "docs", "TOOL_CONTRACT.md"),
+	};
+}
+
 export default function agentBrowserExtension(pi: ExtensionAPI) {
 	const ephemeralSessionSeed = createEphemeralSessionSeed();
 	const hasBraveApiKey = hasUsableBraveApiKey();
-	const toolPromptGuidelines = buildToolPromptGuidelines({ includeBraveSearch: hasBraveApiKey });
+	const toolPromptGuidelines = buildToolPromptGuidelines({ includeBraveSearch: hasBraveApiKey, docs: getInstalledDocsPaths() });
 	const implicitSessionIdleTimeoutMs = String(getImplicitSessionIdleTimeoutMs());
 	const implicitSessionCloseTimeoutMs = getImplicitSessionCloseTimeoutMs();
 	let managedSessionActive = false;
@@ -3457,12 +3894,29 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 			const runTool = async (): Promise<AgentBrowserToolResult> => {
 				const sessionMode = params.sessionMode ?? DEFAULT_SESSION_MODE;
 				const freshSessionName = createFreshSessionName(managedSessionBaseName, ephemeralSessionSeed, freshSessionOrdinal + 1);
-				const executionPlan = buildExecutionPlan(preparedArgs.args, {
+				let executionPlan = buildExecutionPlan(preparedArgs.args, {
 					freshSessionName,
 					managedSessionActive,
 					managedSessionName,
 					sessionMode,
 				});
+				let semanticActionVisibleRefResolution: SemanticActionVisibleRefResolution | undefined;
+				if (!executionPlan.validationError && executionPlan.managedSessionName !== freshSessionName) {
+					semanticActionVisibleRefResolution = await resolveSemanticActionVisibleRefArgs({
+						compiled: compiledSemanticAction,
+						cwd: ctx.cwd,
+						sessionName: executionPlan.sessionName,
+						signal,
+					});
+					if (semanticActionVisibleRefResolution) {
+						executionPlan = buildExecutionPlan(semanticActionVisibleRefResolution.args, {
+							freshSessionName,
+							managedSessionActive,
+							managedSessionName,
+							sessionMode,
+						});
+					}
+				}
 				const redactedEffectiveArgs = redactInvocationArgs(executionPlan.effectiveArgs);
 				const redactedRecoveryHint = redactRecoveryHint(executionPlan.recoveryHint);
 				const compatibilityWorkaround: CompatibilityWorkaround | undefined = executionPlan.compatibilityWorkaround;
@@ -3490,7 +3944,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 					};
 				}
 
-				const commandTokens = extractCommandTokens(preparedArgs.args);
+				const commandTokens = semanticActionVisibleRefResolution ? extractCommandTokens(semanticActionVisibleRefResolution.args) : extractCommandTokens(preparedArgs.args);
 				const exactSensitiveValues = getExactSensitiveStdinValues({
 					command: executionPlan.commandInfo.command,
 					commandTokens,
@@ -3560,10 +4014,13 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 				const priorSessionTabTargetState = executionPlan.sessionName ? sessionTabTargets.get(executionPlan.sessionName) : undefined;
 				const priorSessionTabTarget = priorSessionTabTargetState?.target;
 				const priorRefSnapshotState = executionPlan.sessionName ? sessionRefSnapshots.get(executionPlan.sessionName) : undefined;
+				const resolvedSemanticActionRefSnapshot = semanticActionVisibleRefResolution?.snapshot
+					? { ...semanticActionVisibleRefResolution.snapshot, target: semanticActionVisibleRefResolution.snapshot.target ?? priorSessionTabTarget }
+					: undefined;
 				const staleRefPreflight = buildStaleRefPreflight({
 					commandTokens,
 					currentTarget: priorSessionTabTarget,
-					refSnapshot: priorRefSnapshotState,
+					refSnapshot: resolvedSemanticActionRefSnapshot ?? priorRefSnapshotState,
 					stdin: toolStdin,
 				});
 				if (staleRefPreflight) {
@@ -3668,6 +4125,14 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 					}
 				}
 				const redactedProcessArgs = redactInvocationArgs(processArgs);
+				const shouldProbeScrollNoop = executionPlan.commandInfo.command === "scroll" && executionPlan.startupScopedFlags.length === 0;
+				const scrollPositionBefore = shouldProbeScrollNoop
+					? await collectScrollPositionSnapshot({
+							cwd: ctx.cwd,
+							sessionName: executionPlan.sessionName,
+							signal,
+					  })
+					: undefined;
 
 				onUpdate?.({
 					content: [{ type: "text", text: `Running agent-browser ${buildInvocationPreview(redactedProcessArgs)}` }],
@@ -3921,6 +4386,31 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 							signal,
 						});
 					}
+					const comboboxFocusDiagnostic = succeeded
+						? await collectComboboxFocusDiagnostic({
+								command: executionPlan.commandInfo.command,
+								commandTokens,
+								cwd: ctx.cwd,
+								semanticAction: compiledSemanticAction,
+								sessionName: executionPlan.sessionName,
+								signal,
+						  })
+						: undefined;
+					const recordingDependencyWarning = await collectRecordingDependencyWarning({
+						command: executionPlan.commandInfo.command,
+						commandTokens,
+						succeeded,
+					});
+					const scrollNoopDiagnostic = succeeded && shouldProbeScrollNoop
+						? buildScrollNoopDiagnostic(
+							scrollPositionBefore,
+							await collectScrollPositionSnapshot({
+								cwd: ctx.cwd,
+								sessionName: executionPlan.sessionName,
+								signal,
+							}),
+						)
+						: undefined;
 					let currentRefSnapshot: SessionRefSnapshot | undefined;
 					if (executionPlan.sessionName) {
 						const activeSessionTabTargetState = sessionTabTargets.get(executionPlan.sessionName);
@@ -3937,7 +4427,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 				? extractRefSnapshotFromData(presentationEnvelope?.data)
 				: executionPlan.commandInfo.command === "batch"
 					? extractRefSnapshotFromBatchResults(presentationEnvelope?.data)
-					: overlayBlockerDiagnostic?.snapshot
+					: resolvedSemanticActionRefSnapshot ?? overlayBlockerDiagnostic?.snapshot
 			: undefined;
 						if (refSnapshot && shouldApplySessionTabTargetUpdate({ current: sessionRefSnapshots.get(executionPlan.sessionName), updateOrder: tabTargetUpdateOrder })) {
 							currentRefSnapshot = { ...refSnapshot, target: refSnapshot.target ?? currentSessionTabTarget };
@@ -4082,8 +4572,9 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 						stdin: toolStdin,
 					});
 					const resultArtifactManifest = presentation.artifactManifest ?? artifactManifest;
-					const artifactCleanup = getArtifactCleanupGuidance({
+					const artifactCleanup = await getArtifactCleanupGuidance({
 						command: executionPlan.commandInfo.command,
+						cwd: ctx.cwd,
 						manifest: resultArtifactManifest,
 						succeeded,
 					});
@@ -4147,6 +4638,12 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 					if (selectorTextVisibilityDiagnostics.length > 0) {
 						(nextActions ??= []).push(...buildSelectorTextVisibilityNextActions({ diagnostics: selectorTextVisibilityDiagnostics, sessionName: executionPlan.sessionName }));
 					}
+					if (scrollNoopDiagnostic) {
+						(nextActions ??= []).push(...buildScrollNoopNextActions(executionPlan.sessionName));
+					}
+					if (comboboxFocusDiagnostic) {
+						(nextActions ??= []).push(...buildComboboxFocusNextActions(executionPlan.sessionName));
+					}
 					if (categoryDetails.failureCategory === "stale-ref" && redactedCompiledSemanticAction) {
 						(nextActions ??= []).push({
 							id: "retry-semantic-action-after-stale-ref",
@@ -4156,6 +4653,9 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 							tool: "agent_browser" as const,
 						});
 					}
+					const pageChangeSummary = (scrollNoopDiagnostic || comboboxFocusDiagnostic) && presentation.pageChangeSummary
+						? { ...presentation.pageChangeSummary, nextActionIds: nextActions?.map((action) => action.id) }
+						: presentation.pageChangeSummary;
 					const details = {
 						args: redactedArgs,
 						compiledJob: redactedCompiledJob,
@@ -4189,8 +4689,11 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 						imagePath: presentation.imagePath,
 						imagePaths: presentation.imagePaths,
 						nextActions,
-						pageChangeSummary: presentation.pageChangeSummary,
+						pageChangeSummary,
 						overlayBlockers: overlayBlockerDiagnostic,
+						comboboxFocus: comboboxFocusDiagnostic,
+						recordingDependencyWarning,
+						scrollNoop: scrollNoopDiagnostic,
 						qaPreset,
 						selectorTextVisibility: selectorTextVisibilityDiagnostics[0],
 						selectorTextVisibilityAll: selectorTextVisibilityDiagnostics.length > 1 ? selectorTextVisibilityDiagnostics : undefined,
@@ -4218,11 +4721,14 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
 					const semanticActionCandidateText = nextActions ? formatSemanticActionCandidateText(nextActions) : undefined;
 					const overlayBlockerText = overlayBlockerDiagnostic ? formatOverlayBlockerText(overlayBlockerDiagnostic) : undefined;
 					const selectorTextVisibilityText = formatSelectorTextVisibilityText(selectorTextVisibilityDiagnostics);
+					const scrollNoopDiagnosticText = formatScrollNoopDiagnosticText(scrollNoopDiagnostic);
+					const comboboxFocusDiagnosticText = formatComboboxFocusDiagnosticText(comboboxFocusDiagnostic);
+					const recordingDependencyWarningText = formatRecordingDependencyWarningText(recordingDependencyWarning);
 					const evalStdinHintText = formatEvalStdinHintText(evalStdinHint);
 					const artifactCleanupText = formatArtifactCleanupGuidanceText(artifactCleanup);
 					const timeoutPartialProgressText = timeoutPartialProgress ? formatTimeoutPartialProgressText(timeoutPartialProgress) : undefined;
 					const managedSessionOutcomeText = formatManagedSessionOutcomeText(managedSessionOutcome);
-					const rawAppendedDiagnosticText = [semanticActionCandidateText, overlayBlockerText, selectorTextVisibilityText, evalStdinHintText, artifactCleanupText, timeoutPartialProgressText, managedSessionOutcomeText].filter((item): item is string => item !== undefined).join("\n\n");
+					const rawAppendedDiagnosticText = [semanticActionCandidateText, overlayBlockerText, selectorTextVisibilityText, scrollNoopDiagnosticText, comboboxFocusDiagnosticText, recordingDependencyWarningText, evalStdinHintText, artifactCleanupText, timeoutPartialProgressText, managedSessionOutcomeText].filter((item): item is string => item !== undefined).join("\n\n");
 					const appendedDiagnosticText = redactSensitiveText(redactExactSensitiveText(rawAppendedDiagnosticText, exactSensitiveValues));
 					const shouldAppendDiagnosticText = appendedDiagnosticText.length > 0 && (!userRequestedJson || plainTextInspection);
 					const content = shouldAppendDiagnosticText && redactedContent[0]?.type === "text"