pi-agent-browser-native 0.2.33 → 0.2.34

package/extensions/agent-browser/lib/input-modes.ts

@@ -7,8 +7,11 @@
 export { AGENT_BROWSER_PARAMS } from "./input-modes/params.js";
 export {
 	analyzeQaPresetResults,
+	buildQaCompactPassText,
 	compileAgentBrowserJob,
 	compileAgentBrowserQaPreset,
+	extractQaPageContext,
+	isHttpOrHttpsUrl,
 } from "./input-modes/job.js";
 export {
 	analyzeNetworkSourceLookupResults,

package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts

@@ -4,9 +4,10 @@ import { delimiter, isAbsolute, join, resolve } from "node:path";
 
 import type { ElectronLaunchRecord } from "../../electron/launch.js";
 import type { AgentBrowserSourceLookupAnalysis, CompiledAgentBrowserJob, CompiledAgentBrowserSemanticAction } from "../../input-modes.js";
+import { isHttpOrHttpsUrl } from "../../input-modes/job.js";
 import type { AgentBrowserNextAction } from "../../results.js";
 import { formatSessionArtifactRetentionSummary } from "../../results/artifact-manifest.js";
-import { withOptionalSessionArgs } from "../../results/next-actions.js";
+import { buildNextToolAction, withOptionalSessionArgs } from "../../results/next-actions.js";
 import { buildVisibleRefFallbackDiagnosticFromSnapshot, getVisibleRefFallbackTarget, type VisibleRefFallbackDiagnostic } from "../../results/selector-recovery.js";
 import { extractRefSnapshotFromData, normalizeComparableUrl, type SessionTabTarget } from "../../session-page-state.js";
 import { redactInvocationArgs, redactSensitiveText, type CommandInfo } from "../../runtime.js";
@@ -29,6 +30,7 @@ import type {
 	NavigationSummary,
 	OverlayBlockerCandidate,
 	OverlayBlockerDiagnostic,
+	QaAttachedPreconditionFailure,
 	QaAttachedTarget,
 	RecordingDependencyWarning,
 	ScrollNoopDiagnostic,
@@ -491,6 +493,18 @@ async function collectManagedSessionCommandData(options: { args: string[]; cwd:
 	try { return { data: await runSessionCommandData(options) }; } catch (error) { return { error: error instanceof Error ? error.message : String(error) }; }
 }
 
+async function collectElectronManagedSessionUrl(options: { cwd: string; sessionName: string; signal?: AbortSignal; timeoutMs?: number }): Promise<{ error?: string; url?: string }> {
+	const urlResult = await collectManagedSessionCommandData({
+		args: ["get", "url"],
+		cwd: options.cwd,
+		sessionName: options.sessionName,
+		signal: options.signal,
+		timeoutMs: options.timeoutMs,
+	});
+	const url = boundElectronProbeString(extractStringResultField(urlResult.data, "result") ?? extractStringResultField(urlResult.data, "url"), 300);
+	return urlResult.error ? { error: urlResult.error } : { url };
+}
+
 async function collectElectronManagedSessionTarget(options: { cwd: string; sessionName?: string; signal?: AbortSignal; timeoutMs?: number }): Promise<QaAttachedTarget | undefined> {
 	if (!options.sessionName) return undefined;
 	const [titleResult, urlResult] = await Promise.all([
@@ -514,6 +528,58 @@ export function formatQaAttachedTargetText(target: QaAttachedTarget | undefined)
 	return ["QA attached target:", target.sessionName, target.title, target.url].filter((part): part is string => typeof part === "string" && part.length > 0).join(" — ");
 }
 
+export function buildQaAttachedRecoveryNextActions(sessionName: string | undefined): AgentBrowserNextAction[] {
+	const sessionArgs = (args: string[]) => withOptionalSessionArgs(sessionName, args);
+	return [
+		buildNextToolAction({
+			args: sessionArgs(["tab", "list"]),
+			id: "list-tabs-before-qa-attached",
+			reason: "Inspect the connected session tabs before retrying qa.attached.",
+			safety: "Read-only tab listing for the attached session.",
+		}),
+		buildNextToolAction({
+			args: sessionArgs(["snapshot", "-i"]),
+			id: "snapshot-before-qa-attached",
+			reason: "Capture interactive refs on the active http(s) page before retrying qa.attached.",
+			safety: "Read-only snapshot; confirms a renderable page is selected.",
+		}),
+	];
+}
+
+export async function validateQaAttachedPrecondition(options: {
+	cwd: string;
+	sessionName?: string;
+	signal?: AbortSignal;
+}): Promise<QaAttachedPreconditionFailure | undefined> {
+	if (!options.sessionName) {
+		return {
+			error: "qa.attached requires an active attached session with a resolvable session name.",
+			nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
+		};
+	}
+	const urlProbe = await collectElectronManagedSessionUrl({ cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
+	if (urlProbe.error) {
+		return {
+			error: `qa.attached could not read the attached session URL: ${urlProbe.error}. Run tab list or snapshot -i before retrying qa.attached.`,
+			nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
+		};
+	}
+	const url = urlProbe.url?.trim();
+	if (!url) {
+		return {
+			error: "qa.attached requires an attached session with a readable http(s) page URL. Run tab list, select a stable tab, then snapshot -i before retrying.",
+			nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
+		};
+	}
+	if (!isHttpOrHttpsUrl(url)) {
+		return {
+			error: `qa.attached requires an http(s) page URL; the current attached URL is "${url}". Use tab list and snapshot -i to recover a web surface before retrying.`,
+			nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
+		};
+	}
+	return undefined;
+}
+
 function getTopLevelFillInvocation(commandTokens: string[]): { expected: string; selector: string } | undefined {
 	if (commandTokens[0] !== "fill" || commandTokens.length < 3) return undefined;
 	const selector = commandTokens[1];

package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts

@@ -27,7 +27,7 @@ import {
 	shouldPinSessionTabForCommand,
 } from "./session-state.js";
 import { buildElectronHostFailureResult, getElectronLaunchFailureCategory, redactRecoveryHint } from "./final-result.js";
-import { collectScrollPositionSnapshot } from "./diagnostics.js";
+import { collectScrollPositionSnapshot, validateQaAttachedPrecondition } from "./diagnostics.js";
 import type {
 	BrowserRunInputFields,
 	BrowserRunOptions,
@@ -555,6 +555,31 @@ export async function prepareBrowserRun(options: BrowserRunOptions): Promise<Pre
 		} };
 	}
 
+	if (compiledQaPreset?.checks.attached) {
+		const qaAttachedPrecondition = await validateQaAttachedPrecondition({
+			cwd,
+			sessionName: executionPlan.sessionName,
+			signal,
+		});
+		if (qaAttachedPrecondition) {
+			return { kind: "early-result", statePatch, result: {
+				content: [{ type: "text", text: qaAttachedPrecondition.error }],
+				details: {
+					args: redactedArgs,
+					compiledQaPreset: redactedCompiledQaPreset,
+					compatibilityWorkaround,
+					effectiveArgs: redactedEffectiveArgs,
+					nextActions: qaAttachedPrecondition.nextActions,
+					sessionMode,
+					...buildAgentBrowserResultCategoryDetails({ args: redactedEffectiveArgs, command: executionPlan.commandInfo.command, errorText: qaAttachedPrecondition.error, succeeded: false, validationError: qaAttachedPrecondition.error }),
+					validationError: qaAttachedPrecondition.error,
+					...buildSessionDetailFields(executionPlan.sessionName, executionPlan.usedImplicitSession),
+				},
+				isError: true,
+			} };
+		}
+	}
+
 	let pinnedBatchUnwrapMode: PreparedBrowserRun["pinnedBatchUnwrapMode"];
 	let includePinnedNavigationSummary = false;
 	let sessionTabCorrection: PreparedBrowserRun["sessionTabCorrection"];

package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts

@@ -6,6 +6,8 @@ import {
 	analyzeNetworkSourceLookupResults,
 	analyzeQaPresetResults,
 	analyzeSourceLookupResults,
+	buildQaCompactPassText,
+	extractQaPageContext,
 	redactNetworkSourceLookupAnalysis,
 } from "../../input-modes.js";
 import {
@@ -20,6 +22,7 @@ import {
 	mergeSessionArtifactManifest,
 } from "../../results/artifact-manifest.js";
 import type { SessionArtifactManifest } from "../../results/contracts.js";
+import { shouldCaptureSemanticActionNavigationSummary } from "../../results/presentation/semantic-action.js";
 import {
 	commandExplicitlyTargetsAboutBlank,
 	deriveSessionTabTarget,
@@ -205,7 +208,12 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		const inspectionText = plainTextInspection ? processResult.stdout.trim() : undefined;
 		updateTraceOwnerState({ command: prepared.executionPlan.commandInfo.command, sessionName: prepared.executionPlan.sessionName, subcommand: prepared.executionPlan.commandInfo.subcommand, succeeded, traceOwners });
 
-		if (succeeded && !navigationSummary && shouldCaptureNavigationSummary(prepared.executionPlan.commandInfo.command, presentationEnvelope?.data)) {
+		if (
+			succeeded &&
+			!navigationSummary &&
+			(shouldCaptureNavigationSummary(prepared.executionPlan.commandInfo.command, presentationEnvelope?.data) ||
+				shouldCaptureSemanticActionNavigationSummary(prepared.compiledSemanticAction, presentationEnvelope?.data))
+		) {
 			navigationSummary = await collectNavigationSummary({ cwd, sessionName: prepared.executionPlan.sessionName, signal });
 		}
 		if (navigationSummary && presentationEnvelope) presentationEnvelope = { ...presentationEnvelope, data: mergeNavigationSummaryIntoData(presentationEnvelope.data, navigationSummary) };
@@ -341,7 +349,7 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		}
 
 		const errorText = getAgentBrowserErrorText({ aborted: processResult.aborted, command: prepared.executionPlan.commandInfo.command, effectiveArgs: prepared.redactedProcessArgs, envelope: presentationEnvelope, exitCode: processResult.exitCode, parseError, plainTextInspection, staleRefArgs: getStaleRefArgs(prepared.commandTokens, prepared.runtimeToolStdin), spawnError: processResult.spawnError, stderr: processResult.stderr, timedOut: processResult.timedOut, timeoutMs: processResult.timeoutMs, wrapperRecoveryHint: buildWrapperRecoveryHint({ pinnedBatchUnwrapMode: prepared.pinnedBatchUnwrapMode, sessionTabCorrection }) });
-		const presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text" as const, text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, cwd, envelope: presentationEnvelope, errorText, persistentArtifactStore, sessionName: prepared.executionPlan.sessionName });
+		const presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text" as const, text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd, envelope: presentationEnvelope, errorText, persistentArtifactStore, sessionName: prepared.executionPlan.sessionName });
 		if (parseFailureOutput.artifactManifest) { presentation.artifactManifest = parseFailureOutput.artifactManifest; presentation.artifactRetentionSummary = parseFailureOutput.artifactRetentionSummary; }
 		if (parseFailureOutput.fullOutputPath || parseFailureOutput.fullOutputUnavailable) {
 			const existingText = presentation.content[0]?.type === "text" ? presentation.content[0].text : "";
@@ -351,7 +359,9 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		}
 		if (presentation.artifactManifest) artifactManifest = presentation.artifactManifest;
 		const qaPreset = prepared.compiledQaPreset ? analyzeQaPresetResults(presentationEnvelope?.data) : undefined;
-		const qaAttachedTarget = prepared.compiledQaPreset?.checks.attached ? await collectQaAttachedTarget({ currentTarget: currentSessionTabTarget ?? prepared.priorSessionTabTarget, cwd, sessionName: prepared.executionPlan.sessionName, signal }) : undefined;
+		let qaAttachedTarget = prepared.compiledQaPreset?.checks.attached
+			? await collectQaAttachedTarget({ currentTarget: currentSessionTabTarget ?? prepared.priorSessionTabTarget, cwd, sessionName: prepared.executionPlan.sessionName, signal })
+			: undefined;
 		const sourceLookupElectronContext = prepared.compiledSourceLookup ? getSourceLookupElectronContext({ currentTarget: currentSessionTabTarget, electronLaunchRecords, priorTarget: prepared.priorSessionTabTarget, sessionName: prepared.executionPlan.sessionName }) : undefined;
 		const sourceLookup = prepared.compiledSourceLookup ? await analyzeSourceLookupResults(presentationEnvelope?.data, prepared.compiledSourceLookup, cwd, { electronContext: sourceLookupElectronContext, workspaceRoot: cwd }) : undefined;
 		const networkSourceLookup = prepared.compiledNetworkSourceLookup ? redactNetworkSourceLookupAnalysis(await analyzeNetworkSourceLookupResults(presentationEnvelope?.data, prepared.compiledNetworkSourceLookup, cwd)) : undefined;
@@ -359,15 +369,32 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		else if (networkSourceLookup) presentation.content.unshift({ type: "text", text: networkSourceLookup.summary });
 		if (sourceLookup && presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${sourceLookup.summary}\n\n${presentation.content[0].text}` };
 		else if (sourceLookup) presentation.content.unshift({ type: "text", text: sourceLookup.summary });
-		if (qaPreset && (!qaPreset.passed || qaPreset.warnings.length > 0)) {
-			if (!qaPreset.passed) { succeeded = false; presentation.failureCategory = "qa-failure"; }
+		if (qaPreset && !qaPreset.passed) {
+			succeeded = false;
+			presentation.failureCategory = "qa-failure";
 			presentation.summary = qaPreset.summary;
 			if (presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${qaPreset.summary}\n\n${presentation.content[0].text}` };
 			else presentation.content.unshift({ type: "text", text: qaPreset.summary });
+		} else if (qaPreset?.passed && prepared.compiledQaPreset) {
+			const compactText = buildQaCompactPassText({
+				artifactVerification: presentation.artifactVerification,
+				batchStepCount: presentation.batchSteps?.length ?? prepared.compiledQaPreset.steps.length,
+				checks: prepared.compiledQaPreset.checks,
+				page: extractQaPageContext({
+					attachedTarget: qaAttachedTarget,
+					batchData: presentationEnvelope?.data,
+					compiled: prepared.compiledQaPreset,
+				}),
+				qaPreset,
+			});
+			presentation.summary = qaPreset.summary;
+			const nonTextContent = presentation.content.filter((item) => item.type !== "text");
+			presentation.content = [{ type: "text", text: compactText }, ...nonTextContent];
 		}
 		const qaAttachedTargetText = formatQaAttachedTargetText(qaAttachedTarget);
-		if (qaAttachedTargetText && presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${qaAttachedTargetText}\n\n${presentation.content[0].text}` };
-		else if (qaAttachedTargetText) presentation.content.unshift({ type: "text", text: qaAttachedTargetText });
+		const skipAttachedTargetBanner = qaPreset?.passed && prepared.compiledQaPreset?.checks.attached;
+		if (!skipAttachedTargetBanner && qaAttachedTargetText && presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${qaAttachedTargetText}\n\n${presentation.content[0].text}` };
+		else if (!skipAttachedTargetBanner && qaAttachedTargetText) presentation.content.unshift({ type: "text", text: qaAttachedTargetText });
 		if (managedSessionOutcome && managedSessionOutcome.succeeded !== succeeded) managedSessionOutcome = { ...managedSessionOutcome, succeeded };
 		const evalStdinHint = getEvalStdinHint({ command: prepared.executionPlan.commandInfo.command, data: presentationEnvelope?.data, stdin: prepared.runtimeToolStdin });
 		const resultArtifactManifest = presentation.artifactManifest ?? artifactManifest;

package/extensions/agent-browser/lib/orchestration/browser-run/types.ts

@@ -140,11 +140,17 @@ export interface ElectronBroadGetTextScopeDiagnostic {
 }
 
 export interface QaAttachedTarget {
+	error?: string;
 	sessionName: string;
 	title?: string;
 	url?: string;
 }
 
+export interface QaAttachedPreconditionFailure {
+	error: string;
+	nextActions: AgentBrowserNextAction[];
+}
+
 export interface TimeoutArtifactEvidence {
 	absolutePath: string;
 	exists: boolean;

package/extensions/agent-browser/lib/playbook.ts

@@ -18,11 +18,12 @@ export function buildInstalledDocsGuideline(paths: { readmePath: string; command
 }
 
 export const QUICK_START_GUIDELINES = [
-	"Quick start mental model: use exactly one of args (exact agent-browser CLI args after the binary), semanticAction (a thin shorthand compiled to find argv for locator actions or select argv for native dropdowns), job (a constrained short-workflow schema compiled to batch), qa (a lightweight QA preset built on job/batch, including qa.attached for current sessions), electron (desktop Electron list/launch/status/cleanup/probe), or the experimental sourceLookup / networkSourceLookup helpers (each compiled to batch); stdin is only for batch, eval --stdin, auth save --password-stdin, and wrapper-generated batch stdin from job, qa, sourceLookup, or networkSourceLookup, and is rejected with electron; sessionMode=fresh switches the extension-managed pi-scoped session to a fresh upstream launch when you need new --profile, --session-name, --cdp, --state, --auto-connect, --init-script, --enable, -p/--provider, or iOS --device state.",
+	"Quick start mental model: use exactly one of args (exact agent-browser CLI args after the binary), semanticAction (a thin shorthand compiled to find argv for locator actions or select argv for native dropdowns), job (a constrained short-workflow schema compiled to batch), qa (a lightweight QA preset built on job/batch, including qa.attached for current sessions), electron (desktop Electron list/launch/status/cleanup/probe), or the experimental sourceLookup / networkSourceLookup helpers (candidates only; each compiled to batch); stdin is only for batch, eval --stdin, auth save --password-stdin, and wrapper-generated batch stdin from job, qa, sourceLookup, or networkSourceLookup, and is rejected with electron; sessionMode=fresh switches the extension-managed pi-scoped session to a fresh upstream launch when you need new --profile, --session-name, --cdp, --state, --auto-connect, --init-script, --enable, -p/--provider, or iOS --device state. Do not pass --json in args; the wrapper injects it.",
 	"There is no first-class reusable named browser recipe runtime above top-level job, the qa preset, and raw batch stdin; keep recurring flows in documentation examples or those inputs (closed RQ-0068; see docs/ARCHITECTURE.md#no-reusable-recipe-layer-yet).",
-	"Common first calls: { args: [\"open\", \"https://example.com\"] } then { args: [\"snapshot\", \"-i\"] }; after navigation, use { args: [\"click\", \"@e2\"] } then { args: [\"snapshot\", \"-i\"] }.",
+	"Common first calls (first-call recipe): { args: [\"open\", \"<url>\"] } → { args: [\"snapshot\", \"-i\"] } → { args: [\"click\", \"@eN\"] } or { args: [\"fill\", \"@eN\", \"<text>\"] } using @refs and visible labels from that snapshot, then { args: [\"snapshot\", \"-i\"] } after navigation or DOM changes. On https://example.com/ the main link label is Learn more (use exact snapshot text, not guessed link copy).",
 	"Locator-first clicks/fills and native select changes without hand-building argv: { semanticAction: { action: \"click\", locator: \"text\", value: \"Close\" } }, { semanticAction: { action: \"fill\", locator: \"label\", value: \"Email\", text: \"user@example.com\" } }, or { semanticAction: { action: \"select\", selector: \"#flavor\", value: \"chocolate\" } }; add semanticAction.session when targeting a named upstream browser session; details.compiledSemanticAction shows the semantic target, while details.effectiveArgs may show a resolved current @ref for active-session role/name click/check/uncheck actions to avoid hidden duplicate matches; selector-not-found failures may append bounded click try-*-candidate next actions or, for fill misses with current editable refs, details.richInputRecovery with focus/click actions that do not copy fill text; stale-ref failures can return retry-semantic-action-after-stale-ref for compiled find actions when retry safety is provable.",
-	"Common advanced calls: { args: [\"batch\"], stdin: \"[[\\\"open\\\",\\\"https://example.com\\\"],[\\\"snapshot\\\",\\\"-i\\\"]]\" }, { job: { steps: [{ action: \"open\", url: \"https://example.com\" }, { action: \"assertText\", text: \"Example Domain\" }, { action: \"screenshot\", path: \".dogfood/example.png\" }] } }, { qa: { url: \"https://example.com\", expectedText: \"Example Domain\", screenshotPath: \".dogfood/qa-example.png\" } }, { electron: { action: \"list\", query: \"code\" } }, { electron: { action: \"launch\", appName: \"Visual Studio Code\", handoff: \"snapshot\" } }, { electron: { action: \"probe\" } }, { qa: { attached: true, expectedText: \"Explorer\" } }, { args: [\"eval\", \"--stdin\"], stdin: \"document.title\" }, { args: [\"auth\", \"save\", \"name\", \"--password-stdin\"], stdin: \"<password from user-approved secret source>\" }, { args: [\"--profile\", \"Default\", \"open\", \"https://example.com/account\"], sessionMode: \"fresh\" }, and { args: [\"open\", \"--enable\", \"react-devtools\", \"https://example.com\"], sessionMode: \"fresh\" }. For app pages with a native dropdown, job steps can include { action: \"select\", selector: \"#flavor\", value: \"chocolate\" } before the dependent assertion.",
+	`Common advanced calls: { args: ["batch"], stdin: "[[\"open\",\"https://example.com\"],[\"snapshot\",\"-i\"]]" }, { job: { steps: [{ action: "open", url: "https://example.com" }, { action: "assertText", text: "Example Domain" }, { action: "screenshot", path: ".dogfood/example.png" }] } }, { qa: { url: "https://example.com", expectedText: "Example Domain", screenshotPath: ".dogfood/qa-example.png" } } (example.com smoke only; elsewhere match exact visible text from snapshot -i), { electron: { action: "list", query: "code" } }, { electron: { action: "launch", appName: "Visual Studio Code", handoff: "snapshot" } }, { electron: { action: "probe" } }, { qa: { attached: true, expectedText: "Explorer" } }, { args: ["eval", "--stdin"], stdin: "document.title" }, { args: ["auth", "save", "name", "--password-stdin"], stdin: "<password from user-approved secret source>" }, { args: ["--profile", "Default", "open", "https://example.com/account"], sessionMode: "fresh" }, and { args: ["open", "--enable", "react-devtools", "https://example.com"], sessionMode: "fresh" }. For app pages with a native dropdown, job steps can include { action: "select", selector: "#flavor", value: "chocolate" } before the dependent assertion.`, 
+	"Constrained job navigation is explicit only: click (and select/submit flows that may navigate) does not prove the next page loaded; add assertUrl and/or assertText after navigation-prone steps before screenshot or later interactions. Example: { job: { steps: [{ action: \"open\", url: \"https://shop.example/checkout\" }, { action: \"fill\", selector: \"#email\", text: \"user@example.com\" }, { action: \"click\", selector: \"#continue\" }, { action: \"assertUrl\", url: \"**/shipping\" }, { action: \"assertText\", text: \"Shipping address\" }, { action: \"screenshot\", path: \".dogfood/shipping.png\" }] } }. Top-level click may add navigationSummary hints, but job never auto-inserts post-click asserts.",
 	"High-value command reference: select <selector> <value...> changes native dropdown values; download <selector> <path> saves a file triggered by a click; get title/url/text/html/value/attr/count reads page state; screenshot [path] captures an image; pdf <path> saves a PDF; tab list and tab <tab-id-or-label> inspect or recover the active tab; react tree/inspect/renders/suspense introspect React after --enable react-devtools; vitals [url] measures Core Web Vitals; pushstate <url> performs SPA navigation.",
 	"For artifact-producing commands, read the visible artifact block and details.artifactVerification before using files: check requested path, absolute path, existence, size bytes, artifact kind, optional mediaType, status, optional limitation, and verified/missing/pending/unverified counts. details.artifacts contains per-file metadata. Browser close does not delete explicit saved files; if close reports details.artifactCleanup, use host file tools to remove paths listed in explicitArtifactPaths (when non-empty) after inspection. For annotated screenshots inside batch, put --annotate in top-level args (for example { args: [\"--annotate\", \"batch\"], stdin: \"[[\\\"screenshot\\\",\\\"/tmp/page.png\\\"]]\" }) rather than inside the screenshot step.",
 	"When details.nextActions is present, prefer those exact native agent_browser follow-up payloads over prose guidance; they may include args, stdin, sessionMode, networkSourceLookup, safety notes, or artifactPath for saved files.",
@@ -34,7 +35,7 @@ export const BRAVE_SEARCH_PROMPT_GUIDELINE =
 export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
 	"Standard workflow: open the page, snapshot -i, interact using current @refs from that snapshot, and re-snapshot after navigation, scrolling, rerendering, or other major DOM changes because refs are page-scoped; the wrapper fails mutation-prone stale/recycled refs before upstream can silently target a different current-page element.",
 	"For ordinary forms from one snapshot, batch multiple fill @refs before the submit/click step to avoid serial tool calls; if a fill may autosubmit, navigate, or rerender later fields, split the flow and refresh refs first.",
-	"When snapshot -i compacts because the tree is oversized, scan visible output for Omitted high-value controls and optional details.data.highValueControlRefIds before opening the spill file: those list bounded searchboxes, textboxes, comboboxes, buttons, tabs, checkboxes, radios, options, and menuitems that did not fit the key/other ref previews.",
+	"Snapshot choice: prefer snapshot -i for routine clicks/fills (interactive @refs, main-content-first). Use snapshot --compact when you need a denser same-page tree without full spill; use full snapshot (no -i) only when you need the complete accessibility tree. Re-snapshot after navigation or major DOM changes. When snapshot -i compacts because the tree is oversized, scan visible output for Omitted high-value controls and optional details.data.highValueControlRefIds before opening the spill file: those list bounded searchboxes, textboxes, comboboxes, buttons, tabs, checkboxes, radios, options, and menuitems that did not fit the key/other ref previews.",
 	"When a visible text or accessible-name target should survive ref churn, prefer find locators such as role, text, label, placeholder, alt, title, or testid with the intended action instead of guessing a CSS selector.",
 	"For desktop or host-controlled rich inputs, if semanticAction fill misses, refresh refs and prefer a current editable @ref from details.richInputRecovery or the latest snapshot; focus or click that ref, then use keyboard inserttext or keyboard type with the intended text. Do not auto-submit with Enter or a submit button unless the user flow explicitly calls for it.",
 	"Do not assume Playwright selector dialects such as text=Close or button:has-text('Close') are supported wrapper syntax unless current upstream agent-browser behavior has been verified.",
@@ -59,13 +60,16 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
 	"When using eval --stdin for extraction, return the value you want instead of relying on console.log as the primary result channel. Prefer plain expressions like ({ title: document.title }) or explicitly invoked functions like (() => ({ title: document.title }))(); if a function-shaped snippet returns {}, details.evalStdinHint may warn that the function was serialized instead of called. If get text on a CSS selector surfaces details.selectorTextVisibility or selectorTextVisibilityAll, prefer a visible @ref, a more specific selector, or the inspect-visible-text-candidates nextAction over hidden tab content.",
 	"When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.",
 	"When commands save or spill files (screenshots, downloads, PDFs, traces, recordings, HAR, large snapshot spills), use the user's exact requested paths when given and treat paths as provisional until details.artifactVerification shows every row verified: branch on missingCount, pendingCount, unverifiedCount, per-entry state, and optional limitation before downstream file use or PASS/FAIL reporting.",
+	"For evidence-only screenshots, QA captures, or other audit artifacts, save to an explicit path and branch on details.artifactVerification plus details.artifacts before reporting PASS/FAIL; do not require vision review of inline image attachments unless the user asked for visual inspection.",
+	"Respect explicit user stop boundaries: if the user says to stop before order/post/purchase/submit, do not click that final action.",
+	"Successful record stop needs ffmpeg on PATH; the wrapper may warn after record start when ffmpeg is missing.",
 	"Do not call --help or other exploratory inspection commands unless the user explicitly asks for them or debugging the browser integration is necessary.",
 ] as const;
 
 export const TOOL_PROMPT_GUIDELINES_SUFFIX = [
 	"Prefer agent_browser over bash for opening sites, reading docs on the web, clicking, filling, screenshots, eval, and batch workflows.",
 	"Do not fall back to osascript, AppleScript, or generic browser-driving bash commands when agent_browser can do the job.",
-	"Pass exact agent-browser CLI arguments in args when you are not using semanticAction, job, or qa, excluding the binary name.",
+	"Pass exact agent-browser CLI arguments in args when you are not using semanticAction, job, or qa, excluding the binary name and --json (the wrapper injects --json automatically).",
 	"Use stdin only for eval --stdin, batch, auth save --password-stdin, or wrapper-generated job/qa batches instead of shell heredocs or password args; other command/stdin combinations are rejected before launch.",
 	"Let the extension-managed session handle the common path unless you explicitly need a fresh launch for upstream flags like --profile, --session-name, --cdp, --state, --auto-connect, --init-script, --enable, -p/--provider, or iOS --device.",
 	"Use sessionMode=fresh when switching from an existing implicit session to a new profile/debug/init-script/provider launch without inventing a fixed explicit session name; later auto calls will follow that new session.",
@@ -91,17 +95,14 @@ export function buildSharedBrowserPlaybookGuidelines(options: { includeBraveSear
 	];
 }
 
-const RUNTIME_PROMPT_GUIDELINES = [
-	"Use exactly one input mode: args, semanticAction, job, qa, sourceLookup, networkSourceLookup, or electron. Use stdin only for batch, eval --stdin, auth save --password-stdin, or wrapper-generated batch modes; electron rejects caller stdin.",
-	"Common flow: open, snapshot -i, interact with current @refs or semanticAction, then re-snapshot after navigation, scrolling, rerenders, or DOM changes. Batch same-snapshot form fills unless a fill may submit, navigate, or rerender. If semanticAction fill misses on a rich host input, use richInputRecovery/current editable refs, focus/click, then keyboard inserttext/type; do not auto-submit. Respect explicit stop boundaries: if the user says to stop before order/post/purchase/submit, do not click that final action.",
-	"Prefer stable locators for visible text/names: semanticAction or find role/text/label/placeholder/alt/title/testid. For native selects, use select <selector> <value...> or semanticAction/job select. Use current @refs only from the latest same-page snapshot.",
-	"For signed-in/account-specific content on the web, start with --profile Default plus sessionMode=fresh unless asked otherwise; visible content is model-visible. For desktop/Electron apps, use electron.list first. electron.launch is isolated; for signed-in desktop content, host-launch with --remote-debugging-port, connect, tab list, select the stable tab id, then condition-wait or snapshot -i. Use sessionMode=fresh for other launch-scoped state; otherwise keep implicit continuity.",
-	"For screenshots, recordings, downloads, PDFs, or HARs, save the exact user path and check details.artifactVerification before claiming success. record stop needs ffmpeg on PATH. close does not delete saved files; cleanup is host-owned. Electron cleanup only covers wrapper-launched launchIds. For desktop readiness, prefer condition waits, qa.attached, electron.probe/status, tab list/tab tN, snapshot, or screenshot; wait 30000 is blocked, and \"waited\":\"timeout\" is not completion evidence.",
-	"When details.nextActions is present, prefer those exact follow-up payloads over prose or guessed selectors; diagnostics may include request detail, networkSourceLookup, HAR capture, Electron status/cleanup, tab selection, rich-input focus/click recovery, or snapshot actions.",
-	"For dense snapshots, check Omitted high-value controls and details.data.highValueControlRefIds before opening large spill files.",
-	"For dashboards, verify scroll with screenshot/snapshot; if nothing moved, use scrollintoview <@ref> or target the real scroll region. Native selects need select/semanticAction/job select; custom comboboxes may need re-snapshot, type, Enter/arrows, or visible option refs.",
-	"For extraction, prefer get title/url/text/html/value/attr/count or eval --stdin with a plain expression; do not rely on console.log. When reading several known refs/selectors, use batch JSON-array stdin (for example [[\"get\",\"text\",\"@e1\"]]) or eval --stdin instead of many serial gets. If selector visibility warnings appear, prefer visible @refs or nextActions.",
-	"For non-core debugging, pass upstream commands through args: network, diff, trace/profiler/record, console/errors, stream, dashboard, chat, react, vitals, pushstate, dialog, frame, tab.",
+/** Tier A: always-on tool promptGuidelines (keep small; Tier B lives in SHARED_BROWSER_PLAYBOOK_GUIDELINES and docs). */
+export const RUNTIME_PROMPT_GUIDELINES = [
+	"Use exactly one input mode: args (open→snapshot -i→@refs), semanticAction, job, qa, sourceLookup/networkSourceLookup (candidate hints), or electron. stdin only for batch/eval/auth or wrapper batch; electron rejects stdin. Do not pass --json in args; wrapper injects it.",
+	"Common flow: open, snapshot -i, use current @refs or semanticAction, then re-snapshot after navigation/scroll/rerender/DOM change. Batch same-snapshot fills unless they may submit/navigate/rerender. Respect explicit stop boundaries: if the user says stop before order/post/purchase/submit, do not click the final action.",
+	"Use sessionMode=fresh for launch-scoped flags on an active implicit session. For signed-in/account-specific content, start with --profile Default plus sessionMode=fresh unless asked otherwise; visible content is model-visible.",
+	"For artifacts, save the exact user path and check details.artifactVerification/details.artifacts before claiming success. record stop needs ffmpeg on PATH; close does not delete saved files; \"waited\":\"timeout\" is not proof.",
+	"When details.nextActions is present, prefer exact payloads over prose/guessed selectors. For dense snapshots, check Omitted high-value controls/details.data.highValueControlRefIds. For dashboards, verify scroll with screenshot/snapshot; if nothing moved, target the real scroll region.",
+	"For extraction, prefer get title/url/text/html/value/attr/count or eval --stdin with plain expression, not console.log. Batch three or more known refs/selectors (e.g. [[\"get\",\"text\",\"@e1\"],[\"get\",\"text\",\"@e2\"]]); selector visibility warnings → visible @refs/nextActions.",
 ] as const;
 
 export function buildToolPromptGuidelines(options: { includeBraveSearch: boolean; docs?: { readmePath: string; commandReferencePath: string; toolContractPath: string } }): string[] {

package/extensions/agent-browser/lib/results/categories.ts

@@ -52,6 +52,7 @@ export function classifyAgentBrowserFailureCategory(options: {
 	if (/aborted/i.test(text)) return "aborted";
 	if (/policy[- ]blocked|blocked by caller policy|caller deny policy|caller allow policy/i.test(text)) return "policy-blocked";
 	if (/cleanup failed|cleanup.*partial|partial cleanup|remaining resources/i.test(text)) return "cleanup-failed";
+	if (options.validationError) return "validation-error";
 	if (options.tabDrift || /could not re-select the intended tab|about:blank|selected tab looks wrong|tab drift|tab.*wrong/i.test(text)) return "tab-drift";
 	if (/\bUnknown ref\b|\bstale ref\b|@ref may be stale|\bref\b.*\b(?:not found|missing|expired)\b/i.test(text)) return "stale-ref";
 	if (usedRef && /could not locate element|element not found|no element/i.test(text)) return "stale-ref";
@@ -72,7 +73,6 @@ export function classifyAgentBrowserFailureCategory(options: {
 	if ((command === "download" || text.includes("wait --download") || /\bdownload\b/i.test(text)) && /missing|not verified|not found|failed|timeout|timed out/i.test(text)) {
 		return "download-not-verified";
 	}
-	if (options.validationError) return "validation-error";
 	return "upstream-error";
 }
 

package/extensions/agent-browser/lib/results/presentation/registry.ts

@@ -1,3 +1,4 @@
+import type { CompiledAgentBrowserSemanticAction } from "../../input-modes/types.js";
 import { isRecord } from "../../parsing.js";
 import type { CommandInfo } from "../../runtime.js";
 import { detectConfirmationRequired, type ConfirmationRequiredPresentation } from "../confirmation.js";
@@ -14,6 +15,11 @@ import {
 	getNavigationSummary,
 	isNavigationObservableCommand,
 } from "./navigation.js";
+import {
+	formatSemanticActionPresentationSummary,
+	formatSemanticActionPresentationText,
+	resolvePresentationCommandInfo,
+} from "./semantic-action.js";
 
 function getPageSummary(data: Record<string, unknown>): string | undefined {
 	const title = typeof data.title === "string" ? data.title : undefined;
@@ -91,20 +97,32 @@ function formatBatchSummary(data: unknown): string | undefined {
 		: `Batch failed: ${successCount}/${data.length} succeeded`;
 }
 
-export function formatPresentationSummary(commandInfo: CommandInfo, data: unknown): string {
+export function formatPresentationSummary(
+	commandInfo: CommandInfo,
+	data: unknown,
+	compiledSemanticAction?: CompiledAgentBrowserSemanticAction,
+): string {
 	const confirmationRequired = detectConfirmationRequired(data);
 	if (confirmationRequired) return formatConfirmationRequiredSummary(confirmationRequired);
 
+	const presentationCommandInfo = resolvePresentationCommandInfo(commandInfo, compiledSemanticAction);
+
 	if (commandInfo.command === "batch") {
 		const batchSummary = formatBatchSummary(data);
 		if (batchSummary) return batchSummary;
 	}
 
 	if (isRecord(data)) {
+		if (compiledSemanticAction) {
+			const semanticSummary = formatSemanticActionPresentationSummary(compiledSemanticAction, data);
+			if (semanticSummary) return semanticSummary;
+		}
 		const navigationSummary = getNavigationSummary(data);
-		if (navigationSummary && isNavigationObservableCommand(commandInfo.command)) {
+		if (navigationSummary && isNavigationObservableCommand(presentationCommandInfo.command)) {
 			const navigationText = formatNavigationSummary(navigationSummary);
-			if (navigationText) return `${commandInfo.command ?? "navigation"} → ${navigationText.split("\n", 1)[0] ?? navigationText}`;
+			if (navigationText) {
+				return `${presentationCommandInfo.command ?? "navigation"} → ${navigationText.split("\n", 1)[0] ?? navigationText}`;
+			}
 		}
 	}
 
@@ -121,10 +139,14 @@ export function formatPresentationSummary(commandInfo: CommandInfo, data: unknow
 	}
 
 	if (typeof data === "string" && data.length > 0) return data.split("\n", 1)[0] ?? data;
-	return `${commandInfo.command ?? "agent-browser"} completed`;
+	return `${presentationCommandInfo.command ?? commandInfo.command ?? "agent-browser"} completed`;
 }
 
-export function formatPresentationContentText(commandInfo: CommandInfo, data: unknown): string {
+export function formatPresentationContentText(
+	commandInfo: CommandInfo,
+	data: unknown,
+	compiledSemanticAction?: CompiledAgentBrowserSemanticAction,
+): string {
 	const confirmationRequired = detectConfirmationRequired(data);
 	if (confirmationRequired) return formatConfirmationRequiredText(confirmationRequired);
 
@@ -135,8 +157,14 @@ export function formatPresentationContentText(commandInfo: CommandInfo, data: un
 	if (typeof data === "number" || typeof data === "boolean") return String(data);
 	if (!isRecord(data)) return stringifyModelFacing(data);
 
+	if (compiledSemanticAction) {
+		const semanticText = formatSemanticActionPresentationText(compiledSemanticAction, data);
+		if (semanticText) return semanticText;
+	}
+
+	const presentationCommandInfo = resolvePresentationCommandInfo(commandInfo, compiledSemanticAction);
 	const navigationSummary = getNavigationSummary(data);
-	if (navigationSummary && isNavigationObservableCommand(commandInfo.command)) {
+	if (navigationSummary && isNavigationObservableCommand(presentationCommandInfo.command)) {
 		const navigationText = formatNavigationSummary(navigationSummary);
 		if (navigationText) {
 			const actionText = formatNavigationActionResult(data);

package/extensions/agent-browser/lib/results/presentation/semantic-action.ts

@@ -0,0 +1,133 @@
+/**
+ * Purpose: Map successful semanticAction results to the same presentation signals as direct ref commands.
+ * Responsibilities: Resolve presentation command names, compact action prose, and navigation-summary probe gates.
+ * Scope: semanticAction success presentation only.
+ */
+
+import {
+	getCompiledSemanticActionCommandIndex,
+	isCompiledSemanticActionFindCommand,
+} from "../../input-modes/semantic-action.js";
+import type { CompiledAgentBrowserSemanticAction } from "../../input-modes/types.js";
+import { isRecord } from "../../parsing.js";
+import type { CommandInfo } from "../../runtime.js";
+import {
+	formatNavigationSummary,
+	getNavigationSummary,
+	isNavigationObservableCommand,
+} from "./navigation.js";
+import { redactModelFacingText } from "./common.js";
+
+const SEMANTIC_NAVIGATION_PROBE_ACTIONS = new Set(["check", "click", "uncheck"]);
+
+const SEMANTIC_PRESENTATION_ACTIONS = new Set(["check", "click", "fill", "select", "uncheck"]);
+
+function getPageSummary(data: Record<string, unknown>): string | undefined {
+	const title = typeof data.title === "string" ? data.title : undefined;
+	const url = typeof data.url === "string" ? data.url : undefined;
+	if (!title && !url) return undefined;
+	if (title && url) return `${title}\n${url}`;
+	return title ?? url;
+}
+
+function formatSemanticActionTarget(compiled: CompiledAgentBrowserSemanticAction): string {
+	if (compiled.action === "select") {
+		const selector = compiled.selector ?? "selector";
+		const values = compiled.values?.length ? compiled.values.join(", ") : "";
+		return values ? `${selector} → ${values}` : selector;
+	}
+	const commandIndex = getCompiledSemanticActionCommandIndex(compiled);
+	const locator = compiled.locator ?? compiled.args[commandIndex + 1] ?? "locator";
+	const locatorValue = compiled.args[commandIndex + 2];
+	const nameIndex = compiled.args.indexOf("--name");
+	const name = nameIndex >= 0 ? compiled.args[nameIndex + 1] : undefined;
+	const quotedValue = JSON.stringify(locatorValue ?? "");
+	const target = `${locator} ${quotedValue}`;
+	return name ? `${target} (name ${JSON.stringify(name)})` : target;
+}
+
+export function formatSemanticActionCompactLine(compiled: CompiledAgentBrowserSemanticAction): string {
+	const target = formatSemanticActionTarget(compiled);
+	switch (compiled.action) {
+		case "click":
+			return `Clicked: ${target}`;
+		case "fill":
+			return `Filled: ${target}`;
+		case "check":
+			return `Checked: ${target}`;
+		case "uncheck":
+			return `Unchecked: ${target}`;
+		case "select":
+			return `Selected: ${target}`;
+		default:
+			return `${compiled.action}: ${target}`;
+	}
+}
+
+export function resolveSemanticPresentationCommand(
+	compiled: CompiledAgentBrowserSemanticAction | undefined,
+): string | undefined {
+	if (!compiled || !SEMANTIC_PRESENTATION_ACTIONS.has(compiled.action)) return undefined;
+	if (compiled.action === "select") return "select";
+	if (isCompiledSemanticActionFindCommand(compiled)) return compiled.action;
+	return undefined;
+}
+
+export function resolvePresentationCommandInfo(
+	commandInfo: CommandInfo,
+	compiledSemanticAction?: CompiledAgentBrowserSemanticAction,
+): CommandInfo {
+	const presentationCommand = resolveSemanticPresentationCommand(compiledSemanticAction);
+	if (!presentationCommand) return commandInfo;
+	return { ...commandInfo, command: presentationCommand };
+}
+
+export function shouldCaptureSemanticActionNavigationSummary(
+	compiled: CompiledAgentBrowserSemanticAction | undefined,
+	data: unknown,
+): boolean {
+	if (!compiled || !SEMANTIC_NAVIGATION_PROBE_ACTIONS.has(compiled.action)) return false;
+	if (!isCompiledSemanticActionFindCommand(compiled)) return false;
+	return !isRecord(data) || (typeof data.title !== "string" && typeof data.url !== "string");
+}
+
+export function formatSemanticActionPresentationText(
+	compiled: CompiledAgentBrowserSemanticAction,
+	data: Record<string, unknown>,
+): string | undefined {
+	const presentationCommand = resolveSemanticPresentationCommand(compiled);
+	if (!presentationCommand) return undefined;
+
+	const actionLine = formatSemanticActionCompactLine(compiled);
+	const navigationSummary = getNavigationSummary(data);
+	if (navigationSummary && isNavigationObservableCommand(presentationCommand)) {
+		const navigationText = formatNavigationSummary(navigationSummary);
+		if (navigationText) return `${actionLine}\n\nCurrent page:\n${navigationText}`;
+	}
+
+	const pageSummary = getPageSummary(data);
+	if (pageSummary) return `${actionLine}\n\nCurrent page:\n${redactModelFacingText(pageSummary)}`;
+
+	return actionLine;
+}
+
+export function formatSemanticActionPresentationSummary(
+	compiled: CompiledAgentBrowserSemanticAction,
+	data: Record<string, unknown>,
+): string | undefined {
+	const presentationCommand = resolveSemanticPresentationCommand(compiled);
+	if (!presentationCommand) return undefined;
+
+	const navigationSummary = getNavigationSummary(data);
+	if (navigationSummary && isNavigationObservableCommand(presentationCommand)) {
+		const navigationText = formatNavigationSummary(navigationSummary);
+		if (navigationText) {
+			return `${presentationCommand} → ${navigationText.split("\n", 1)[0] ?? navigationText}`;
+		}
+	}
+
+	const pageSummary = getPageSummary(data);
+	if (pageSummary) return `${presentationCommand} → ${pageSummary.split("\n", 1)[0] ?? pageSummary}`;
+
+	return `${presentationCommand} → ${formatSemanticActionTarget(compiled)}`;
+}