pi-agent-browser-native 0.2.33 → 0.2.35

package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts

@@ -1,11 +1,14 @@
 import { readFile, rm } from "node:fs/promises";
 
+import { isCloseCommand, isOpenNavigationCommand } from "../../command-taxonomy.js";
 import { cleanupElectronLaunchResources, inspectElectronLaunchStatus, type ElectronCleanupResult } from "../../electron/cleanup.js";
 import type { ElectronLaunchRecord } from "../../electron/launch.js";
 import {
 	analyzeNetworkSourceLookupResults,
 	analyzeQaPresetResults,
 	analyzeSourceLookupResults,
+	buildQaCompactPassText,
+	extractQaPageContext,
 	redactNetworkSourceLookupAnalysis,
 } from "../../input-modes.js";
 import {
@@ -20,6 +23,7 @@ import {
 	mergeSessionArtifactManifest,
 } from "../../results/artifact-manifest.js";
 import type { SessionArtifactManifest } from "../../results/contracts.js";
+import { shouldCaptureSemanticActionNavigationSummary } from "../../results/presentation/semantic-action.js";
 import {
 	commandExplicitlyTargetsAboutBlank,
 	deriveSessionTabTarget,
@@ -35,7 +39,7 @@ import {
 import type { PersistentSessionArtifactEviction, PersistentSessionArtifactStore } from "../../temp.js";
 import { writePersistentSessionArtifactFile, writeSecureTempFile } from "../../temp.js";
 import { isRecord } from "../../parsing.js";
-import { hasLaunchScopedTabCorrectionFlag, resolveManagedSessionState } from "../../runtime.js";
+import { createFreshSessionName, hasLaunchScopedTabCorrectionFlag, resolveManagedSessionState } from "../../runtime.js";
 import {
 	applyOpenResultTabCorrection,
 	buildAboutBlankRecoveryHint,
@@ -59,6 +63,7 @@ import {
 	unwrapPinnedSessionBatchEnvelope,
 	updateTraceOwnerState,
 } from "./session-state.js";
+import { collectClickDispatchDiagnostic } from "./click-dispatch.js";
 import {
 	buildScrollNoopDiagnostic,
 	collectComboboxFocusDiagnostic,
@@ -174,6 +179,7 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 	const { prepared, processResult } = input;
 	const { electronChildProcesses, electronLaunchRecords, sessionPageState, traceOwners } = state;
 	let artifactManifest = state.artifactManifest;
+	let freshSessionOrdinal = state.freshSessionOrdinal;
 	let managedSessionActive = state.managedSessionActive;
 	let managedSessionCwd = state.managedSessionCwd;
 	let managedSessionName = state.managedSessionName;
@@ -205,14 +211,28 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		const inspectionText = plainTextInspection ? processResult.stdout.trim() : undefined;
 		updateTraceOwnerState({ command: prepared.executionPlan.commandInfo.command, sessionName: prepared.executionPlan.sessionName, subcommand: prepared.executionPlan.commandInfo.subcommand, succeeded, traceOwners });
 
-		if (succeeded && !navigationSummary && shouldCaptureNavigationSummary(prepared.executionPlan.commandInfo.command, presentationEnvelope?.data)) {
+		let clickDispatchDiagnostic: Awaited<ReturnType<typeof collectClickDispatchDiagnostic>>;
+		if (succeeded && prepared.clickDispatchProbe) {
+			clickDispatchDiagnostic = await collectClickDispatchDiagnostic({ cwd, probe: prepared.clickDispatchProbe, sessionName: prepared.executionPlan.sessionName, signal });
+			if (clickDispatchDiagnostic) {
+				succeeded = false;
+				presentationEnvelope = { ...(presentationEnvelope ?? {}), error: clickDispatchDiagnostic.summary, success: false };
+			}
+		}
+
+		if (
+			succeeded &&
+			!navigationSummary &&
+			(shouldCaptureNavigationSummary(prepared.executionPlan.commandInfo.command, presentationEnvelope?.data) ||
+				shouldCaptureSemanticActionNavigationSummary(prepared.compiledSemanticAction, presentationEnvelope?.data))
+		) {
 			navigationSummary = await collectNavigationSummary({ cwd, sessionName: prepared.executionPlan.sessionName, signal });
 		}
 		if (navigationSummary && presentationEnvelope) presentationEnvelope = { ...presentationEnvelope, data: mergeNavigationSummaryIntoData(presentationEnvelope.data, navigationSummary) };
 		let overlayBlockerDiagnostic: Awaited<ReturnType<typeof collectOverlayBlockerDiagnostic>>;
 
 		let openResultTabCorrection: Awaited<ReturnType<typeof collectOpenResultTabCorrection>>;
-		if (succeeded && prepared.executionPlan.sessionName && hasLaunchScopedTabCorrectionFlag(prepared.runtimeToolArgs) && ["goto", "navigate", "open"].includes(prepared.executionPlan.commandInfo.command ?? "")) {
+		if (succeeded && prepared.executionPlan.sessionName && hasLaunchScopedTabCorrectionFlag(prepared.runtimeToolArgs) && isOpenNavigationCommand(prepared.executionPlan.commandInfo.command)) {
 			const targetTitle = extractStringResultField(presentationEnvelope?.data, "title");
 			const targetUrl = extractStringResultField(presentationEnvelope?.data, "url");
 			const plannedTabCorrection = await collectOpenResultTabCorrection({ cwd, sessionName: prepared.executionPlan.sessionName, signal, targetTitle, targetUrl });
@@ -267,7 +287,7 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 			fillVerificationDiagnostic = await collectFillVerificationDiagnostic({ commandTokens: prepared.commandTokens, cwd, sessionName: prepared.executionPlan.sessionName, signal });
 			electronRefFreshnessDiagnostic = buildElectronRefFreshnessDiagnostic({ command: prepared.executionPlan.commandInfo.command, commandTokens: prepared.commandTokens, record: electronRecordForCommand, sessionName: prepared.executionPlan.sessionName, stdin: prepared.runtimeToolStdin });
 		}
-		if (succeeded && !sessionTabCorrection && !aboutBlankSessionMismatch && !electronRecordForCommand) overlayBlockerDiagnostic = await collectOverlayBlockerDiagnostic({ command: prepared.executionPlan.commandInfo.command, cwd, data: presentationEnvelope?.data, navigationSummary, priorTarget: prepared.priorSessionTabTarget, sessionName: prepared.executionPlan.sessionName, signal });
+		if (succeeded && !sessionTabCorrection && !aboutBlankSessionMismatch && !electronRecordForCommand && !clickDispatchDiagnostic) overlayBlockerDiagnostic = await collectOverlayBlockerDiagnostic({ command: prepared.executionPlan.commandInfo.command, cwd, data: presentationEnvelope?.data, navigationSummary, priorTarget: prepared.priorSessionTabTarget, sessionName: prepared.executionPlan.sessionName, signal });
 		if (succeeded) {
 			selectorTextVisibilityDiagnostics = await collectSelectorTextVisibilityDiagnostics({ commandInfo: prepared.executionPlan.commandInfo, commandTokens: prepared.commandTokens, cwd, data: presentationEnvelope?.data, sessionName: prepared.executionPlan.sessionName, signal });
 			electronBroadGetTextScopeDiagnostics = collectElectronBroadGetTextScopeDiagnostics({ commandInfo: prepared.executionPlan.commandInfo, commandTokens: prepared.commandTokens, currentTarget: currentSessionTabTarget, data: presentationEnvelope?.data, electronLaunchRecords, priorTarget: prepared.priorSessionTabTarget, sessionName: prepared.executionPlan.sessionName });
@@ -279,8 +299,10 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		let currentRefSnapshotInvalidation: SessionRefSnapshotInvalidation | undefined;
 		const batchRefSnapshotState = prepared.executionPlan.commandInfo.command === "batch" ? extractLatestRefSnapshotStateFromBatchResults(presentationEnvelope?.data) : undefined;
 		if (prepared.executionPlan.sessionName) {
-			if (prepared.executionPlan.commandInfo.command === "close" && succeeded) sessionPageState.clearSession(prepared.executionPlan.sessionName);
-			else if (currentSessionTabTarget) {
+			if (isCloseCommand(prepared.executionPlan.commandInfo.command) && succeeded) {
+				sessionPageState.clearSession(prepared.executionPlan.sessionName);
+				state.closedManagedSessionNames.add(prepared.executionPlan.sessionName);
+			} else if (currentSessionTabTarget) {
 				const tabUpdate = sessionPageState.applyTabTarget({ sessionName: prepared.executionPlan.sessionName, target: currentSessionTabTarget, update: sessionPageStateUpdate });
 				if (!tabUpdate.applied && succeeded) sessionPageState.markPinning(prepared.executionPlan.sessionName, "drift");
 			}
@@ -299,11 +321,19 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		const priorManagedSessionActive = managedSessionActive;
 		const priorManagedSessionCwd = managedSessionCwd;
 		const priorManagedSessionName = managedSessionName;
-		const managedSessionState = resolveManagedSessionState({ command: prepared.executionPlan.commandInfo.command, managedSessionName: prepared.executionPlan.managedSessionName, priorActive: priorManagedSessionActive, priorSessionName: priorManagedSessionName, succeeded });
+		const commandClosesSession = isCloseCommand(prepared.executionPlan.commandInfo.command);
+		const managedCloseSessionName = commandClosesSession && succeeded && prepared.executionPlan.sessionName === priorManagedSessionName
+			? prepared.executionPlan.sessionName
+			: prepared.executionPlan.managedSessionName;
+		const managedSessionState = resolveManagedSessionState({ command: prepared.executionPlan.commandInfo.command, managedSessionName: managedCloseSessionName, priorActive: priorManagedSessionActive, priorSessionName: priorManagedSessionName, succeeded });
 		const replacedManagedSessionName = managedSessionState.replacedSessionName;
 		managedSessionActive = managedSessionState.active;
 		managedSessionName = managedSessionState.sessionName;
-		let managedSessionOutcome = buildManagedSessionOutcome({ activeAfter: managedSessionActive, activeBefore: priorManagedSessionActive, attemptedSessionName: prepared.executionPlan.managedSessionName, command: prepared.executionPlan.commandInfo.command, currentSessionName: managedSessionName, previousSessionName: priorManagedSessionName, replacedSessionName: replacedManagedSessionName, sessionMode: prepared.sessionMode, succeeded });
+		if (commandClosesSession && succeeded && managedCloseSessionName === priorManagedSessionName && !managedSessionActive) {
+			freshSessionOrdinal += 1;
+			managedSessionName = createFreshSessionName(state.managedSessionBaseName, state.ephemeralSessionSeed, freshSessionOrdinal);
+		}
+		let managedSessionOutcome = buildManagedSessionOutcome({ activeAfter: managedSessionActive, activeBefore: priorManagedSessionActive, attemptedSessionName: managedCloseSessionName, command: prepared.executionPlan.commandInfo.command, currentSessionName: managedSessionName, previousSessionName: priorManagedSessionName, replacedSessionName: replacedManagedSessionName, sessionMode: prepared.sessionMode, succeeded });
 		if (prepared.executionPlan.managedSessionName && succeeded) managedSessionCwd = cwd;
 		if (prepared.executionPlan.sessionName && succeeded) {
 			if (openResultTabCorrection || sessionTabCorrection || aboutBlankSessionMismatch?.recoveryApplied) sessionPageState.markPinning(prepared.executionPlan.sessionName, "drift");
@@ -311,7 +341,8 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		}
 		if (replacedManagedSessionName) {
 			sessionPageState.clearSession(replacedManagedSessionName);
-			await closeManagedSession({ cwd: priorManagedSessionCwd, sessionName: replacedManagedSessionName, timeoutMs: implicitSessionCloseTimeoutMs });
+			const replacedCloseError = await closeManagedSession({ cwd: priorManagedSessionCwd, sessionName: replacedManagedSessionName, timeoutMs: implicitSessionCloseTimeoutMs });
+			if (!replacedCloseError) state.closedManagedSessionNames.add(replacedManagedSessionName);
 		}
 
 		let electronLaunchRecord: ElectronLaunchRecord | undefined;
@@ -341,7 +372,7 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		}
 
 		const errorText = getAgentBrowserErrorText({ aborted: processResult.aborted, command: prepared.executionPlan.commandInfo.command, effectiveArgs: prepared.redactedProcessArgs, envelope: presentationEnvelope, exitCode: processResult.exitCode, parseError, plainTextInspection, staleRefArgs: getStaleRefArgs(prepared.commandTokens, prepared.runtimeToolStdin), spawnError: processResult.spawnError, stderr: processResult.stderr, timedOut: processResult.timedOut, timeoutMs: processResult.timeoutMs, wrapperRecoveryHint: buildWrapperRecoveryHint({ pinnedBatchUnwrapMode: prepared.pinnedBatchUnwrapMode, sessionTabCorrection }) });
-		const presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text" as const, text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, cwd, envelope: presentationEnvelope, errorText, persistentArtifactStore, sessionName: prepared.executionPlan.sessionName });
+		const presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text" as const, text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd, envelope: presentationEnvelope, errorText, persistentArtifactStore, sessionName: prepared.executionPlan.sessionName });
 		if (parseFailureOutput.artifactManifest) { presentation.artifactManifest = parseFailureOutput.artifactManifest; presentation.artifactRetentionSummary = parseFailureOutput.artifactRetentionSummary; }
 		if (parseFailureOutput.fullOutputPath || parseFailureOutput.fullOutputUnavailable) {
 			const existingText = presentation.content[0]?.type === "text" ? presentation.content[0].text : "";
@@ -351,7 +382,9 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		}
 		if (presentation.artifactManifest) artifactManifest = presentation.artifactManifest;
 		const qaPreset = prepared.compiledQaPreset ? analyzeQaPresetResults(presentationEnvelope?.data) : undefined;
-		const qaAttachedTarget = prepared.compiledQaPreset?.checks.attached ? await collectQaAttachedTarget({ currentTarget: currentSessionTabTarget ?? prepared.priorSessionTabTarget, cwd, sessionName: prepared.executionPlan.sessionName, signal }) : undefined;
+		let qaAttachedTarget = prepared.compiledQaPreset?.checks.attached
+			? await collectQaAttachedTarget({ currentTarget: currentSessionTabTarget ?? prepared.priorSessionTabTarget, cwd, sessionName: prepared.executionPlan.sessionName, signal })
+			: undefined;
 		const sourceLookupElectronContext = prepared.compiledSourceLookup ? getSourceLookupElectronContext({ currentTarget: currentSessionTabTarget, electronLaunchRecords, priorTarget: prepared.priorSessionTabTarget, sessionName: prepared.executionPlan.sessionName }) : undefined;
 		const sourceLookup = prepared.compiledSourceLookup ? await analyzeSourceLookupResults(presentationEnvelope?.data, prepared.compiledSourceLookup, cwd, { electronContext: sourceLookupElectronContext, workspaceRoot: cwd }) : undefined;
 		const networkSourceLookup = prepared.compiledNetworkSourceLookup ? redactNetworkSourceLookupAnalysis(await analyzeNetworkSourceLookupResults(presentationEnvelope?.data, prepared.compiledNetworkSourceLookup, cwd)) : undefined;
@@ -359,15 +392,32 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		else if (networkSourceLookup) presentation.content.unshift({ type: "text", text: networkSourceLookup.summary });
 		if (sourceLookup && presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${sourceLookup.summary}\n\n${presentation.content[0].text}` };
 		else if (sourceLookup) presentation.content.unshift({ type: "text", text: sourceLookup.summary });
-		if (qaPreset && (!qaPreset.passed || qaPreset.warnings.length > 0)) {
-			if (!qaPreset.passed) { succeeded = false; presentation.failureCategory = "qa-failure"; }
+		if (qaPreset && !qaPreset.passed) {
+			succeeded = false;
+			presentation.failureCategory = "qa-failure";
 			presentation.summary = qaPreset.summary;
 			if (presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${qaPreset.summary}\n\n${presentation.content[0].text}` };
 			else presentation.content.unshift({ type: "text", text: qaPreset.summary });
+		} else if (qaPreset?.passed && prepared.compiledQaPreset) {
+			const compactText = buildQaCompactPassText({
+				artifactVerification: presentation.artifactVerification,
+				batchStepCount: presentation.batchSteps?.length ?? prepared.compiledQaPreset.steps.length,
+				checks: prepared.compiledQaPreset.checks,
+				page: extractQaPageContext({
+					attachedTarget: qaAttachedTarget,
+					batchData: presentationEnvelope?.data,
+					compiled: prepared.compiledQaPreset,
+				}),
+				qaPreset,
+			});
+			presentation.summary = qaPreset.summary;
+			const nonTextContent = presentation.content.filter((item) => item.type !== "text");
+			presentation.content = [{ type: "text", text: compactText }, ...nonTextContent];
 		}
 		const qaAttachedTargetText = formatQaAttachedTargetText(qaAttachedTarget);
-		if (qaAttachedTargetText && presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${qaAttachedTargetText}\n\n${presentation.content[0].text}` };
-		else if (qaAttachedTargetText) presentation.content.unshift({ type: "text", text: qaAttachedTargetText });
+		const skipAttachedTargetBanner = qaPreset?.passed && prepared.compiledQaPreset?.checks.attached;
+		if (!skipAttachedTargetBanner && qaAttachedTargetText && presentation.content[0]?.type === "text") presentation.content[0] = { ...presentation.content[0], text: `${qaAttachedTargetText}\n\n${presentation.content[0].text}` };
+		else if (!skipAttachedTargetBanner && qaAttachedTargetText) presentation.content.unshift({ type: "text", text: qaAttachedTargetText });
 		if (managedSessionOutcome && managedSessionOutcome.succeeded !== succeeded) managedSessionOutcome = { ...managedSessionOutcome, succeeded };
 		const evalStdinHint = getEvalStdinHint({ command: prepared.executionPlan.commandInfo.command, data: presentationEnvelope?.data, stdin: prepared.runtimeToolStdin });
 		const resultArtifactManifest = presentation.artifactManifest ?? artifactManifest;
@@ -377,8 +427,8 @@ export async function processBrowserOutput(input: ProcessBrowserOutputInput): Pr
 		const finalRecoveryState = await prepareFinalResultRecoveryState({ aboutBlankSessionMismatch, batchRefSnapshotState, commandTokens: prepared.commandTokens, compiledSemanticAction: prepared.compiledSemanticAction, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, cwd, electronPostCommandHealth, errorText, executionPlan: prepared.executionPlan, parseError, plainTextInspection, presentation, processResult, redactedProcessArgs: prepared.redactedProcessArgs, runtimeToolArgs: prepared.runtimeToolArgs, sessionPageState, sessionPageStateUpdate, sessionTabCorrection, signal, succeeded });
 		currentRefSnapshot = finalRecoveryState.currentRefSnapshot;
 		currentRefSnapshotInvalidation = finalRecoveryState.currentRefSnapshotInvalidation;
-		const result = buildFinalAgentBrowserToolResult({ aboutBlankSessionMismatch, artifactCleanup, categoryDetails: finalRecoveryState.categoryDetails, comboboxFocusDiagnostic, compiledNetworkSourceLookup: prepared.compiledNetworkSourceLookup, compiledSemanticAction: prepared.compiledSemanticAction, compatibilityWorkaround: prepared.compatibilityWorkaround, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, electronBroadGetTextScopeDiagnostics, electronFailedConnectCleanup, electronHandoff, electronLaunch: prepared.electronLaunch, electronLaunchRecord, electronLaunchRecords, electronPostCommandHealth, electronProfileIsolationDetails: input.electronProfileIsolationDetails, electronRefFreshnessDiagnostic, electronSessionMismatch, errorText, evalStdinHint, exactSensitiveValues: prepared.exactSensitiveValues, executionPlan: prepared.executionPlan, fillVerificationDiagnostic, inspectionText, managedSessionOutcome, navigationSummary, networkSourceLookup, noActivePageSnapshotFailure: finalRecoveryState.noActivePageSnapshotFailure, openResultTabCorrection, overlayBlockerDiagnostic, parseError, parseFailureOutput, parseSucceeded, plainTextInspection, presentation, presentationEnvelope, priorSessionTabTarget: prepared.priorSessionTabTarget, processResult, qaAttachedTarget, qaPreset, recordingDependencyWarning, redactedArgs: prepared.redactedArgs, redactedCompiledElectron: prepared.redactedCompiledElectron, redactedCompiledJob: prepared.redactedCompiledJob, redactedCompiledNetworkSourceLookup: prepared.redactedCompiledNetworkSourceLookup, redactedCompiledQaPreset: prepared.redactedCompiledQaPreset, redactedCompiledSemanticAction: prepared.redactedCompiledSemanticAction, redactedCompiledSourceLookup: prepared.redactedCompiledSourceLookup, redactedContent, redactedProcessArgs: prepared.redactedProcessArgs, redactedRecoveryHint: prepared.redactedRecoveryHint, resultArtifactManifest, richInputRecoveryDiagnostic: finalRecoveryState.richInputRecoveryDiagnostic, scrollNoopDiagnostic, selectorTextVisibilityDiagnostics, sessionMode: prepared.sessionMode, sessionTabCorrection, sourceLookup, succeeded, timeoutPartialProgress, userRequestedJson: prepared.userRequestedJson, visibleRefFallbackDiagnostic: finalRecoveryState.visibleRefFallbackDiagnostic, visibleRefFallbackSessionName: finalRecoveryState.visibleRefFallbackSessionName });
-		const statePatch: BrowserRunStatePatch = { artifactManifest, managedSessionActive, managedSessionCwd, managedSessionName };
+		const result = buildFinalAgentBrowserToolResult({ aboutBlankSessionMismatch, artifactCleanup, categoryDetails: finalRecoveryState.categoryDetails, clickDispatchDiagnostic, commandTokens: prepared.commandTokens, comboboxFocusDiagnostic, compiledNetworkSourceLookup: prepared.compiledNetworkSourceLookup, compiledSemanticAction: prepared.compiledSemanticAction, compatibilityWorkaround: prepared.compatibilityWorkaround, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, electronBroadGetTextScopeDiagnostics, electronFailedConnectCleanup, electronHandoff, electronLaunch: prepared.electronLaunch, electronLaunchRecord, electronLaunchRecords, electronPostCommandHealth, electronProfileIsolationDetails: input.electronProfileIsolationDetails, electronRefFreshnessDiagnostic, electronSessionMismatch, errorText, evalStdinHint, exactSensitiveValues: prepared.exactSensitiveValues, executionPlan: prepared.executionPlan, fillVerificationDiagnostic, inspectionText, managedSessionOutcome, navigationSummary, networkSourceLookup, noActivePageSnapshotFailure: finalRecoveryState.noActivePageSnapshotFailure, openResultTabCorrection, overlayBlockerDiagnostic, parseError, parseFailureOutput, parseSucceeded, plainTextInspection, presentation, presentationEnvelope, priorSessionTabTarget: prepared.priorSessionTabTarget, processResult, qaAttachedTarget, qaPreset, recordingDependencyWarning, redactedArgs: prepared.redactedArgs, redactedCompiledElectron: prepared.redactedCompiledElectron, redactedCompiledJob: prepared.redactedCompiledJob, redactedCompiledNetworkSourceLookup: prepared.redactedCompiledNetworkSourceLookup, redactedCompiledQaPreset: prepared.redactedCompiledQaPreset, redactedCompiledSemanticAction: prepared.redactedCompiledSemanticAction, redactedCompiledSourceLookup: prepared.redactedCompiledSourceLookup, redactedContent, redactedProcessArgs: prepared.redactedProcessArgs, redactedRecoveryHint: prepared.redactedRecoveryHint, resultArtifactManifest, richInputRecoveryDiagnostic: finalRecoveryState.richInputRecoveryDiagnostic, scrollNoopDiagnostic, selectorTextVisibilityDiagnostics, sessionMode: prepared.sessionMode, sessionTabCorrection, sourceLookup, succeeded, timeoutPartialProgress, userRequestedJson: prepared.userRequestedJson, visibleRefFallbackDiagnostic: finalRecoveryState.visibleRefFallbackDiagnostic, visibleRefFallbackSessionName: finalRecoveryState.visibleRefFallbackSessionName });
+		const statePatch: BrowserRunStatePatch = { artifactManifest, freshSessionOrdinal, managedSessionActive, managedSessionCwd, managedSessionName };
 		return { result, statePatch };
 	} finally {
 		if (processResult.stdoutSpillPath) await rm(processResult.stdoutSpillPath, { force: true }).catch(() => undefined);

package/extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.ts

@@ -0,0 +1,93 @@
+import { isAbsolute, resolve } from "node:path";
+
+import { isCloseCommand } from "../../command-taxonomy.js";
+import { executableExistsOnPath } from "../../executable-path.js";
+import type { SessionArtifactManifest } from "../../results/contracts.js";
+import type { PromptPolicy, PromptRequestedArtifact } from "../../prompt-policy.js";
+import type { SessionRefSnapshot } from "../../session-page-state.js";
+import { findBlockedFinalizingAction, STOP_BOUNDARY_GUARD_SCOPE, type BrowserFinalizingAction } from "./browser-action-model.js";
+
+export interface StopBoundaryViolation {
+	action: BrowserFinalizingAction;
+	command: string[];
+	message: string;
+	reason: "explicit-user-stop-boundary";
+	stepIndex?: number;
+	target?: string;
+}
+
+export interface RequestedArtifactCloseViolation {
+	message: string;
+	missingArtifacts: PromptRequestedArtifact[];
+	reason: "requested-artifacts-missing-before-close";
+}
+
+function formatStopBoundaryActionPhrase(action: BrowserFinalizingAction): string {
+	if (action.kind === "keyboard-submit") return "keyboard submit (Enter/Return)";
+	return "click-like action";
+}
+
+export function findStopBoundaryViolation(options: { commandTokens: string[]; promptPolicy: PromptPolicy; refSnapshot?: SessionRefSnapshot; stdin?: string }): StopBoundaryViolation | undefined {
+	if (!options.promptPolicy.stopBoundary) return undefined;
+	const blocked = findBlockedFinalizingAction({
+		commandTokens: options.commandTokens,
+		refSnapshot: options.refSnapshot,
+		stdin: options.stdin,
+	});
+	if (!blocked) return undefined;
+	const target = blocked.targetLabel;
+	const actionPhrase = formatStopBoundaryActionPhrase(blocked);
+	const scopeNote = `Best-effort guard scope covers ${STOP_BOUNDARY_GUARD_SCOPE.covered.join(", ")}; it does not block ${STOP_BOUNDARY_GUARD_SCOPE.excluded.join(", ")}.`;
+	if (blocked.stepIndex === undefined) {
+		return {
+			action: blocked,
+			command: blocked.command,
+			message: `Blocked likely final submit/order ${actionPhrase} (${target}) because the latest user prompt set an explicit stop boundary. Gather evidence on the current page instead of activating the final action. ${scopeNote}`,
+			reason: "explicit-user-stop-boundary",
+			target,
+		};
+	}
+	return {
+		action: blocked,
+		command: blocked.command,
+		message: `Blocked likely final submit/order ${actionPhrase} in batch step ${blocked.stepIndex + 1} (${target}) because the latest user prompt set an explicit stop boundary. Gather evidence on the current page instead of activating the final action. ${scopeNote}`,
+		reason: "explicit-user-stop-boundary",
+		stepIndex: blocked.stepIndex,
+		target,
+	};
+}
+
+function resolveArtifactPath(cwd: string, path: string): string {
+	return isAbsolute(path) ? path : resolve(cwd, path);
+}
+
+function manifestContainsArtifact(manifest: SessionArtifactManifest | undefined, cwd: string, artifact: PromptRequestedArtifact): boolean {
+	if (!manifest) return false;
+	const requestedAbsolutePath = resolveArtifactPath(cwd, artifact.path);
+	const expectedKind = artifact.kind === "screenshot" ? "image" : "video";
+	return manifest.entries.some((entry) => {
+		const entryAbsolutePath = entry.absolutePath ?? resolveArtifactPath(cwd, entry.path);
+		return entry.storageScope === "explicit-path" && entry.kind === expectedKind && entryAbsolutePath === requestedAbsolutePath && entry.retentionState === "live" && entry.exists === true;
+	});
+}
+
+async function isArtifactRequired(artifact: PromptRequestedArtifact): Promise<boolean> {
+	if (artifact.required) return true;
+	return artifact.kind === "recording" && await executableExistsOnPath("ffmpeg");
+}
+
+export async function findRequestedArtifactCloseViolation(options: { artifactManifest?: SessionArtifactManifest; command: string | undefined; cwd: string; promptPolicy: PromptPolicy }): Promise<RequestedArtifactCloseViolation | undefined> {
+	if (!isCloseCommand(options.command)) return undefined;
+	const missingArtifacts: PromptRequestedArtifact[] = [];
+	for (const artifact of options.promptPolicy.requestedArtifacts) {
+		if (!await isArtifactRequired(artifact)) continue;
+		if (!manifestContainsArtifact(options.artifactManifest, options.cwd, artifact)) missingArtifacts.push(artifact);
+	}
+	if (missingArtifacts.length === 0) return undefined;
+	const missingList = missingArtifacts.map((artifact) => `${artifact.kind}: ${artifact.path}`).join(", ");
+	return {
+		message: `Blocked browser close because requested artifact path${missingArtifacts.length === 1 ? " is" : "s are"} missing or unverified: ${missingList}. Save the requested artifact path first, or report why an optional artifact is unavailable before closing.`,
+		missingArtifacts,
+		reason: "requested-artifacts-missing-before-close",
+	};
+}

package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts

@@ -14,8 +14,18 @@ import {
 	type SessionRefSnapshotInvalidation,
 	type SessionTabTarget,
 } from "../../session-page-state.js";
+import {
+	isCloseCommand,
+	isElectronPostCommandHealthCommand,
+	isNavigationObservableCommandName,
+	isRefGuardedCommand,
+	isRefInvalidatingBatchCommand,
+	isSessionTabPinningExcludedCommand,
+	isSessionTabPostCommandCorrectionExcludedCommand,
+} from "../../command-taxonomy.js";
 import { chooseOpenResultTabCorrection, redactInvocationArgs, type OpenResultTabCorrection } from "../../runtime.js";
 import { isRecord } from "../../parsing.js";
+import { parseUserBatchStdin } from "../batch-stdin.js";
 import type {
 	AboutBlankSessionMismatch,
 	BatchCommandStep,
@@ -35,65 +45,8 @@ import type {
 	TraceOwner,
 } from "./types.js";
 
-export const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
 export const NAVIGATION_SUMMARY_EVAL = `({ title: document.title, url: location.href })`;
 
-const SESSION_TAB_PINNING_EXCLUDED_COMMANDS = new Set(["close", "goto", "navigate", "open", "session", "tab"]);
-const SESSION_TAB_POST_COMMAND_CORRECTION_EXCLUDED_COMMANDS = new Set(["batch", "close", "session", "tab"]);
-const REF_INVALIDATING_BATCH_COMMANDS = new Set([
-	"back",
-	"check",
-	"click",
-	"dblclick",
-	"drag",
-	"forward",
-	"goto",
-	"keyboard",
-	"mouse",
-	"navigate",
-	"open",
-	"press",
-	"reload",
-	"select",
-	"type",
-	"uncheck",
-	"upload",
-]);
-const REF_GUARDED_COMMANDS = new Set([
-	"check",
-	"click",
-	"dblclick",
-	"download",
-	"drag",
-	"fill",
-	"focus",
-	"hover",
-	"keyboard",
-	"mouse",
-	"press",
-	"scrollintoview",
-	"select",
-	"type",
-	"uncheck",
-	"upload",
-]);
-const ELECTRON_POST_COMMAND_HEALTH_COMMANDS = new Set([
-	"back",
-	"check",
-	"click",
-	"dblclick",
-	"fill",
-	"find",
-	"forward",
-	"keyboard",
-	"mouse",
-	"press",
-	"reload",
-	"select",
-	"type",
-	"uncheck",
-]);
-
 export function applyBrowserRunStatePatch(state: BrowserRunState, patch: BrowserRunStatePatch | undefined): void {
 	if (!patch) return;
 	if ("artifactManifest" in patch) state.artifactManifest = patch.artifactManifest;
@@ -103,10 +56,6 @@ export function applyBrowserRunStatePatch(state: BrowserRunState, patch: Browser
 	if (patch.managedSessionName !== undefined) state.managedSessionName = patch.managedSessionName;
 }
 
-export function mergeBrowserRunStatePatch(left: BrowserRunStatePatch | undefined, right: BrowserRunStatePatch | undefined): BrowserRunStatePatch {
-	return { ...(left ?? {}), ...(right ?? {}) };
-}
-
 export function buildSessionDetailFields(sessionName: string | undefined, usedImplicitSession: boolean): Record<string, unknown> {
 	return sessionName ? { sessionName, usedImplicitSession } : {};
 }
@@ -126,7 +75,7 @@ export function buildManagedSessionOutcome(options: {
 	if (!attemptedSessionName) return undefined;
 	let status: ManagedSessionOutcome["status"];
 	let summary: string;
-	if (command === "close") {
+	if (isCloseCommand(command)) {
 		status = succeeded ? "closed" : activeBefore ? "preserved" : "abandoned";
 		summary = succeeded
 			? `Managed session ${attemptedSessionName} was closed.`
@@ -238,8 +187,7 @@ export function extractNavigationSummaryFromData(data: unknown): NavigationSumma
 
 export function shouldCaptureNavigationSummary(command: string | undefined, data: unknown): boolean {
 	return (
-		command !== undefined &&
-		NAVIGATION_SUMMARY_COMMANDS.has(command) &&
+		isNavigationObservableCommandName(command) &&
 		(!isRecord(data) || (typeof data.title !== "string" && typeof data.url !== "string"))
 	);
 }
@@ -263,58 +211,6 @@ function extractBatchResultCommand(item: Record<string, unknown>): string[] {
 	return Array.isArray(item.command) ? item.command.filter((token): token is string => typeof token === "string") : [];
 }
 
-function validateUserBatchStep(
-	step: unknown,
-	index: number,
-):
-	| { ok: true; step: BatchCommandStep }
-	| { ok: false; error: string } {
-	if (!Array.isArray(step)) {
-		return {
-			ok: false,
-			error: `agent_browser batch stdin step ${index} must be a non-empty array of string command tokens.`,
-		};
-	}
-	if (step.length === 0) {
-		return {
-			ok: false,
-			error: `agent_browser batch stdin step ${index} must not be empty.`,
-		};
-	}
-	const invalidTokenIndex = step.findIndex((token) => typeof token !== "string");
-	if (invalidTokenIndex !== -1) {
-		return {
-			ok: false,
-			error: `agent_browser batch stdin step ${index} token ${invalidTokenIndex} must be a string.`,
-		};
-	}
-	return { ok: true, step: step as BatchCommandStep };
-}
-
-function parseUserBatchStdin(stdin: string | undefined): { error?: string; steps?: BatchCommandStep[] } {
-	if (stdin === undefined) {
-		return { steps: [] };
-	}
-	try {
-		const parsed = JSON.parse(stdin) as unknown;
-		if (!Array.isArray(parsed)) {
-			return { error: "agent_browser batch stdin must be a JSON array of command steps." };
-		}
-		const steps: BatchCommandStep[] = [];
-		for (const [index, rawStep] of parsed.entries()) {
-			const validated = validateUserBatchStep(rawStep, index);
-			if (!validated.ok) {
-				return { error: validated.error };
-			}
-			steps.push(validated.step);
-		}
-		return { steps };
-	} catch (error) {
-		const message = error instanceof Error ? error.message : String(error);
-		return { error: `agent_browser batch stdin could not be parsed as JSON: ${message}` };
-	}
-}
-
 export function getStaleRefArgs(commandTokens: string[], stdin?: string): string[] {
 	if (commandTokens[0] !== "batch" || stdin === undefined) {
 		return commandTokens;
@@ -331,7 +227,7 @@ function collectRefsFromTokens(tokens: string[]): string[] {
 }
 
 export function getGuardedRefUsage(commandTokens: string[], stdin?: string, options: { includeRefsAfterBatchSnapshot?: boolean } = {}): string[] {
-	const collectFromStep = (step: string[]) => REF_GUARDED_COMMANDS.has(step[0] ?? "") ? collectRefsFromTokens(step) : [];
+	const collectFromStep = (step: string[]) => isRefGuardedCommand(step[0]) ? collectRefsFromTokens(step) : [];
 	if (commandTokens[0] !== "batch" || stdin === undefined) {
 		return collectFromStep(commandTokens);
 	}
@@ -357,10 +253,10 @@ function getBatchRefInvalidationMessage(commandTokens: string[], stdin?: string)
 			priorStepInvalidatesRefs = false;
 		}
 		const refIds = collectRefsFromTokens(step);
-		if (refIds.length > 0 && REF_GUARDED_COMMANDS.has(step[0] ?? "") && priorStepInvalidatesRefs) {
+		if (refIds.length > 0 && isRefGuardedCommand(step[0]) && priorStepInvalidatesRefs) {
 			return `Batch step ${step[0]} uses page-scoped ref ${refIds.map((refId) => `@${refId}`).join(", ")} after an earlier batch step can navigate or mutate the page. Split the batch, run snapshot -i after the page-changing step, then retry with current refs.`;
 		}
-		if (REF_INVALIDATING_BATCH_COMMANDS.has(step[0] ?? "")) {
+		if (isRefInvalidatingBatchCommand(step[0])) {
 			priorStepInvalidatesRefs = true;
 		}
 	}
@@ -438,7 +334,7 @@ export function shouldPinSessionTabForCommand(options: {
 		options.pinningRequired === true &&
 		options.sessionName !== undefined &&
 		options.command !== undefined &&
-		!SESSION_TAB_PINNING_EXCLUDED_COMMANDS.has(options.command) &&
+		!isSessionTabPinningExcludedCommand(options.command) &&
 		supportsPinnedStdinCommand(options)
 	);
 }
@@ -464,7 +360,7 @@ export function buildPinnedBatchPlan(options: {
 	if (options.commandTokens.length === 0) {
 		return undefined;
 	}
-	const includeNavigationSummary = options.command !== undefined && NAVIGATION_SUMMARY_COMMANDS.has(options.command);
+	const includeNavigationSummary = isNavigationObservableCommandName(options.command);
 	const tabSelectionStep: BatchCommandStep = ["tab", options.selectedTab];
 	const commandStep = options.commandTokens as BatchCommandStep;
 	const navigationSummarySteps: BatchCommandStep[] = includeNavigationSummary ? [["eval", NAVIGATION_SUMMARY_EVAL]] : [];
@@ -480,7 +376,7 @@ export function shouldCorrectSessionTabAfterCommand(options: { command?: string;
 		options.pinningRequired === true &&
 		options.sessionName !== undefined &&
 		options.command !== undefined &&
-		!SESSION_TAB_POST_COMMAND_CORRECTION_EXCLUDED_COMMANDS.has(options.command)
+		!isSessionTabPostCommandCorrectionExcludedCommand(options.command)
 	);
 }
 
@@ -747,7 +643,7 @@ export function formatElectronSessionMismatchText(mismatch: ElectronSessionMisma
 }
 
 export function shouldInspectElectronPostCommandHealth(command: string | undefined): boolean {
-	return command !== undefined && ELECTRON_POST_COMMAND_HEALTH_COMMANDS.has(command);
+	return isElectronPostCommandHealthCommand(command);
 }
 
 export function buildElectronLifecycleNextActions(record: ElectronLaunchRecord): AgentBrowserNextAction[] {

package/extensions/agent-browser/lib/orchestration/browser-run/types.ts

@@ -20,7 +20,9 @@ import type { SessionArtifactManifest } from "../../results/contracts.js";
 import type { RichInputRecoveryDiagnostic, VisibleRefFallbackDiagnostic } from "../../results/selector-recovery.js";
 import type { SessionPageState, SessionRefSnapshot, SessionRefSnapshotInvalidation, SessionTabTarget } from "../../session-page-state.js";
 import type { buildExecutionPlan, CompatibilityWorkaround, OpenResultTabCorrection } from "../../runtime.js";
+import type { PromptPolicy } from "../../prompt-policy.js";
 import type { AgentBrowserExecuteParams, ResolvedAgentBrowserValidInput } from "../input-plan.js";
+import type { BatchCommandStep } from "../batch-stdin.js";
 
 export type AgentBrowserToolResult = AgentToolResult<unknown> & { isError?: boolean };
 export type AgentBrowserProcessResult = Awaited<ReturnType<typeof runAgentBrowserProcess>>;
@@ -30,7 +32,7 @@ export type AgentBrowserResultCategoryDetails = ReturnType<typeof buildAgentBrow
 
 export type TraceOwner = "profiler" | "trace";
 export type PinnedBatchUnwrapMode = "single-command" | "user-batch";
-export type BatchCommandStep = [string, ...string[]];
+export type { BatchCommandStep } from "../batch-stdin.js";
 
 export interface BrowserRunContext {
 	cwd: string;
@@ -61,6 +63,7 @@ export interface BrowserRunInputFields {
 
 export interface BrowserRunState {
 	artifactManifest?: SessionArtifactManifest;
+	closedManagedSessionNames: Set<string>;
 	electronChildProcesses: Map<string, ChildProcess>;
 	electronLaunchRecords: Map<string, ElectronLaunchRecord>;
 	ephemeralSessionSeed: string;
@@ -91,6 +94,7 @@ export interface BrowserRunOptions {
 	input: ResolvedAgentBrowserValidInput;
 	onUpdate?: (result: AgentToolResult<unknown>) => void;
 	params: AgentBrowserExecuteParams;
+	promptPolicy: PromptPolicy;
 	sessionPageStateUpdate: ReturnType<SessionPageState["beginUpdate"]>;
 	signal?: AbortSignal;
 	state: BrowserRunState;
@@ -120,6 +124,24 @@ export interface OverlayBlockerDiagnostic {
 	summary: string;
 }
 
+export interface ClickDispatchProbeTarget {
+	kind: "selector" | "xpath";
+	selector: string;
+}
+
+export interface ClickDispatchProbe {
+	marker: string;
+	target: ClickDispatchProbeTarget;
+}
+
+export interface ClickDispatchDiagnostic {
+	nativeEventCount: number;
+	reason: "native-click-produced-no-target-dom-event";
+	status: "no-native-event-observed";
+	summary: string;
+	target: ClickDispatchProbeTarget;
+}
+
 export interface SelectorTextVisibilityDiagnostic {
 	firstMatchVisible?: boolean;
 	firstVisibleTextPreview?: string;
@@ -140,11 +162,17 @@ export interface ElectronBroadGetTextScopeDiagnostic {
 }
 
 export interface QaAttachedTarget {
+	error?: string;
 	sessionName: string;
 	title?: string;
 	url?: string;
 }
 
+export interface QaAttachedPreconditionFailure {
+	error: string;
+	nextActions: AgentBrowserNextAction[];
+}
+
 export interface TimeoutArtifactEvidence {
 	absolutePath: string;
 	exists: boolean;
@@ -347,6 +375,7 @@ export interface PreparedBrowserRun {
 	exactSensitiveValues: string[];
 	executionPlan: AgentBrowserExecutionPlan;
 	includePinnedNavigationSummary: boolean;
+	clickDispatchProbe?: ClickDispatchProbe;
 	pinnedBatchUnwrapMode?: PinnedBatchUnwrapMode;
 	preparedArgs: PreparedAgentBrowserArgs;
 	priorRefSnapshotState?: SessionRefSnapshot;
@@ -411,6 +440,8 @@ export interface FinalResultInput {
 	aboutBlankSessionMismatch?: AboutBlankSessionMismatch;
 	artifactCleanup?: ArtifactCleanupGuidance;
 	categoryDetails: AgentBrowserResultCategoryDetails;
+	clickDispatchDiagnostic?: ClickDispatchDiagnostic;
+	commandTokens: string[];
 	comboboxFocusDiagnostic?: ComboboxFocusDiagnostic;
 	compiledNetworkSourceLookup?: CompiledAgentBrowserNetworkSourceLookup;
 	compiledSemanticAction?: CompiledAgentBrowserSemanticAction;