pi-agent-browser-native 0.2.34 → 0.2.36

package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts

@@ -4,6 +4,7 @@ import type { ElectronLaunchStatus } from "../../electron/cleanup.js";
 import type { ElectronCdpTarget, ElectronLaunchRecord } from "../../electron/launch.js";
 import { runAgentBrowserProcess } from "../../process.js";
 import { buildAgentBrowserNextActions, getAgentBrowserErrorText, parseAgentBrowserEnvelope, type AgentBrowserBatchResult, type AgentBrowserEnvelope, type AgentBrowserNextAction } from "../../results.js";
+import { buildNextToolAction, withOptionalSessionArgs } from "../../results/next-actions.js";
 import {
 	extractRefSnapshotFromData,
 	isAboutBlankUrl,
@@ -14,8 +15,18 @@ import {
 	type SessionRefSnapshotInvalidation,
 	type SessionTabTarget,
 } from "../../session-page-state.js";
+import {
+	isCloseCommand,
+	isElectronPostCommandHealthCommand,
+	isNavigationObservableCommandName,
+	isRefGuardedCommand,
+	isRefInvalidatingBatchCommand,
+	isSessionTabPinningExcludedCommand,
+	isSessionTabPostCommandCorrectionExcludedCommand,
+} from "../../command-taxonomy.js";
 import { chooseOpenResultTabCorrection, redactInvocationArgs, type OpenResultTabCorrection } from "../../runtime.js";
 import { isRecord } from "../../parsing.js";
+import { parseUserBatchStdin } from "../batch-stdin.js";
 import type {
 	AboutBlankSessionMismatch,
 	BatchCommandStep,
@@ -35,65 +46,8 @@ import type {
 	TraceOwner,
 } from "./types.js";
 
-export const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
 export const NAVIGATION_SUMMARY_EVAL = `({ title: document.title, url: location.href })`;
 
-const SESSION_TAB_PINNING_EXCLUDED_COMMANDS = new Set(["close", "goto", "navigate", "open", "session", "tab"]);
-const SESSION_TAB_POST_COMMAND_CORRECTION_EXCLUDED_COMMANDS = new Set(["batch", "close", "session", "tab"]);
-const REF_INVALIDATING_BATCH_COMMANDS = new Set([
-	"back",
-	"check",
-	"click",
-	"dblclick",
-	"drag",
-	"forward",
-	"goto",
-	"keyboard",
-	"mouse",
-	"navigate",
-	"open",
-	"press",
-	"reload",
-	"select",
-	"type",
-	"uncheck",
-	"upload",
-]);
-const REF_GUARDED_COMMANDS = new Set([
-	"check",
-	"click",
-	"dblclick",
-	"download",
-	"drag",
-	"fill",
-	"focus",
-	"hover",
-	"keyboard",
-	"mouse",
-	"press",
-	"scrollintoview",
-	"select",
-	"type",
-	"uncheck",
-	"upload",
-]);
-const ELECTRON_POST_COMMAND_HEALTH_COMMANDS = new Set([
-	"back",
-	"check",
-	"click",
-	"dblclick",
-	"fill",
-	"find",
-	"forward",
-	"keyboard",
-	"mouse",
-	"press",
-	"reload",
-	"select",
-	"type",
-	"uncheck",
-]);
-
 export function applyBrowserRunStatePatch(state: BrowserRunState, patch: BrowserRunStatePatch | undefined): void {
 	if (!patch) return;
 	if ("artifactManifest" in patch) state.artifactManifest = patch.artifactManifest;
@@ -103,10 +57,6 @@ export function applyBrowserRunStatePatch(state: BrowserRunState, patch: Browser
 	if (patch.managedSessionName !== undefined) state.managedSessionName = patch.managedSessionName;
 }
 
-export function mergeBrowserRunStatePatch(left: BrowserRunStatePatch | undefined, right: BrowserRunStatePatch | undefined): BrowserRunStatePatch {
-	return { ...(left ?? {}), ...(right ?? {}) };
-}
-
 export function buildSessionDetailFields(sessionName: string | undefined, usedImplicitSession: boolean): Record<string, unknown> {
 	return sessionName ? { sessionName, usedImplicitSession } : {};
 }
@@ -126,7 +76,7 @@ export function buildManagedSessionOutcome(options: {
 	if (!attemptedSessionName) return undefined;
 	let status: ManagedSessionOutcome["status"];
 	let summary: string;
-	if (command === "close") {
+	if (isCloseCommand(command)) {
 		status = succeeded ? "closed" : activeBefore ? "preserved" : "abandoned";
 		summary = succeeded
 			? `Managed session ${attemptedSessionName} was closed.`
@@ -169,8 +119,85 @@ export function buildManagedSessionOutcome(options: {
 	};
 }
 
+function isFreshPostLaunchFailure(outcome: ManagedSessionOutcome): boolean {
+	return !outcome.succeeded && outcome.sessionMode === "fresh" && outcome.activeAfter && !!outcome.currentSessionName && (outcome.status === "created" || outcome.status === "replaced" || outcome.status === "unchanged");
+}
+
+function formatManagedSessionOutcomeHeadline(outcome: ManagedSessionOutcome): string {
+	if (outcome.status === "preserved") {
+		return "Managed session outcome: Fresh launch failed; your previous browser session is still active.";
+	}
+	if (outcome.status === "abandoned") {
+		return "Managed session outcome: Fresh launch failed; no managed browser session is current.";
+	}
+	if (isFreshPostLaunchFailure(outcome)) {
+		return "Managed session outcome: Fresh launch became current, but this tool call failed after launch.";
+	}
+	return `Managed session outcome: ${outcome.summary}`;
+}
+
+function formatManagedSessionOutcomeRecoveryGuidance(outcome: ManagedSessionOutcome): string {
+	const lines = ["Recovery:"];
+	if (outcome.status === "preserved") {
+		lines.push('- Continue with sessionMode "auto" on the current session, or retry the intended launch with sessionMode "fresh".');
+		lines.push("- Run doctor to verify agent-browser install and environment when failures persist.");
+	} else if (outcome.status === "abandoned") {
+		lines.push('- Retry with sessionMode "fresh" (for example args: ["open", "<url>"]) after verifying agent-browser is on PATH.');
+		lines.push("- Run doctor when install or environment issues are suspected.");
+	} else if (isFreshPostLaunchFailure(outcome)) {
+		lines.push('- Continue with sessionMode "auto" on the current session, or inspect failureCategory / qaPreset to fix the post-launch failure.');
+		lines.push("- Run doctor only if later browser commands also fail.");
+	} else {
+		lines.push('- Retry with sessionMode "fresh" when launch-scoped flags must apply, or run doctor to verify the environment.');
+	}
+	lines.push("- Full session names and transition details remain in details.managedSessionOutcome.");
+	return lines.join("\n");
+}
+
 export function formatManagedSessionOutcomeText(outcome: ManagedSessionOutcome | undefined): string | undefined {
-	return outcome && !outcome.succeeded && outcome.sessionMode === "fresh" ? `Managed session outcome: ${outcome.summary}` : undefined;
+	if (!outcome || outcome.succeeded || outcome.sessionMode !== "fresh") return undefined;
+	return [formatManagedSessionOutcomeHeadline(outcome), formatManagedSessionOutcomeRecoveryGuidance(outcome)].join("\n");
+}
+
+export function buildManagedSessionFreshFailureNextActions(outcome: ManagedSessionOutcome | undefined): AgentBrowserNextAction[] {
+	if (!outcome || outcome.succeeded || outcome.sessionMode !== "fresh") return [];
+	const actions: AgentBrowserNextAction[] = [];
+	if (!isFreshPostLaunchFailure(outcome)) {
+		actions.push(buildNextToolAction({
+			args: ["doctor"],
+			id: "run-agent-browser-doctor",
+			reason: "Verify agent-browser install, PATH, and environment after a failed fresh launch.",
+			safety: "Read-only local diagnostics; does not mutate browser state.",
+		}));
+	}
+	if ((outcome.status === "preserved" || isFreshPostLaunchFailure(outcome)) && outcome.activeAfter && outcome.currentSessionName) {
+		const sessionLabel = isFreshPostLaunchFailure(outcome) ? "current managed session" : "preserved managed session";
+		actions.push(
+			buildNextToolAction({
+				args: withOptionalSessionArgs(outcome.currentSessionName, ["get", "url"]),
+				id: "verify-current-managed-session",
+				reason: `Confirm the ${sessionLabel} before continuing with sessionMode auto.`,
+				safety: `Read-only URL check on the ${sessionLabel}.`,
+			}),
+			buildNextToolAction({
+				args: withOptionalSessionArgs(outcome.currentSessionName, ["snapshot", "-i"]),
+				id: "snapshot-current-managed-session",
+				reason: `Refresh interactive refs on the ${sessionLabel} before retrying the workflow.`,
+				safety: "Read-only snapshot; no navigation.",
+			}),
+		);
+	} else {
+		actions.push(
+			buildNextToolAction({
+				args: ["open", "about:blank"],
+				id: "retry-fresh-managed-session",
+				reason: "Start a new managed browser session after the failed fresh launch.",
+				safety: "Replace about:blank with the intended URL from your workflow.",
+				sessionMode: "fresh",
+			}),
+		);
+	}
+	return actions;
 }
 
 function getTraceOwner(command: string | undefined): TraceOwner | undefined {
@@ -238,8 +265,7 @@ export function extractNavigationSummaryFromData(data: unknown): NavigationSumma
 
 export function shouldCaptureNavigationSummary(command: string | undefined, data: unknown): boolean {
 	return (
-		command !== undefined &&
-		NAVIGATION_SUMMARY_COMMANDS.has(command) &&
+		isNavigationObservableCommandName(command) &&
 		(!isRecord(data) || (typeof data.title !== "string" && typeof data.url !== "string"))
 	);
 }
@@ -263,58 +289,6 @@ function extractBatchResultCommand(item: Record<string, unknown>): string[] {
 	return Array.isArray(item.command) ? item.command.filter((token): token is string => typeof token === "string") : [];
 }
 
-function validateUserBatchStep(
-	step: unknown,
-	index: number,
-):
-	| { ok: true; step: BatchCommandStep }
-	| { ok: false; error: string } {
-	if (!Array.isArray(step)) {
-		return {
-			ok: false,
-			error: `agent_browser batch stdin step ${index} must be a non-empty array of string command tokens.`,
-		};
-	}
-	if (step.length === 0) {
-		return {
-			ok: false,
-			error: `agent_browser batch stdin step ${index} must not be empty.`,
-		};
-	}
-	const invalidTokenIndex = step.findIndex((token) => typeof token !== "string");
-	if (invalidTokenIndex !== -1) {
-		return {
-			ok: false,
-			error: `agent_browser batch stdin step ${index} token ${invalidTokenIndex} must be a string.`,
-		};
-	}
-	return { ok: true, step: step as BatchCommandStep };
-}
-
-function parseUserBatchStdin(stdin: string | undefined): { error?: string; steps?: BatchCommandStep[] } {
-	if (stdin === undefined) {
-		return { steps: [] };
-	}
-	try {
-		const parsed = JSON.parse(stdin) as unknown;
-		if (!Array.isArray(parsed)) {
-			return { error: "agent_browser batch stdin must be a JSON array of command steps." };
-		}
-		const steps: BatchCommandStep[] = [];
-		for (const [index, rawStep] of parsed.entries()) {
-			const validated = validateUserBatchStep(rawStep, index);
-			if (!validated.ok) {
-				return { error: validated.error };
-			}
-			steps.push(validated.step);
-		}
-		return { steps };
-	} catch (error) {
-		const message = error instanceof Error ? error.message : String(error);
-		return { error: `agent_browser batch stdin could not be parsed as JSON: ${message}` };
-	}
-}
-
 export function getStaleRefArgs(commandTokens: string[], stdin?: string): string[] {
 	if (commandTokens[0] !== "batch" || stdin === undefined) {
 		return commandTokens;
@@ -331,7 +305,7 @@ function collectRefsFromTokens(tokens: string[]): string[] {
 }
 
 export function getGuardedRefUsage(commandTokens: string[], stdin?: string, options: { includeRefsAfterBatchSnapshot?: boolean } = {}): string[] {
-	const collectFromStep = (step: string[]) => REF_GUARDED_COMMANDS.has(step[0] ?? "") ? collectRefsFromTokens(step) : [];
+	const collectFromStep = (step: string[]) => isRefGuardedCommand(step[0]) ? collectRefsFromTokens(step) : [];
 	if (commandTokens[0] !== "batch" || stdin === undefined) {
 		return collectFromStep(commandTokens);
 	}
@@ -357,10 +331,10 @@ function getBatchRefInvalidationMessage(commandTokens: string[], stdin?: string)
 			priorStepInvalidatesRefs = false;
 		}
 		const refIds = collectRefsFromTokens(step);
-		if (refIds.length > 0 && REF_GUARDED_COMMANDS.has(step[0] ?? "") && priorStepInvalidatesRefs) {
+		if (refIds.length > 0 && isRefGuardedCommand(step[0]) && priorStepInvalidatesRefs) {
 			return `Batch step ${step[0]} uses page-scoped ref ${refIds.map((refId) => `@${refId}`).join(", ")} after an earlier batch step can navigate or mutate the page. Split the batch, run snapshot -i after the page-changing step, then retry with current refs.`;
 		}
-		if (REF_INVALIDATING_BATCH_COMMANDS.has(step[0] ?? "")) {
+		if (isRefInvalidatingBatchCommand(step[0])) {
 			priorStepInvalidatesRefs = true;
 		}
 	}
@@ -438,7 +412,7 @@ export function shouldPinSessionTabForCommand(options: {
 		options.pinningRequired === true &&
 		options.sessionName !== undefined &&
 		options.command !== undefined &&
-		!SESSION_TAB_PINNING_EXCLUDED_COMMANDS.has(options.command) &&
+		!isSessionTabPinningExcludedCommand(options.command) &&
 		supportsPinnedStdinCommand(options)
 	);
 }
@@ -464,7 +438,7 @@ export function buildPinnedBatchPlan(options: {
 	if (options.commandTokens.length === 0) {
 		return undefined;
 	}
-	const includeNavigationSummary = options.command !== undefined && NAVIGATION_SUMMARY_COMMANDS.has(options.command);
+	const includeNavigationSummary = isNavigationObservableCommandName(options.command);
 	const tabSelectionStep: BatchCommandStep = ["tab", options.selectedTab];
 	const commandStep = options.commandTokens as BatchCommandStep;
 	const navigationSummarySteps: BatchCommandStep[] = includeNavigationSummary ? [["eval", NAVIGATION_SUMMARY_EVAL]] : [];
@@ -480,7 +454,7 @@ export function shouldCorrectSessionTabAfterCommand(options: { command?: string;
 		options.pinningRequired === true &&
 		options.sessionName !== undefined &&
 		options.command !== undefined &&
-		!SESSION_TAB_POST_COMMAND_CORRECTION_EXCLUDED_COMMANDS.has(options.command)
+		!isSessionTabPostCommandCorrectionExcludedCommand(options.command)
 	);
 }
 
@@ -747,7 +721,7 @@ export function formatElectronSessionMismatchText(mismatch: ElectronSessionMisma
 }
 
 export function shouldInspectElectronPostCommandHealth(command: string | undefined): boolean {
-	return command !== undefined && ELECTRON_POST_COMMAND_HEALTH_COMMANDS.has(command);
+	return isElectronPostCommandHealthCommand(command);
 }
 
 export function buildElectronLifecycleNextActions(record: ElectronLaunchRecord): AgentBrowserNextAction[] {

package/extensions/agent-browser/lib/orchestration/browser-run/types.ts

@@ -20,7 +20,9 @@ import type { SessionArtifactManifest } from "../../results/contracts.js";
 import type { RichInputRecoveryDiagnostic, VisibleRefFallbackDiagnostic } from "../../results/selector-recovery.js";
 import type { SessionPageState, SessionRefSnapshot, SessionRefSnapshotInvalidation, SessionTabTarget } from "../../session-page-state.js";
 import type { buildExecutionPlan, CompatibilityWorkaround, OpenResultTabCorrection } from "../../runtime.js";
+import type { PromptPolicy } from "../../prompt-policy.js";
 import type { AgentBrowserExecuteParams, ResolvedAgentBrowserValidInput } from "../input-plan.js";
+import type { BatchCommandStep } from "../batch-stdin.js";
 
 export type AgentBrowserToolResult = AgentToolResult<unknown> & { isError?: boolean };
 export type AgentBrowserProcessResult = Awaited<ReturnType<typeof runAgentBrowserProcess>>;
@@ -30,7 +32,7 @@ export type AgentBrowserResultCategoryDetails = ReturnType<typeof buildAgentBrow
 
 export type TraceOwner = "profiler" | "trace";
 export type PinnedBatchUnwrapMode = "single-command" | "user-batch";
-export type BatchCommandStep = [string, ...string[]];
+export type { BatchCommandStep } from "../batch-stdin.js";
 
 export interface BrowserRunContext {
 	cwd: string;
@@ -61,6 +63,7 @@ export interface BrowserRunInputFields {
 
 export interface BrowserRunState {
 	artifactManifest?: SessionArtifactManifest;
+	closedManagedSessionNames: Set<string>;
 	electronChildProcesses: Map<string, ChildProcess>;
 	electronLaunchRecords: Map<string, ElectronLaunchRecord>;
 	ephemeralSessionSeed: string;
@@ -91,6 +94,7 @@ export interface BrowserRunOptions {
 	input: ResolvedAgentBrowserValidInput;
 	onUpdate?: (result: AgentToolResult<unknown>) => void;
 	params: AgentBrowserExecuteParams;
+	promptPolicy: PromptPolicy;
 	sessionPageStateUpdate: ReturnType<SessionPageState["beginUpdate"]>;
 	signal?: AbortSignal;
 	state: BrowserRunState;
@@ -120,12 +124,38 @@ export interface OverlayBlockerDiagnostic {
 	summary: string;
 }
 
+export interface ClickDispatchProbeTarget {
+	kind: "selector" | "xpath";
+	selector: string;
+}
+
+export interface ClickDispatchProbe {
+	marker: string;
+	target: ClickDispatchProbeTarget;
+}
+
+export interface ClickDispatchDiagnostic {
+	nativeEventCount: number;
+	reason: "native-click-produced-no-target-dom-event";
+	status: "no-native-event-observed";
+	summary: string;
+	target: ClickDispatchProbeTarget;
+}
+
+export interface SelectorTextVisibilityCandidate {
+	index: number;
+	role?: string;
+	tagName: string;
+	textPreview?: string;
+}
+
 export interface SelectorTextVisibilityDiagnostic {
 	firstMatchVisible?: boolean;
 	firstVisibleTextPreview?: string;
 	matchCount: number;
 	selector: string;
 	summary: string;
+	visibleCandidates?: SelectorTextVisibilityCandidate[];
 	visibleCount: number;
 }
 
@@ -174,6 +204,11 @@ export interface EvalStdinHint {
 	suggestion: string;
 }
 
+export interface EvalResultWarning {
+	reason: string;
+	suggestion: string;
+}
+
 export interface ArtifactCleanupGuidance {
 	explicitArtifactPaths: string[];
 	note: string;
@@ -353,6 +388,7 @@ export interface PreparedBrowserRun {
 	exactSensitiveValues: string[];
 	executionPlan: AgentBrowserExecutionPlan;
 	includePinnedNavigationSummary: boolean;
+	clickDispatchProbe?: ClickDispatchProbe;
 	pinnedBatchUnwrapMode?: PinnedBatchUnwrapMode;
 	preparedArgs: PreparedAgentBrowserArgs;
 	priorRefSnapshotState?: SessionRefSnapshot;
@@ -417,6 +453,8 @@ export interface FinalResultInput {
 	aboutBlankSessionMismatch?: AboutBlankSessionMismatch;
 	artifactCleanup?: ArtifactCleanupGuidance;
 	categoryDetails: AgentBrowserResultCategoryDetails;
+	clickDispatchDiagnostic?: ClickDispatchDiagnostic;
+	commandTokens: string[];
 	comboboxFocusDiagnostic?: ComboboxFocusDiagnostic;
 	compiledNetworkSourceLookup?: CompiledAgentBrowserNetworkSourceLookup;
 	compiledSemanticAction?: CompiledAgentBrowserSemanticAction;
@@ -436,6 +474,7 @@ export interface FinalResultInput {
 	electronSessionMismatch?: ElectronSessionMismatch;
 	errorText?: string;
 	evalStdinHint?: EvalStdinHint;
+	evalResultWarning?: EvalResultWarning;
 	exactSensitiveValues: string[];
 	executionPlan: AgentBrowserExecutionPlan;
 	fillVerificationDiagnostic?: FillVerificationDiagnostic;