pi-agent-browser-native 0.2.44 → 0.2.46

package/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.ts

@@ -71,7 +71,48 @@ function buildClickDispatchProbeInstallScript(probe: ClickDispatchProbe): string
 const marker = ${JSON.stringify(probe.marker)};
 const element = ${resolveTarget};
 if (!element) return { status: "target-not-found", marker };
-const state = { events: [], target: { tagName: element.tagName.toLowerCase() } };
+const cssEscape = (value) => {
+  if (window.CSS && typeof window.CSS.escape === "function") return window.CSS.escape(value);
+  return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");
+};
+const getSelector = (node) => {
+  if (!(node instanceof Element)) return undefined;
+  if (node.id) return "#" + cssEscape(node.id);
+  const testId = node.getAttribute("data-testid") || node.getAttribute("data-test-id");
+  if (testId) return '[data-testid="' + cssEscape(testId) + '"]';
+  const parts = [];
+  let current = node;
+  while (current && current !== document.body && parts.length < 4) {
+    const tag = current.tagName.toLowerCase();
+    const parent = current.parentElement;
+    if (!parent) break;
+    const siblings = Array.from(parent.children).filter((child) => child.tagName === current.tagName);
+    const index = siblings.indexOf(current) + 1;
+    parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag);
+    current = parent;
+  }
+  return parts.length > 0 ? parts.join(" > ") : undefined;
+};
+const rectInfo = (rect) => ({ bottom: rect.bottom, left: rect.left, right: rect.right, top: rect.top });
+const targetRect = element.getBoundingClientRect();
+const targetOutsideViewport = targetRect.bottom < 0 || targetRect.right < 0 || targetRect.top > window.innerHeight || targetRect.left > window.innerWidth;
+let nearestScrollContainer;
+for (let current = element.parentElement; current && current !== document.body; current = current.parentElement) {
+  if (current.scrollHeight > current.clientHeight + 1 || current.scrollWidth > current.clientWidth + 1) {
+    const containerRect = current.getBoundingClientRect();
+    nearestScrollContainer = {
+      selector: getSelector(current),
+      tagName: current.tagName.toLowerCase(),
+      targetOutsideContainer: targetRect.bottom < containerRect.top || targetRect.top > containerRect.bottom || targetRect.right < containerRect.left || targetRect.left > containerRect.right,
+      targetOutsideViewport,
+      rect: rectInfo(containerRect),
+      scrollLeft: current.scrollLeft,
+      scrollTop: current.scrollTop,
+    };
+    break;
+  }
+}
+const state = { events: [], target: { tagName: element.tagName.toLowerCase(), nearestScrollContainer, rect: rectInfo(targetRect), targetOutsideViewport } };
 const eventTypes = ["pointerdown", "mousedown", "pointerup", "mouseup", "click"];
 const listeners = eventTypes.map((type) => {
   const listener = (event) => {
@@ -126,9 +167,9 @@ export function formatClickDispatchDiagnosticText(diagnostic: ClickDispatchDiagn
 	return `Click dispatch diagnostic: ${diagnostic.summary}`;
 }
 
-export function buildClickDispatchNextActions(options: { commandTokens: string[]; sessionName?: string }): AgentBrowserNextAction[] {
+export function buildClickDispatchNextActions(options: { commandTokens: string[]; diagnostic?: ClickDispatchDiagnostic; sessionName?: string }): AgentBrowserNextAction[] {
 	const retryArgs = options.commandTokens[0] === "click" ? options.commandTokens : ["click", ...options.commandTokens];
-	return [
+	const actions: AgentBrowserNextAction[] = [
 		{
 			id: "inspect-click-dispatch-miss",
 			params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) },
@@ -136,14 +177,26 @@ export function buildClickDispatchNextActions(options: { commandTokens: string[]
 			safety: "Read-only snapshot; the wrapper does not replay clicks in-page when upstream reports success without DOM events.",
 			tool: "agent_browser",
 		},
-		{
-			id: "retry-click-after-dispatch-miss",
-			params: { args: withOptionalSessionArgs(options.sessionName, retryArgs) },
-			reason: "Retry the same upstream click after confirming the target is visible; do not assume the prior success mutated the page.",
-			safety: "Only retry when the target is still intended; use page-change evidence or a fresh snapshot before continuing the workflow.",
-			tool: "agent_browser",
-		},
 	];
+	if (options.diagnostic?.scrollContainer) {
+		actions.push({
+			id: "scroll-target-into-view-after-dispatch-miss",
+			params: { args: withOptionalSessionArgs(options.sessionName, ["scrollintoview", retryArgs[1]].filter((item): item is string => typeof item === "string")) },
+			reason: options.diagnostic.scrollContainer.selector
+				? `The target may be outside nested scroll container ${options.diagnostic.scrollContainer.selector}; scroll the target into view before retrying the click.`
+				: "The target may be inside an offscreen nested scroll container; scroll the target into view before retrying the click.",
+			safety: "Use only for the same current page and target; run snapshot -i again if the page rerendered.",
+			tool: "agent_browser",
+		});
+	}
+	actions.push({
+		id: "retry-click-after-dispatch-miss",
+		params: { args: withOptionalSessionArgs(options.sessionName, retryArgs) },
+		reason: "Retry the same upstream click after confirming the target is visible; do not assume the prior success mutated the page.",
+		safety: "Only retry when the target is still intended; use page-change evidence or a fresh snapshot before continuing the workflow.",
+		tool: "agent_browser",
+	});
+	return actions;
 }
 
 export async function prepareClickDispatchProbe(options: { commandTokens: string[]; cwd: string; refSnapshot?: SessionRefSnapshot; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchProbe | undefined> {
@@ -156,6 +209,20 @@ export async function prepareClickDispatchProbe(options: { commandTokens: string
 	return installResult?.status === "installed" ? probe : undefined;
 }
 
+function getClickDispatchScrollContainerDiagnostic(result: Record<string, unknown>): ClickDispatchDiagnostic["scrollContainer"] {
+	const target = isRecord(result.target) ? result.target : undefined;
+	const scrollContainer = isRecord(target?.nearestScrollContainer) ? target.nearestScrollContainer : undefined;
+	const targetOutsideViewport = typeof target?.targetOutsideViewport === "boolean" ? target.targetOutsideViewport : undefined;
+	const targetOutsideContainer = typeof scrollContainer?.targetOutsideContainer === "boolean" ? scrollContainer.targetOutsideContainer : undefined;
+	if (!scrollContainer && !targetOutsideViewport) return undefined;
+	if (targetOutsideContainer !== true && targetOutsideViewport !== true) return undefined;
+	const selector = typeof scrollContainer?.selector === "string" ? redactSensitiveText(scrollContainer.selector) : undefined;
+	const summary = selector
+		? `Target appears outside nested scroll container ${selector}; use scrollintoview on the target or scroll that container before retrying.`
+		: "Target appears outside the viewport or a nested scroll container; use scrollintoview on the target before retrying.";
+	return { selector, summary, targetOutsideContainer, targetOutsideViewport };
+}
+
 export async function collectClickDispatchDiagnostic(options: { cwd: string; probe?: ClickDispatchProbe; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchDiagnostic | undefined> {
 	if (!options.probe || !options.sessionName) return undefined;
 	const data = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeCheckScript(options.probe) });
@@ -164,10 +231,14 @@ export async function collectClickDispatchDiagnostic(options: { cwd: string; pro
 	const status = typeof result.status === "string" ? result.status : undefined;
 	if (status !== "no-native-event-observed") return undefined;
 	const nativeEventCount = typeof result.nativeEventCount === "number" ? result.nativeEventCount : 0;
-	const summary = "Upstream click reported success but no trusted DOM event reached the selected element. Gather evidence with snapshot or page-change checks, then retry upstream click or report the workflow issue; the wrapper does not replay clicks in-page.";
+	const scrollContainer = getClickDispatchScrollContainerDiagnostic(result);
+	const summary = scrollContainer
+		? `Upstream click reported success but no trusted DOM event reached the selected element. ${scrollContainer.summary}`
+		: "Upstream click reported success but no trusted DOM event reached the selected element. Gather evidence with snapshot or page-change checks, then retry upstream click or report the workflow issue; the wrapper does not replay clicks in-page.";
 	return {
 		nativeEventCount,
 		reason: "native-click-produced-no-target-dom-event",
+		...(scrollContainer ? { scrollContainer } : {}),
 		status,
 		summary,
 		target: redactClickDispatchTarget(options.probe.target),

package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts

@@ -3,6 +3,7 @@ import { isAbsolute, resolve } from "node:path";
 
 import { isCloseCommand, isOpenNavigationCommand } from "../../command-taxonomy.js";
 import type { ElectronLaunchRecord } from "../../electron/launch.js";
+import { boundElectronProbeString } from "../../electron/text.js";
 import { executableExistsOnPath } from "../../executable-path.js";
 import type { AgentBrowserSourceLookupAnalysis, CompiledAgentBrowserJob, CompiledAgentBrowserSemanticAction } from "../../input-modes.js";
 import { isHttpOrHttpsUrl } from "../../input-modes/job.js";
@@ -10,7 +11,7 @@ import type { AgentBrowserNextAction } from "../../results.js";
 import { formatSessionArtifactRetentionSummary } from "../../results/artifact-manifest.js";
 import { buildNextToolAction, withOptionalSessionArgs } from "../../results/next-actions.js";
 import { buildVisibleRefFallbackDiagnosticFromSnapshot, getVisibleRefFallbackTarget, type VisibleRefFallbackDiagnostic } from "../../results/selector-recovery.js";
-import { extractRefSnapshotFromData, normalizeComparableUrl, type SessionTabTarget } from "../../session-page-state.js";
+import { extractRefSnapshotFromData, normalizeComparableUrl, type SessionRefSnapshot, type SessionTabTarget } from "../../session-page-state.js";
 import { redactInvocationArgs, redactSensitiveText, type CommandInfo } from "../../runtime.js";
 import { isRecord } from "../../parsing.js";
 import {
@@ -41,6 +42,7 @@ import type {
 	SelectorTextVisibilityDiagnostic,
 	TimeoutArtifactEvidence,
 	TimeoutPartialProgress,
+	TimeoutProgressStep,
 } from "./types.js";
 import type { SessionArtifactManifest } from "../../results/contracts.js";
 
@@ -50,12 +52,6 @@ export function sleepMs(ms: number): Promise<void> {
 	return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-function boundElectronProbeString(value: string | undefined, maxLength = 240): string | undefined {
-	const trimmed = value?.trim();
-	if (!trimmed) return undefined;
-	return trimmed.length > maxLength ? `${trimmed.slice(0, Math.max(0, maxLength - 3))}...` : trimmed;
-}
-
 export async function collectNavigationSummary(options: {
 	cwd: string;
 	sessionName?: string;
@@ -288,16 +284,22 @@ export function buildOverlayBlockerNextActions(options: { diagnostic: OverlayBlo
 	return [{ id: "inspect-overlay-state", params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) }, reason: "Refresh interactive refs and inspect whether an overlay, banner, modal, or dialog is blocking the intended click.", safety: "Read-only inspection; use current refs from this snapshot before interacting.", tool: "agent_browser" }, ...options.diagnostic.candidates.map((candidate, index) => ({ id: `try-overlay-blocker-candidate-${index + 1}`, params: { args: withOptionalSessionArgs(options.sessionName, candidate.args) }, reason: candidate.reason, safety: "Only click this if the candidate is clearly a close/dismiss control for an overlay that blocks the intended workflow.", tool: "agent_browser" as const }))];
 }
 
+export function collectSnapshotOverlayBlockerDiagnostic(data: unknown): OverlayBlockerDiagnostic | undefined {
+	const candidates = getOverlayBlockerCandidates(data);
+	const snapshot = extractRefSnapshotFromData(data);
+	if (candidates.length === 0 || !snapshot) return undefined;
+	return { candidates, snapshot, summary: "Snapshot contains dialog/modal context plus likely close or dismiss controls; treat covered controls as potentially obstructed until the overlay state is resolved." };
+}
+
 export async function collectOverlayBlockerDiagnostic(options: { command?: string; cwd: string; data: unknown; navigationSummary?: NavigationSummary; priorTarget?: SessionTabTarget; sessionName?: string; signal?: AbortSignal }): Promise<OverlayBlockerDiagnostic | undefined> {
 	if (options.command !== "click" || !isRecord(options.data) || typeof options.data.clicked !== "string") return undefined;
 	const priorUrl = normalizeComparableUrl(options.priorTarget?.url);
 	const currentUrl = normalizeComparableUrl(options.navigationSummary?.url);
 	if (!priorUrl || !currentUrl || priorUrl !== currentUrl) return undefined;
 	const snapshotData = await runSessionCommandData({ args: ["snapshot", "-i"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
-	const candidates = getOverlayBlockerCandidates(snapshotData);
-	const snapshot = extractRefSnapshotFromData(snapshotData);
-	if (candidates.length === 0 || !snapshot) return undefined;
-	return { candidates, snapshot, summary: `Click completed but the page stayed on ${currentUrl}; a fresh snapshot contains likely overlay close/dismiss controls.` };
+	const diagnostic = collectSnapshotOverlayBlockerDiagnostic(snapshotData);
+	if (!diagnostic) return undefined;
+	return { ...diagnostic, summary: `Click completed but the page stayed on ${currentUrl}; a fresh snapshot contains likely overlay close/dismiss controls.` };
 }
 
 const SELECTOR_TEXT_VISIBILITY_CANDIDATE_LIMIT = 8;
@@ -605,17 +607,25 @@ export async function validateQaAttachedPrecondition(options: {
 	return undefined;
 }
 
-function getTopLevelFillInvocation(commandTokens: string[]): { expected: string; selector: string } | undefined {
+function getTopLevelFillInvocation(commandTokens: string[]): { expected: string; refId?: string; selector: string } | undefined {
 	if (commandTokens[0] !== "fill" || commandTokens.length < 3) return undefined;
 	const selector = commandTokens[1];
 	const expected = commandTokens.slice(2).join(" ");
-	return selector && expected.length > 0 ? { expected, selector } : undefined;
+	const refId = selector?.match(/^@?(e\d+)$/)?.[1];
+	return selector && expected.length > 0 ? { expected, ...(refId ? { refId } : {}), selector } : undefined;
+}
+
+function shouldVerifyContenteditableFill(fill: { refId?: string } | undefined, refSnapshot?: SessionRefSnapshot): boolean {
+	if (!fill?.refId) return false;
+	const ref = refSnapshot?.refs?.[fill.refId];
+	if (!ref) return false;
+	return ref.isContentEditable === true && (ref.role === "generic" || ref.role === "unknown" || ref.role === "textbox");
 }
 
 export function buildFillVerificationNextActions(diagnostic: FillVerificationDiagnostic, sessionName: string | undefined): AgentBrowserNextAction[] {
 	return [
-		{ id: "inspect-after-fill-verification", params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) }, reason: "Refresh the UI after a fill that reported success but did not appear to update the input value.", safety: "Read-only snapshot; use current refs before retrying.", tool: "agent_browser" },
-		{ id: "verify-filled-value", params: { args: withOptionalSessionArgs(sessionName, ["get", "value", diagnostic.selector]) }, reason: "Check the target input value directly before submitting or creating files.", safety: "Read-only value check; selector may still be stale if the Electron UI rerendered.", tool: "agent_browser" },
+		{ id: "inspect-after-fill-verification", params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) }, reason: "Refresh the UI after a fill that reported success but did not appear to update the target.", safety: "Read-only snapshot; use current refs before retrying.", tool: "agent_browser" },
+		{ id: "verify-filled-value", params: { args: withOptionalSessionArgs(sessionName, ["get", diagnostic.method, diagnostic.selector]) }, reason: `Check the target ${diagnostic.method} directly before submitting or creating files.`, safety: "Read-only check; selector may still be stale if the UI rerendered.", tool: "agent_browser" },
 	];
 }
 
@@ -627,22 +637,30 @@ function extractFillVerificationValue(data: unknown): string | undefined {
 	return undefined;
 }
 
-export async function collectFillVerificationDiagnostic(options: { commandTokens: string[]; cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<FillVerificationDiagnostic | undefined> {
+export async function collectFillVerificationDiagnostic(options: { commandTokens: string[]; cwd: string; forceValueVerification?: boolean; refSnapshot?: SessionRefSnapshot; sessionName?: string; signal?: AbortSignal }): Promise<FillVerificationDiagnostic | undefined> {
 	const fill = getTopLevelFillInvocation(options.commandTokens);
 	if (!fill || !options.sessionName) return undefined;
+	const contenteditable = shouldVerifyContenteditableFill(fill, options.refSnapshot);
+	if (!contenteditable && !options.forceValueVerification) return undefined;
+	const method = contenteditable ? "text" : "value";
 	let valueData: unknown | undefined;
-	try { valueData = await runSessionCommandData({ args: ["get", "value", fill.selector], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, timeoutMs: ELECTRON_FILL_VERIFICATION_TIMEOUT_MS }); } catch { return undefined; }
+	try { valueData = await runSessionCommandData({ args: ["get", method, fill.selector], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, timeoutMs: ELECTRON_FILL_VERIFICATION_TIMEOUT_MS }); } catch { return undefined; }
 	const actual = extractFillVerificationValue(valueData);
 	if (actual === undefined || actual === fill.expected) return undefined;
-	const diagnostic: FillVerificationDiagnostic = { actual: actual.length > 0 ? boundElectronProbeString(actual, 160) : "", expected: boundElectronProbeString(fill.expected, 160) ?? fill.expected, nextActionIds: [], selector: fill.selector, status: "mismatch", summary: `Fill verification warning: fill ${fill.selector} reported success, but get value returned ${actual.length > 0 ? `"${boundElectronProbeString(actual, 80)}"` : "an empty value"}.` };
+	const reason = contenteditable ? "contenteditable-fill-mismatch" : "value-fill-mismatch";
+	const actualPreview = actual.length > 0 ? `"${boundElectronProbeString(actual, 80)}"` : `an empty ${method}`;
+	const diagnostic: FillVerificationDiagnostic = { actual: actual.length > 0 ? boundElectronProbeString(actual, 160) : "", expected: boundElectronProbeString(fill.expected, 160) ?? fill.expected, method, nextActionIds: [], reason, selector: fill.selector, status: "mismatch", summary: `Fill verification warning: fill ${fill.selector} reported success, but get ${method} returned ${actualPreview}.` };
 	diagnostic.nextActionIds = buildFillVerificationNextActions(diagnostic, options.sessionName).map((action) => action.id);
 	return diagnostic;
 }
 
 export function formatFillVerificationText(diagnostic: FillVerificationDiagnostic | undefined): string | undefined {
 	if (!diagnostic) return undefined;
-	const actual = diagnostic.actual !== undefined ? `actual "${diagnostic.actual}"` : "actual value unavailable";
-	return `${diagnostic.summary}\nExpected: "${diagnostic.expected}"; ${actual}.\nNext: re-run snapshot -i, then prefer click/focus plus keyboard type for custom Electron quick-input controls before submitting.`;
+	const actual = diagnostic.actual !== undefined ? `actual "${diagnostic.actual}"` : `actual ${diagnostic.method} unavailable`;
+	const recovery = diagnostic.reason === "contenteditable-fill-mismatch"
+		? "Contenteditable fill may append or prepend instead of replacing. Re-run snapshot -i, then prefer focus/click plus keyboard shortcut selection or direct keyboard insertion only after verifying the editor state."
+		: "Re-run snapshot -i, then prefer click/focus plus keyboard type for custom quick-input controls before submitting.";
+	return `${diagnostic.summary}\nExpected: "${diagnostic.expected}"; ${actual}.\nNext: ${recovery}`;
 }
 
 export async function collectVisibleRefFallbackDiagnostic(options: { commandTokens: string[]; compiledSemanticAction?: CompiledAgentBrowserSemanticAction; cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<VisibleRefFallbackDiagnostic | undefined> {
@@ -669,8 +687,8 @@ export async function collectElectronHandoff(options: { cwd: string; handoff: "c
 	return { handoff: "snapshot", refSnapshot, snapshot, ...(snapshotRetryCount > 0 ? { snapshotRetryCount } : {}), tabs };
 }
 
-function getTimeoutProgressSteps(compiledJob: CompiledAgentBrowserJob | undefined, command: string | undefined, stdin: string | undefined): Array<{ args: string[]; index: number }> {
-	if (compiledJob) return compiledJob.steps.map((step, index) => ({ args: step.args, index: index + 1 }));
+function getTimeoutProgressSteps(compiledJob: CompiledAgentBrowserJob | undefined, command: string | undefined, stdin: string | undefined): Array<{ args: string[]; generatedFrom?: string; index: number }> {
+	if (compiledJob) return compiledJob.steps.map((step, index) => ({ args: step.args, generatedFrom: step.generatedFrom, index: index + 1 }));
 	if (command !== "batch" || !stdin) return [];
 	return parseValidBatchStepEntries(stdin).map(({ index, step }) => ({ args: step, index: index + 1 }));
 }
@@ -721,8 +739,8 @@ async function collectTimeoutArtifactEvidence(cwd: string, steps: Array<{ args:
 		const absolutePath = isAbsolute(path) ? path : resolve(cwd, path);
 		const artifact = await statTimeoutArtifactPath(absolutePath);
 		evidence.push(artifact.exists
-			? { absolutePath, exists: true, path, sizeBytes: artifact.sizeBytes, stepIndex: step.index }
-			: { absolutePath, exists: false, path, stepIndex: step.index });
+			? { absolutePath, exists: true, path, sizeBytes: artifact.sizeBytes, state: "verified", stepIndex: step.index }
+			: { absolutePath, exists: false, path, state: "missing", stepIndex: step.index });
 	}
 	return evidence;
 }
@@ -735,18 +753,122 @@ function getPlannedCurrentPageUrl(steps: Array<{ args: string[]; index: number }
 	return undefined;
 }
 
+const TIMEOUT_RETRYABLE_COMMANDS = new Set([
+	"console",
+	"diff",
+	"errors",
+	"get",
+	"goto",
+	"navigate",
+	"network",
+	"open",
+	"pdf",
+	"pushstate",
+	"screenshot",
+	"snapshot",
+	"tab",
+	"vitals",
+	"wait",
+]);
+
+function getTimeoutStepRetry(step: { args: string[] }): { args: string[] } | undefined {
+	const command = step.args[0];
+	return command && TIMEOUT_RETRYABLE_COMMANDS.has(command) ? { args: step.args } : undefined;
+}
+
+function normalizeUrlForTimeoutComparison(url: string | undefined): URL | undefined {
+	if (!url) return undefined;
+	try {
+		return new URL(url);
+	} catch {
+		return undefined;
+	}
+}
+
+function currentUrlMatchesNavigationStep(currentUrl: string | undefined, plannedUrl: string | undefined): boolean {
+	if (!currentUrl || !plannedUrl) return false;
+	if (currentUrl === plannedUrl) return true;
+	const current = normalizeUrlForTimeoutComparison(currentUrl);
+	const planned = normalizeUrlForTimeoutComparison(plannedUrl);
+	if (!current || !planned || current.origin !== planned.origin) return false;
+	const plannedPath = planned.pathname.endsWith("/") ? planned.pathname : `${planned.pathname}/`;
+	const currentPath = current.pathname.endsWith("/") ? current.pathname : `${current.pathname}/`;
+	return planned.pathname === "/" || currentPath.startsWith(plannedPath);
+}
+
+function buildTimeoutProgressSteps(options: {
+	artifacts: TimeoutArtifactEvidence[];
+	currentPageSource?: "live" | "planned";
+	currentPageUrl?: string;
+	steps: Array<{ args: string[]; generatedFrom?: string; index: number }>;
+}): { openedButPostOpenTimedOut?: boolean; retryStep?: TimeoutProgressStep; steps: TimeoutProgressStep[] } {
+	let retryStep: TimeoutProgressStep | undefined;
+	let lastCompletedNavigationIndex: number | undefined;
+	const progressSteps = options.steps.map((step): TimeoutProgressStep => {
+		const stepArtifacts = options.artifacts.filter((artifact) => artifact.stepIndex === step.index);
+		const command = step.args[0];
+		const navigationUrl = isOpenNavigationCommand(command) || command === "pushstate" ? getLastPositionalToken(step.args) : undefined;
+		if (stepArtifacts.some((artifact) => artifact.exists)) {
+			return { ...step, reason: "Declared artifact exists on disk after timeout.", status: "completed" };
+		}
+		if (options.currentPageSource === "live" && currentUrlMatchesNavigationStep(options.currentPageUrl, navigationUrl)) {
+			lastCompletedNavigationIndex = step.index;
+			return { ...step, reason: "Live page URL was recovered after timeout.", status: "completed" };
+		}
+		return { ...step, reason: stepArtifacts.length > 0 ? "Declared artifact was not present when the watchdog fired." : undefined, status: "unknown" };
+	});
+	const highestCompletedIndex = Math.max(0, ...progressSteps.filter((step) => step.status === "completed").map((step) => step.index));
+	for (const step of progressSteps) {
+		if (step.status === "unknown" && step.index < highestCompletedIndex) {
+			step.status = "completed";
+			step.reason = "Later step completion evidence indicates the batch advanced past this step before timeout.";
+		}
+	}
+	for (const step of progressSteps) {
+		const command = step.args[0];
+		if (step.status === "completed" && (isOpenNavigationCommand(command) || command === "pushstate")) {
+			lastCompletedNavigationIndex = Math.max(lastCompletedNavigationIndex ?? 0, step.index);
+		}
+	}
+	for (const step of progressSteps) {
+		if (step.status === "completed") continue;
+		if (!retryStep) {
+			const retry = getTimeoutStepRetry(step);
+			retryStep = {
+				...step,
+				reason: step.reason ?? (retry ? "Likely active when the wrapper watchdog fired." : "Likely active when the wrapper watchdog fired; executable retry omitted because this step may have already mutated page state."),
+				retry,
+				status: "failed",
+			};
+			Object.assign(step, retryStep);
+			continue;
+		}
+		step.status = "pending";
+		step.reason = step.reason ?? `Pending behind timed-out step ${retryStep.index}.`;
+	}
+	return {
+		openedButPostOpenTimedOut: lastCompletedNavigationIndex !== undefined && retryStep !== undefined && retryStep.index > lastCompletedNavigationIndex,
+		retryStep,
+		steps: progressSteps,
+	};
+}
+
 export async function collectTimeoutPartialProgress(options: { command?: string; compiledJob?: CompiledAgentBrowserJob; cwd: string; sessionName?: string; stdin?: string }): Promise<TimeoutPartialProgress | undefined> {
-	const steps = getTimeoutProgressSteps(options.compiledJob, options.command, options.stdin);
-	const artifacts = await collectTimeoutArtifactEvidence(options.cwd, steps);
+	const rawSteps = getTimeoutProgressSteps(options.compiledJob, options.command, options.stdin);
+	const artifacts = await collectTimeoutArtifactEvidence(options.cwd, rawSteps);
 	const [urlData, titleData] = await Promise.all([runSessionCommandData({ args: ["get", "url"], cwd: options.cwd, sessionName: options.sessionName }), runSessionCommandData({ args: ["get", "title"], cwd: options.cwd, sessionName: options.sessionName })]);
 	const recoveredUrl = extractStringResultField(urlData, "result") ?? extractStringResultField(urlData, "url");
 	const title = extractStringResultField(titleData, "result") ?? extractStringResultField(titleData, "title");
-	const plannedUrl = recoveredUrl ? undefined : getPlannedCurrentPageUrl(steps);
+	const plannedUrl = recoveredUrl ? undefined : getPlannedCurrentPageUrl(rawSteps);
 	const url = recoveredUrl ?? plannedUrl;
-	if (steps.length === 0 && artifacts.length === 0 && !url && !title) return undefined;
+	const currentPageSource = recoveredUrl ? "live" as const : plannedUrl ? "planned" as const : title ? "live" as const : undefined;
+	const stepProgress = buildTimeoutProgressSteps({ artifacts, currentPageSource: recoveredUrl ? "live" : undefined, currentPageUrl: recoveredUrl, steps: rawSteps });
+	if (rawSteps.length === 0 && artifacts.length === 0 && !url && !title) return undefined;
 	const foundArtifacts = artifacts.filter((artifact) => artifact.exists).length;
+	const completedSteps = stepProgress.steps.filter((step) => step.status === "completed").length;
 	const pageStateSummary = recoveredUrl || title ? " and current page state" : plannedUrl ? " and planned page URL" : "";
-	return { artifacts, currentPage: url || title ? { title, url } : undefined, steps: steps.length > 0 ? steps : undefined, summary: `Timed out before upstream returned final results; recovered ${foundArtifacts}/${artifacts.length} declared artifact path${artifacts.length === 1 ? "" : "s"}${pageStateSummary}.` };
+	const retrySummary = stepProgress.retryStep ? ` Retry step ${stepProgress.retryStep.index} is the first incomplete step.` : "";
+	return { artifacts, currentPage: url || title ? { source: currentPageSource, title, url } : undefined, liveUrlRecovered: recoveredUrl !== undefined, openedButPostOpenTimedOut: stepProgress.openedButPostOpenTimedOut, retryStep: stepProgress.retryStep, steps: stepProgress.steps.length > 0 ? stepProgress.steps : undefined, summary: `Timed out before upstream returned final results; recovered ${completedSteps}/${rawSteps.length} planned step state${rawSteps.length === 1 ? "" : "s"} and ${foundArtifacts}/${artifacts.length} declared artifact path${artifacts.length === 1 ? "" : "s"}${pageStateSummary}.${retrySummary}` };
 }
 
 function redactSensitivePathSegmentsForDiagnostic(path: string): string {
@@ -775,9 +897,16 @@ export function formatTimeoutPartialProgressText(progress: TimeoutPartialProgres
 	if (progress.steps && progress.steps.length > 0) {
 		const shownSteps = progress.steps.slice(0, 6);
 		lines.push("Planned steps:");
-		for (const step of shownSteps) lines.push(`- Step ${step.index}: ${redactSensitivePathSegmentsForDiagnostic(redactInvocationArgs(step.args).join(" "))}`);
+		for (const step of shownSteps) {
+			const commandText = redactSensitivePathSegmentsForDiagnostic(redactInvocationArgs(step.args).join(" "));
+			const generatedFrom = step.generatedFrom ? `, generated from ${step.generatedFrom}` : "";
+			lines.push(`- Step ${step.index} [${step.status}${generatedFrom}]: ${commandText}${step.reason ? ` — ${redactSensitivePathSegmentsForDiagnostic(redactSensitiveText(step.reason))}` : ""}`);
+		}
 		if (progress.steps.length > shownSteps.length) lines.push(`- ... ${progress.steps.length - shownSteps.length} more step${progress.steps.length - shownSteps.length === 1 ? "" : "s"} omitted`);
 	}
+	if (progress.retryStep?.retry?.args) {
+		lines.push(`Retry failed step: ${JSON.stringify({ args: redactInvocationArgs(progress.retryStep.retry.args) })}`);
+	}
 	for (const artifact of progress.artifacts) lines.push(`Artifact from step ${artifact.stepIndex}: ${redactSensitivePathSegmentsForDiagnostic(artifact.path)} (${artifact.exists ? `exists${typeof artifact.sizeBytes === "number" ? `, ${artifact.sizeBytes} bytes` : ""}` : "missing"})`);
 	return lines.join("\n");
 }

package/extensions/agent-browser/lib/orchestration/browser-run/final-result.ts

@@ -19,6 +19,7 @@ import {
 	AgentBrowserNextActionCollector,
 	alignPageChangeSummaryNextActionIds,
 	isStandaloneSnapshotNextAction,
+	withOptionalSessionArgs,
 } from "../../results/next-actions.js";
 import {
 	buildConnectedSessionNextActions,
@@ -282,7 +283,8 @@ export async function prepareFinalResultRecoveryState(options: {
 	let visibleRefFallbackDiagnostic: FinalRecoveryState["visibleRefFallbackDiagnostic"];
 	const visibleRefFallbackSessionName = options.executionPlan.sessionName ?? extractExplicitSessionName(options.runtimeToolArgs);
 	if (categoryDetails.failureCategory === "selector-not-found") {
-		visibleRefFallbackDiagnostic = await collectVisibleRefFallbackDiagnostic({ commandTokens: options.commandTokens, compiledSemanticAction: options.compiledSemanticAction, cwd: options.cwd, sessionName: visibleRefFallbackSessionName, signal: options.signal });
+		const selectorRecoveryCommandTokens = options.presentation.batchFailure?.failedStep.command ?? options.commandTokens;
+		visibleRefFallbackDiagnostic = await collectVisibleRefFallbackDiagnostic({ commandTokens: selectorRecoveryCommandTokens, compiledSemanticAction: options.compiledSemanticAction, cwd: options.cwd, sessionName: visibleRefFallbackSessionName, signal: options.signal });
 		if (visibleRefFallbackDiagnostic && visibleRefFallbackSessionName) {
 			const refUpdate = options.sessionPageState.applyRefSnapshot({ fallbackTarget: options.currentSessionTabTarget, sessionName: visibleRefFallbackSessionName, snapshot: visibleRefFallbackDiagnostic.snapshot, update: options.sessionPageStateUpdate });
 			currentRefSnapshot = refUpdate.refSnapshot;
@@ -299,6 +301,51 @@ export async function prepareFinalResultRecoveryState(options: {
 	return { categoryDetails, currentRefSnapshot, currentRefSnapshotInvalidation, noActivePageSnapshotFailure, richInputRecoveryDiagnostic, visibleRefFallbackDiagnostic, visibleRefFallbackSessionName };
 }
 
+function buildTimeoutPartialProgressNextActions(options: FinalResultInput): AgentBrowserNextAction[] {
+	const retryArgs = options.timeoutPartialProgress?.retryStep?.retry?.args;
+	if (!retryArgs) return [];
+	const stepIndex = options.timeoutPartialProgress?.retryStep?.index;
+	const freshSessionAbandoned = options.sessionMode === "fresh" && options.timeoutPartialProgress?.liveUrlRecovered !== true;
+	return [{
+		id: "retry-timeout-step",
+		params: freshSessionAbandoned
+			? { args: retryArgs, sessionMode: "fresh" }
+			: { args: withOptionalSessionArgs(options.executionPlan.sessionName, retryArgs) },
+		reason: freshSessionAbandoned
+			? `Retry the first incomplete timed-out step${stepIndex === undefined ? "" : ` ${stepIndex}`} in a fresh browser session because the timed-out fresh session was not proven live.`
+			: `Retry the first incomplete timed-out step${stepIndex === undefined ? "" : ` ${stepIndex}`} against the current browser session.`,
+		safety: "Only read-only or idempotent timeout steps get executable retry args; inspect current page/artifact state before using the action.",
+		tool: "agent_browser" as const,
+	}];
+}
+
+function buildDialogTimeoutNextActions(options: { command?: string; sessionName?: string }): AgentBrowserNextAction[] {
+	if (options.command !== "dialog" && options.command !== "click" && options.command !== "tap" && options.command !== "find" && options.command !== "eval") return [];
+	return [
+		{
+			id: "inspect-dialog-after-timeout",
+			params: { args: withOptionalSessionArgs(options.sessionName, ["dialog", "status"]) },
+			reason: "Check whether a blocking JavaScript dialog is pending after the timed-out interaction.",
+			safety: "Read-only dialog status; this wrapper bounds dialog commands so recovery attempts do not wait for the full default watchdog.",
+			tool: "agent_browser" as const,
+		},
+		{
+			id: "dismiss-dialog-after-timeout",
+			params: { args: withOptionalSessionArgs(options.sessionName, ["dialog", "dismiss"]) },
+			reason: "Dismiss a pending alert/confirm/prompt when the workflow can safely abandon the dialog.",
+			safety: "Only run when dismissing/canceling the dialog is acceptable for the user flow.",
+			tool: "agent_browser" as const,
+		},
+		{
+			id: "recover-fresh-session-after-dialog-timeout",
+			params: { args: ["open", "about:blank"], sessionMode: "fresh" as const },
+			reason: "Start a clean browser session if the current session remains blocked behind a JavaScript dialog.",
+			safety: "Replace about:blank with the intended recovery URL; this abandons the blocked managed session.",
+			tool: "agent_browser" as const,
+		},
+	];
+}
+
 function buildResultNextActions(options: FinalResultInput): AgentBrowserNextAction[] | undefined {
 	const nextActionCollector = new AgentBrowserNextActionCollector(options.presentation.nextActions);
 	if (options.categoryDetails.resultCategory === "success" && options.executionPlan.commandInfo.command === "connect" && !options.electronLaunchRecord) nextActionCollector.appendUnique(buildConnectedSessionNextActions(options.executionPlan.sessionName));
@@ -322,10 +369,14 @@ function buildResultNextActions(options: FinalResultInput): AgentBrowserNextActi
 	if (options.selectorTextVisibilityDiagnostics.length > 0) nextActionCollector.append(buildSelectorTextVisibilityNextActions({ diagnostics: options.selectorTextVisibilityDiagnostics, sessionName: options.executionPlan.sessionName }));
 	if (options.electronBroadGetTextScopeDiagnostics.length > 0) nextActionCollector.append(buildElectronBroadGetTextScopeNextActions({ diagnostics: options.electronBroadGetTextScopeDiagnostics, sessionName: options.executionPlan.sessionName }));
 	if (options.sourceLookup?.electronContext) nextActionCollector.appendUnique(buildSourceLookupElectronNextActions(options.sourceLookup));
-	if (options.clickDispatchDiagnostic) nextActionCollector.append(buildClickDispatchNextActions({ commandTokens: options.commandTokens, sessionName: options.executionPlan.sessionName }));
+	if (options.clickDispatchDiagnostic) nextActionCollector.append(buildClickDispatchNextActions({ commandTokens: options.commandTokens, diagnostic: options.clickDispatchDiagnostic, sessionName: options.executionPlan.sessionName }));
 	if (options.scrollNoopDiagnostic) nextActionCollector.append(buildScrollNoopNextActions(options.executionPlan.sessionName));
 	if (options.comboboxFocusDiagnostic) nextActionCollector.append(buildComboboxFocusNextActions(options.executionPlan.sessionName));
 	if (options.managedSessionOutcome) nextActionCollector.appendUnique(buildManagedSessionFreshFailureNextActions(options.managedSessionOutcome));
+	if (options.categoryDetails.failureCategory === "timeout" && options.processResult.timedOut) {
+		nextActionCollector.appendUnique(buildTimeoutPartialProgressNextActions(options));
+		nextActionCollector.appendUnique(buildDialogTimeoutNextActions({ command: options.executionPlan.commandInfo.command, sessionName: options.executionPlan.sessionName }));
+	}
 	if (options.categoryDetails.failureCategory === "stale-ref" && options.redactedCompiledSemanticAction && isCompiledSemanticActionFindCommand(options.compiledSemanticAction)) nextActionCollector.append([{ id: "retry-semantic-action-after-stale-ref", params: { args: options.redactedCompiledSemanticAction.args }, reason: "Retry the same semantic target via its compiled find command after the upstream stale-ref failure proves the prior action did not execute.", safety: "Use only for the same intended target; direct stale @refs still require a fresh snapshot or stable locator before retrying.", tool: "agent_browser" as const }]);
 	if (options.electronLaunchRecord) nextActionCollector.append(buildAgentBrowserNextActions({ electron: { launchId: options.electronLaunchRecord.launchId, sessionName: options.electronLaunchRecord.sessionName, status: options.electronLaunchRecord.cleanupState }, failureCategory: options.categoryDetails.failureCategory, resultCategory: options.categoryDetails.resultCategory, successCategory: options.categoryDetails.successCategory }));
 	return nextActionCollector.toArray();

package/extensions/agent-browser/lib/orchestration/browser-run/index.ts

@@ -24,6 +24,7 @@ export async function runAgentBrowserTool(options: BrowserRunOptions): Promise<A
 			env: prepared.executionPlan.managedSessionName ? { AGENT_BROWSER_IDLE_TIMEOUT_MS: options.implicitSessionIdleTimeoutMs } : undefined,
 			signal: options.signal,
 			stdin: prepared.processStdin,
+			timeoutMs: prepared.processTimeoutMs,
 		});
 
 		const missingBinaryResult = await buildMissingBinaryFailureResult({