pi-agent-browser-native 0.2.44 → 0.2.45

package/extensions/agent-browser/lib/input-modes/job.ts

@@ -8,8 +8,10 @@ import type { ArtifactVerificationSummary } from "../results/contracts.js";
 import { isRecord } from "../parsing.js";
 import { summarizeNetworkFailures } from "../results/network.js";
 import { getBatchResultItems, getCommandNameFromBatchItem, getSelectValues } from "./shared.js";
+import { compileAgentBrowserSemanticAction } from "./semantic-action.js";
 import {
 	AGENT_BROWSER_JOB_STEP_ACTIONS,
+	AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS,
 	AGENT_BROWSER_QA_LOAD_STATES,
 	type AgentBrowserJobStepAction,
 	type AgentBrowserQaLoadState,
@@ -27,10 +29,88 @@ function getRequiredJobString(step: Record<string, unknown>, field: "path" | "se
 	return { value };
 }
 
+function compileJobClickOrFillStep(step: Record<string, unknown>, action: "click" | "fill"): { args?: string[]; error?: string } {
+	const hasSelector = typeof step.selector === "string" && step.selector.trim().length > 0;
+	const hasLocator = step.locator !== undefined || step.role !== undefined || step.name !== undefined || step.value !== undefined;
+	if (hasSelector && hasLocator) {
+		return { error: `job step ${action} must use either selector or semantic locator fields, not both.` };
+	}
+	if (hasSelector) {
+		if (action === "click") return { args: ["click", step.selector as string] };
+		const text = getRequiredJobString(step, "text", action);
+		if (text.error) return { error: text.error };
+		return { args: ["fill", step.selector as string, text.value as string] };
+	}
+	if (!hasLocator) {
+		return { error: `job step ${action} requires either a non-empty selector string or semantic locator fields.` };
+	}
+	const compiled = compileAgentBrowserSemanticAction({
+		action,
+		locator: step.locator,
+		name: step.name,
+		role: step.role,
+		text: step.text,
+		value: step.value,
+	});
+	if (compiled.error) return { error: compiled.error.replaceAll("semanticAction", `job step ${action}`) };
+	return { args: compiled.compiled?.args };
+}
+
+function getUnsupportedJobStepField(step: Record<string, unknown>, allowedFields: ReadonlySet<string>): string | undefined {
+	return Object.keys(step).find((field) => !allowedFields.has(field));
+}
+
+const JOB_TYPE_ALLOWED_FIELDS = new Set(["action", "delayMs", "press", "selector", "text"]);
+
+function compileJobTypeSteps(step: Record<string, unknown>): { error?: string; steps?: CompiledAgentBrowserJobStep[] } {
+	const unsupportedField = getUnsupportedJobStepField(step, JOB_TYPE_ALLOWED_FIELDS);
+	if (unsupportedField) return { error: `job step type does not support ${unsupportedField}; supported fields are selector, text, delayMs, and press.` };
+	const text = getRequiredJobString(step, "text", "type");
+	if (text.error) return { error: text.error };
+	const selector = step.selector;
+	if (selector !== undefined && (typeof selector !== "string" || selector.trim().length === 0)) {
+		return { error: "job step type selector must be a non-empty string when provided." };
+	}
+	if (step.locator !== undefined || step.role !== undefined || step.name !== undefined || step.value !== undefined || step.values !== undefined) {
+		return { error: "job step type supports selector, text, delayMs, and press only; focus the target first or use click/fill semantic locator fields in a separate step." };
+	}
+	const delayMs = step.delayMs;
+	if (delayMs !== undefined && (typeof delayMs !== "number" || !Number.isInteger(delayMs) || delayMs <= 0)) {
+		return { error: "job step type delayMs must be a positive integer when provided." };
+	}
+	const press = step.press;
+	if (press !== undefined && (typeof press !== "string" || press.trim().length === 0)) {
+		return { error: "job step type press must be a non-empty key string when provided." };
+	}
+	const typedText = text.value as string;
+	const typedChars = Array.from(typedText);
+	if (typedChars.length === 0) return { error: "job step type requires non-empty text." };
+	if (delayMs !== undefined && typedChars.length > AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS) {
+		return { error: `job step type delayMs supports at most ${AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS} characters; split longer text into shorter calls or omit delayMs.` };
+	}
+	const compiledSteps: CompiledAgentBrowserJobStep[] = [];
+	if (delayMs === undefined) {
+		compiledSteps.push({ action: "type", args: typeof selector === "string" ? ["type", selector, typedText] : ["keyboard", "type", typedText] });
+	} else {
+		if (typeof selector === "string") compiledSteps.push({ action: "type", args: ["focus", selector], generatedFrom: "type.selector" });
+		for (const [index, char] of typedChars.entries()) {
+			compiledSteps.push({ action: "type", args: ["keyboard", "type", char], generatedFrom: "type.delayMs" });
+			if (index < typedChars.length - 1) compiledSteps.push({ action: "wait", args: ["wait", String(delayMs)], generatedFrom: "type.delayMs" });
+		}
+	}
+	if (typeof press === "string") compiledSteps.push({ action: "type", args: ["press", press], generatedFrom: "type.press" });
+	return { steps: compiledSteps };
+}
+
 export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAgentBrowserJob; error?: string } {
 	if (!isRecord(input)) {
 		return { error: "job must be an object." };
 	}
+	const rawFailFast = input.failFast;
+	if (rawFailFast !== undefined && typeof rawFailFast !== "boolean") {
+		return { error: "job.failFast must be a boolean when provided." };
+	}
+	const failFast = rawFailFast !== false;
 	const rawSteps = input.steps;
 	if (!Array.isArray(rawSteps) || rawSteps.length === 0) {
 		return { error: "job.steps must be a non-empty array." };
@@ -46,20 +126,29 @@ export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAge
 		}
 		const jobAction = action as AgentBrowserJobStepAction;
 		let args: string[];
+		let generatedFrom: string | undefined;
+		let extraSteps: CompiledAgentBrowserJobStep[] = [];
 		if (jobAction === "open") {
 			const result = getRequiredJobString(rawStep, "url", jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
 			args = ["open", result.value as string];
-		} else if (jobAction === "click") {
-			const result = getRequiredJobString(rawStep, "selector", jobAction);
+			if (rawStep.loadState !== undefined) {
+				if (typeof rawStep.loadState !== "string" || !AGENT_BROWSER_QA_LOAD_STATES.includes(rawStep.loadState as AgentBrowserQaLoadState)) {
+					return { error: `job.steps[${index}].loadState must be one of: ${AGENT_BROWSER_QA_LOAD_STATES.join(", ")}.` };
+				}
+				extraSteps = [{ action: "wait", args: ["wait", "--load", rawStep.loadState], generatedFrom: "open.loadState" }];
+			}
+		} else if (jobAction === "click" || jobAction === "fill") {
+			const result = compileJobClickOrFillStep(rawStep, jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
-			args = ["click", result.value as string];
-		} else if (jobAction === "fill") {
-			const selector = getRequiredJobString(rawStep, "selector", jobAction);
-			if (selector.error) return { error: `job.steps[${index}]: ${selector.error}` };
-			const text = getRequiredJobString(rawStep, "text", jobAction);
-			if (text.error) return { error: `job.steps[${index}]: ${text.error}` };
-			args = ["fill", selector.value as string, text.value as string];
+			args = result.args as string[];
+		} else if (jobAction === "type") {
+			const result = compileJobTypeSteps(rawStep);
+			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
+			const [firstStep, ...restSteps] = result.steps as CompiledAgentBrowserJobStep[];
+			args = firstStep.args;
+			generatedFrom = firstStep.generatedFrom;
+			extraSteps = restSteps;
 		} else if (jobAction === "select") {
 			const selector = getRequiredJobString(rawStep, "selector", jobAction);
 			if (selector.error) return { error: `job.steps[${index}]: ${selector.error}` };
@@ -84,14 +173,16 @@ export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAge
 			const result = getRequiredJobString(rawStep, "path", jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
 			args = ["wait", "--download", result.value as string];
+		} else if (jobAction === "snapshot") {
+			args = ["snapshot", "-i"];
 		} else {
 			const result = getRequiredJobString(rawStep, "path", jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
 			args = ["screenshot", result.value as string];
 		}
-		steps.push({ action: jobAction, args });
+		steps.push({ action: jobAction, args, generatedFrom }, ...extraSteps);
 	}
-	return { compiled: { args: ["batch"], stdin: JSON.stringify(steps.map((step) => step.args)), steps } };
+	return { compiled: { args: failFast ? ["batch", "--bail"] : ["batch"], failFast, stdin: JSON.stringify(steps.map((step) => step.args)), steps } };
 }
 
 export function isHttpOrHttpsUrl(url: string): boolean {
@@ -110,6 +201,8 @@ function describeQaChecksRun(checks: CompiledAgentBrowserQaPreset["checks"]): st
 	if (checks.checkNetwork) parts.push("network");
 	if (checks.checkConsole) parts.push("console");
 	if (checks.checkErrors) parts.push("errors");
+	if (checks.diagnosticsResetAtStart) parts.push("diagnostics-reset");
+	else if (checks.checkNetwork || checks.checkConsole || checks.checkErrors) parts.push("attached-diagnostics-preserved");
 	if (checks.screenshotPath) parts.push("screenshot");
 	return parts.join(", ");
 }
@@ -145,6 +238,9 @@ export function buildQaCompactPassText(options: {
 	const pageParts = [options.page?.title, options.page?.url].filter((part): part is string => typeof part === "string" && part.length > 0);
 	if (pageParts.length > 0) lines.push(`Page: ${pageParts.join(" — ")}`);
 	lines.push(`Checks run: ${describeQaChecksRun(options.checks)} (${options.batchStepCount} batch step${options.batchStepCount === 1 ? "" : "s"})`);
+	if (options.checks.attached && !options.checks.diagnosticsResetAtStart && (options.checks.checkNetwork || options.checks.checkConsole || options.checks.checkErrors)) {
+		lines.push("Attached diagnostics: existing upstream session console/network/error buffers were preserved; rows may include events from before qa.attached started.");
+	}
 	if (options.checks.screenshotPath) {
 		const verification = options.artifactVerification;
 		lines.push(verification
@@ -155,7 +251,83 @@ export function buildQaCompactPassText(options: {
 	return lines.join("\n");
 }
 
-export function analyzeQaPresetResults(data: unknown): AgentBrowserQaPresetAnalysis | undefined {
+const QA_VISIBLE_TEXT_TIMEOUT_MS = 5_000;
+
+function formatQaExpectedTextPreview(text: string): string {
+	return JSON.stringify(text.length > 80 ? `${text.slice(0, 77)}...` : text);
+}
+
+function buildQaVisibleTextPredicate(text: string): string {
+	return `(() => {
+  const expected = ${JSON.stringify(text)}.replace(/\\s+/g, " ").trim();
+  if (!expected) return false;
+  const root = document.body || document.documentElement;
+  if (!root) return false;
+  const skipTags = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "SVG"]);
+  const normalize = (value) => String(value ?? "").replace(/\\s+/g, " ").trim();
+  const isVisibleElement = (element) => {
+    if (!(element instanceof HTMLElement)) return false;
+    if (skipTags.has(element.tagName)) return false;
+    const style = window.getComputedStyle(element);
+    if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return false;
+    return element.getClientRects().length > 0;
+  };
+  const hasVisibleAncestors = (node) => {
+    for (let element = node.parentElement; element; element = element.parentElement) {
+      if (!isVisibleElement(element)) return false;
+      if (element === root) break;
+    }
+    return true;
+  };
+  const textWalker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
+  let visitedText = 0;
+  for (let node = textWalker.nextNode(); node && visitedText < 6000; node = textWalker.nextNode(), visitedText += 1) {
+    if (!hasVisibleAncestors(node)) continue;
+    if (normalize(node.nodeValue).includes(expected)) return true;
+  }
+  const elementWalker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
+  let visitedElements = 0;
+  for (let node = elementWalker.nextNode(); node && visitedElements < 3000; node = elementWalker.nextNode(), visitedElements += 1) {
+    const element = node;
+    if (!isVisibleElement(element) || !("value" in element)) continue;
+    if (normalize(element.value).includes(expected)) return true;
+  }
+  return false;
+})()`;
+}
+
+function qaVisibleTextWaitPassed(item: ReturnType<typeof getBatchResultItems>[number] | undefined, step: CompiledAgentBrowserJobStep): boolean | undefined {
+	if (step.args[0] !== "wait" || step.args[1] !== "--fn") return undefined;
+	if (!item || item.success === false) return false;
+	if (typeof item.result === "boolean") return item.result;
+	if (isRecord(item.result) && typeof item.result.result === "boolean") return item.result.result;
+	return true;
+}
+
+function extractQaTextAssertionResultText(item: ReturnType<typeof getBatchResultItems>[number] | undefined): string | undefined {
+	if (!item || item.success === false) return undefined;
+	const result = item.result;
+	if (typeof result === "string") return result;
+	if (!isRecord(result)) return undefined;
+	for (const key of ["result", "text", "value"] as const) {
+		const value = result[key];
+		if (typeof value === "string") return value;
+	}
+	return undefined;
+}
+
+export function analyzeQaPresetTimeout(compiled: CompiledAgentBrowserQaPreset): AgentBrowserQaPresetAnalysis | undefined {
+	if (compiled.checks.expectedText.length === 0) return undefined;
+	const failedChecks = compiled.checks.expectedText.map((text) => `expected text was not verified before timeout: ${formatQaExpectedTextPreview(text)}`);
+	return {
+		failedChecks,
+		passed: false,
+		summary: `QA preset failed: ${failedChecks.join("; ")}.`,
+		warnings: ["The wrapper timed out before expected-text evidence could be verified; inspect timeoutPartialProgress and retry with a narrower readiness condition if the page was still loading."],
+	};
+}
+
+export function analyzeQaPresetResults(data: unknown, compiled?: CompiledAgentBrowserQaPreset): AgentBrowserQaPresetAnalysis | undefined {
 	const items = getBatchResultItems(data);
 	if (items.length === 0) return undefined;
 	const failedChecks: string[] = [];
@@ -179,6 +351,18 @@ export function analyzeQaPresetResults(data: unknown): AgentBrowserQaPresetAnaly
 			if (networkFailures.benignCount > 0) warnings.push(`${networkFailures.benignCount} benign network request failure(s) ignored`);
 		}
 	}
+	if (compiled?.checks.expectedText.length) {
+		let expectedTextIndex = 0;
+		compiled.steps.forEach((step, index) => {
+			if (step.action !== "assertText") return;
+			const expected = compiled.checks.expectedText[expectedTextIndex++];
+			if (!expected) return;
+			const visibleTextPassed = qaVisibleTextWaitPassed(items[index], step);
+			if (visibleTextPassed === true) return;
+			const actual = extractQaTextAssertionResultText(items[index]);
+			if (!actual || !actual.includes(expected)) failedChecks.push(`expected text not found: ${formatQaExpectedTextPreview(expected)}`);
+		});
+	}
 	const uniqueFailures = [...new Set(failedChecks)];
 	const uniqueWarnings = [...new Set(warnings)];
 	return {
@@ -234,18 +418,19 @@ export function compileAgentBrowserQaPreset(input: unknown): { compiled?: Compil
 	if (rawLoadState !== undefined && (typeof rawLoadState !== "string" || !AGENT_BROWSER_QA_LOAD_STATES.includes(rawLoadState as AgentBrowserQaLoadState))) {
 		return { error: `qa.loadState must be one of: ${AGENT_BROWSER_QA_LOAD_STATES.join(", ")}.` };
 	}
-	const checkConsole = input.checkConsole !== false;
-	const checkErrors = input.checkErrors !== false;
-	const checkNetwork = input.checkNetwork !== false;
+	const checkConsole = typeof input.checkConsole === "boolean" ? input.checkConsole : !attached;
+	const checkErrors = typeof input.checkErrors === "boolean" ? input.checkErrors : !attached;
+	const checkNetwork = typeof input.checkNetwork === "boolean" ? input.checkNetwork : !attached;
 	const loadState = (rawLoadState as AgentBrowserQaLoadState | undefined) ?? "domcontentloaded";
+	const diagnosticsResetAtStart = !attached;
 	const steps: CompiledAgentBrowserJobStep[] = [];
-	if (checkNetwork) steps.push({ action: "wait", args: ["network", "requests", "--clear"] });
-	if (checkConsole) steps.push({ action: "wait", args: ["console", "--clear"] });
-	if (checkErrors) steps.push({ action: "wait", args: ["errors", "--clear"] });
+	if (diagnosticsResetAtStart && checkNetwork) steps.push({ action: "wait", args: ["network", "requests", "--clear"] });
+	if (diagnosticsResetAtStart && checkConsole) steps.push({ action: "wait", args: ["console", "--clear"] });
+	if (diagnosticsResetAtStart && checkErrors) steps.push({ action: "wait", args: ["errors", "--clear"] });
 	if (!attached && normalizedUrl) steps.push({ action: "open", args: ["open", normalizedUrl] });
 	steps.push({ action: "wait", args: ["wait", "--load", loadState] });
 	for (const text of expectedText) {
-		steps.push({ action: "assertText", args: ["wait", "--text", text] });
+		steps.push({ action: "assertText", args: ["wait", "--fn", buildQaVisibleTextPredicate(text), "--timeout", String(QA_VISIBLE_TEXT_TIMEOUT_MS)] });
 	}
 	if (typeof expectedSelector === "string") {
 		steps.push({ action: "wait", args: ["wait", expectedSelector] });
@@ -256,8 +441,9 @@ export function compileAgentBrowserQaPreset(input: unknown): { compiled?: Compil
 	if (typeof screenshotPath === "string") steps.push({ action: "screenshot", args: ["screenshot", screenshotPath] });
 	return {
 		compiled: {
-			args: ["batch"],
-			checks: { attached, checkConsole, checkErrors, checkNetwork, expectedSelector, expectedText, loadState, screenshotPath, url: normalizedUrl },
+			args: ["batch", "--bail"],
+			checks: { attached, checkConsole, checkErrors, checkNetwork, diagnosticsResetAtStart, expectedSelector, expectedText, loadState, screenshotPath, url: normalizedUrl },
+			failFast: true,
 			stdin: JSON.stringify(steps.map((step) => step.args)),
 			steps,
 		},

package/extensions/agent-browser/lib/input-modes/params.ts

@@ -15,6 +15,7 @@ import {
 	AGENT_BROWSER_ELECTRON_HANDOFFS,
 	AGENT_BROWSER_ELECTRON_TARGET_TYPES,
 	AGENT_BROWSER_JOB_STEP_ACTIONS,
+	AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS,
 	AGENT_BROWSER_QA_LOAD_STATES,
 	AGENT_BROWSER_SEMANTIC_ACTIONS,
 	AGENT_BROWSER_SEMANTIC_LOCATORS,
@@ -33,14 +34,14 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 	semanticAction: Type.Optional(
 		Type.Object({
 			action: StringEnum(AGENT_BROWSER_SEMANTIC_ACTIONS, {
-				description: "Intent action to compile to an existing agent-browser find command, or to upstream select when action=select.",
+				description: "Intent action to compile to an existing agent-browser find command, direct selector/ref command, or upstream select when action=select.",
 			}),
 			locator: Type.Optional(StringEnum(AGENT_BROWSER_SEMANTIC_LOCATORS, {
 				description: "Upstream find locator family to use for check/click/fill actions.",
 			})),
 			value: Type.Optional(Type.String({ description: "Locator value for find actions, or a single option value for select actions. For locator=role, role may be supplied instead." })),
 			values: Type.Optional(Type.Array(Type.String({ description: "Option value for select actions." }), { description: "One or more option values for select actions.", minItems: 1 })),
-			selector: Type.Optional(Type.String({ description: "Selector or @ref for select actions; compiled to select <selector> <value...>." })),
+			selector: Type.Optional(Type.String({ description: "Selector or @ref for direct click/check/fill actions, or for select actions compiled to select <selector> <value...>." })),
 			text: Type.Optional(Type.String({ description: "Text/value argument for fill actions." })),
 			role: Type.Optional(Type.String({ description: "Role locator value for locator=role. May be used instead of value; when both are set they must match." })),
 			name: Type.Optional(Type.String({ description: "Accessible name filter for locator=role; compiles to --name <name>." })),
@@ -54,9 +55,9 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 				expectedText: Type.Optional(Type.Union([Type.String(), Type.Array(Type.String())], { description: "Text that must appear on the page." })),
 				expectedSelector: Type.Optional(Type.String({ description: "Selector or @ref that must appear on the page." })),
 				screenshotPath: Type.Optional(Type.String({ description: "Optional evidence screenshot path captured at the end of the QA preset." })),
-				checkConsole: Type.Optional(Type.Boolean({ description: "Whether to fail on console error messages. Defaults to true." })),
-				checkErrors: Type.Optional(Type.Boolean({ description: "Whether to fail on page errors. Defaults to true." })),
-				checkNetwork: Type.Optional(Type.Boolean({ description: "Whether to inspect network requests and fail on actionable request failures; benign icon misses warn. Defaults to true." })),
+				checkConsole: Type.Optional(Type.Boolean({ description: "Whether to inspect console messages and fail on console errors. Defaults to false for qa.attached because upstream buffers may predate the check." })),
+				checkErrors: Type.Optional(Type.Boolean({ description: "Whether to inspect page errors and fail when errors are present. Defaults to false for qa.attached because upstream buffers may predate the check." })),
+				checkNetwork: Type.Optional(Type.Boolean({ description: "Whether to inspect network requests and fail on actionable request failures; benign icon misses warn. Defaults to false for qa.attached because upstream buffers may predate the check." })),
 				loadState: Type.Optional(StringEnum(AGENT_BROWSER_QA_LOAD_STATES, { description: "Page readiness state for the QA preset before assertions and diagnostics. Defaults to domcontentloaded; use networkidle only for pages without long-lived background requests." })),
 			}, { additionalProperties: false }),
 			Type.Object({
@@ -160,17 +161,24 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 	),
 	job: Type.Optional(
 		Type.Object({
+			failFast: Type.Optional(Type.Boolean({ description: "Stop the compiled batch on the first failed job step. Defaults to true so later mutating steps do not run after setup/assertion failures." })),
 			steps: Type.Array(
 				Type.Object({
 					action: StringEnum(AGENT_BROWSER_JOB_STEP_ACTIONS, {
 						description: "Constrained one-call job step compiled to existing upstream batch commands.",
 					}),
 					url: Type.Optional(Type.String({ description: "URL for open steps, or URL pattern for assertUrl steps." })),
-					selector: Type.Optional(Type.String({ description: "Selector or @ref for click/fill/select-like steps." })),
+					loadState: Type.Optional(StringEnum(AGENT_BROWSER_QA_LOAD_STATES, { description: "Optional readiness wait to insert immediately after an open step; use domcontentloaded/load/networkidle when the next job step needs page hydration evidence before clicking or reading." })),
+					selector: Type.Optional(Type.String({ description: "Selector or @ref for click/fill/type/select-like steps; omit when using semantic locator fields on click/fill steps." })),
+					locator: Type.Optional(StringEnum(AGENT_BROWSER_SEMANTIC_LOCATORS, { description: "Semantic locator for click/fill steps when selector is omitted." })),
+					role: Type.Optional(Type.String({ description: "Role locator value for click/fill steps when locator is role." })),
+					name: Type.Optional(Type.String({ description: "Accessible name filter for role locator click/fill steps." })),
 					text: Type.Optional(Type.String({ description: "Text for fill steps or visible text for assertText steps." })),
-					value: Type.Optional(Type.String({ description: "Single option value for select steps." })),
+					value: Type.Optional(Type.String({ description: "Single option value for select steps, or locator value for semantic click/fill steps." })),
 					values: Type.Optional(Type.Array(Type.String({ description: "Option value for select steps." }), { description: "One or more option values for select steps.", minItems: 1 })),
 					path: Type.Optional(Type.String({ description: "Artifact/download path for waitForDownload or screenshot steps." })),
+					delayMs: Type.Optional(Type.Integer({ description: `Optional per-character delay for type steps; when set, the job compiles to focus/keyboard type/wait steps instead of instant fill-like typing, capped at ${AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS} characters.`, minimum: 1 })),
+					press: Type.Optional(Type.String({ description: "Optional key to press after a type step, for example Enter." })),
 					milliseconds: Type.Optional(Type.Number({ description: "Milliseconds for wait steps." })),
 				}, { additionalProperties: false }),
 				{ minItems: 1 },
@@ -178,6 +186,8 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 		}, { additionalProperties: false }),
 	),
 	stdin: Type.Optional(Type.String({ description: "Optional raw stdin content; only supported for batch, eval --stdin, auth save --password-stdin, and is generated internally by job, qa, sourceLookup, or networkSourceLookup mode. Do not use with electron mode." })),
+	outputPath: Type.Optional(Type.String({ description: "Optional workspace-relative or absolute file path that receives the model-facing command data/result after the browser command completes. Useful for eval/get/snapshot captures that should become durable local artifacts.", minLength: 1 })),
+	timeoutMs: Type.Optional(Type.Integer({ description: "Optional per-call wrapper subprocess watchdog in milliseconds for browser CLI args/job/qa/source lookup calls. Use for long opens or large output captures; fixed wait steps still must stay below the upstream IPC wait budget. Electron actions use electron.timeoutMs instead.", minimum: 1 })),
 	sessionMode: Type.Optional(
 		StringEnum(["auto", "fresh"] as const, {
 			description:

package/extensions/agent-browser/lib/input-modes/semantic-action.ts

@@ -62,8 +62,28 @@ export function compileAgentBrowserSemanticAction(input: unknown): { compiled?:
 		const args = typeof session === "string" ? ["--session", session, "select", selector, ...(selectedValues.values as string[])] : ["select", selector, ...(selectedValues.values as string[])];
 		return { compiled: { action: "select", selector, values: selectedValues.values, args } };
 	}
-	if (selector !== undefined || values !== undefined) {
-		return { error: "semanticAction.selector and values are only supported for select actions." };
+	if (values !== undefined) {
+		return { error: "semanticAction.values is only supported for select actions." };
+	}
+	if (selector !== undefined) {
+		if (typeof selector !== "string" || selector.trim().length === 0) {
+			return { error: "semanticAction.selector must be a non-empty string when provided." };
+		}
+		if (locator !== undefined || value !== undefined || role !== undefined || name !== undefined) {
+			return { error: "semanticAction.selector cannot be combined with locator, value, role, or name; use selector for a direct click/check/fill target or locator fields for find-based actions." };
+		}
+		if (text !== undefined && typeof text !== "string") {
+			return { error: "semanticAction.text must be a string when provided." };
+		}
+		if (action === "fill" && (typeof text !== "string" || text.length === 0)) {
+			return { error: `semanticAction.text is required for ${action}.` };
+		}
+		if (action !== "fill" && text !== undefined) {
+			return { error: "semanticAction.text is only supported for fill actions." };
+		}
+		const directArgs = typeof session === "string" ? ["--session", session, action, selector] : [action, selector];
+		if (action === "fill") directArgs.push(text as string);
+		return { compiled: { action: action as AgentBrowserSemanticActionName, selector, args: directArgs } };
 	}
 	if (typeof locator !== "string" || !AGENT_BROWSER_SEMANTIC_LOCATORS.includes(locator as AgentBrowserSemanticLocator)) {
 		return { error: `semanticAction.locator must be one of: ${AGENT_BROWSER_SEMANTIC_LOCATORS.join(", ")}.` };

package/extensions/agent-browser/lib/input-modes/types.ts

@@ -10,7 +10,8 @@ export const AGENT_BROWSER_SEMANTIC_ACTIONS = ["check", "click", "fill", "select
 
 export const AGENT_BROWSER_SEMANTIC_LOCATORS = ["alt", "label", "placeholder", "role", "testid", "text", "title"] as const;
 
-export const AGENT_BROWSER_JOB_STEP_ACTIONS = ["open", "click", "fill", "select", "wait", "assertText", "assertUrl", "waitForDownload", "screenshot"] as const;
+export const AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS = 200;
+export const AGENT_BROWSER_JOB_STEP_ACTIONS = ["open", "click", "fill", "type", "select", "wait", "assertText", "assertUrl", "waitForDownload", "screenshot", "snapshot"] as const;
 
 export const AGENT_BROWSER_QA_LOAD_STATES = ["domcontentloaded", "load", "networkidle"] as const;
 
@@ -102,10 +103,12 @@ export interface CompiledAgentBrowserSemanticAction {
 export interface CompiledAgentBrowserJobStep {
 	action: AgentBrowserJobStepAction;
 	args: string[];
+	generatedFrom?: string;
 }
 
 export interface CompiledAgentBrowserJob {
 	args: string[];
+	failFast: boolean;
 	stdin: string;
 	steps: CompiledAgentBrowserJobStep[];
 }
@@ -115,6 +118,7 @@ export interface CompiledAgentBrowserQaPreset extends CompiledAgentBrowserJob {
 		checkConsole: boolean;
 		checkErrors: boolean;
 		checkNetwork: boolean;
+		diagnosticsResetAtStart: boolean;
 		loadState: AgentBrowserQaLoadState;
 		expectedText: string[];
 		expectedSelector?: string;

package/extensions/agent-browser/lib/input-modes.ts

@@ -7,6 +7,7 @@
 export { AGENT_BROWSER_PARAMS } from "./input-modes/params.js";
 export {
 	analyzeQaPresetResults,
+	analyzeQaPresetTimeout,
 	buildQaCompactPassText,
 	compileAgentBrowserJob,
 	compileAgentBrowserQaPreset,

package/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.ts

@@ -71,7 +71,48 @@ function buildClickDispatchProbeInstallScript(probe: ClickDispatchProbe): string
 const marker = ${JSON.stringify(probe.marker)};
 const element = ${resolveTarget};
 if (!element) return { status: "target-not-found", marker };
-const state = { events: [], target: { tagName: element.tagName.toLowerCase() } };
+const cssEscape = (value) => {
+  if (window.CSS && typeof window.CSS.escape === "function") return window.CSS.escape(value);
+  return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");
+};
+const getSelector = (node) => {
+  if (!(node instanceof Element)) return undefined;
+  if (node.id) return "#" + cssEscape(node.id);
+  const testId = node.getAttribute("data-testid") || node.getAttribute("data-test-id");
+  if (testId) return '[data-testid="' + cssEscape(testId) + '"]';
+  const parts = [];
+  let current = node;
+  while (current && current !== document.body && parts.length < 4) {
+    const tag = current.tagName.toLowerCase();
+    const parent = current.parentElement;
+    if (!parent) break;
+    const siblings = Array.from(parent.children).filter((child) => child.tagName === current.tagName);
+    const index = siblings.indexOf(current) + 1;
+    parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag);
+    current = parent;
+  }
+  return parts.length > 0 ? parts.join(" > ") : undefined;
+};
+const rectInfo = (rect) => ({ bottom: rect.bottom, left: rect.left, right: rect.right, top: rect.top });
+const targetRect = element.getBoundingClientRect();
+const targetOutsideViewport = targetRect.bottom < 0 || targetRect.right < 0 || targetRect.top > window.innerHeight || targetRect.left > window.innerWidth;
+let nearestScrollContainer;
+for (let current = element.parentElement; current && current !== document.body; current = current.parentElement) {
+  if (current.scrollHeight > current.clientHeight + 1 || current.scrollWidth > current.clientWidth + 1) {
+    const containerRect = current.getBoundingClientRect();
+    nearestScrollContainer = {
+      selector: getSelector(current),
+      tagName: current.tagName.toLowerCase(),
+      targetOutsideContainer: targetRect.bottom < containerRect.top || targetRect.top > containerRect.bottom || targetRect.right < containerRect.left || targetRect.left > containerRect.right,
+      targetOutsideViewport,
+      rect: rectInfo(containerRect),
+      scrollLeft: current.scrollLeft,
+      scrollTop: current.scrollTop,
+    };
+    break;
+  }
+}
+const state = { events: [], target: { tagName: element.tagName.toLowerCase(), nearestScrollContainer, rect: rectInfo(targetRect), targetOutsideViewport } };
 const eventTypes = ["pointerdown", "mousedown", "pointerup", "mouseup", "click"];
 const listeners = eventTypes.map((type) => {
   const listener = (event) => {
@@ -126,9 +167,9 @@ export function formatClickDispatchDiagnosticText(diagnostic: ClickDispatchDiagn
 	return `Click dispatch diagnostic: ${diagnostic.summary}`;
 }
 
-export function buildClickDispatchNextActions(options: { commandTokens: string[]; sessionName?: string }): AgentBrowserNextAction[] {
+export function buildClickDispatchNextActions(options: { commandTokens: string[]; diagnostic?: ClickDispatchDiagnostic; sessionName?: string }): AgentBrowserNextAction[] {
 	const retryArgs = options.commandTokens[0] === "click" ? options.commandTokens : ["click", ...options.commandTokens];
-	return [
+	const actions: AgentBrowserNextAction[] = [
 		{
 			id: "inspect-click-dispatch-miss",
 			params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) },
@@ -136,14 +177,26 @@ export function buildClickDispatchNextActions(options: { commandTokens: string[]
 			safety: "Read-only snapshot; the wrapper does not replay clicks in-page when upstream reports success without DOM events.",
 			tool: "agent_browser",
 		},
-		{
-			id: "retry-click-after-dispatch-miss",
-			params: { args: withOptionalSessionArgs(options.sessionName, retryArgs) },
-			reason: "Retry the same upstream click after confirming the target is visible; do not assume the prior success mutated the page.",
-			safety: "Only retry when the target is still intended; use page-change evidence or a fresh snapshot before continuing the workflow.",
-			tool: "agent_browser",
-		},
 	];
+	if (options.diagnostic?.scrollContainer) {
+		actions.push({
+			id: "scroll-target-into-view-after-dispatch-miss",
+			params: { args: withOptionalSessionArgs(options.sessionName, ["scrollintoview", retryArgs[1]].filter((item): item is string => typeof item === "string")) },
+			reason: options.diagnostic.scrollContainer.selector
+				? `The target may be outside nested scroll container ${options.diagnostic.scrollContainer.selector}; scroll the target into view before retrying the click.`
+				: "The target may be inside an offscreen nested scroll container; scroll the target into view before retrying the click.",
+			safety: "Use only for the same current page and target; run snapshot -i again if the page rerendered.",
+			tool: "agent_browser",
+		});
+	}
+	actions.push({
+		id: "retry-click-after-dispatch-miss",
+		params: { args: withOptionalSessionArgs(options.sessionName, retryArgs) },
+		reason: "Retry the same upstream click after confirming the target is visible; do not assume the prior success mutated the page.",
+		safety: "Only retry when the target is still intended; use page-change evidence or a fresh snapshot before continuing the workflow.",
+		tool: "agent_browser",
+	});
+	return actions;
 }
 
 export async function prepareClickDispatchProbe(options: { commandTokens: string[]; cwd: string; refSnapshot?: SessionRefSnapshot; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchProbe | undefined> {
@@ -156,6 +209,20 @@ export async function prepareClickDispatchProbe(options: { commandTokens: string
 	return installResult?.status === "installed" ? probe : undefined;
 }
 
+function getClickDispatchScrollContainerDiagnostic(result: Record<string, unknown>): ClickDispatchDiagnostic["scrollContainer"] {
+	const target = isRecord(result.target) ? result.target : undefined;
+	const scrollContainer = isRecord(target?.nearestScrollContainer) ? target.nearestScrollContainer : undefined;
+	const targetOutsideViewport = typeof target?.targetOutsideViewport === "boolean" ? target.targetOutsideViewport : undefined;
+	const targetOutsideContainer = typeof scrollContainer?.targetOutsideContainer === "boolean" ? scrollContainer.targetOutsideContainer : undefined;
+	if (!scrollContainer && !targetOutsideViewport) return undefined;
+	if (targetOutsideContainer !== true && targetOutsideViewport !== true) return undefined;
+	const selector = typeof scrollContainer?.selector === "string" ? redactSensitiveText(scrollContainer.selector) : undefined;
+	const summary = selector
+		? `Target appears outside nested scroll container ${selector}; use scrollintoview on the target or scroll that container before retrying.`
+		: "Target appears outside the viewport or a nested scroll container; use scrollintoview on the target before retrying.";
+	return { selector, summary, targetOutsideContainer, targetOutsideViewport };
+}
+
 export async function collectClickDispatchDiagnostic(options: { cwd: string; probe?: ClickDispatchProbe; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchDiagnostic | undefined> {
 	if (!options.probe || !options.sessionName) return undefined;
 	const data = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeCheckScript(options.probe) });
@@ -164,10 +231,14 @@ export async function collectClickDispatchDiagnostic(options: { cwd: string; pro
 	const status = typeof result.status === "string" ? result.status : undefined;
 	if (status !== "no-native-event-observed") return undefined;
 	const nativeEventCount = typeof result.nativeEventCount === "number" ? result.nativeEventCount : 0;
-	const summary = "Upstream click reported success but no trusted DOM event reached the selected element. Gather evidence with snapshot or page-change checks, then retry upstream click or report the workflow issue; the wrapper does not replay clicks in-page.";
+	const scrollContainer = getClickDispatchScrollContainerDiagnostic(result);
+	const summary = scrollContainer
+		? `Upstream click reported success but no trusted DOM event reached the selected element. ${scrollContainer.summary}`
+		: "Upstream click reported success but no trusted DOM event reached the selected element. Gather evidence with snapshot or page-change checks, then retry upstream click or report the workflow issue; the wrapper does not replay clicks in-page.";
 	return {
 		nativeEventCount,
 		reason: "native-click-produced-no-target-dom-event",
+		...(scrollContainer ? { scrollContainer } : {}),
 		status,
 		summary,
 		target: redactClickDispatchTarget(options.probe.target),