pi-agent-browser-native 0.2.44 → 0.2.46

package/extensions/agent-browser/lib/input-modes/job.ts

@@ -8,8 +8,10 @@ import type { ArtifactVerificationSummary } from "../results/contracts.js";
 import { isRecord } from "../parsing.js";
 import { summarizeNetworkFailures } from "../results/network.js";
 import { getBatchResultItems, getCommandNameFromBatchItem, getSelectValues } from "./shared.js";
+import { compileAgentBrowserSemanticAction } from "./semantic-action.js";
 import {
 	AGENT_BROWSER_JOB_STEP_ACTIONS,
+	AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS,
 	AGENT_BROWSER_QA_LOAD_STATES,
 	type AgentBrowserJobStepAction,
 	type AgentBrowserQaLoadState,
@@ -27,10 +29,88 @@ function getRequiredJobString(step: Record<string, unknown>, field: "path" | "se
 	return { value };
 }
 
+function compileJobClickOrFillStep(step: Record<string, unknown>, action: "click" | "fill"): { args?: string[]; error?: string } {
+	const hasSelector = typeof step.selector === "string" && step.selector.trim().length > 0;
+	const hasLocator = step.locator !== undefined || step.role !== undefined || step.name !== undefined || step.value !== undefined;
+	if (hasSelector && hasLocator) {
+		return { error: `job step ${action} must use either selector or semantic locator fields, not both.` };
+	}
+	if (hasSelector) {
+		if (action === "click") return { args: ["click", step.selector as string] };
+		const text = getRequiredJobString(step, "text", action);
+		if (text.error) return { error: text.error };
+		return { args: ["fill", step.selector as string, text.value as string] };
+	}
+	if (!hasLocator) {
+		return { error: `job step ${action} requires either a non-empty selector string or semantic locator fields.` };
+	}
+	const compiled = compileAgentBrowserSemanticAction({
+		action,
+		locator: step.locator,
+		name: step.name,
+		role: step.role,
+		text: step.text,
+		value: step.value,
+	});
+	if (compiled.error) return { error: compiled.error.replaceAll("semanticAction", `job step ${action}`) };
+	return { args: compiled.compiled?.args };
+}
+
+function getUnsupportedJobStepField(step: Record<string, unknown>, allowedFields: ReadonlySet<string>): string | undefined {
+	return Object.keys(step).find((field) => !allowedFields.has(field));
+}
+
+const JOB_TYPE_ALLOWED_FIELDS = new Set(["action", "delayMs", "press", "selector", "text"]);
+
+function compileJobTypeSteps(step: Record<string, unknown>): { error?: string; steps?: CompiledAgentBrowserJobStep[] } {
+	const unsupportedField = getUnsupportedJobStepField(step, JOB_TYPE_ALLOWED_FIELDS);
+	if (unsupportedField) return { error: `job step type does not support ${unsupportedField}; supported fields are selector, text, delayMs, and press.` };
+	const text = getRequiredJobString(step, "text", "type");
+	if (text.error) return { error: text.error };
+	const selector = step.selector;
+	if (selector !== undefined && (typeof selector !== "string" || selector.trim().length === 0)) {
+		return { error: "job step type selector must be a non-empty string when provided." };
+	}
+	if (step.locator !== undefined || step.role !== undefined || step.name !== undefined || step.value !== undefined || step.values !== undefined) {
+		return { error: "job step type supports selector, text, delayMs, and press only; focus the target first or use click/fill semantic locator fields in a separate step." };
+	}
+	const delayMs = step.delayMs;
+	if (delayMs !== undefined && (typeof delayMs !== "number" || !Number.isInteger(delayMs) || delayMs <= 0)) {
+		return { error: "job step type delayMs must be a positive integer when provided." };
+	}
+	const press = step.press;
+	if (press !== undefined && (typeof press !== "string" || press.trim().length === 0)) {
+		return { error: "job step type press must be a non-empty key string when provided." };
+	}
+	const typedText = text.value as string;
+	const typedChars = Array.from(typedText);
+	if (typedChars.length === 0) return { error: "job step type requires non-empty text." };
+	if (delayMs !== undefined && typedChars.length > AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS) {
+		return { error: `job step type delayMs supports at most ${AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS} characters; split longer text into shorter calls or omit delayMs.` };
+	}
+	const compiledSteps: CompiledAgentBrowserJobStep[] = [];
+	if (delayMs === undefined) {
+		compiledSteps.push({ action: "type", args: typeof selector === "string" ? ["type", selector, typedText] : ["keyboard", "type", typedText] });
+	} else {
+		if (typeof selector === "string") compiledSteps.push({ action: "type", args: ["focus", selector], generatedFrom: "type.selector" });
+		for (const [index, char] of typedChars.entries()) {
+			compiledSteps.push({ action: "type", args: ["keyboard", "type", char], generatedFrom: "type.delayMs" });
+			if (index < typedChars.length - 1) compiledSteps.push({ action: "wait", args: ["wait", String(delayMs)], generatedFrom: "type.delayMs" });
+		}
+	}
+	if (typeof press === "string") compiledSteps.push({ action: "type", args: ["press", press], generatedFrom: "type.press" });
+	return { steps: compiledSteps };
+}
+
 export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAgentBrowserJob; error?: string } {
 	if (!isRecord(input)) {
 		return { error: "job must be an object." };
 	}
+	const rawFailFast = input.failFast;
+	if (rawFailFast !== undefined && typeof rawFailFast !== "boolean") {
+		return { error: "job.failFast must be a boolean when provided." };
+	}
+	const failFast = rawFailFast !== false;
 	const rawSteps = input.steps;
 	if (!Array.isArray(rawSteps) || rawSteps.length === 0) {
 		return { error: "job.steps must be a non-empty array." };
@@ -46,20 +126,29 @@ export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAge
 		}
 		const jobAction = action as AgentBrowserJobStepAction;
 		let args: string[];
+		let generatedFrom: string | undefined;
+		let extraSteps: CompiledAgentBrowserJobStep[] = [];
 		if (jobAction === "open") {
 			const result = getRequiredJobString(rawStep, "url", jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
 			args = ["open", result.value as string];
-		} else if (jobAction === "click") {
-			const result = getRequiredJobString(rawStep, "selector", jobAction);
+			if (rawStep.loadState !== undefined) {
+				if (typeof rawStep.loadState !== "string" || !AGENT_BROWSER_QA_LOAD_STATES.includes(rawStep.loadState as AgentBrowserQaLoadState)) {
+					return { error: `job.steps[${index}].loadState must be one of: ${AGENT_BROWSER_QA_LOAD_STATES.join(", ")}.` };
+				}
+				extraSteps = [{ action: "wait", args: ["wait", "--load", rawStep.loadState], generatedFrom: "open.loadState" }];
+			}
+		} else if (jobAction === "click" || jobAction === "fill") {
+			const result = compileJobClickOrFillStep(rawStep, jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
-			args = ["click", result.value as string];
-		} else if (jobAction === "fill") {
-			const selector = getRequiredJobString(rawStep, "selector", jobAction);
-			if (selector.error) return { error: `job.steps[${index}]: ${selector.error}` };
-			const text = getRequiredJobString(rawStep, "text", jobAction);
-			if (text.error) return { error: `job.steps[${index}]: ${text.error}` };
-			args = ["fill", selector.value as string, text.value as string];
+			args = result.args as string[];
+		} else if (jobAction === "type") {
+			const result = compileJobTypeSteps(rawStep);
+			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
+			const [firstStep, ...restSteps] = result.steps as CompiledAgentBrowserJobStep[];
+			args = firstStep.args;
+			generatedFrom = firstStep.generatedFrom;
+			extraSteps = restSteps;
 		} else if (jobAction === "select") {
 			const selector = getRequiredJobString(rawStep, "selector", jobAction);
 			if (selector.error) return { error: `job.steps[${index}]: ${selector.error}` };
@@ -84,14 +173,16 @@ export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAge
 			const result = getRequiredJobString(rawStep, "path", jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
 			args = ["wait", "--download", result.value as string];
+		} else if (jobAction === "snapshot") {
+			args = ["snapshot", "-i"];
 		} else {
 			const result = getRequiredJobString(rawStep, "path", jobAction);
 			if (result.error) return { error: `job.steps[${index}]: ${result.error}` };
 			args = ["screenshot", result.value as string];
 		}
-		steps.push({ action: jobAction, args });
+		steps.push({ action: jobAction, args, generatedFrom }, ...extraSteps);
 	}
-	return { compiled: { args: ["batch"], stdin: JSON.stringify(steps.map((step) => step.args)), steps } };
+	return { compiled: { args: failFast ? ["batch", "--bail"] : ["batch"], failFast, stdin: JSON.stringify(steps.map((step) => step.args)), steps } };
 }
 
 export function isHttpOrHttpsUrl(url: string): boolean {
@@ -110,6 +201,8 @@ function describeQaChecksRun(checks: CompiledAgentBrowserQaPreset["checks"]): st
 	if (checks.checkNetwork) parts.push("network");
 	if (checks.checkConsole) parts.push("console");
 	if (checks.checkErrors) parts.push("errors");
+	if (checks.diagnosticsResetAtStart) parts.push("diagnostics-reset");
+	else if (checks.checkNetwork || checks.checkConsole || checks.checkErrors) parts.push("attached-diagnostics-preserved");
 	if (checks.screenshotPath) parts.push("screenshot");
 	return parts.join(", ");
 }
@@ -145,6 +238,9 @@ export function buildQaCompactPassText(options: {
 	const pageParts = [options.page?.title, options.page?.url].filter((part): part is string => typeof part === "string" && part.length > 0);
 	if (pageParts.length > 0) lines.push(`Page: ${pageParts.join(" — ")}`);
 	lines.push(`Checks run: ${describeQaChecksRun(options.checks)} (${options.batchStepCount} batch step${options.batchStepCount === 1 ? "" : "s"})`);
+	if (options.checks.attached && !options.checks.diagnosticsResetAtStart && (options.checks.checkNetwork || options.checks.checkConsole || options.checks.checkErrors)) {
+		lines.push("Attached diagnostics: existing upstream session console/network/error buffers were preserved; rows may include events from before qa.attached started.");
+	}
 	if (options.checks.screenshotPath) {
 		const verification = options.artifactVerification;
 		lines.push(verification
@@ -155,7 +251,83 @@ export function buildQaCompactPassText(options: {
 	return lines.join("\n");
 }
 
-export function analyzeQaPresetResults(data: unknown): AgentBrowserQaPresetAnalysis | undefined {
+const QA_VISIBLE_TEXT_TIMEOUT_MS = 5_000;
+
+function formatQaExpectedTextPreview(text: string): string {
+	return JSON.stringify(text.length > 80 ? `${text.slice(0, 77)}...` : text);
+}
+
+function buildQaVisibleTextPredicate(text: string): string {
+	return `(() => {
+  const expected = ${JSON.stringify(text)}.replace(/\\s+/g, " ").trim();
+  if (!expected) return false;
+  const root = document.body || document.documentElement;
+  if (!root) return false;
+  const skipTags = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "SVG"]);
+  const normalize = (value) => String(value ?? "").replace(/\\s+/g, " ").trim();
+  const isVisibleElement = (element) => {
+    if (!(element instanceof HTMLElement)) return false;
+    if (skipTags.has(element.tagName)) return false;
+    const style = window.getComputedStyle(element);
+    if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return false;
+    return element.getClientRects().length > 0;
+  };
+  const hasVisibleAncestors = (node) => {
+    for (let element = node.parentElement; element; element = element.parentElement) {
+      if (!isVisibleElement(element)) return false;
+      if (element === root) break;
+    }
+    return true;
+  };
+  const textWalker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
+  let visitedText = 0;
+  for (let node = textWalker.nextNode(); node && visitedText < 6000; node = textWalker.nextNode(), visitedText += 1) {
+    if (!hasVisibleAncestors(node)) continue;
+    if (normalize(node.nodeValue).includes(expected)) return true;
+  }
+  const elementWalker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
+  let visitedElements = 0;
+  for (let node = elementWalker.nextNode(); node && visitedElements < 3000; node = elementWalker.nextNode(), visitedElements += 1) {
+    const element = node;
+    if (!isVisibleElement(element) || !("value" in element)) continue;
+    if (normalize(element.value).includes(expected)) return true;
+  }
+  return false;
+})()`;
+}
+
+function qaVisibleTextWaitPassed(item: ReturnType<typeof getBatchResultItems>[number] | undefined, step: CompiledAgentBrowserJobStep): boolean | undefined {
+	if (step.args[0] !== "wait" || step.args[1] !== "--fn") return undefined;
+	if (!item || item.success === false) return false;
+	if (typeof item.result === "boolean") return item.result;
+	if (isRecord(item.result) && typeof item.result.result === "boolean") return item.result.result;
+	return true;
+}
+
+function extractQaTextAssertionResultText(item: ReturnType<typeof getBatchResultItems>[number] | undefined): string | undefined {
+	if (!item || item.success === false) return undefined;
+	const result = item.result;
+	if (typeof result === "string") return result;
+	if (!isRecord(result)) return undefined;
+	for (const key of ["result", "text", "value"] as const) {
+		const value = result[key];
+		if (typeof value === "string") return value;
+	}
+	return undefined;
+}
+
+export function analyzeQaPresetTimeout(compiled: CompiledAgentBrowserQaPreset): AgentBrowserQaPresetAnalysis | undefined {
+	if (compiled.checks.expectedText.length === 0) return undefined;
+	const failedChecks = compiled.checks.expectedText.map((text) => `expected text was not verified before timeout: ${formatQaExpectedTextPreview(text)}`);
+	return {
+		failedChecks,
+		passed: false,
+		summary: `QA preset failed: ${failedChecks.join("; ")}.`,
+		warnings: ["The wrapper timed out before expected-text evidence could be verified; inspect timeoutPartialProgress and retry with a narrower readiness condition if the page was still loading."],
+	};
+}
+
+export function analyzeQaPresetResults(data: unknown, compiled?: CompiledAgentBrowserQaPreset): AgentBrowserQaPresetAnalysis | undefined {
 	const items = getBatchResultItems(data);
 	if (items.length === 0) return undefined;
 	const failedChecks: string[] = [];
@@ -179,6 +351,18 @@ export function analyzeQaPresetResults(data: unknown): AgentBrowserQaPresetAnaly
 			if (networkFailures.benignCount > 0) warnings.push(`${networkFailures.benignCount} benign network request failure(s) ignored`);
 		}
 	}
+	if (compiled?.checks.expectedText.length) {
+		let expectedTextIndex = 0;
+		compiled.steps.forEach((step, index) => {
+			if (step.action !== "assertText") return;
+			const expected = compiled.checks.expectedText[expectedTextIndex++];
+			if (!expected) return;
+			const visibleTextPassed = qaVisibleTextWaitPassed(items[index], step);
+			if (visibleTextPassed === true) return;
+			const actual = extractQaTextAssertionResultText(items[index]);
+			if (!actual || !actual.includes(expected)) failedChecks.push(`expected text not found: ${formatQaExpectedTextPreview(expected)}`);
+		});
+	}
 	const uniqueFailures = [...new Set(failedChecks)];
 	const uniqueWarnings = [...new Set(warnings)];
 	return {
@@ -234,18 +418,19 @@ export function compileAgentBrowserQaPreset(input: unknown): { compiled?: Compil
 	if (rawLoadState !== undefined && (typeof rawLoadState !== "string" || !AGENT_BROWSER_QA_LOAD_STATES.includes(rawLoadState as AgentBrowserQaLoadState))) {
 		return { error: `qa.loadState must be one of: ${AGENT_BROWSER_QA_LOAD_STATES.join(", ")}.` };
 	}
-	const checkConsole = input.checkConsole !== false;
-	const checkErrors = input.checkErrors !== false;
-	const checkNetwork = input.checkNetwork !== false;
+	const checkConsole = typeof input.checkConsole === "boolean" ? input.checkConsole : !attached;
+	const checkErrors = typeof input.checkErrors === "boolean" ? input.checkErrors : !attached;
+	const checkNetwork = typeof input.checkNetwork === "boolean" ? input.checkNetwork : !attached;
 	const loadState = (rawLoadState as AgentBrowserQaLoadState | undefined) ?? "domcontentloaded";
+	const diagnosticsResetAtStart = !attached;
 	const steps: CompiledAgentBrowserJobStep[] = [];
-	if (checkNetwork) steps.push({ action: "wait", args: ["network", "requests", "--clear"] });
-	if (checkConsole) steps.push({ action: "wait", args: ["console", "--clear"] });
-	if (checkErrors) steps.push({ action: "wait", args: ["errors", "--clear"] });
+	if (diagnosticsResetAtStart && checkNetwork) steps.push({ action: "wait", args: ["network", "requests", "--clear"] });
+	if (diagnosticsResetAtStart && checkConsole) steps.push({ action: "wait", args: ["console", "--clear"] });
+	if (diagnosticsResetAtStart && checkErrors) steps.push({ action: "wait", args: ["errors", "--clear"] });
 	if (!attached && normalizedUrl) steps.push({ action: "open", args: ["open", normalizedUrl] });
 	steps.push({ action: "wait", args: ["wait", "--load", loadState] });
 	for (const text of expectedText) {
-		steps.push({ action: "assertText", args: ["wait", "--text", text] });
+		steps.push({ action: "assertText", args: ["wait", "--fn", buildQaVisibleTextPredicate(text), "--timeout", String(QA_VISIBLE_TEXT_TIMEOUT_MS)] });
 	}
 	if (typeof expectedSelector === "string") {
 		steps.push({ action: "wait", args: ["wait", expectedSelector] });
@@ -256,8 +441,9 @@ export function compileAgentBrowserQaPreset(input: unknown): { compiled?: Compil
 	if (typeof screenshotPath === "string") steps.push({ action: "screenshot", args: ["screenshot", screenshotPath] });
 	return {
 		compiled: {
-			args: ["batch"],
-			checks: { attached, checkConsole, checkErrors, checkNetwork, expectedSelector, expectedText, loadState, screenshotPath, url: normalizedUrl },
+			args: ["batch", "--bail"],
+			checks: { attached, checkConsole, checkErrors, checkNetwork, diagnosticsResetAtStart, expectedSelector, expectedText, loadState, screenshotPath, url: normalizedUrl },
+			failFast: true,
 			stdin: JSON.stringify(steps.map((step) => step.args)),
 			steps,
 		},

package/extensions/agent-browser/lib/input-modes/params.ts

@@ -4,8 +4,8 @@
  * Scope: Schema-only; behavioral validation lives in the mode compilers.
  */
 
-import { StringEnum } from "@earendil-works/pi-ai";
-import { Type } from "typebox";
+import { JsonSchema, type JsonSchemaBuilder } from "../json-schema.js";
+import { StringEnum as localStringEnum, type StringEnumBuilder } from "../string-enum-schema.js";
 
 import {
 	ELECTRON_DISCOVERY_DEFAULT_MAX_RESULTS,
@@ -15,6 +15,7 @@ import {
 	AGENT_BROWSER_ELECTRON_HANDOFFS,
 	AGENT_BROWSER_ELECTRON_TARGET_TYPES,
 	AGENT_BROWSER_JOB_STEP_ACTIONS,
+	AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS,
 	AGENT_BROWSER_QA_LOAD_STATES,
 	AGENT_BROWSER_SEMANTIC_ACTIONS,
 	AGENT_BROWSER_SEMANTIC_LOCATORS,
@@ -22,7 +23,11 @@ import {
 	SOURCE_LOOKUP_MAX_WORKSPACE_FILES,
 } from "./types.js";
 
-export const AGENT_BROWSER_PARAMS = Type.Object({
+export function createAgentBrowserParamsSchema(
+	Type: JsonSchemaBuilder = JsonSchema,
+	StringEnum: StringEnumBuilder = localStringEnum,
+) {
+	return Type.Object({
 
 	args: Type.Optional(
 		Type.Array(Type.String({ description: "Exact agent-browser CLI arguments, excluding the binary name. Do not pass --json; the wrapper injects it. First-call recipe: open → snapshot -i → click/fill @eN → snapshot -i." }), {
@@ -33,14 +38,14 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 	semanticAction: Type.Optional(
 		Type.Object({
 			action: StringEnum(AGENT_BROWSER_SEMANTIC_ACTIONS, {
-				description: "Intent action to compile to an existing agent-browser find command, or to upstream select when action=select.",
+				description: "Intent action to compile to an existing agent-browser find command, direct selector/ref command, or upstream select when action=select.",
 			}),
 			locator: Type.Optional(StringEnum(AGENT_BROWSER_SEMANTIC_LOCATORS, {
 				description: "Upstream find locator family to use for check/click/fill actions.",
 			})),
 			value: Type.Optional(Type.String({ description: "Locator value for find actions, or a single option value for select actions. For locator=role, role may be supplied instead." })),
 			values: Type.Optional(Type.Array(Type.String({ description: "Option value for select actions." }), { description: "One or more option values for select actions.", minItems: 1 })),
-			selector: Type.Optional(Type.String({ description: "Selector or @ref for select actions; compiled to select <selector> <value...>." })),
+			selector: Type.Optional(Type.String({ description: "Selector or @ref for direct click/check/fill actions, or for select actions compiled to select <selector> <value...>." })),
 			text: Type.Optional(Type.String({ description: "Text/value argument for fill actions." })),
 			role: Type.Optional(Type.String({ description: "Role locator value for locator=role. May be used instead of value; when both are set they must match." })),
 			name: Type.Optional(Type.String({ description: "Accessible name filter for locator=role; compiles to --name <name>." })),
@@ -54,9 +59,9 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 				expectedText: Type.Optional(Type.Union([Type.String(), Type.Array(Type.String())], { description: "Text that must appear on the page." })),
 				expectedSelector: Type.Optional(Type.String({ description: "Selector or @ref that must appear on the page." })),
 				screenshotPath: Type.Optional(Type.String({ description: "Optional evidence screenshot path captured at the end of the QA preset." })),
-				checkConsole: Type.Optional(Type.Boolean({ description: "Whether to fail on console error messages. Defaults to true." })),
-				checkErrors: Type.Optional(Type.Boolean({ description: "Whether to fail on page errors. Defaults to true." })),
-				checkNetwork: Type.Optional(Type.Boolean({ description: "Whether to inspect network requests and fail on actionable request failures; benign icon misses warn. Defaults to true." })),
+				checkConsole: Type.Optional(Type.Boolean({ description: "Whether to inspect console messages and fail on console errors. Defaults to false for qa.attached because upstream buffers may predate the check." })),
+				checkErrors: Type.Optional(Type.Boolean({ description: "Whether to inspect page errors and fail when errors are present. Defaults to false for qa.attached because upstream buffers may predate the check." })),
+				checkNetwork: Type.Optional(Type.Boolean({ description: "Whether to inspect network requests and fail on actionable request failures; benign icon misses warn. Defaults to false for qa.attached because upstream buffers may predate the check." })),
 				loadState: Type.Optional(StringEnum(AGENT_BROWSER_QA_LOAD_STATES, { description: "Page readiness state for the QA preset before assertions and diagnostics. Defaults to domcontentloaded; use networkidle only for pages without long-lived background requests." })),
 			}, { additionalProperties: false }),
 			Type.Object({
@@ -160,17 +165,24 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 	),
 	job: Type.Optional(
 		Type.Object({
+			failFast: Type.Optional(Type.Boolean({ description: "Stop the compiled batch on the first failed job step. Defaults to true so later mutating steps do not run after setup/assertion failures." })),
 			steps: Type.Array(
 				Type.Object({
 					action: StringEnum(AGENT_BROWSER_JOB_STEP_ACTIONS, {
 						description: "Constrained one-call job step compiled to existing upstream batch commands.",
 					}),
 					url: Type.Optional(Type.String({ description: "URL for open steps, or URL pattern for assertUrl steps." })),
-					selector: Type.Optional(Type.String({ description: "Selector or @ref for click/fill/select-like steps." })),
+					loadState: Type.Optional(StringEnum(AGENT_BROWSER_QA_LOAD_STATES, { description: "Optional readiness wait to insert immediately after an open step; use domcontentloaded/load/networkidle when the next job step needs page hydration evidence before clicking or reading." })),
+					selector: Type.Optional(Type.String({ description: "Selector or @ref for click/fill/type/select-like steps; omit when using semantic locator fields on click/fill steps." })),
+					locator: Type.Optional(StringEnum(AGENT_BROWSER_SEMANTIC_LOCATORS, { description: "Semantic locator for click/fill steps when selector is omitted." })),
+					role: Type.Optional(Type.String({ description: "Role locator value for click/fill steps when locator is role." })),
+					name: Type.Optional(Type.String({ description: "Accessible name filter for role locator click/fill steps." })),
 					text: Type.Optional(Type.String({ description: "Text for fill steps or visible text for assertText steps." })),
-					value: Type.Optional(Type.String({ description: "Single option value for select steps." })),
+					value: Type.Optional(Type.String({ description: "Single option value for select steps, or locator value for semantic click/fill steps." })),
 					values: Type.Optional(Type.Array(Type.String({ description: "Option value for select steps." }), { description: "One or more option values for select steps.", minItems: 1 })),
 					path: Type.Optional(Type.String({ description: "Artifact/download path for waitForDownload or screenshot steps." })),
+					delayMs: Type.Optional(Type.Integer({ description: `Optional per-character delay for type steps; when set, the job compiles to focus/keyboard type/wait steps instead of instant fill-like typing, capped at ${AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS} characters.`, minimum: 1 })),
+					press: Type.Optional(Type.String({ description: "Optional key to press after a type step, for example Enter." })),
 					milliseconds: Type.Optional(Type.Number({ description: "Milliseconds for wait steps." })),
 				}, { additionalProperties: false }),
 				{ minItems: 1 },
@@ -178,6 +190,8 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 		}, { additionalProperties: false }),
 	),
 	stdin: Type.Optional(Type.String({ description: "Optional raw stdin content; only supported for batch, eval --stdin, auth save --password-stdin, and is generated internally by job, qa, sourceLookup, or networkSourceLookup mode. Do not use with electron mode." })),
+	outputPath: Type.Optional(Type.String({ description: "Optional workspace-relative or absolute file path that receives the model-facing command data/result after the browser command completes. Useful for eval/get/snapshot captures that should become durable local artifacts.", minLength: 1 })),
+	timeoutMs: Type.Optional(Type.Integer({ description: "Optional per-call wrapper subprocess watchdog in milliseconds for browser CLI args/job/qa/source lookup calls. Use for long opens or large output captures; fixed wait steps still must stay below the upstream IPC wait budget. Electron actions use electron.timeoutMs instead.", minimum: 1 })),
 	sessionMode: Type.Optional(
 		StringEnum(["auto", "fresh"] as const, {
 			description:
@@ -185,4 +199,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
 			default: DEFAULT_SESSION_MODE,
 		}),
 	),
-}, { additionalProperties: false });
+	}, { additionalProperties: false });
+}
+
+export const AGENT_BROWSER_PARAMS = createAgentBrowserParamsSchema();

package/extensions/agent-browser/lib/input-modes/semantic-action.ts

@@ -62,8 +62,28 @@ export function compileAgentBrowserSemanticAction(input: unknown): { compiled?:
 		const args = typeof session === "string" ? ["--session", session, "select", selector, ...(selectedValues.values as string[])] : ["select", selector, ...(selectedValues.values as string[])];
 		return { compiled: { action: "select", selector, values: selectedValues.values, args } };
 	}
-	if (selector !== undefined || values !== undefined) {
-		return { error: "semanticAction.selector and values are only supported for select actions." };
+	if (values !== undefined) {
+		return { error: "semanticAction.values is only supported for select actions." };
+	}
+	if (selector !== undefined) {
+		if (typeof selector !== "string" || selector.trim().length === 0) {
+			return { error: "semanticAction.selector must be a non-empty string when provided." };
+		}
+		if (locator !== undefined || value !== undefined || role !== undefined || name !== undefined) {
+			return { error: "semanticAction.selector cannot be combined with locator, value, role, or name; use selector for a direct click/check/fill target or locator fields for find-based actions." };
+		}
+		if (text !== undefined && typeof text !== "string") {
+			return { error: "semanticAction.text must be a string when provided." };
+		}
+		if (action === "fill" && (typeof text !== "string" || text.length === 0)) {
+			return { error: `semanticAction.text is required for ${action}.` };
+		}
+		if (action !== "fill" && text !== undefined) {
+			return { error: "semanticAction.text is only supported for fill actions." };
+		}
+		const directArgs = typeof session === "string" ? ["--session", session, action, selector] : [action, selector];
+		if (action === "fill") directArgs.push(text as string);
+		return { compiled: { action: action as AgentBrowserSemanticActionName, selector, args: directArgs } };
 	}
 	if (typeof locator !== "string" || !AGENT_BROWSER_SEMANTIC_LOCATORS.includes(locator as AgentBrowserSemanticLocator)) {
 		return { error: `semanticAction.locator must be one of: ${AGENT_BROWSER_SEMANTIC_LOCATORS.join(", ")}.` };

package/extensions/agent-browser/lib/input-modes/types.ts

@@ -10,7 +10,8 @@ export const AGENT_BROWSER_SEMANTIC_ACTIONS = ["check", "click", "fill", "select
 
 export const AGENT_BROWSER_SEMANTIC_LOCATORS = ["alt", "label", "placeholder", "role", "testid", "text", "title"] as const;
 
-export const AGENT_BROWSER_JOB_STEP_ACTIONS = ["open", "click", "fill", "select", "wait", "assertText", "assertUrl", "waitForDownload", "screenshot"] as const;
+export const AGENT_BROWSER_JOB_TYPE_DELAYED_TEXT_MAX_CHARACTERS = 200;
+export const AGENT_BROWSER_JOB_STEP_ACTIONS = ["open", "click", "fill", "type", "select", "wait", "assertText", "assertUrl", "waitForDownload", "screenshot", "snapshot"] as const;
 
 export const AGENT_BROWSER_QA_LOAD_STATES = ["domcontentloaded", "load", "networkidle"] as const;
 
@@ -102,10 +103,12 @@ export interface CompiledAgentBrowserSemanticAction {
 export interface CompiledAgentBrowserJobStep {
 	action: AgentBrowserJobStepAction;
 	args: string[];
+	generatedFrom?: string;
 }
 
 export interface CompiledAgentBrowserJob {
 	args: string[];
+	failFast: boolean;
 	stdin: string;
 	steps: CompiledAgentBrowserJobStep[];
 }
@@ -115,6 +118,7 @@ export interface CompiledAgentBrowserQaPreset extends CompiledAgentBrowserJob {
 		checkConsole: boolean;
 		checkErrors: boolean;
 		checkNetwork: boolean;
+		diagnosticsResetAtStart: boolean;
 		loadState: AgentBrowserQaLoadState;
 		expectedText: string[];
 		expectedSelector?: string;

package/extensions/agent-browser/lib/input-modes.ts

@@ -7,6 +7,7 @@
 export { AGENT_BROWSER_PARAMS } from "./input-modes/params.js";
 export {
 	analyzeQaPresetResults,
+	analyzeQaPresetTimeout,
 	buildQaCompactPassText,
 	compileAgentBrowserJob,
 	compileAgentBrowserQaPreset,

package/extensions/agent-browser/lib/json-schema.ts

@@ -0,0 +1,73 @@
+/**
+ * Purpose: Build the small JSON Schema subset used by Pi tool schemas without importing TypeBox at runtime.
+ * Responsibilities: Preserve plain JSON Schema objects Pi consumes while keeping extension startup cheap.
+ * Scope: Schema construction only; runtime validation still belongs to Pi and the tool input compilers.
+ */
+
+import type { TSchema, TSchemaOptions, TUnsafe } from "typebox";
+
+const OPTIONAL_SCHEMA = Symbol("pi-agent-browser-optional-schema");
+
+type SchemaObject = TSchema & { [OPTIONAL_SCHEMA]?: true };
+type SchemaProperties = Record<string, TSchema>;
+
+function withOptions(schema: Record<string, unknown>, options?: TSchemaOptions): TSchema {
+	return { ...schema, ...(options ?? {}) } as TSchema;
+}
+
+function literalType(value: unknown): "boolean" | "number" | "string" | undefined {
+	const valueType = typeof value;
+	return valueType === "string" || valueType === "number" || valueType === "boolean" ? valueType : undefined;
+}
+
+function propertySchema(schema: TSchema): TSchema {
+	const clone = { ...(schema as SchemaObject & Record<PropertyKey, unknown>) };
+	delete clone[OPTIONAL_SCHEMA];
+	return clone as TSchema;
+}
+
+export const JsonSchema = {
+	Array(items: TSchema, options?: TSchemaOptions): TSchema {
+		return withOptions({ type: "array", items }, options);
+	},
+	Boolean(options?: TSchemaOptions): TSchema {
+		return withOptions({ type: "boolean" }, options);
+	},
+	Integer(options?: TSchemaOptions): TSchema {
+		return withOptions({ type: "integer" }, options);
+	},
+	Literal(value: unknown, options?: TSchemaOptions): TSchema {
+		const type = literalType(value);
+		return withOptions(type ? { type, const: value } : { const: value }, options);
+	},
+	Number(options?: TSchemaOptions): TSchema {
+		return withOptions({ type: "number" }, options);
+	},
+	Object(properties: SchemaProperties, options?: TSchemaOptions): TSchema {
+		const required = globalThis.Object.entries(properties)
+			.filter(([, schema]) => (schema as SchemaObject)[OPTIONAL_SCHEMA] !== true)
+			.map(([key]) => key);
+		return withOptions({
+			type: "object",
+			properties: globalThis.Object.fromEntries(
+				globalThis.Object.entries(properties).map(([key, schema]) => [key, propertySchema(schema)]),
+			),
+			...(required.length > 0 ? { required } : {}),
+		}, options);
+	},
+	Optional(schema: TSchema): TSchema {
+		return { ...(schema as SchemaObject), [OPTIONAL_SCHEMA]: true } as TSchema;
+	},
+	String(options?: TSchemaOptions): TSchema {
+		return withOptions({ type: "string" }, options);
+	},
+	Union(types: TSchema[], options?: TSchemaOptions): TSchema {
+		return withOptions({ anyOf: types }, options);
+	},
+	Unsafe<Value>(schema: TSchema): TUnsafe<Value> {
+		return schema as TUnsafe<Value>;
+	},
+};
+
+export type JsonSchemaBuilder = typeof JsonSchema;
+export type { TSchema, TSchemaOptions, TUnsafe };