pi-agent-browser-native 0.2.31 → 0.2.33

package/extensions/agent-browser/lib/input-modes/semantic-action.ts

@@ -0,0 +1,107 @@
+/**
+ * Purpose: Compile semanticAction shorthand inputs into upstream agent-browser commands.
+ * Responsibilities: Validate shorthand locator/action fields and expose compiled-command helpers.
+ * Scope: semanticAction mode only.
+ */
+
+import { isRecord } from "../parsing.js";
+import { getSelectValues } from "./shared.js";
+import {
+	AGENT_BROWSER_SEMANTIC_ACTIONS,
+	AGENT_BROWSER_SEMANTIC_LOCATORS,
+	type AgentBrowserSemanticActionName,
+	type AgentBrowserSemanticLocator,
+	type CompiledAgentBrowserSemanticAction,
+} from "./types.js";
+
+export function getCompiledSemanticActionCommandIndex(compiled: CompiledAgentBrowserSemanticAction): number {
+	return compiled.args[0] === "--session" ? 2 : 0;
+}
+
+export function getCompiledSemanticActionSessionPrefix(compiled: CompiledAgentBrowserSemanticAction): string[] {
+	const commandIndex = getCompiledSemanticActionCommandIndex(compiled);
+	return commandIndex > 0 ? compiled.args.slice(0, commandIndex) : [];
+}
+
+export function isCompiledSemanticActionFindCommand(compiled: CompiledAgentBrowserSemanticAction | undefined): boolean {
+	if (!compiled) return false;
+	return compiled.args[getCompiledSemanticActionCommandIndex(compiled)] === "find";
+}
+
+export function compileAgentBrowserSemanticAction(input: unknown): { compiled?: CompiledAgentBrowserSemanticAction; error?: string } {
+	if (!isRecord(input)) {
+		return { error: "semanticAction must be an object." };
+	}
+	const action = input.action;
+	const locator = input.locator;
+	const value = input.value;
+	const values = input.values;
+	const selector = input.selector;
+	const text = input.text;
+	const role = input.role;
+	const name = input.name;
+	const session = input.session;
+	if (typeof action !== "string" || !AGENT_BROWSER_SEMANTIC_ACTIONS.includes(action as AgentBrowserSemanticActionName)) {
+		return { error: `semanticAction.action must be one of: ${AGENT_BROWSER_SEMANTIC_ACTIONS.join(", ")}.` };
+	}
+	if (session !== undefined && (typeof session !== "string" || session.trim().length === 0)) {
+		return { error: "semanticAction.session must be a non-empty string when provided." };
+	}
+	if (action === "select") {
+		if (locator !== undefined || role !== undefined || name !== undefined) {
+			return { error: "semanticAction.locator, role, and name are not supported for select; use selector plus value or values." };
+		}
+		if (text !== undefined) {
+			return { error: "semanticAction.text is not supported for select; use value or values for option values." };
+		}
+		if (typeof selector !== "string" || selector.trim().length === 0) {
+			return { error: "semanticAction.selector is required for select." };
+		}
+		const selectedValues = getSelectValues(input, "semanticAction");
+		if (selectedValues.error) return { error: selectedValues.error };
+		const args = typeof session === "string" ? ["--session", session, "select", selector, ...(selectedValues.values as string[])] : ["select", selector, ...(selectedValues.values as string[])];
+		return { compiled: { action: "select", selector, values: selectedValues.values, args } };
+	}
+	if (selector !== undefined || values !== undefined) {
+		return { error: "semanticAction.selector and values are only supported for select actions." };
+	}
+	if (typeof locator !== "string" || !AGENT_BROWSER_SEMANTIC_LOCATORS.includes(locator as AgentBrowserSemanticLocator)) {
+		return { error: `semanticAction.locator must be one of: ${AGENT_BROWSER_SEMANTIC_LOCATORS.join(", ")}.` };
+	}
+	if (value !== undefined && (typeof value !== "string" || value.trim().length === 0)) {
+		return { error: "semanticAction.value must be a non-empty string when provided." };
+	}
+	if (role !== undefined && (typeof role !== "string" || role.trim().length === 0)) {
+		return { error: "semanticAction.role must be a non-empty string when provided." };
+	}
+	const locatorValue = locator === "role" && typeof role === "string" ? role : value;
+	if (typeof locatorValue !== "string" || locatorValue.trim().length === 0) {
+		return { error: locator === "role" ? "semanticAction.value or semanticAction.role must be a non-empty string for locator=role." : "semanticAction.value must be a non-empty string." };
+	}
+	if (text !== undefined && typeof text !== "string") {
+		return { error: "semanticAction.text must be a string when provided." };
+	}
+	if (action === "fill" && (typeof text !== "string" || text.length === 0)) {
+		return { error: `semanticAction.text is required for ${action}.` };
+	}
+	if (action !== "fill" && text !== undefined) {
+		return { error: "semanticAction.text is only supported for fill actions." };
+	}
+	if (role !== undefined && locator !== "role") {
+		return { error: "semanticAction.role is only supported for locator=role." };
+	}
+	if (role !== undefined && value !== undefined && role !== value) {
+		return { error: "semanticAction.role must match value when both are provided for locator=role." };
+	}
+	if (name !== undefined && (locator !== "role" || typeof name !== "string" || name.length === 0)) {
+		return { error: "semanticAction.name is only supported as a non-empty string for locator=role." };
+	}
+	const args = typeof session === "string" ? ["--session", session, "find", locator, locatorValue, action] : ["find", locator, locatorValue, action];
+	if (action === "fill") {
+		args.push(text as string);
+	}
+	if (locator === "role" && typeof name === "string") {
+		args.push("--name", name);
+	}
+	return { compiled: { action: action as AgentBrowserSemanticActionName, locator: locator as AgentBrowserSemanticLocator, args } };
+}

package/extensions/agent-browser/lib/input-modes/shared.ts

@@ -0,0 +1,46 @@
+/**
+ * Purpose: Hold tiny shared parsing helpers for structured agent_browser input modes.
+ * Responsibilities: Normalize common select values, batch result rows, and workspace scan limits.
+ * Scope: Generic input-mode helpers only; mode-specific policy stays in the owning module.
+ */
+
+import { isRecord } from "../parsing.js";
+import { SOURCE_LOOKUP_DEFAULT_MAX_WORKSPACE_FILES, SOURCE_LOOKUP_MAX_WORKSPACE_FILES } from "./types.js";
+
+export function getSelectValues(input: Record<string, unknown>, context: string): { values?: string[]; error?: string } {
+	const rawValue = input.value;
+	const rawValues = input.values;
+	if (rawValue !== undefined && rawValues !== undefined) {
+		return { error: `${context}.value and ${context}.values cannot both be provided for select.` };
+	}
+	if (rawValues !== undefined) {
+		if (!Array.isArray(rawValues) || rawValues.length === 0 || rawValues.some((value) => typeof value !== "string" || value.trim().length === 0)) {
+			return { error: `${context}.values must be a non-empty array of non-empty strings for select.` };
+		}
+		return { values: rawValues };
+	}
+	if (typeof rawValue === "string" && rawValue.trim().length > 0) {
+		return { values: [rawValue] };
+	}
+	return { error: `${context}.value or ${context}.values is required for select.` };
+}
+
+export function getBatchResultItems(data: unknown): Array<Record<string, unknown>> {
+	return Array.isArray(data) ? data.filter(isRecord) : [];
+}
+
+export function getCommandNameFromBatchItem(item: Record<string, unknown>): string | undefined {
+	const command = item.command;
+	return Array.isArray(command) && typeof command[0] === "string" ? command[0] : undefined;
+}
+
+export function validateLookupMaxWorkspaceFiles(value: unknown, fieldName: string): { value?: number; error?: string } {
+	if (value === undefined) return { value: SOURCE_LOOKUP_DEFAULT_MAX_WORKSPACE_FILES };
+	if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
+		return { error: `${fieldName} must be a positive integer when provided.` };
+	}
+	if (value > SOURCE_LOOKUP_MAX_WORKSPACE_FILES) {
+		return { error: `${fieldName} must be ${SOURCE_LOOKUP_MAX_WORKSPACE_FILES} or less.` };
+	}
+	return { value };
+}

package/extensions/agent-browser/lib/input-modes/types.ts

@@ -0,0 +1,221 @@
+/**
+ * Purpose: Define structured agent_browser input-mode constants and TypeScript contracts.
+ * Responsibilities: Share schema enums and compiled input-mode result types across input-mode modules.
+ * Scope: Types and constants only; validation and compilation live in sibling modules.
+ */
+
+export const DEFAULT_SESSION_MODE = "auto" as const;
+
+export const AGENT_BROWSER_SEMANTIC_ACTIONS = ["check", "click", "fill", "select", "uncheck"] as const;
+
+export const AGENT_BROWSER_SEMANTIC_LOCATORS = ["alt", "label", "placeholder", "role", "testid", "text", "title"] as const;
+
+export const AGENT_BROWSER_JOB_STEP_ACTIONS = ["open", "click", "fill", "select", "wait", "assertText", "assertUrl", "waitForDownload", "screenshot"] as const;
+
+export const AGENT_BROWSER_QA_LOAD_STATES = ["domcontentloaded", "load", "networkidle"] as const;
+
+export const AGENT_BROWSER_ELECTRON_ACTIONS = ["list", "launch", "status", "cleanup", "probe"] as const;
+
+export const AGENT_BROWSER_ELECTRON_HANDOFFS = ["connect", "tabs", "snapshot"] as const;
+
+export const AGENT_BROWSER_ELECTRON_TARGET_TYPES = ["page", "webview", "any"] as const;
+
+export const AGENT_BROWSER_ELECTRON_LIST_FIELDS = new Set(["action", "query", "maxResults"]);
+
+export const AGENT_BROWSER_ELECTRON_PROBE_FIELDS = new Set(["action", "launchId", "timeoutMs"]);
+
+export const AGENT_BROWSER_ELECTRON_RESERVED_APP_ARGS = ["--user-data-dir", "--remote-debugging-port", "--remote-debugging-address", "--remote-debugging-pipe"] as const;
+
+export const SOURCE_LOOKUP_WORKSPACE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx"]);
+
+export const SOURCE_LOOKUP_IGNORED_DIRECTORIES = new Set([".git", "node_modules", "dist", "build", "coverage", ".next", "out", "tmp", "temp"]);
+
+export const SOURCE_LOOKUP_DEFAULT_MAX_WORKSPACE_FILES = 2_000;
+
+export const SOURCE_LOOKUP_MAX_WORKSPACE_FILES = 5_000;
+
+export type AgentBrowserSemanticActionName = (typeof AGENT_BROWSER_SEMANTIC_ACTIONS)[number];
+
+export type AgentBrowserSemanticLocator = (typeof AGENT_BROWSER_SEMANTIC_LOCATORS)[number];
+
+export type AgentBrowserJobStepAction = (typeof AGENT_BROWSER_JOB_STEP_ACTIONS)[number];
+
+export type AgentBrowserQaLoadState = (typeof AGENT_BROWSER_QA_LOAD_STATES)[number];
+
+export type AgentBrowserElectronAction = (typeof AGENT_BROWSER_ELECTRON_ACTIONS)[number];
+
+export type AgentBrowserSourceLookupStatus = "candidates-found" | "no-candidates" | "unsupported";
+
+export type AgentBrowserNetworkSourceLookupStatus = "failed-requests-found" | "no-failed-requests" | "no-candidates";
+
+export type CompiledAgentBrowserElectron =
+	| {
+		action: "list";
+		maxResults?: number;
+		query?: string;
+	}
+	| {
+		action: "launch";
+		allow?: string[];
+		appArgs?: string[];
+		deny?: string[];
+		appName?: string;
+		appPath?: string;
+		bundleId?: string;
+		executablePath?: string;
+		handoff: "connect" | "snapshot" | "tabs";
+		targetType: "any" | "page" | "webview";
+		timeoutMs?: number;
+	}
+	| {
+		action: "cleanup" | "status";
+		all?: boolean;
+		launchId?: string;
+		timeoutMs?: number;
+	}
+	| {
+		action: "probe";
+		launchId?: string;
+		timeoutMs?: number;
+	};
+
+export interface AgentBrowserSemanticActionInput {
+	action: AgentBrowserSemanticActionName;
+	locator?: AgentBrowserSemanticLocator;
+	value?: string;
+	values?: string[];
+	selector?: string;
+	text?: string;
+	role?: string;
+	name?: string;
+	session?: string;
+}
+
+export interface CompiledAgentBrowserSemanticAction {
+	action: AgentBrowserSemanticActionName;
+	locator?: AgentBrowserSemanticLocator;
+	selector?: string;
+	values?: string[];
+	args: string[];
+}
+
+export interface CompiledAgentBrowserJobStep {
+	action: AgentBrowserJobStepAction;
+	args: string[];
+}
+
+export interface CompiledAgentBrowserJob {
+	args: string[];
+	stdin: string;
+	steps: CompiledAgentBrowserJobStep[];
+}
+
+export interface CompiledAgentBrowserQaPreset extends CompiledAgentBrowserJob {
+	checks: {
+		checkConsole: boolean;
+		checkErrors: boolean;
+		checkNetwork: boolean;
+		loadState: AgentBrowserQaLoadState;
+		expectedText: string[];
+		expectedSelector?: string;
+		screenshotPath?: string;
+		attached: boolean;
+		url?: string;
+	};
+}
+
+export interface CompiledAgentBrowserSourceLookupStep {
+	action: "dom" | "react";
+	args: string[];
+}
+
+export interface CompiledAgentBrowserSourceLookup {
+	args: string[];
+	stdin: string;
+	steps: CompiledAgentBrowserSourceLookupStep[];
+	query: {
+		componentName?: string;
+		includeDomHints: boolean;
+		maxWorkspaceFiles: number;
+		reactFiberId?: string;
+		selector?: string;
+	};
+}
+
+export interface AgentBrowserSourceLookupCandidate {
+	column?: number;
+	componentName?: string;
+	confidence: "high" | "medium" | "low";
+	evidence: string[];
+	file?: string;
+	line?: number;
+	source: "react-inspect" | "dom-attribute" | "workspace-search";
+}
+
+export interface AgentBrowserSourceLookupElectronContext {
+	appName?: string;
+	appPath?: string;
+	executablePath?: string;
+	launchId?: string;
+	sessionName?: string;
+	url?: string;
+}
+
+export interface AgentBrowserSourceLookupAnalysis {
+	candidates: AgentBrowserSourceLookupCandidate[];
+	electronContext?: AgentBrowserSourceLookupElectronContext;
+	limitations: string[];
+	status: AgentBrowserSourceLookupStatus;
+	summary: string;
+	workspaceRoot?: string;
+}
+
+export interface AgentBrowserSourceLookupAnalysisContext {
+	electronContext?: AgentBrowserSourceLookupElectronContext;
+	workspaceRoot: string;
+}
+
+export interface CompiledAgentBrowserNetworkSourceLookup {
+	args: string[];
+	stdin: string;
+	steps: Array<{ action: "network"; args: string[] }>;
+	query: {
+		filter?: string;
+		maxWorkspaceFiles: number;
+		requestId?: string;
+		session?: string;
+		url?: string;
+	};
+}
+
+export interface AgentBrowserNetworkSourceLookupRequest {
+	error?: string;
+	method?: string;
+	requestId?: string;
+	status?: number;
+	url?: string;
+}
+
+export interface AgentBrowserNetworkSourceLookupCandidate {
+	confidence: "high" | "medium" | "low";
+	evidence: string[];
+	file?: string;
+	line?: number;
+	requestUrl?: string;
+	source: "initiator" | "workspace-search";
+}
+
+export interface AgentBrowserNetworkSourceLookupAnalysis {
+	candidates: AgentBrowserNetworkSourceLookupCandidate[];
+	failedRequests: AgentBrowserNetworkSourceLookupRequest[];
+	limitations: string[];
+	status: AgentBrowserNetworkSourceLookupStatus;
+	summary: string;
+}
+
+export interface AgentBrowserQaPresetAnalysis {
+	failedChecks: string[];
+	passed: boolean;
+	summary: string;
+	warnings: string[];
+}

package/extensions/agent-browser/lib/input-modes.ts

@@ -0,0 +1,41 @@
+/**
+ * Purpose: Re-export focused input-mode schema, compiler, and analysis modules for the agent_browser wrapper.
+ * Responsibilities: Preserve the extension entrypoint's import surface while keeping input-mode code split by concern.
+ * Scope: Barrel only; add behavior to focused files under ./input-modes/.
+ */
+
+export { AGENT_BROWSER_PARAMS } from "./input-modes/params.js";
+export {
+	analyzeQaPresetResults,
+	compileAgentBrowserJob,
+	compileAgentBrowserQaPreset,
+} from "./input-modes/job.js";
+export {
+	analyzeNetworkSourceLookupResults,
+	analyzeSourceLookupResults,
+	compileAgentBrowserNetworkSourceLookup,
+	compileAgentBrowserSourceLookup,
+	redactNetworkSourceLookupAnalysis,
+	redactNetworkSourceLookupArgs,
+	redactNetworkSourceLookupSurface,
+	redactNetworkSourceLookupUrl,
+} from "./input-modes/lookups.js";
+export { compileAgentBrowserElectron } from "./input-modes/electron.js";
+export {
+	compileAgentBrowserSemanticAction,
+	getCompiledSemanticActionCommandIndex,
+	getCompiledSemanticActionSessionPrefix,
+	isCompiledSemanticActionFindCommand,
+} from "./input-modes/semantic-action.js";
+export type {
+	AgentBrowserNetworkSourceLookupAnalysis,
+	AgentBrowserQaPresetAnalysis,
+	AgentBrowserSourceLookupAnalysis,
+	AgentBrowserSourceLookupElectronContext,
+	CompiledAgentBrowserElectron,
+	CompiledAgentBrowserJob,
+	CompiledAgentBrowserNetworkSourceLookup,
+	CompiledAgentBrowserQaPreset,
+	CompiledAgentBrowserSemanticAction,
+	CompiledAgentBrowserSourceLookup,
+} from "./input-modes/types.js";