pi-agent-browser-native 0.2.32 → 0.2.34

package/extensions/agent-browser/lib/results/presentation/errors.ts

@@ -0,0 +1,125 @@
+import type { CommandInfo } from "../../runtime.js";
+import { redactModelFacingText } from "./common.js";
+import { buildAgentBrowserNextActions } from "../action-recommendations.js";
+import { buildAgentBrowserResultCategoryDetails } from "../categories.js";
+import type { AgentBrowserNextAction, ToolPresentation } from "../contracts.js";
+import { withOptionalSessionArgs } from "../next-actions.js";
+
+const STALE_REF_ERROR_HINT = [
+	"Agent-browser hint: This ref may be stale after navigation, scrolling, or re-rendering.",
+	"Run `snapshot -i` again and retry with a current `@e…` ref; for less ref churn, use `find role|text|label|placeholder|alt|title|testid ...` or `scrollintoview` before interacting with off-screen elements.",
+].join(" ");
+
+const SELECTOR_DIALECT_ERROR_HINT = [
+	"Agent-browser hint: This selector may use an unsupported selector dialect.",
+	"Prefer refs from `snapshot -i`, or use supported `find role|text|label|placeholder|alt|title|testid ...` locators; use `scrollintoview` before interacting with off-screen elements.",
+].join(" ");
+
+function getSelectorRecoveryHint(errorText: string): string | undefined {
+	const normalized = errorText.trim();
+	if (normalized.length === 0) return undefined;
+
+	if (/\bUnknown ref\b|\bstale ref\b|\bref\b.*\b(?:not found|missing|expired)\b/i.test(normalized)) {
+		return STALE_REF_ERROR_HINT;
+	}
+
+	const mentionsPlaywrightSelectorDialect = /(?:\btext=|:has-text\(|\bgetByRole\b|\bgetByText\b)/i.test(normalized);
+	const reportsSelectorMatchFailure =
+		/\b(?:no elements? found|failed to find|could not find|unable to find)\b.*\b(?:selector|locator)\b/i.test(normalized) ||
+		/\b(?:selector|locator)\b.*\b(?:no elements? found|not found|missing|failed to find|could not find|unable to find)\b/i.test(normalized);
+
+	if (
+		/\b(?:unsupported|unknown|invalid)\s+(?:selector|locator)\b/i.test(normalized) ||
+		/\bfailed to parse selector\b/i.test(normalized) ||
+		/\bselector\b.*\b(?:parse|syntax|unsupported|invalid)\b/i.test(normalized) ||
+		(mentionsPlaywrightSelectorDialect && reportsSelectorMatchFailure)
+	) {
+		return SELECTOR_DIALECT_ERROR_HINT;
+	}
+
+	return undefined;
+}
+
+interface CommandSuggestion {
+	args?: string[];
+	description: string;
+	id?: string;
+}
+
+const UNKNOWN_COMMAND_SUGGESTIONS: Record<string, CommandSuggestion[]> = {
+	attr: [{ description: "Use `get attr <selector> <name>` to read an attribute from a selector or current `@ref`." }],
+	count: [{ description: "Use `get count <selector>` to count matching elements." }],
+	html: [{ description: "Use `get html <selector>` to read element HTML, or `get html` for the page when upstream supports it." }],
+	text: [{ description: "Use `get text <selector>` to read text from a selector or current `@ref`; run `snapshot -i` first when you need a safe `@ref`." }],
+	title: [{ args: ["get", "title"], description: "Use `get title` to read the current page title.", id: "use-get-title" }],
+	url: [{ args: ["get", "url"], description: "Use `get url` to read the current page URL.", id: "use-get-url" }],
+	value: [{ description: "Use `get value <selector>` to read form control value from a selector or current `@ref`." }],
+};
+
+function getUnknownCommandSuggestions(command: string | undefined, errorText: string): CommandSuggestion[] {
+	if (!command) return [];
+	const normalizedCommand = command.trim().toLowerCase();
+	if (!/\bunknown\s+command\b|\bunknown\s+subcommand\b|\bunrecognized\s+command\b/i.test(errorText)) return [];
+	return UNKNOWN_COMMAND_SUGGESTIONS[normalizedCommand] ?? [];
+}
+
+function formatUnknownCommandSuggestionText(suggestions: CommandSuggestion[]): string | undefined {
+	if (suggestions.length === 0) return undefined;
+	return ["Agent-browser hint: This looks like a getter shortcut, but upstream getter commands are grouped under `get`.", ...suggestions.map((suggestion) => suggestion.description)].join(" ");
+}
+
+function buildUnknownCommandSuggestionActions(suggestions: CommandSuggestion[], sessionName: string | undefined): AgentBrowserNextAction[] | undefined {
+	const actions = suggestions
+		.filter((suggestion): suggestion is CommandSuggestion & { args: string[]; id: string } => suggestion.args !== undefined && suggestion.id !== undefined)
+		.map((suggestion) => ({
+			id: suggestion.id,
+			params: { args: withOptionalSessionArgs(sessionName, suggestion.args) },
+			reason: suggestion.description,
+			safety: "Read-only getter command; safe to retry when you intended to inspect page state.",
+			tool: "agent_browser" as const,
+		}));
+	return actions.length > 0 ? actions : undefined;
+}
+
+export function appendSelectorRecoveryHint(errorText: string): string {
+	const hint = getSelectorRecoveryHint(errorText);
+	if (!hint || errorText.includes("Agent-browser hint:")) return errorText;
+	return `${errorText}\n\n${hint}`;
+}
+
+export function buildErrorPresentation(options: {
+	args?: string[];
+	commandInfo: CommandInfo;
+	errorText: string;
+	sessionName?: string;
+}): ToolPresentation {
+	const { args, commandInfo, errorText, sessionName } = options;
+	const safeErrorText = redactModelFacingText(errorText);
+	const selectorHintedErrorText = appendSelectorRecoveryHint(safeErrorText);
+	const unknownCommandSuggestions = getUnknownCommandSuggestions(commandInfo.command, safeErrorText);
+	const unknownCommandSuggestionText = formatUnknownCommandSuggestionText(unknownCommandSuggestions);
+	const hintedErrorText = unknownCommandSuggestionText && !selectorHintedErrorText.includes("Agent-browser hint:")
+		? `${selectorHintedErrorText}\n\n${unknownCommandSuggestionText}`
+		: selectorHintedErrorText;
+	const categoryDetails = buildAgentBrowserResultCategoryDetails({
+		args: [commandInfo.command, commandInfo.subcommand].filter((item): item is string => item !== undefined),
+		command: commandInfo.command,
+		errorText: hintedErrorText,
+		succeeded: false,
+	});
+	const nextActions = [
+		...(buildUnknownCommandSuggestionActions(unknownCommandSuggestions, sessionName) ?? []),
+		...(buildAgentBrowserNextActions({
+			args,
+			command: commandInfo.command,
+			failureCategory: categoryDetails.failureCategory,
+			resultCategory: "failure",
+		}) ?? []),
+	];
+	return {
+		...categoryDetails,
+		content: [{ type: "text", text: hintedErrorText }],
+		nextActions: nextActions.length > 0 ? nextActions : undefined,
+		summary: hintedErrorText,
+	};
+}

package/extensions/agent-browser/lib/results/presentation/large-output.ts

@@ -0,0 +1,182 @@
+/**
+ * Purpose: Compact oversized model-facing tool output into bounded previews and spill artifacts.
+ * Responsibilities: Write full output to persistent/session temp storage, update artifact manifests, and preserve safe previews.
+ * Scope: Large-output compaction only.
+ */
+
+import type { CommandInfo } from "../../runtime.js";
+import {
+	type PersistentSessionArtifactEviction,
+	type PersistentSessionArtifactStore,
+	writePersistentSessionArtifactFile,
+	writeSecureTempFile,
+} from "../../temp.js";
+import { buildEvictedSessionArtifactEntries } from "../artifact-manifest.js";
+import type { ArtifactStorageScope, SessionArtifactManifest, SessionArtifactManifestEntry, ToolPresentation } from "../contracts.js";
+import { countLines, truncateText } from "../text.js";
+import { applyArtifactManifest } from "./artifacts.js";
+import { getPresentationText } from "./content.js";
+import { redactModelFacingText, stringifyModelFacing } from "./common.js";
+
+const LARGE_OUTPUT_INLINE_MAX_CHARS = 8_000;
+
+const LARGE_OUTPUT_INLINE_MAX_LINES = 120;
+
+const LARGE_OUTPUT_PREVIEW_MAX_CHARS = 2_500;
+
+const LARGE_OUTPUT_PREVIEW_MAX_LINES = 40;
+
+const LARGE_OUTPUT_FILE_PREFIX = "pi-agent-browser-output";
+
+function shouldCompactLargeOutput(text: string): boolean {
+	return text.length > LARGE_OUTPUT_INLINE_MAX_CHARS || countLines(text) > LARGE_OUTPUT_INLINE_MAX_LINES;
+}
+
+function buildLargeOutputPreview(text: string): { omittedLineCount: number; previewText: string } {
+	const lines = text.split("\n");
+	const previewLines: string[] = [];
+	let previewChars = 0;
+	for (const line of lines) {
+		if (previewLines.length >= LARGE_OUTPUT_PREVIEW_MAX_LINES || previewChars >= LARGE_OUTPUT_PREVIEW_MAX_CHARS) {
+			break;
+		}
+		const remainingChars = LARGE_OUTPUT_PREVIEW_MAX_CHARS - previewChars;
+		const previewLine = truncateText(line, Math.max(40, remainingChars));
+		previewLines.push(previewLine);
+		previewChars += previewLine.length + 1;
+	}
+	return {
+		omittedLineCount: Math.max(0, lines.length - previewLines.length),
+		previewText: previewLines.join("\n"),
+	};
+}
+
+interface LargeOutputSpillWriteResult {
+	evictedArtifacts: PersistentSessionArtifactEviction[];
+	path: string;
+	storageScope: ArtifactStorageScope;
+}
+
+async function writeLargeOutputSpillFile(options: {
+	data: unknown;
+	persistentArtifactStore?: PersistentSessionArtifactStore;
+	text: string;
+}): Promise<LargeOutputSpillWriteResult> {
+	const payload =
+		typeof options.data === "string"
+			? redactModelFacingText(options.data)
+			: typeof options.data === "number" || typeof options.data === "boolean"
+				? String(options.data)
+				: options.data === undefined
+					? redactModelFacingText(options.text)
+					: stringifyModelFacing(options.data);
+	const isStructuredPayload = typeof options.data !== "string" && typeof options.data !== "number" && typeof options.data !== "boolean";
+	const fileOptions = {
+		content: payload,
+		prefix: LARGE_OUTPUT_FILE_PREFIX,
+		suffix: isStructuredPayload ? ".json" : ".txt",
+	};
+	if (options.persistentArtifactStore) {
+		const result = await writePersistentSessionArtifactFile({ ...fileOptions, store: options.persistentArtifactStore });
+		return { ...result, storageScope: "persistent-session" };
+	}
+	return { evictedArtifacts: [], path: await writeSecureTempFile(fileOptions), storageScope: "process-temp" };
+}
+
+function buildSpillArtifactEntries(options: {
+	commandInfo: CommandInfo;
+	evictedArtifacts: PersistentSessionArtifactEviction[];
+	path: string;
+	storageScope: ArtifactStorageScope;
+}): SessionArtifactManifestEntry[] {
+	const nowMs = Date.now();
+	return [
+		{
+			command: options.commandInfo.command,
+			createdAtMs: nowMs,
+			kind: "spill",
+			path: options.path,
+			retentionState: options.storageScope === "persistent-session" ? "live" : "ephemeral",
+			storageScope: options.storageScope,
+			subcommand: options.commandInfo.subcommand,
+		},
+		...buildEvictedSessionArtifactEntries(options.evictedArtifacts, nowMs),
+	];
+}
+
+export async function compactLargePresentationOutput(options: {
+	artifactManifest?: SessionArtifactManifest;
+	commandInfo: CommandInfo;
+	data: unknown;
+	persistentArtifactStore?: PersistentSessionArtifactStore;
+	presentation: ToolPresentation;
+}): Promise<ToolPresentation> {
+	const text = getPresentationText(options.presentation);
+	if (text.length === 0 || !shouldCompactLargeOutput(text)) {
+		return options.presentation;
+	}
+
+	let fullOutputPath: string | undefined;
+	let spill: LargeOutputSpillWriteResult | undefined;
+	let spillErrorText: string | undefined;
+	try {
+		spill = await writeLargeOutputSpillFile({
+			data: options.data,
+			persistentArtifactStore: options.persistentArtifactStore,
+			text,
+		});
+		fullOutputPath = spill.path;
+	} catch (error) {
+		spillErrorText = error instanceof Error ? error.message : String(error);
+	}
+
+	const { omittedLineCount, previewText } = buildLargeOutputPreview(text);
+	const commandLabel = options.commandInfo.command ?? "agent-browser";
+	const lines = [
+		`Large ${commandLabel} output compacted.`,
+		"",
+		"Preview:",
+		previewText,
+	];
+	if (omittedLineCount > 0) {
+		lines.push(`- ... (${omittedLineCount} additional lines omitted)`);
+	}
+	lines.push(
+		"",
+		fullOutputPath
+			? `Full output path: ${fullOutputPath}`
+			: `Full output unavailable: ${spillErrorText ?? "spill file could not be created."}`,
+	);
+
+	const firstTextIndex = options.presentation.content.findIndex((part) => part.type === "text");
+	const compactedText = lines.join("\n");
+	if (firstTextIndex >= 0) {
+		options.presentation.content[firstTextIndex] = { type: "text", text: compactedText };
+	} else {
+		options.presentation.content.unshift({ type: "text", text: compactedText });
+	}
+	options.presentation.data = {
+		compacted: true,
+		fullOutputPath,
+		outputCharCount: text.length,
+		outputLineCount: countLines(text),
+		previewCharCount: previewText.length,
+		previewLineCount: countLines(previewText),
+		spillError: spillErrorText,
+	};
+	options.presentation.fullOutputPath = fullOutputPath;
+	options.presentation.summary = `${options.presentation.summary} (compact)`;
+	if (fullOutputPath && spill) {
+		return applyArtifactManifest(
+			options.presentation,
+			options.presentation.artifactManifest ?? options.artifactManifest,
+			buildSpillArtifactEntries({
+				commandInfo: options.commandInfo,
+				evictedArtifacts: spill.evictedArtifacts,
+				path: fullOutputPath,
+				storageScope: spill.storageScope,
+			}),
+		);
+	}
+	return options.presentation;
+}

package/extensions/agent-browser/lib/results/presentation/navigation.ts

@@ -0,0 +1,216 @@
+/**
+ * Purpose: Format scalar extraction results, navigation summaries, and page-change summaries.
+ * Responsibilities: Keep navigation/extraction presentation separate from core tool result orchestration.
+ * Scope: Navigation and get/eval extraction formatting only.
+ */
+
+import { isRecord } from "../../parsing.js";
+import type { CommandInfo } from "../../runtime.js";
+import { detectConfirmationRequired } from "../confirmation.js";
+import type { AgentBrowserPageChangeSummary, FileArtifactMetadata } from "../contracts.js";
+import { redactModelFacingText, stringifyModelFacing } from "./common.js";
+
+const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
+
+const PAGE_CHANGE_SUMMARY_COMMANDS = new Set([
+	"back",
+	"check",
+	"click",
+	"dblclick",
+	"dialog",
+	"download",
+	"fill",
+	"forward",
+	"goto",
+	"hover",
+	"navigate",
+	"open",
+	"pdf",
+	"press",
+	"pushstate",
+	"reload",
+	"screenshot",
+	"scroll",
+	"scrollintoview",
+	"select",
+	"swipe",
+	"tap",
+	"type",
+	"uncheck",
+]);
+
+const NAVIGATION_SUMMARY_FIELD = "navigationSummary";
+
+interface NavigationSummary {
+	title?: string;
+	url?: string;
+}
+
+function getScalarExtractionResult(data: Record<string, unknown>): string | undefined {
+	const { result } = data;
+	if (typeof result === "string") {
+		return result.trim().length > 0 ? result : undefined;
+	}
+	if (typeof result === "number" || typeof result === "boolean") {
+		return String(result);
+	}
+	return undefined;
+}
+
+function getExtractionOrigin(data: Record<string, unknown>): string | undefined {
+	if (typeof data.origin === "string" && data.origin.trim().length > 0) {
+		return data.origin.trim();
+	}
+	if (typeof data.url === "string" && data.url.trim().length > 0) {
+		return data.url.trim();
+	}
+	return undefined;
+}
+
+function formatGetSummaryLabel(subcommand: string | undefined): string {
+	if (!subcommand) {
+		return "Get result";
+	}
+	if (subcommand.toLowerCase() === "url") {
+		return "URL";
+	}
+	return `${subcommand.slice(0, 1).toUpperCase()}${subcommand.slice(1)}`;
+}
+
+export function formatExtractionSummary(commandInfo: CommandInfo, data: Record<string, unknown>): string | undefined {
+	const scalarResult = getScalarExtractionResult(data);
+	if (!scalarResult) {
+		return undefined;
+	}
+	const safeScalarResult = redactModelFacingText(scalarResult);
+	const firstResultLine = safeScalarResult.split("\n", 1)[0] ?? safeScalarResult;
+	if (commandInfo.command === "get") {
+		return `${formatGetSummaryLabel(commandInfo.subcommand)}: ${firstResultLine}`;
+	}
+	if (commandInfo.command === "eval") {
+		return `Eval result: ${firstResultLine}`;
+	}
+	return undefined;
+}
+
+export function formatExtractionText(commandInfo: CommandInfo, data: Record<string, unknown>): string | undefined {
+	if (commandInfo.command !== "get" && commandInfo.command !== "eval") {
+		return undefined;
+	}
+	const scalarResult = getScalarExtractionResult(data);
+	if (!scalarResult) {
+		return undefined;
+	}
+	const origin = getExtractionOrigin(data);
+	const safeScalarResult = redactModelFacingText(scalarResult);
+	const safeOrigin = origin ? redactModelFacingText(origin) : undefined;
+	return safeOrigin && safeOrigin !== safeScalarResult ? `${safeScalarResult}\n\nOrigin: ${safeOrigin}` : safeScalarResult;
+}
+
+export function isNavigationObservableCommand(command: string | undefined): boolean {
+	return command !== undefined && NAVIGATION_SUMMARY_COMMANDS.has(command);
+}
+
+function isNavigationSummary(value: unknown): value is NavigationSummary {
+	return isRecord(value) && (typeof value.title === "string" || typeof value.url === "string");
+}
+
+export function getNavigationSummary(data: Record<string, unknown>): NavigationSummary | undefined {
+	const candidate = data[NAVIGATION_SUMMARY_FIELD];
+	return isNavigationSummary(candidate) ? candidate : undefined;
+}
+
+function getTopLevelNavigationSummary(data: Record<string, unknown>): NavigationSummary | undefined {
+	return isNavigationSummary(data)
+		? {
+			title: typeof data.title === "string" ? data.title : undefined,
+			url: typeof data.url === "string" ? data.url : undefined,
+		}
+		: undefined;
+}
+
+function getNormalizedNavigationSummary(summary: NavigationSummary | undefined): { title?: string; url?: string } | undefined {
+	const title = typeof summary?.title === "string" && summary.title.trim().length > 0 ? summary.title.trim() : undefined;
+	const url = typeof summary?.url === "string" && summary.url.trim().length > 0 ? summary.url.trim() : undefined;
+	return title || url ? { title, url } : undefined;
+}
+
+export function formatNavigationSummary(summary: NavigationSummary): string | undefined {
+	const normalized = getNormalizedNavigationSummary(summary);
+	if (!normalized) return undefined;
+	if (normalized.title && normalized.url) return `${normalized.title}\n${normalized.url}`;
+	return normalized.title ?? normalized.url;
+}
+
+function isPageChangeSummaryCommand(command: string | undefined): boolean {
+	return command !== undefined && PAGE_CHANGE_SUMMARY_COMMANDS.has(command);
+}
+
+export function buildPageChangeSummary(options: {
+	artifacts?: FileArtifactMetadata[];
+	commandInfo: CommandInfo;
+	data: unknown;
+	nextActions?: Array<{ id: string }>;
+	savedFilePath?: string;
+	summary: string;
+}): AgentBrowserPageChangeSummary | undefined {
+	const { artifacts, commandInfo, data, nextActions, savedFilePath } = options;
+	const artifactCount = artifacts?.length ?? 0;
+	const navigation = isRecord(data)
+		? getNormalizedNavigationSummary(getNavigationSummary(data) ?? (isPageChangeSummaryCommand(commandInfo.command) ? getTopLevelNavigationSummary(data) : undefined))
+		: undefined;
+	const confirmationRequired = detectConfirmationRequired(data) !== undefined;
+	if (!navigation && !confirmationRequired && artifactCount === 0 && !savedFilePath && !isPageChangeSummaryCommand(commandInfo.command)) {
+		return undefined;
+	}
+	const changeType: AgentBrowserPageChangeSummary["changeType"] = savedFilePath || artifactCount > 0
+		? "artifact"
+		: navigation
+			? "navigation"
+			: confirmationRequired
+				? "confirmation"
+				: "mutation";
+	const parts = [commandInfo.command ?? "agent-browser", changeType];
+	if (navigation?.title) parts.push(navigation.title);
+	if (navigation?.url) parts.push(navigation.url);
+	if (savedFilePath) parts.push(savedFilePath);
+	else if (artifactCount > 0) parts.push(`${artifactCount} artifact${artifactCount === 1 ? "" : "s"}`);
+	return {
+		...(artifactCount > 0 ? { artifactCount } : {}),
+		changeType,
+		...(commandInfo.command ? { command: commandInfo.command } : {}),
+		...(nextActions ? { nextActionIds: nextActions.map((action) => action.id) } : {}),
+		...(savedFilePath ? { savedFilePath } : {}),
+		summary: parts.join(" → "),
+		...(navigation?.title ? { title: navigation.title } : {}),
+		...(navigation?.url ? { url: navigation.url } : {}),
+	};
+}
+
+function stripNavigationSummary(data: Record<string, unknown>): Record<string, unknown> {
+	const { [NAVIGATION_SUMMARY_FIELD]: _navigationSummary, ...rest } = data;
+	return rest;
+}
+
+export function formatNavigationActionResult(data: Record<string, unknown>): string | undefined {
+	const actionData = stripNavigationSummary(data);
+	const lines: string[] = [];
+	if (typeof actionData.clicked === "string" || typeof actionData.clicked === "boolean") {
+		lines.push(`Clicked: ${String(actionData.clicked)}`);
+	}
+	if (typeof actionData.href === "string") {
+		lines.push(`Href: ${redactModelFacingText(actionData.href)}`);
+	}
+	if (typeof actionData.navigated === "boolean") {
+		lines.push(`Navigated: ${actionData.navigated}`);
+	}
+	if (lines.length > 0) {
+		return lines.join("\n");
+	}
+
+	const actionText = stringifyModelFacing(actionData).trim();
+	if (actionText.length === 0 || actionText === "{}") {
+		return undefined;
+	}
+	return actionText;
+}