pi-agent-browser-native 0.1.6 → 0.2.1

package/extensions/agent-browser/lib/results/presentation.ts

@@ -9,9 +9,18 @@
 import { readFile, stat } from "node:fs/promises";
 import { resolve } from "node:path";
 
-import type { CommandInfo } from "../runtime.js";
+import { parseCommandInfo, type CommandInfo } from "../runtime.js";
 import { buildSnapshotPresentation, formatRawSnapshotText, formatSnapshotSummary } from "./snapshot.js";
-import { type AgentBrowserBatchResult, type AgentBrowserEnvelope, type ToolPresentation, isRecord, parsePositiveInteger, stringifyUnknown } from "./shared.js";
+import {
+	type AgentBrowserBatchResult,
+	type AgentBrowserEnvelope,
+	type BatchFailurePresentationDetails,
+	type BatchStepPresentationDetails,
+	type ToolPresentation,
+	isRecord,
+	parsePositiveInteger,
+	stringifyUnknown,
+} from "./shared.js";
 
 const IMAGE_EXTENSION_TO_MIME_TYPE: Record<string, string> = {
 	".gif": "image/gif",
@@ -23,6 +32,13 @@ const IMAGE_EXTENSION_TO_MIME_TYPE: Record<string, string> = {
 
 const INLINE_IMAGE_MAX_BYTES_ENV = "PI_AGENT_BROWSER_INLINE_IMAGE_MAX_BYTES";
 const DEFAULT_INLINE_IMAGE_MAX_BYTES = 5 * 1_024 * 1_024;
+const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
+const NAVIGATION_SUMMARY_FIELD = "navigationSummary";
+
+interface NavigationSummary {
+	title?: string;
+	url?: string;
+}
 
 function getImageMimeType(filePath: string): string | undefined {
 	const extension = filePath.toLowerCase().slice(filePath.lastIndexOf("."));
@@ -90,24 +106,241 @@ function getScreenshotSummary(data: Record<string, unknown>): string | undefined
 	return typeof data.path === "string" ? `Saved image: ${data.path}` : undefined;
 }
 
-function formatBatchContent(data: AgentBrowserBatchResult[]): string {
-	return data
-		.map((item, index) => {
-			const command = Array.isArray(item.command) ? item.command.join(" ") : `step-${index + 1}`;
-			if (item.success === false) {
-				return `${command}\nError: ${stringifyUnknown(item.error)}`;
-			}
-			return `${command}\n${stringifyUnknown(item.result)}`;
-		})
+function isNavigationObservableCommand(command: string | undefined): boolean {
+	return command !== undefined && NAVIGATION_SUMMARY_COMMANDS.has(command);
+}
+
+function isNavigationSummary(value: unknown): value is NavigationSummary {
+	return isRecord(value) && (typeof value.title === "string" || typeof value.url === "string");
+}
+
+function getNavigationSummary(data: Record<string, unknown>): NavigationSummary | undefined {
+	const candidate = data[NAVIGATION_SUMMARY_FIELD];
+	return isNavigationSummary(candidate) ? candidate : undefined;
+}
+
+function formatNavigationSummary(summary: NavigationSummary): string | undefined {
+	const title = typeof summary.title === "string" && summary.title.trim().length > 0 ? summary.title.trim() : undefined;
+	const url = typeof summary.url === "string" && summary.url.trim().length > 0 ? summary.url.trim() : undefined;
+	if (!title && !url) return undefined;
+	if (title && url) return `${title}\n${url}`;
+	return title ?? url;
+}
+
+function stripNavigationSummary(data: Record<string, unknown>): Record<string, unknown> {
+	const { [NAVIGATION_SUMMARY_FIELD]: _navigationSummary, ...rest } = data;
+	return rest;
+}
+
+function formatNavigationActionResult(data: Record<string, unknown>): string | undefined {
+	const actionData = stripNavigationSummary(data);
+	const lines: string[] = [];
+	if (typeof actionData.clicked === "string" || typeof actionData.clicked === "boolean") {
+		lines.push(`Clicked: ${String(actionData.clicked)}`);
+	}
+	if (typeof actionData.href === "string") {
+		lines.push(`Href: ${actionData.href}`);
+	}
+	if (typeof actionData.navigated === "boolean") {
+		lines.push(`Navigated: ${actionData.navigated}`);
+	}
+	if (lines.length > 0) {
+		return lines.join("\n");
+	}
+
+	const actionText = stringifyUnknown(actionData).trim();
+	if (actionText.length === 0 || actionText === "{}") {
+		return undefined;
+	}
+	return actionText;
+}
+
+function isStringArray(value: unknown): value is string[] {
+	return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+
+function getPresentationText(presentation: ToolPresentation): string {
+	return presentation.content
+		.filter((part): part is Extract<ToolPresentation["content"][number], { type: "text" }> => part.type === "text")
+		.map((part) => part.text.trim())
+		.filter((text) => text.length > 0)
 		.join("\n\n");
 }
 
+function getPresentationImages(presentation: ToolPresentation): Array<Extract<ToolPresentation["content"][number], { type: "image" }>> {
+	return presentation.content.filter(
+		(part): part is Extract<ToolPresentation["content"][number], { type: "image" }> => part.type === "image",
+	);
+}
+
+function getPresentationPaths(options: {
+	primaryPath?: string;
+	secondaryPaths?: string[];
+}): string[] {
+	return options.secondaryPaths ?? (options.primaryPath ? [options.primaryPath] : []);
+}
+
+function formatBatchStepCommand(command: string[] | undefined, index: number): string {
+	return command && command.length > 0 ? command.join(" ") : `step-${index + 1}`;
+}
+
+function formatBatchStepError(error: unknown): string {
+	const errorText = stringifyUnknown(error).trim();
+	return errorText.length > 0 ? `Error: ${errorText}` : "Error: batch step failed.";
+}
+
+function getBatchFailureDetails(steps: Array<{ details: BatchStepPresentationDetails }>): BatchFailurePresentationDetails | undefined {
+	const failedSteps = steps.filter((step) => step.details.success === false);
+	if (failedSteps.length === 0) {
+		return undefined;
+	}
+	const successCount = steps.length - failedSteps.length;
+	return {
+		failedStep: failedSteps[0].details,
+		failureCount: failedSteps.length,
+		successCount,
+		totalCount: steps.length,
+	};
+}
+
+async function buildBatchStepPresentation(options: {
+	cwd: string;
+	index: number;
+	item: AgentBrowserBatchResult;
+}): Promise<{ details: BatchStepPresentationDetails; presentation: ToolPresentation }> {
+	const { cwd, index, item } = options;
+	const command = isStringArray(item.command) ? item.command : undefined;
+	const commandText = formatBatchStepCommand(command, index);
+
+	if (item.success === false) {
+		const errorText = formatBatchStepError(item.error);
+		const presentation: ToolPresentation = {
+			content: [{ type: "text", text: errorText }],
+			summary: errorText,
+		};
+		return {
+			details: {
+				command,
+				commandText,
+				data: item.error,
+				index,
+				success: false,
+				summary: errorText,
+				text: errorText,
+			},
+			presentation,
+		};
+	}
+
+	const presentation = await buildToolPresentation({
+		commandInfo: parseCommandInfo(command ?? []),
+		cwd,
+		envelope: { data: item.result, success: true },
+	});
+	const fullOutputPaths = getPresentationPaths({
+		primaryPath: presentation.fullOutputPath,
+		secondaryPaths: presentation.fullOutputPaths,
+	});
+	const imagePaths = getPresentationPaths({
+		primaryPath: presentation.imagePath,
+		secondaryPaths: presentation.imagePaths,
+	});
+	const text = getPresentationText(presentation) || presentation.summary;
+
+	return {
+		details: {
+			command,
+			commandText,
+			data: presentation.data,
+			fullOutputPath: fullOutputPaths[0],
+			fullOutputPaths: fullOutputPaths.length > 0 ? fullOutputPaths : undefined,
+			imagePath: imagePaths[0],
+			imagePaths: imagePaths.length > 0 ? imagePaths : undefined,
+			index,
+			success: true,
+			summary: presentation.summary,
+			text,
+		},
+		presentation,
+	};
+}
+
+async function buildBatchPresentation(options: {
+	cwd: string;
+	data: AgentBrowserBatchResult[];
+	summary: string;
+}): Promise<ToolPresentation> {
+	const { cwd, data, summary } = options;
+	const steps: Array<{ details: BatchStepPresentationDetails; presentation: ToolPresentation }> = [];
+	for (const [index, item] of data.entries()) {
+		steps.push(await buildBatchStepPresentation({ cwd, index, item }));
+	}
+
+	const batchFailure = getBatchFailureDetails(steps);
+	const images = steps.flatMap((step) => getPresentationImages(step.presentation));
+	const fullOutputPaths = steps.flatMap((step) => getPresentationPaths({
+		primaryPath: step.presentation.fullOutputPath,
+		secondaryPaths: step.presentation.fullOutputPaths,
+	}));
+	const imagePaths = steps.flatMap((step) => getPresentationPaths({
+		primaryPath: step.presentation.imagePath,
+		secondaryPaths: step.presentation.imagePaths,
+	}));
+	const stepText =
+		steps.length === 0
+			? "(no batch steps)"
+			: steps
+				.map(({ details, presentation }) => {
+					const inlineImageCount = getPresentationImages(presentation).length;
+					const status = details.success ? "succeeded" : "failed";
+					const lines = [`Step ${details.index + 1} — ${details.commandText} (${status})`];
+					if (details.text.length > 0) {
+						lines.push(details.text);
+					}
+					if (inlineImageCount > 0) {
+						lines.push(`(${inlineImageCount} inline image attachment${inlineImageCount === 1 ? "" : "s"} below)`);
+					}
+					return lines.join("\n");
+				})
+				.join("\n\n");
+	const failureHeader =
+		batchFailure === undefined
+			? undefined
+			: [
+					summary,
+					`First failing step: ${batchFailure.failedStep.index + 1} — ${batchFailure.failedStep.commandText}`,
+					batchFailure.failureCount > 1
+						? `${batchFailure.failureCount} steps failed. See the per-step results below.`
+						: "See the per-step results below.",
+				].join("\n");
+	const text = failureHeader ? `${failureHeader}\n\n${stepText}` : stepText;
+
+	return {
+		batchFailure,
+		batchSteps: steps.map((step) => step.details),
+		content: [{ type: "text", text }, ...images],
+		data,
+		fullOutputPath: fullOutputPaths[0],
+		fullOutputPaths: fullOutputPaths.length > 0 ? fullOutputPaths : undefined,
+		imagePath: imagePaths[0],
+		imagePaths: imagePaths.length > 0 ? imagePaths : undefined,
+		summary,
+	};
+}
+
 function formatSummary(commandInfo: CommandInfo, data: unknown): string {
 	if (Array.isArray(data) && commandInfo.command === "batch") {
 		const successCount = data.filter((item) => isRecord(item) && item.success !== false).length;
-		return `Batch: ${successCount}/${data.length} succeeded`;
+		return successCount === data.length ? `Batch: ${successCount}/${data.length} succeeded` : `Batch failed: ${successCount}/${data.length} succeeded`;
 	}
 	if (isRecord(data)) {
+		const navigationSummary = getNavigationSummary(data);
+		if (navigationSummary && isNavigationObservableCommand(commandInfo.command)) {
+			const navigationText = formatNavigationSummary(navigationSummary);
+			if (navigationText) {
+				return `${commandInfo.command ?? "navigation"} → ${navigationText.split("\n", 1)[0] ?? navigationText}`;
+			}
+		}
 		if (commandInfo.command === "snapshot") {
 			return formatSnapshotSummary(data);
 		}
@@ -136,9 +369,6 @@ function formatSummary(commandInfo: CommandInfo, data: unknown): string {
 }
 
 function formatContentText(commandInfo: CommandInfo, data: unknown): string {
-	if (Array.isArray(data) && commandInfo.command === "batch") {
-		return formatBatchContent(data as AgentBrowserBatchResult[]);
-	}
 	if (typeof data === "string") {
 		return data;
 	}
@@ -149,6 +379,15 @@ function formatContentText(commandInfo: CommandInfo, data: unknown): string {
 		return stringifyUnknown(data);
 	}
 
+	const navigationSummary = getNavigationSummary(data);
+	if (navigationSummary && isNavigationObservableCommand(commandInfo.command)) {
+		const navigationText = formatNavigationSummary(navigationSummary);
+		if (navigationText) {
+			const actionText = formatNavigationActionResult(data);
+			return actionText ? `${actionText}\n\nCurrent page:\n${navigationText}` : `Current page:\n${navigationText}`;
+		}
+	}
+
 	if (commandInfo.command === "snapshot") {
 		return formatRawSnapshotText(data);
 	}
@@ -232,13 +471,15 @@ export async function buildToolPresentation(options: {
 	const data = envelope?.data;
 	const summary = formatSummary(commandInfo, data);
 	const presentation =
-		commandInfo.command === "snapshot" && isRecord(data)
-			? await buildSnapshotPresentation(data)
-			: {
-					content: [{ type: "text" as const, text: formatContentText(commandInfo, data) }],
-					data,
-					summary,
-			  };
+		commandInfo.command === "batch" && Array.isArray(data)
+			? await buildBatchPresentation({ cwd, data: data as AgentBrowserBatchResult[], summary })
+			: commandInfo.command === "snapshot" && isRecord(data)
+				? await buildSnapshotPresentation(data)
+				: {
+						content: [{ type: "text" as const, text: formatContentText(commandInfo, data) }],
+						data,
+						summary,
+				  };
 
 	const imagePath = extractImagePath(cwd, data);
 	if (!imagePath) {

package/extensions/agent-browser/lib/results/shared.ts

@@ -19,11 +19,36 @@ export interface AgentBrowserBatchResult {
 	success?: boolean;
 }
 
+export interface BatchStepPresentationDetails {
+	command?: string[];
+	commandText: string;
+	data?: unknown;
+	fullOutputPath?: string;
+	fullOutputPaths?: string[];
+	imagePath?: string;
+	imagePaths?: string[];
+	index: number;
+	success: boolean;
+	summary: string;
+	text: string;
+}
+
+export interface BatchFailurePresentationDetails {
+	failedStep: BatchStepPresentationDetails;
+	failureCount: number;
+	successCount: number;
+	totalCount: number;
+}
+
 export interface ToolPresentation {
+	batchFailure?: BatchFailurePresentationDetails;
+	batchSteps?: BatchStepPresentationDetails[];
 	content: Array<{ text: string; type: "text" } | { data: string; mimeType: string; type: "image" }>;
 	data?: unknown;
 	fullOutputPath?: string;
+	fullOutputPaths?: string[];
 	imagePath?: string;
+	imagePaths?: string[];
 	summary: string;
 }
 

package/extensions/agent-browser/lib/results/snapshot.ts

@@ -12,11 +12,13 @@ import { type ToolPresentation, compareRefIds, countLines, isRecord, normalizeWh
 const SNAPSHOT_INLINE_MAX_CHARS = 6_000;
 const SNAPSHOT_INLINE_MAX_LINES = 80;
 const SNAPSHOT_INLINE_MAX_REFS = 60;
-const SNAPSHOT_PRIMARY_PREVIEW_LINES = 10;
-const SNAPSHOT_SECTION_PREVIEW_LINES = 4;
-const SNAPSHOT_MAX_ADDITIONAL_SECTIONS = 5;
-const SNAPSHOT_KEY_REF_MAX_LINES = 20;
-const SNAPSHOT_OTHER_REF_MAX_LINES = 12;
+const SNAPSHOT_PRIMARY_PREVIEW_LINES = 8;
+const SNAPSHOT_SECTION_PREVIEW_LINES = 2;
+const SNAPSHOT_MAX_ADDITIONAL_SECTIONS = 2;
+const SNAPSHOT_KEY_REF_MAX_LINES = 8;
+const SNAPSHOT_OTHER_REF_MAX_LINES = 4;
+const SNAPSHOT_ROLE_COUNT_MAX_ENTRIES = 4;
+const SNAPSHOT_FALLBACK_PREVIEW_MAX_LINES = 12;
 const SNAPSHOT_NAME_MAX_CHARS = 96;
 const SNAPSHOT_LINE_MAX_CHARS = 140;
 const SNAPSHOT_SPILL_FILE_PREFIX = "pi-agent-browser-snapshot";
@@ -165,7 +167,12 @@ function formatRoleCounts(roleCounts: Record<string, number>): string | undefine
 		if (right[1] !== left[1]) return right[1] - left[1];
 		return getRolePriority(left[0]) - getRolePriority(right[0]);
 	});
-	return ordered.map(([role, count]) => `${role} ${count}`).join(", ");
+	const visibleEntries = ordered.slice(0, SNAPSHOT_ROLE_COUNT_MAX_ENTRIES).map(([role, count]) => `${role} ${count}`);
+	const omittedEntries = Math.max(0, ordered.length - visibleEntries.length);
+	if (omittedEntries > 0) {
+		visibleEntries.push(`+${omittedEntries} more`);
+	}
+	return visibleEntries.join(", ");
 }
 
 function parseSnapshotLines(snapshot: string): SnapshotLine[] {
@@ -373,10 +380,10 @@ function buildSegmentPreview(segment: SnapshotSegment, maxLines: number): Snapsh
 
 function buildFallbackSnapshotOutline(snapshotLines: SnapshotLine[]): SnapshotPreview {
 	const selected = new Set<number>();
-	for (let index = 0; index < snapshotLines.length && selected.size < 6; index += 1) {
+	for (let index = 0; index < snapshotLines.length && selected.size < 4; index += 1) {
 		if (!isNoiseSnapshotLine(snapshotLines[index])) selected.add(index);
 	}
-	for (let index = 0; index < snapshotLines.length && selected.size < 18; index += 1) {
+	for (let index = 0; index < snapshotLines.length && selected.size < SNAPSHOT_FALLBACK_PREVIEW_MAX_LINES; index += 1) {
 		const line = snapshotLines[index];
 		if (isNoiseSnapshotLine(line)) continue;
 		if (SNAPSHOT_SIGNAL_ROLES.has(line.role) || line.ref || line.name.length > 0) {
@@ -385,7 +392,7 @@ function buildFallbackSnapshotOutline(snapshotLines: SnapshotLine[]): SnapshotPr
 	}
 	const chosenLines = [...selected]
 		.sort((left, right) => left - right)
-		.slice(0, 18)
+		.slice(0, SNAPSHOT_FALLBACK_PREVIEW_MAX_LINES)
 		.map((index) => snapshotLines[index]);
 	return {
 		omittedCount: Math.max(0, snapshotLines.length - chosenLines.length),
@@ -508,6 +515,8 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
 	const snapshotSegments = useStructuredPreview ? buildSnapshotSegments(snapshotLines) : [];
 	const primarySegment = useStructuredPreview ? choosePrimarySegment(snapshotSegments) : undefined;
 	const additionalSegments = useStructuredPreview ? chooseAdditionalSegments(snapshotSegments, primarySegment) : [];
+	const additionalSegmentCount = useStructuredPreview && primarySegment ? Math.max(0, snapshotSegments.length - 1) : 0;
+	const omittedAdditionalSectionCount = Math.max(0, additionalSegmentCount - additionalSegments.length);
 	const primaryPreview = primarySegment ? buildSegmentPreview(primarySegment, SNAPSHOT_PRIMARY_PREVIEW_LINES) : undefined;
 	const additionalPreviews = additionalSegments
 		.map((segment) => ({
@@ -548,11 +557,9 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
 	const lines: string[] = [
 		`Origin: ${origin}`,
 		`Refs: ${refEntries.length}`,
-		...(roleCountsText ? [`Roles: ${roleCountsText}`] : []),
+		...(roleCountsText ? [`Top roles: ${roleCountsText}`] : []),
 		"",
-		fullOutputPath
-			? `Compact snapshot view. Full raw snapshot: ${fullOutputPath}`
-			: `Compact snapshot view. Full raw snapshot unavailable: ${spillErrorText ?? "temp spill file could not be created."}`,
+		"Compact snapshot view.",
 	];
 
 	if (fallbackPreview) {
@@ -581,6 +588,9 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
 					lines.push(`- ... (${preview.omittedCount} more lines in this section)`);
 				}
 			});
+			if (omittedAdditionalSectionCount > 0) {
+				lines.push(`- ... (${omittedAdditionalSectionCount} more sections omitted)`);
+			}
 		}
 	}
 
@@ -589,11 +599,16 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
 		lines.push("", "Other refs:", ...otherRefEntries.map(formatCompactRef));
 	}
 	if (omittedOtherRefs > 0) {
-		lines.push(
-			`- ... (${omittedOtherRefs} additional refs ${fullOutputPath ? "in the full snapshot file" : "were omitted with the full raw snapshot"})`,
-		);
+		lines.push(`- ... (${omittedOtherRefs} additional refs omitted)`);
 	}
 
+	lines.push(
+		"",
+		fullOutputPath
+			? "Full raw snapshot path is available in details.fullOutputPath."
+			: `Full raw snapshot unavailable: ${spillErrorText ?? "temp spill file could not be created."}`,
+	);
+
 	return {
 		content: [{ type: "text", text: lines.join("\n") }],
 		data: {
@@ -603,6 +618,7 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
 			previewMode: fallbackPreview ? "outline" : "structured",
 			spillError: spillErrorText,
 			previewRefIds: [...previewRefIds],
+			additionalSectionsOmitted: omittedAdditionalSectionCount,
 			previewSections: [
 				...(primarySegment
 					? [