pi-agent-browser-native 0.2.32 → 0.2.33

package/extensions/agent-browser/lib/results/contracts.ts

@@ -0,0 +1,220 @@
+/**
+ * Purpose: Define stable result-rendering data contracts shared across focused result modules.
+ * Responsibilities: Keep upstream envelope, presentation, artifact, category, and network shapes in one type-only surface.
+ * Scope: Types only; runtime classifiers, manifests, network rules, and text helpers live in neighboring modules.
+ * Usage: Imported with `import type` by result modules and re-exported by the public results facade.
+ * Invariants/Assumptions: This file has no runtime policy so adding fields cannot hide behavior in a catch-all module.
+ */
+
+import type { AgentBrowserNextAction } from "./next-actions.js";
+
+export type { AgentBrowserNextAction } from "./next-actions.js";
+
+export interface AgentBrowserEnvelope {
+	data?: unknown;
+	error?: unknown;
+	success: boolean;
+}
+
+export interface AgentBrowserBatchResult {
+	command?: string[];
+	error?: unknown;
+	result?: unknown;
+	success?: boolean;
+}
+
+export type AgentBrowserResultCategory = "failure" | "success";
+
+export type AgentBrowserSuccessCategory = "artifact-saved" | "artifact-unverified" | "completed" | "inspection";
+
+export type AgentBrowserFailureCategory =
+	| "aborted"
+	| "cleanup-failed"
+	| "confirmation-required"
+	| "download-not-verified"
+	| "missing-binary"
+	| "parse-failure"
+	| "policy-blocked"
+	| "qa-failure"
+	| "selector-not-found"
+	| "selector-unsupported"
+	| "stale-ref"
+	| "tab-drift"
+	| "timeout"
+	| "upstream-error"
+	| "validation-error";
+
+export interface AgentBrowserResultCategoryDetails {
+	failureCategory?: AgentBrowserFailureCategory;
+	resultCategory: AgentBrowserResultCategory;
+	successCategory?: AgentBrowserSuccessCategory;
+}
+
+export interface AgentBrowserPageChangeSummary {
+	artifactCount?: number;
+	changeType: "artifact" | "confirmation" | "mutation" | "navigation";
+	command?: string;
+	nextActionIds?: string[];
+	savedFilePath?: string;
+	summary: string;
+	title?: string;
+	url?: string;
+}
+
+export type FileArtifactKind = "download" | "file" | "har" | "image" | "pdf" | "profile" | "trace" | "video";
+
+export type FileArtifactStatus = "missing" | "repaired-from-temp" | "saved" | "upstream-temp-only";
+
+export interface FileArtifactMetadata {
+	absolutePath: string;
+	artifactType?: FileArtifactKind;
+	command?: string;
+	cwd?: string;
+	exists?: boolean;
+	extension?: string;
+	kind: FileArtifactKind;
+	mediaType?: string;
+	path: string;
+	requestedPath?: string;
+	session?: string;
+	sizeBytes?: number;
+	status?: FileArtifactStatus;
+	subcommand?: string;
+	tempPath?: string;
+}
+
+export type ArtifactVerificationState = "missing" | "pending" | "unverified" | "verified";
+
+export interface ArtifactVerificationEntry {
+	absolutePath?: string;
+	exists?: boolean;
+	kind: FileArtifactKind | "spill";
+	limitation?: string;
+	mediaType?: string;
+	path: string;
+	requestedPath?: string;
+	retentionState?: ArtifactRetentionState;
+	sizeBytes?: number;
+	state: ArtifactVerificationState;
+	status?: FileArtifactStatus;
+	storageScope?: ArtifactStorageScope;
+}
+
+export interface ArtifactVerificationSummary {
+	artifacts: ArtifactVerificationEntry[];
+	missingCount: number;
+	pendingCount: number;
+	unverifiedCount: number;
+	verified: boolean;
+	verifiedCount: number;
+}
+
+export interface SavedFilePresentationDetails {
+	command: "download" | "pdf" | "wait";
+	kind: "download" | "pdf";
+	metadata?: Record<string, unknown>;
+	path: string;
+	subcommand?: string;
+}
+
+export type ArtifactRetentionState = "evicted" | "ephemeral" | "live" | "missing";
+
+export type ArtifactStorageScope = "explicit-path" | "persistent-session" | "process-temp";
+
+export interface SessionArtifactManifestEntry {
+	absolutePath?: string;
+	command?: string;
+	createdAtMs: number;
+	cwd?: string;
+	evictedAtMs?: number;
+	exists?: boolean;
+	extension?: string;
+	kind: FileArtifactKind | "spill";
+	mediaType?: string;
+	path: string;
+	requestedPath?: string;
+	retentionState: ArtifactRetentionState;
+	session?: string;
+	sizeBytes?: number;
+	storageScope: ArtifactStorageScope;
+	subcommand?: string;
+}
+
+export interface SessionArtifactManifest {
+	entries: SessionArtifactManifestEntry[];
+	evictedCount: number;
+	liveCount: number;
+	maxEntries: number;
+	updatedAtMs: number;
+	version: 1;
+}
+
+export interface BatchStepPresentationDetails {
+	artifactVerification?: ArtifactVerificationSummary;
+	artifacts?: FileArtifactMetadata[];
+	command?: string[];
+	commandText: string;
+	data?: unknown;
+	failureCategory?: AgentBrowserFailureCategory;
+	fullOutputPath?: string;
+	fullOutputPaths?: string[];
+	imagePath?: string;
+	imagePaths?: string[];
+	index: number;
+	nextActions?: AgentBrowserNextAction[];
+	pageChangeSummary?: AgentBrowserPageChangeSummary;
+	resultCategory: AgentBrowserResultCategory;
+	savedFile?: SavedFilePresentationDetails;
+	savedFilePath?: string;
+	success: boolean;
+	successCategory?: AgentBrowserSuccessCategory;
+	summary: string;
+	text: string;
+}
+
+export interface BatchFailurePresentationDetails {
+	failedStep: BatchStepPresentationDetails;
+	failureCount: number;
+	successCount: number;
+	totalCount: number;
+}
+
+export interface ToolPresentation {
+	artifactManifest?: SessionArtifactManifest;
+	artifactRetentionSummary?: string;
+	artifactVerification?: ArtifactVerificationSummary;
+	artifacts?: FileArtifactMetadata[];
+	batchFailure?: BatchFailurePresentationDetails;
+	batchSteps?: BatchStepPresentationDetails[];
+	content: Array<{ text: string; type: "text" } | { data: string; mimeType: string; type: "image" }>;
+	data?: unknown;
+	failureCategory?: AgentBrowserFailureCategory;
+	fullOutputPath?: string;
+	fullOutputPaths?: string[];
+	imagePath?: string;
+	imagePaths?: string[];
+	nextActions?: AgentBrowserNextAction[];
+	pageChangeSummary?: AgentBrowserPageChangeSummary;
+	resultCategory?: AgentBrowserResultCategory;
+	savedFile?: SavedFilePresentationDetails;
+	savedFilePath?: string;
+	successCategory?: AgentBrowserSuccessCategory;
+	summary: string;
+}
+
+export type NetworkFailureImpact = "actionable" | "benign";
+
+export interface NetworkFailureClassification {
+	impact: NetworkFailureImpact;
+	reason: string;
+	resourceType?: string;
+	status?: number;
+	url?: string;
+}
+
+export interface NetworkFailureSummary {
+	actionableCount: number;
+	benignCount: number;
+	failures: NetworkFailureClassification[];
+	totalCount: number;
+}

package/extensions/agent-browser/lib/results/editable-ref-evidence.ts

@@ -0,0 +1,72 @@
+/**
+ * Purpose: Detect whether upstream ref metadata or snapshot text proves an element is editable.
+ * Responsibilities: Parse structured ref flags and snapshot-line contenteditable/editable markers into a conservative boolean/unknown signal.
+ * Scope: Editable evidence only; callers decide how to rank controls or build recovery actions from the signal.
+ * Usage: Imported by snapshot compaction and selector/fill recovery diagnostics.
+ * Invariants/Assumptions: Explicit false evidence wins over positive hints; unknown remains undefined.
+ */
+
+const EDITABLE_REF_EVIDENCE_KEYS = ["editable", "contentEditable", "contenteditable", "isContentEditable"] as const;
+const EDITABLE_FALSE_TEXT_PATTERN = /\b(?:contenteditable|editable)\s*=\s*["']?(?:false|0)["']?/i;
+const EDITABLE_ASSIGNMENT_TEXT_PATTERN = /\b(contenteditable|editable)\s*=\s*("[^"]*"|'[^']*'|[^\s,\]]+)/gi;
+const EDITABLE_BARE_TEXT_PATTERN = /\b(?:contenteditable|editable)\b(?!\s*=)/i;
+
+function parseEditableEvidenceValue(value: unknown): boolean | undefined {
+	if (typeof value === "boolean") return value;
+	if (typeof value === "number") {
+		if (value === 1) return true;
+		if (value === 0) return false;
+		return undefined;
+	}
+	if (typeof value !== "string") return undefined;
+	const normalized = value.trim().replace(/^["']|["']$/g, "").toLowerCase();
+	if (["false", "0", "no"].includes(normalized)) return false;
+	if (["", "true", "1", "yes", "plaintext-only"].includes(normalized)) return true;
+	return undefined;
+}
+
+function stripLeadingSnapshotAccessibleName(text: string): string {
+	return text.replace(/^(\s*[-*]\s+\S+\s+)(?:"[^"]*"|'[^']*')/, "$1");
+}
+
+function parseEditableEvidenceText(text: string | undefined): boolean | undefined {
+	if (!text) return undefined;
+	const markerText = stripLeadingSnapshotAccessibleName(text);
+	if (EDITABLE_FALSE_TEXT_PATTERN.test(markerText)) return false;
+	let hasPositiveAssignment = false;
+	for (const match of markerText.matchAll(EDITABLE_ASSIGNMENT_TEXT_PATTERN)) {
+		const key = match[1]?.toLowerCase();
+		const evidence = parseEditableEvidenceValue(match[2]);
+		if (evidence === false) return false;
+		if (evidence === true && (key === "contenteditable" || key === "editable")) {
+			hasPositiveAssignment = true;
+		}
+	}
+	if (hasPositiveAssignment) return true;
+	return EDITABLE_BARE_TEXT_PATTERN.test(markerText) ? true : undefined;
+}
+
+export function getEditableRefEvidence(options: {
+	ref?: Record<string, unknown>;
+	text?: string;
+}): boolean | undefined {
+	let hasPositiveEvidence = false;
+	if (options.ref) {
+		for (const key of EDITABLE_REF_EVIDENCE_KEYS) {
+			const evidence = parseEditableEvidenceValue(options.ref[key]);
+			if (evidence === false) return false;
+			if (evidence === true) hasPositiveEvidence = true;
+		}
+	}
+	const textEvidence = parseEditableEvidenceText(options.text);
+	if (textEvidence === false) return false;
+	if (textEvidence === true) hasPositiveEvidence = true;
+	return hasPositiveEvidence ? true : undefined;
+}
+
+export function hasPositiveEditableRefEvidence(options: {
+	ref?: Record<string, unknown>;
+	text?: string;
+}): boolean {
+	return getEditableRefEvidence(options) === true;
+}

package/extensions/agent-browser/lib/results/envelope.ts

@@ -10,7 +10,8 @@ import { readFile } from "node:fs/promises";
 
 import { isRecord } from "../parsing.js";
 import { detectConfirmationRequired } from "./confirmation.js";
-import { type AgentBrowserBatchResult, type AgentBrowserEnvelope, stringifyUnknown } from "./shared.js";
+import type { AgentBrowserBatchResult, AgentBrowserEnvelope } from "./contracts.js";
+import { stringifyUnknown } from "./text.js";
 
 function hasStructuredBatchStepFailure(data: unknown): data is AgentBrowserBatchResult[] {
 	return Array.isArray(data) && data.some((item) => isRecord(item) && item.success === false);

package/extensions/agent-browser/lib/results/network.ts

@@ -0,0 +1,64 @@
+/**
+ * Purpose: Classify failed network requests into actionable vs benign diagnostics.
+ * Responsibilities: Recognize failed request rows, de-prioritize browser icon misses, and summarize failure counts.
+ * Scope: Network diagnostic classification only.
+ * Usage: QA preset analysis and presentation network summaries.
+ * Invariants/Assumptions: Browser favicon/apple-touch icon misses are warnings; API/document/script failures are actionable.
+ */
+
+import { isRecord } from "../parsing.js";
+import type { NetworkFailureClassification, NetworkFailureSummary } from "./contracts.js";
+
+function getStringRecordField(value: Record<string, unknown>, key: string): string | undefined {
+	const field = value[key];
+	return typeof field === "string" && field.trim().length > 0 ? field.trim() : undefined;
+}
+
+function getNetworkRequestUrlPath(url: string | undefined): string | undefined {
+	if (!url) return undefined;
+	try {
+		return new URL(url).pathname;
+	} catch {
+		const withoutQuery = url.split(/[?#]/, 1)[0];
+		return withoutQuery.length > 0 ? withoutQuery : undefined;
+	}
+}
+
+function isFailedNetworkRequest(request: Record<string, unknown>): boolean {
+	return (typeof request.status === "number" && request.status >= 400) || request.failed === true || typeof request.error === "string";
+}
+
+function isBenignAssetFailure(request: Record<string, unknown>, url: string | undefined, resourceType: string | undefined): boolean {
+	const path = getNetworkRequestUrlPath(url);
+	if (!path) return false;
+	const normalizedResourceType = resourceType?.toLowerCase();
+	return /(?:^|\/)(?:favicon(?:[-.\w]*)?\.(?:ico|png|svg)|apple-touch-icon(?:[-.\w]*)?\.png)$/i.test(path)
+		&& (request.status === 404 || request.failed === true || typeof request.error === "string")
+		&& (!normalizedResourceType || ["image", "img", "other"].includes(normalizedResourceType) || normalizedResourceType.startsWith("image/"));
+}
+
+export function classifyNetworkRequestFailure(request: Record<string, unknown>): NetworkFailureClassification | undefined {
+	if (!isFailedNetworkRequest(request)) return undefined;
+	const url = getStringRecordField(request, "url");
+	const resourceType = getStringRecordField(request, "resourceType") ?? getStringRecordField(request, "mimeType");
+	const status = typeof request.status === "number" ? request.status : undefined;
+	if (isBenignAssetFailure(request, url, resourceType)) {
+		return { impact: "benign", reason: "low-impact browser icon asset", resourceType, status, url };
+	}
+	return { impact: "actionable", reason: "document, script, API, or non-benign request failure", resourceType, status, url };
+}
+
+export function summarizeNetworkFailures(requests: unknown[]): NetworkFailureSummary {
+	const failures = requests.flatMap((request) => {
+		if (!isRecord(request)) return [];
+		const classification = classifyNetworkRequestFailure(request);
+		return classification ? [classification] : [];
+	});
+	const benignCount = failures.filter((failure) => failure.impact === "benign").length;
+	return {
+		actionableCount: failures.length - benignCount,
+		benignCount,
+		failures,
+		totalCount: failures.length,
+	};
+}

package/extensions/agent-browser/lib/results/next-actions.ts

@@ -0,0 +1,117 @@
+/**
+ * Purpose: Own machine-readable agent_browser next-action contracts and merge policy.
+ * Responsibilities: Define the stable nextAction shape, build basic argv follow-ups, and provide deterministic action-list collection helpers.
+ * Scope: Result follow-up action mechanics only; command-specific recovery and artifact policies live in neighboring modules.
+ * Usage: Imported by result presentation helpers and the extension entrypoint when attaching details.nextActions.
+ * Invariants/Assumptions: Action ids are stable machine-readable contracts; dedupe preserves first occurrence order.
+ */
+
+export interface AgentBrowserNextAction {
+	artifactPath?: string;
+	id: string;
+	params?: {
+		args?: string[];
+		electron?: {
+			action: "cleanup" | "list" | "launch" | "probe" | "status";
+			all?: boolean;
+			handoff?: "connect" | "snapshot" | "tabs";
+			launchId?: string;
+		};
+		networkSourceLookup?: {
+			filter?: string;
+			requestId?: string;
+			session?: string;
+			url?: string;
+		};
+		sessionMode?: "auto" | "fresh";
+		stdin?: string;
+	};
+	reason: string;
+	safety?: string;
+	tool: "agent_browser";
+}
+
+export function withOptionalSessionArgs(sessionName: string | undefined, args: string[]): string[] {
+	return sessionName && args[0] !== "--session" ? ["--session", sessionName, ...args] : args;
+}
+
+export function buildNextToolAction(options: {
+	args: string[];
+	id: string;
+	reason: string;
+	safety?: string;
+	sessionMode?: "auto" | "fresh";
+	stdin?: string;
+}): AgentBrowserNextAction {
+	return {
+		id: options.id,
+		params: {
+			args: options.args,
+			...(options.sessionMode ? { sessionMode: options.sessionMode } : {}),
+			...(options.stdin ? { stdin: options.stdin } : {}),
+		},
+		reason: options.reason,
+		...(options.safety ? { safety: options.safety } : {}),
+		tool: "agent_browser",
+	};
+}
+
+export function appendUniqueAgentBrowserNextActions(
+	target: AgentBrowserNextAction[],
+	additions: AgentBrowserNextAction[] | undefined,
+): AgentBrowserNextAction[] {
+	if (!additions || additions.length === 0) return target;
+	const existingIds = new Set(target.map((action) => action.id));
+	for (const action of additions) {
+		if (existingIds.has(action.id)) continue;
+		target.push(action);
+		existingIds.add(action.id);
+	}
+	return target;
+}
+
+export function isStandaloneSnapshotNextAction(action: AgentBrowserNextAction): boolean {
+	const args = action.params?.args;
+	if (!args || action.params?.stdin) return false;
+	const commandIndex = args[0] === "--session" ? 2 : 0;
+	return args[commandIndex] === "snapshot";
+}
+
+export function alignPageChangeSummaryNextActionIds<T extends { nextActionIds?: string[] }>(
+	summary: T | undefined,
+	nextActions: AgentBrowserNextAction[] | undefined,
+): T | undefined {
+	if (!summary?.nextActionIds || !nextActions) return summary;
+	const nextActionIds = new Set(nextActions.map((action) => action.id));
+	const alignedIds = summary.nextActionIds.filter((id) => nextActionIds.has(id));
+	return alignedIds.length > 0 ? { ...summary, nextActionIds: alignedIds } : { ...summary, nextActionIds: undefined };
+}
+
+export class AgentBrowserNextActionCollector {
+	private actions: AgentBrowserNextAction[];
+
+	constructor(initialActions: AgentBrowserNextAction[] | undefined = undefined) {
+		this.actions = initialActions ? [...initialActions] : [];
+	}
+
+	append(actions: AgentBrowserNextAction[] | undefined): void {
+		if (!actions || actions.length === 0) return;
+		this.actions.push(...actions);
+	}
+
+	appendUnique(actions: AgentBrowserNextAction[] | undefined): void {
+		appendUniqueAgentBrowserNextActions(this.actions, actions);
+	}
+
+	replace(actions: AgentBrowserNextAction[] | undefined): void {
+		this.actions = actions ? [...actions] : [];
+	}
+
+	removeWhere(predicate: (action: AgentBrowserNextAction) => boolean): void {
+		this.actions = this.actions.filter((action) => !predicate(action));
+	}
+
+	toArray(): AgentBrowserNextAction[] | undefined {
+		return this.actions.length > 0 ? [...this.actions] : undefined;
+	}
+}