pi-agent-browser-native 0.2.32 → 0.2.33

package/extensions/agent-browser/lib/results/action-recommendations.ts

@@ -0,0 +1,269 @@
+/**
+ * Purpose: Build generic nextAction recommendations from result categories, artifacts, Electron lifecycle state, and recovery context.
+ * Responsibilities: Preserve stable action ids/order while keeping recommendation policy out of generic shared helpers.
+ * Scope: Generic result-level recommendations only; feature-specific diagnostics append their own actions in the extension entrypoint.
+ * Usage: Called by presentation and extension result assembly.
+ * Invariants/Assumptions: Action ids are public machine-readable contracts; preserve first-observed order.
+ */
+
+import { isPendingRecordingArtifact } from "./artifact-state.js";
+import type {
+	AgentBrowserFailureCategory,
+	AgentBrowserResultCategory,
+	AgentBrowserSuccessCategory,
+	FileArtifactMetadata,
+} from "./contracts.js";
+import { buildNextToolAction, type AgentBrowserNextAction } from "./next-actions.js";
+import {
+	AGENT_BROWSER_RECOVERY_NEXT_ACTION_IDS,
+	buildRecoveryNextActions,
+	type AgentBrowserRecoveryContext,
+} from "./recovery-actions.js";
+
+function buildArtifactAction(path: string): AgentBrowserNextAction {
+	return {
+		artifactPath: path,
+		id: "use-saved-artifact",
+		reason: "Use the saved artifact path from the structured result instead of scraping it from text.",
+		safety: "Verify artifact metadata such as exists/status before treating the file as durable.",
+		tool: "agent_browser",
+	};
+}
+
+function buildArtifactVerificationAction(artifact: FileArtifactMetadata): AgentBrowserNextAction {
+	return {
+		artifactPath: artifact.path,
+		id: "verify-artifact-path",
+		reason: "The wrapper has artifact metadata but did not verify this file as present on disk.",
+		safety: "Check details.artifactVerification and the filesystem before treating the artifact as durable.",
+		tool: "agent_browser",
+	};
+}
+
+function buildElectronToolAction(options: {
+	action: "cleanup" | "probe" | "status";
+	id: string;
+	launchId: string;
+	reason: string;
+	safety?: string;
+}): AgentBrowserNextAction {
+	return {
+		id: options.id,
+		params: { electron: { action: options.action, launchId: options.launchId } },
+		reason: options.reason,
+		...(options.safety ? { safety: options.safety } : {}),
+		tool: "agent_browser",
+	};
+}
+
+const MUTATING_COMMANDS = new Set([
+	"back",
+	"check",
+	"click",
+	"dblclick",
+	"dialog",
+	"fill",
+	"forward",
+	"hover",
+	"press",
+	"pushstate",
+	"reload",
+	"scroll",
+	"scrollintoview",
+	"select",
+	"swipe",
+	"tap",
+	"type",
+	"uncheck",
+]);
+
+function getDownloadRetryPath(args: string[] | undefined, fallback: string | undefined): string | undefined {
+	if (fallback) return fallback;
+	if (!args || args.length === 0) return undefined;
+	const downloadFlagIndex = args.indexOf("--download");
+	if (downloadFlagIndex >= 0) {
+		const candidate = args[downloadFlagIndex + 1];
+		return candidate && !candidate.startsWith("-") ? candidate : undefined;
+	}
+	const downloadCommandIndex = args.indexOf("download");
+	if (downloadCommandIndex >= 0 && args.length > downloadCommandIndex + 2) {
+		return args[args.length - 1];
+	}
+	return undefined;
+}
+
+export function buildAgentBrowserNextActions(options: {
+	artifacts?: FileArtifactMetadata[];
+	args?: string[];
+	command?: string;
+	confirmationId?: string;
+	electron?: {
+		launchId?: string;
+		sessionName?: string;
+		status?: "active" | "cleaned" | "dead" | "failed" | "partial" | "succeeded";
+	};
+	failureCategory?: AgentBrowserFailureCategory;
+	resultCategory: AgentBrowserResultCategory;
+	recovery?: AgentBrowserRecoveryContext;
+	savedFilePath?: string;
+	successCategory?: AgentBrowserSuccessCategory;
+}): AgentBrowserNextAction[] | undefined {
+	const actions: AgentBrowserNextAction[] = [];
+	if (options.recovery) {
+		actions.push(...buildRecoveryNextActions(options.recovery));
+	}
+	if (options.electron?.launchId) {
+		const { launchId, sessionName, status } = options.electron;
+		if (options.resultCategory === "success" && status !== "cleaned") {
+			actions.push(
+				buildElectronToolAction({
+					action: "status",
+					id: "status-electron-launch",
+					launchId,
+					reason: "Check the wrapper-tracked Electron launch liveness and current CDP targets without mutating the app.",
+				}),
+				buildElectronToolAction({
+					action: "probe",
+					id: "probe-electron-launch",
+					launchId,
+					reason: "Probe the attached Electron managed session and carry the wrapper launchId for follow-up diagnostics.",
+				}),
+				buildElectronToolAction({
+					action: "cleanup",
+					id: "cleanup-electron-launch",
+					launchId,
+					reason: "Clean the wrapper-owned Electron process and isolated userDataDir when the run is complete.",
+					safety: "Only operates on the launchId created by electron.launch; explicit artifacts and manually launched apps remain host-owned.",
+				}),
+			);
+			if (sessionName) {
+				actions.push(
+					buildNextToolAction({
+						args: ["--session", sessionName, "tab", "list"],
+						id: "list-electron-tabs",
+						reason: "Inspect attached Electron page/webview targets before choosing the active tab.",
+					}),
+					buildNextToolAction({
+						args: ["--session", sessionName, "snapshot", "-i"],
+						id: "snapshot-electron-session",
+						reason: "Refresh interactive refs for the attached Electron session.",
+						safety: "Use current Electron refs only after a fresh snapshot for this session.",
+					}),
+				);
+			}
+		} else if (options.resultCategory === "failure" && options.failureCategory === "cleanup-failed") {
+			actions.push(
+				buildElectronToolAction({
+					action: "status",
+					id: "status-electron-launch",
+					launchId,
+					reason: "Inspect which wrapper-tracked Electron resources remain after partial cleanup.",
+				}),
+				buildElectronToolAction({
+					action: "cleanup",
+					id: "retry-electron-cleanup",
+					launchId,
+					reason: "Retry cleanup for the same wrapper-owned Electron launch after reviewing remaining resources.",
+					safety: "Only retry for the same launchId; do not use cleanup for manually launched Electron apps.",
+				}),
+			);
+		}
+	}
+	if (options.resultCategory === "success") {
+		if (options.command === "open") {
+			actions.push(buildNextToolAction({
+				args: ["snapshot", "-i"],
+				id: "inspect-opened-page",
+				reason: "Inspect the opened page before choosing interactive refs.",
+			}));
+		} else if (options.command && MUTATING_COMMANDS.has(options.command)) {
+			actions.push(buildNextToolAction({
+				args: ["snapshot", "-i"],
+				id: "inspect-after-mutation",
+				reason: "Refresh interactive refs after a browser mutation, navigation, scroll, or rerender.",
+				safety: "Do not reuse prior @refs until a fresh snapshot confirms they still exist.",
+			}));
+		}
+		const artifacts = options.artifacts ?? [];
+		const savedFileArtifact = options.savedFilePath ? artifacts.find((artifact) => artifact.path === options.savedFilePath) : undefined;
+		if (options.savedFilePath && savedFileArtifact?.exists !== false) {
+			actions.push(buildArtifactAction(options.savedFilePath));
+		}
+		for (const artifact of artifacts) {
+			if (isPendingRecordingArtifact(artifact)) {
+				continue;
+			}
+			if (artifact.exists === false) {
+				if (artifact.kind === "download") {
+					actions.push(buildNextToolAction({
+						args: ["wait", "--download", artifact.path],
+						id: "wait-for-download",
+						reason: "Upstream reported a download path, but the wrapper did not verify the file on disk.",
+						safety: "Use a bounded wait timeout that stays below the native wrapper IPC budget.",
+					}));
+				} else {
+					actions.push(buildArtifactVerificationAction(artifact));
+				}
+				continue;
+			}
+			if (artifact.path !== options.savedFilePath) {
+				actions.push(buildArtifactAction(artifact.path));
+			}
+		}
+	} else {
+		switch (options.failureCategory) {
+			case "confirmation-required":
+				if (options.confirmationId) {
+					actions.push(
+						buildNextToolAction({
+							args: ["confirm", options.confirmationId],
+							id: "approve-confirmation",
+							reason: "Approve the pending upstream confirmation when the requested action is safe.",
+							safety: "Only confirm after reviewing the guarded action shown in the result.",
+						}),
+						buildNextToolAction({
+							args: ["deny", options.confirmationId],
+							id: "deny-confirmation",
+							reason: "Deny the pending upstream confirmation when the guarded action is unsafe or unintended.",
+						}),
+					);
+				}
+				break;
+			case "stale-ref":
+			case "selector-not-found":
+			case "selector-unsupported":
+				actions.push(buildNextToolAction({
+					args: ["snapshot", "-i"],
+					id: "refresh-interactive-refs",
+					reason: "Get current interactive refs before retrying the element action.",
+					safety: "Prefer a current @ref or a stable find locator; do not retry stale refs blindly.",
+				}));
+				break;
+			case "download-not-verified":
+				{
+					const retryPath = getDownloadRetryPath(options.args, options.savedFilePath);
+					actions.push(buildNextToolAction({
+						args: retryPath ? ["wait", "--download", retryPath] : ["wait", "--download"],
+						id: "wait-for-download",
+						reason: "Wait for the browser download and let the wrapper verify saved-file metadata.",
+						safety: "Use a bounded wait timeout that stays below the native wrapper IPC budget.",
+					}));
+				}
+				break;
+			case "tab-drift":
+				if (options.recovery?.kind === "about-blank" || options.recovery?.kind === "tab-drift") {
+					break;
+				}
+				actions.push(
+					buildNextToolAction({
+						args: ["tab", "list"],
+						id: AGENT_BROWSER_RECOVERY_NEXT_ACTION_IDS.genericTabDriftListTabs,
+						reason: "Inspect available tabs before selecting the intended target.",
+						safety: "Read-only. Retry snapshot only after selecting or confirming the intended stable tab.",
+					}),
+				);
+				break;
+		}
+	}
+	return actions.length > 0 ? actions : undefined;
+}

package/extensions/agent-browser/lib/results/artifact-manifest.ts

@@ -0,0 +1,114 @@
+/**
+ * Purpose: Own persistent session artifact manifest merge, retention, and validation logic.
+ * Responsibilities: Parse manifest bounds, recognize manifest entries, merge new artifact rows, and format retention summaries.
+ * Scope: Manifest accounting only; artifact detection and presentation live in presentation modules.
+ * Usage: Imported by presentation and snapshot artifact persistence paths.
+ * Invariants/Assumptions: Explicit-path artifacts are host-owned while persistent-session spill files are bounded by the manifest cap.
+ */
+
+import type { SessionArtifactManifest, SessionArtifactManifestEntry } from "./contracts.js";
+
+export const SESSION_ARTIFACT_MANIFEST_VERSION = 1;
+export const SESSION_ARTIFACT_MANIFEST_MAX_ENTRIES_ENV = "PI_AGENT_BROWSER_SESSION_ARTIFACT_MANIFEST_MAX_ENTRIES";
+export const DEFAULT_SESSION_ARTIFACT_MANIFEST_MAX_ENTRIES = 100;
+
+function parsePositiveSafeInteger(value: string | undefined): number | undefined {
+	if (value === undefined) return undefined;
+	const parsed = Number(value);
+	if (!Number.isSafeInteger(parsed) || parsed <= 0) return undefined;
+	return parsed;
+}
+
+export function getSessionArtifactManifestMaxEntries(env: NodeJS.ProcessEnv = process.env): number {
+	return parsePositiveSafeInteger(env[SESSION_ARTIFACT_MANIFEST_MAX_ENTRIES_ENV]) ?? DEFAULT_SESSION_ARTIFACT_MANIFEST_MAX_ENTRIES;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}
+
+function isManifestEntry(value: unknown): value is SessionArtifactManifestEntry {
+	if (!isRecord(value)) return false;
+	if (typeof value.path !== "string" || value.path.trim().length === 0) return false;
+	if (typeof value.createdAtMs !== "number" || !Number.isFinite(value.createdAtMs)) return false;
+	if (!["evicted", "ephemeral", "live", "missing"].includes(String(value.retentionState))) return false;
+	if (!["explicit-path", "persistent-session", "process-temp"].includes(String(value.storageScope))) return false;
+	if (typeof value.kind !== "string" || value.kind.trim().length === 0) return false;
+	return true;
+}
+
+export function isSessionArtifactManifest(value: unknown): value is SessionArtifactManifest {
+	if (!isRecord(value)) return false;
+	if (value.version !== SESSION_ARTIFACT_MANIFEST_VERSION) return false;
+	if (!Array.isArray(value.entries) || !value.entries.every(isManifestEntry)) return false;
+	if (typeof value.updatedAtMs !== "number" || !Number.isFinite(value.updatedAtMs)) return false;
+	if (typeof value.maxEntries !== "number" || !Number.isSafeInteger(value.maxEntries) || value.maxEntries <= 0) return false;
+	if (typeof value.liveCount !== "number" || !Number.isSafeInteger(value.liveCount) || value.liveCount < 0) return false;
+	if (typeof value.evictedCount !== "number" || !Number.isSafeInteger(value.evictedCount) || value.evictedCount < 0) return false;
+	return true;
+}
+
+export function buildEvictedSessionArtifactEntries(
+	evictedArtifacts: Array<{ mtimeMs: number; path: string; sizeBytes: number }>,
+	nowMs: number,
+): SessionArtifactManifestEntry[] {
+	return evictedArtifacts.map((artifact) => ({
+		createdAtMs: artifact.mtimeMs,
+		evictedAtMs: nowMs,
+		kind: "spill",
+		path: artifact.path,
+		retentionState: "evicted",
+		sizeBytes: artifact.sizeBytes,
+		storageScope: "persistent-session",
+	}));
+}
+
+export function formatSessionArtifactRetentionSummary(manifest: SessionArtifactManifest): string {
+	const ephemeralCount = manifest.entries.filter((entry) => entry.retentionState === "ephemeral").length;
+	const missingCount = manifest.entries.filter((entry) => entry.retentionState === "missing").length;
+	const parts = [`${manifest.liveCount} live`, `${manifest.evictedCount} evicted`];
+	if (ephemeralCount > 0) parts.push(`${ephemeralCount} ephemeral`);
+	if (missingCount > 0) parts.push(`${missingCount} missing`);
+	return `Session artifacts: ${parts.join(", ")} (${manifest.entries.length}/${manifest.maxEntries} recent).`;
+}
+
+export function mergeSessionArtifactManifest(options: {
+	base?: SessionArtifactManifest;
+	entries?: SessionArtifactManifestEntry[];
+	nowMs?: number;
+}): SessionArtifactManifest | undefined {
+	const nowMs = options.nowMs ?? Date.now();
+	const maxEntries = getSessionArtifactManifestMaxEntries();
+	const getEntryKey = (entry: SessionArtifactManifestEntry) =>
+		entry.storageScope === "explicit-path" && entry.absolutePath ? `${entry.storageScope}:${entry.absolutePath}` : `${entry.storageScope}:${entry.path}`;
+	const byPath = new Map<string, SessionArtifactManifestEntry>();
+	for (const entry of options.base?.entries ?? []) {
+		byPath.set(getEntryKey(entry), entry);
+	}
+	for (const entry of options.entries ?? []) {
+		const key = getEntryKey(entry);
+		const existing = byPath.get(key);
+		byPath.set(key, {
+			...existing,
+			...entry,
+			createdAtMs: existing?.createdAtMs ?? entry.createdAtMs,
+			evictedAtMs: entry.retentionState === "evicted" ? (entry.evictedAtMs ?? nowMs) : entry.evictedAtMs,
+		});
+	}
+	if (byPath.size === 0) return undefined;
+	const entries = [...byPath.values()]
+		.sort((left, right) => {
+			const leftTime = left.evictedAtMs ?? left.createdAtMs;
+			const rightTime = right.evictedAtMs ?? right.createdAtMs;
+			return rightTime - leftTime || left.path.localeCompare(right.path);
+		})
+		.slice(0, maxEntries);
+	return {
+		entries,
+		evictedCount: entries.filter((entry) => entry.retentionState === "evicted").length,
+		liveCount: entries.filter((entry) => entry.retentionState === "live").length,
+		maxEntries,
+		updatedAtMs: nowMs,
+		version: SESSION_ARTIFACT_MANIFEST_VERSION,
+	};
+}

package/extensions/agent-browser/lib/results/artifact-state.ts

@@ -0,0 +1,13 @@
+/**
+ * Purpose: Centralize small artifact state predicates shared by result classifiers, recommendations, and presentation.
+ * Responsibilities: Identify pending recording artifacts whose output is not durable until record stop completes.
+ * Scope: Artifact predicates only; verification summaries, manifests, and user-facing formatting live in neighboring modules.
+ * Usage: Imported by categories, action recommendations, and presentation to avoid divergent artifact-state rules.
+ * Invariants/Assumptions: `record start` video artifacts are pending and should not be treated like verified saved files.
+ */
+
+import type { FileArtifactMetadata } from "./contracts.js";
+
+export function isPendingRecordingArtifact(artifact: FileArtifactMetadata): boolean {
+	return artifact.command === "record" && artifact.subcommand === "start" && artifact.kind === "video";
+}

package/extensions/agent-browser/lib/results/categories.ts

@@ -0,0 +1,106 @@
+/**
+ * Purpose: Classify successful and failed agent-browser outcomes into stable result categories.
+ * Responsibilities: Map artifacts, inspection calls, errors, refs, selectors, downloads, drift, and timeouts to small enums.
+ * Scope: Category policy only; next-action recommendations and presentation formatting live elsewhere.
+ * Usage: Called by presentation and extension result assembly before details are exposed to Pi.
+ * Invariants/Assumptions: Category strings are public machine-readable contracts covered by tests and docs.
+ */
+
+import type {
+	AgentBrowserFailureCategory,
+	AgentBrowserResultCategoryDetails,
+	AgentBrowserSuccessCategory,
+	FileArtifactMetadata,
+	SavedFilePresentationDetails,
+} from "./contracts.js";
+import { isPendingRecordingArtifact } from "./artifact-state.js";
+
+function hasUnverifiedFileArtifact(artifacts: FileArtifactMetadata[] | undefined): boolean {
+	return (artifacts ?? []).some((artifact) => !isPendingRecordingArtifact(artifact) && artifact.exists !== true);
+}
+
+export function classifyAgentBrowserSuccessCategory(options: {
+	artifacts?: FileArtifactMetadata[];
+	inspection?: boolean;
+	savedFile?: SavedFilePresentationDetails;
+}): AgentBrowserSuccessCategory {
+	if (options.inspection) return "inspection";
+	if ((options.artifacts ?? []).length > 0) return hasUnverifiedFileArtifact(options.artifacts) ? "artifact-unverified" : "artifact-saved";
+	if (options.savedFile) return "artifact-saved";
+	return "completed";
+}
+
+export function classifyAgentBrowserFailureCategory(options: {
+	args?: string[];
+	command?: string;
+	confirmationRequired?: boolean;
+	errorText?: string;
+	parseError?: string;
+	spawnError?: string;
+	stderr?: string;
+	tabDrift?: boolean;
+	timedOut?: boolean;
+	validationError?: string;
+}): AgentBrowserFailureCategory {
+	const text = [options.errorText, options.validationError, options.parseError, options.spawnError, options.stderr].filter(Boolean).join("\n");
+	const command = options.command ?? "";
+	const usedRef = options.args?.some((arg) => /^@e\d+\b/.test(arg)) ?? false;
+	if (options.confirmationRequired || /confirmation required|pending confirmation|requires confirmation/i.test(text)) return "confirmation-required";
+	if (options.timedOut || /timeout|timed out|watchdog|IPC read timeout|must stay under its 30s IPC read timeout/i.test(text)) return "timeout";
+	if (/ENOENT|not found on PATH|could not find.*agent-browser|agent-browser is required but was not found/i.test(text)) return "missing-binary";
+	if (options.parseError || /invalid JSON|missing boolean success|success field must be boolean|returned no JSON output/i.test(text)) return "parse-failure";
+	if (/aborted/i.test(text)) return "aborted";
+	if (/policy[- ]blocked|blocked by caller policy|caller deny policy|caller allow policy/i.test(text)) return "policy-blocked";
+	if (/cleanup failed|cleanup.*partial|partial cleanup|remaining resources/i.test(text)) return "cleanup-failed";
+	if (options.tabDrift || /could not re-select the intended tab|about:blank|selected tab looks wrong|tab drift|tab.*wrong/i.test(text)) return "tab-drift";
+	if (/\bUnknown ref\b|\bstale ref\b|@ref may be stale|\bref\b.*\b(?:not found|missing|expired)\b/i.test(text)) return "stale-ref";
+	if (usedRef && /could not locate element|element not found|no element/i.test(text)) return "stale-ref";
+	const mentionsPlaywrightSelectorDialect = /(?:\btext=|:has-text\(|\bgetByRole\b|\bgetByText\b)/i.test(text);
+	const reportsSelectorMatchFailure =
+		/\b(?:no elements? found|failed to find|could not find|unable to find)\b.*\b(?:selector|locator)\b/i.test(text) ||
+		/\b(?:selector|locator)\b.*\b(?:no elements? found|not found|missing|failed to find|could not find|unable to find)\b/i.test(text);
+	if (
+		/\b(?:unsupported|unknown|invalid)\s+(?:selector|locator)\b/i.test(text) ||
+		/\bfailed to parse selector\b/i.test(text) ||
+		/\bselector\b.*\b(?:parse|syntax|unsupported|invalid)\b/i.test(text) ||
+		(mentionsPlaywrightSelectorDialect && reportsSelectorMatchFailure)
+	) {
+		return "selector-unsupported";
+	}
+	if (command === "find" && /could not locate element|element not found|no elements? found|unable to find/i.test(text)) return "selector-not-found";
+	if (reportsSelectorMatchFailure) return "selector-not-found";
+	if ((command === "download" || text.includes("wait --download") || /\bdownload\b/i.test(text)) && /missing|not verified|not found|failed|timeout|timed out/i.test(text)) {
+		return "download-not-verified";
+	}
+	if (options.validationError) return "validation-error";
+	return "upstream-error";
+}
+
+
+export function buildAgentBrowserResultCategoryDetails(options: {
+	artifacts?: FileArtifactMetadata[];
+	args?: string[];
+	command?: string;
+	confirmationRequired?: boolean;
+	errorText?: string;
+	failureCategory?: AgentBrowserFailureCategory;
+	inspection?: boolean;
+	parseError?: string;
+	savedFile?: SavedFilePresentationDetails;
+	spawnError?: string;
+	succeeded: boolean;
+	tabDrift?: boolean;
+	timedOut?: boolean;
+	validationError?: string;
+}): AgentBrowserResultCategoryDetails {
+	if (options.succeeded) {
+		return {
+			resultCategory: "success",
+			successCategory: classifyAgentBrowserSuccessCategory(options),
+		};
+	}
+	return {
+		failureCategory: options.failureCategory ?? classifyAgentBrowserFailureCategory(options),
+		resultCategory: "failure",
+	};
+}