pi-agent-browser-native 0.2.48 → 0.2.50

package/dist/extensions/agent-browser/lib/results/presentation/batch.js

@@ -0,0 +1,397 @@
+import { isRecord } from "../../parsing.js";
+import { extractCommandTokens, parseCommandInfo, redactInvocationArgs, redactSensitiveText, redactSensitiveValue } from "../../runtime.js";
+import { buildAgentBrowserNextActions } from "../action-recommendations.js";
+import { formatSessionArtifactRetentionSummary } from "../artifact-manifest.js";
+import { classifyAgentBrowserFailureCategory } from "../categories.js";
+import { detectConfirmationRequired } from "../confirmation.js";
+import { applyNetworkRouteRecords, buildNetworkRouteDiagnostics } from "../network-routes.js";
+import { withOptionalSessionArgs } from "../next-actions.js";
+import { stringifyModelFacing } from "./common.js";
+import { buildArtifactVerificationSummary, classifyPresentationSuccessCategory, manifestHasNewNoticeWorthyEntries } from "./artifacts.js";
+import { formatBatchStepCommand, getPresentationImages, getPresentationPaths, getPresentationText, isStringArray } from "./content.js";
+import { buildPageChangeSummary } from "./navigation.js";
+import { appendSelectorRecoveryHint, getClipboardWritePayloadCandidates, redactClipboardPermissionErrorValue } from "./errors.js";
+export function isAgentBrowserBatchResultArray(value) {
+    return Array.isArray(value) && value.every(isRecord);
+}
+function isWaitTextAssertionCommand(command) {
+    return command?.[0] === "wait" && command.includes("--text");
+}
+function buildWaitTextAssertionFailureNextAction(sessionName) {
+    return {
+        id: "inspect-after-text-assertion-failure",
+        params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) },
+        reason: "Inspect the current page after the text assertion failed before concluding the expected text is absent.",
+        safety: "Read-only snapshot; use current refs or visible text from this page before retrying the assertion.",
+        tool: "agent_browser",
+    };
+}
+function mergePresentationNextActions(...groups) {
+    const actions = [];
+    const seen = new Set();
+    for (const group of groups) {
+        for (const action of group ?? []) {
+            if (seen.has(action.id))
+                continue;
+            actions.push(action);
+            seen.add(action.id);
+        }
+    }
+    return actions.length > 0 ? actions : undefined;
+}
+function formatBatchStepError(error) {
+    const errorText = stringifyModelFacing(error).trim();
+    const formattedErrorText = errorText.length > 0 ? `Error: ${errorText}` : "Error: batch step failed.";
+    return appendSelectorRecoveryHint(formattedErrorText);
+}
+function getBatchFailureDetails(steps) {
+    const failedSteps = steps.filter((step) => step.details.success === false);
+    if (failedSteps.length === 0)
+        return undefined;
+    const successCount = steps.length - failedSteps.length;
+    return {
+        failedStep: failedSteps[0].details,
+        failureCount: failedSteps.length,
+        successCount,
+        totalCount: steps.length,
+    };
+}
+function hasModelFacingArgRedaction(args) {
+    return args?.some((arg) => arg === "[REDACTED]" || arg.includes("%5BREDACTED%5D") || arg.includes("[REDACTED]")) === true;
+}
+function getStatefulCommandSensitiveValues(command) {
+    if (!command)
+        return [];
+    const tokens = extractCommandTokens(command);
+    const values = [];
+    if (tokens[0] === "cookies" && tokens[1] === "set" && tokens[3])
+        values.push(tokens[3]);
+    if (tokens[0] === "storage" && ["local", "session"].includes(tokens[1] ?? "") && tokens[2] === "set" && tokens[4])
+        values.push(tokens[4]);
+    for (let index = 0; index < tokens.length; index += 1) {
+        const token = tokens[index];
+        if (token === "--password" && tokens[index + 1])
+            values.push(tokens[index + 1]);
+        else if (token?.startsWith("--password="))
+            values.push(token.slice("--password=".length));
+    }
+    return values.filter((value) => value.length > 0);
+}
+function redactExactValues(value, sensitiveValues) {
+    if (sensitiveValues.length === 0)
+        return redactSensitiveValue(value);
+    if (typeof value === "string") {
+        let redacted = value;
+        for (const sensitiveValue of sensitiveValues)
+            redacted = redacted.split(sensitiveValue).join("[REDACTED]");
+        return redactSensitiveText(redacted);
+    }
+    if (Array.isArray(value))
+        return value.map((item) => redactExactValues(item, sensitiveValues));
+    if (!isRecord(value))
+        return value;
+    return redactSensitiveValue(Object.fromEntries(Object.entries(value).map(([key, entryValue]) => [key, redactExactValues(entryValue, sensitiveValues)])));
+}
+function getTypedTextLength(command) {
+    return command?.[0] === "keyboard" && command[1] === "type" && typeof command[2] === "string"
+        ? Array.from(command[2]).length
+        : undefined;
+}
+function getWaitDelayMs(command) {
+    return command?.[0] === "wait" && typeof command[1] === "string" && /^\d+$/.test(command[1]) ? command[1] : undefined;
+}
+function formatBatchStepDetails(details, presentation) {
+    const inlineImageCount = getPresentationImages(presentation).length;
+    const status = details.success ? "succeeded" : "failed";
+    const lines = [`Step ${details.index + 1} — ${details.commandText} (${status})`];
+    if (details.text.length > 0)
+        lines.push(details.text);
+    if (inlineImageCount > 0)
+        lines.push(`(${inlineImageCount} inline image attachment${inlineImageCount === 1 ? "" : "s"} below)`);
+    return lines.join("\n");
+}
+function formatTypedSequenceSummary(steps, startIndex) {
+    let index = startIndex;
+    const firstDetails = steps[index]?.details;
+    if (!firstDetails?.success)
+        return undefined;
+    let target;
+    if (firstDetails.command?.[0] === "focus" && typeof firstDetails.command[1] === "string") {
+        target = firstDetails.command[1];
+        index += 1;
+    }
+    let typedCharCount = 0;
+    let typedStepCount = 0;
+    let delayMs;
+    while (index < steps.length) {
+        const details = steps[index]?.details;
+        if (!details?.success)
+            break;
+        const typedLength = getTypedTextLength(details.command);
+        if (typedLength === undefined)
+            break;
+        typedCharCount += typedLength;
+        typedStepCount += 1;
+        index += 1;
+        const nextDelay = getWaitDelayMs(steps[index]?.details.command);
+        const followingTypedLength = getTypedTextLength(steps[index + 1]?.details.command);
+        if (nextDelay !== undefined && followingTypedLength !== undefined && steps[index]?.details.success && steps[index + 1]?.details.success) {
+            if (delayMs !== undefined && delayMs !== nextDelay)
+                return undefined;
+            delayMs = nextDelay;
+            index += 1;
+        }
+    }
+    if (typedStepCount < 2)
+        return undefined;
+    let pressedKey;
+    const pressDetails = steps[index]?.details;
+    if (pressDetails?.success && pressDetails.command?.[0] === "press" && typeof pressDetails.command[1] === "string") {
+        pressedKey = pressDetails.command[1];
+        index += 1;
+    }
+    const firstStep = firstDetails.index + 1;
+    const lastStep = steps[index - 1]?.details.index + 1;
+    const stepRange = lastStep && lastStep > firstStep ? `${firstStep}-${lastStep}` : String(firstStep);
+    const commandLabel = target ? `type ${target}` : "keyboard type";
+    const lines = [
+        `Step ${stepRange} — ${commandLabel} (succeeded)`,
+        `Typed ${typedCharCount} char${typedCharCount === 1 ? "" : "s"}${delayMs ? ` with delayMs=${delayMs}` : ""}.`,
+    ];
+    if (pressedKey)
+        lines.push(`Pressed ${pressedKey}.`);
+    return { nextIndex: index, text: lines.join("\n") };
+}
+function formatBatchStepsText(steps) {
+    if (steps.length === 0)
+        return "(no batch steps)";
+    const lines = [];
+    for (let index = 0; index < steps.length;) {
+        const typedSequence = formatTypedSequenceSummary(steps, index);
+        if (typedSequence) {
+            lines.push(typedSequence.text);
+            index = typedSequence.nextIndex;
+            continue;
+        }
+        const step = steps[index];
+        if (step)
+            lines.push(formatBatchStepDetails(step.details, step.presentation));
+        index += 1;
+    }
+    return lines.join("\n\n");
+}
+async function buildBatchStepPresentation(options) {
+    const { artifactManifest, artifactRequest, buildNestedToolPresentation, cwd, index, item, networkRoutes, persistentArtifactStore, sessionName } = options;
+    const command = isStringArray(item.command) ? item.command : undefined;
+    const redactedCommand = command ? redactInvocationArgs(command) : undefined;
+    const commandText = formatBatchStepCommand(hasModelFacingArgRedaction(redactedCommand) ? redactedCommand : command, index);
+    if (item.success === false) {
+        const redactedErrorData = command?.[0] === "clipboard"
+            ? redactSensitiveValue(redactClipboardPermissionErrorValue({ command: "clipboard", subcommand: command[1] }, item.error, getClipboardWritePayloadCandidates(command)))
+            : redactExactValues(item.error, getStatefulCommandSensitiveValues(command));
+        const errorText = formatBatchStepError(redactedErrorData);
+        const failureCategory = classifyAgentBrowserFailureCategory({
+            args: command,
+            command: command?.[0],
+            errorText,
+        });
+        const confirmationRequired = detectConfirmationRequired(item.error);
+        const nextActions = mergePresentationNextActions(buildAgentBrowserNextActions({
+            args: command,
+            command: command?.[0],
+            confirmationId: confirmationRequired?.id,
+            failureCategory,
+            resultCategory: "failure",
+        }), isWaitTextAssertionCommand(command) ? [buildWaitTextAssertionFailureNextAction(sessionName)] : undefined);
+        const presentation = {
+            content: [{ type: "text", text: errorText }],
+            failureCategory,
+            nextActions,
+            resultCategory: "failure",
+            summary: errorText,
+        };
+        return {
+            details: {
+                artifactVerification: presentation.artifactVerification,
+                artifacts: presentation.artifacts,
+                command: redactedCommand,
+                commandText,
+                data: redactedErrorData,
+                failureCategory,
+                index,
+                nextActions,
+                resultCategory: "failure",
+                success: false,
+                summary: errorText,
+                text: errorText,
+            },
+            presentation,
+        };
+    }
+    const commandInfo = parseCommandInfo(command ?? []);
+    const networkRouteDiagnostics = commandInfo.command === "network" && commandInfo.subcommand === "requests"
+        ? buildNetworkRouteDiagnostics(item.result, networkRoutes)
+        : undefined;
+    const presentation = await buildNestedToolPresentation({
+        artifactManifest,
+        artifactRequest,
+        commandInfo,
+        cwd,
+        args: command,
+        envelope: { data: item.result, success: true },
+        networkRouteDiagnostics,
+        persistentArtifactStore,
+        sessionName,
+    });
+    const fullOutputPaths = getPresentationPaths({
+        primaryPath: presentation.fullOutputPath,
+        secondaryPaths: presentation.fullOutputPaths,
+    });
+    const imagePaths = getPresentationPaths({
+        primaryPath: presentation.imagePath,
+        secondaryPaths: presentation.imagePaths,
+    });
+    const text = getPresentationText(presentation) || presentation.summary;
+    const stepSucceeded = presentation.resultCategory !== "failure";
+    const nextActions = presentation.nextActions ?? buildAgentBrowserNextActions({
+        artifacts: presentation.artifacts,
+        args: command,
+        command: command?.[0],
+        failureCategory: presentation.failureCategory,
+        resultCategory: stepSucceeded ? "success" : "failure",
+        savedFilePath: presentation.savedFilePath,
+        successCategory: presentation.successCategory,
+    });
+    const pageChangeSummary = buildPageChangeSummary({
+        artifacts: presentation.artifacts,
+        commandInfo,
+        data: presentation.data,
+        nextActions,
+        savedFilePath: presentation.savedFilePath,
+        summary: presentation.summary,
+    });
+    return {
+        details: {
+            artifactVerification: presentation.artifactVerification,
+            artifacts: presentation.artifacts,
+            command: redactedCommand,
+            commandText,
+            data: presentation.data,
+            failureCategory: stepSucceeded ? undefined : presentation.failureCategory,
+            fullOutputPath: fullOutputPaths[0],
+            fullOutputPaths: fullOutputPaths.length > 0 ? fullOutputPaths : undefined,
+            imagePath: imagePaths[0],
+            imagePaths: imagePaths.length > 0 ? imagePaths : undefined,
+            index,
+            networkRouteDiagnostics: presentation.networkRouteDiagnostics,
+            nextActions,
+            pageChangeSummary,
+            resultCategory: stepSucceeded ? "success" : "failure",
+            savedFile: presentation.savedFile,
+            savedFilePath: presentation.savedFilePath,
+            success: stepSucceeded,
+            successCategory: stepSucceeded ? classifyPresentationSuccessCategory({ artifactVerification: presentation.artifactVerification, artifacts: presentation.artifacts, savedFile: presentation.savedFile }) : undefined,
+            summary: presentation.summary,
+            text,
+        },
+        presentation,
+    };
+}
+export async function buildBatchPresentation(options) {
+    const { artifactRequests, buildNestedToolPresentation, cwd, data, networkRoutes, persistentArtifactStore, sessionName, summary } = options;
+    const steps = [];
+    const protectedPersistentPaths = [];
+    let currentArtifactManifest = options.artifactManifest;
+    let currentNetworkRoutes = networkRoutes;
+    for (const [index, item] of data.entries()) {
+        const step = await buildBatchStepPresentation({
+            artifactManifest: currentArtifactManifest,
+            artifactRequest: artifactRequests?.[index],
+            buildNestedToolPresentation,
+            cwd,
+            index,
+            item,
+            networkRoutes: currentNetworkRoutes,
+            persistentArtifactStore: persistentArtifactStore ? { ...persistentArtifactStore, protectedPaths: protectedPersistentPaths } : undefined,
+            sessionName,
+        });
+        steps.push(step);
+        currentArtifactManifest = step.presentation.artifactManifest ?? currentArtifactManifest;
+        currentNetworkRoutes = applyNetworkRouteRecords(currentNetworkRoutes, isStringArray(item.command) ? extractCommandTokens(item.command) : undefined, item.success !== false && step.details.success);
+        protectedPersistentPaths.push(...getPresentationPaths({
+            primaryPath: step.presentation.fullOutputPath,
+            secondaryPaths: step.presentation.fullOutputPaths,
+        }));
+    }
+    const batchFailure = getBatchFailureDetails(steps);
+    const images = steps.flatMap((step) => getPresentationImages(step.presentation));
+    const artifacts = steps.flatMap((step) => step.presentation.artifacts ?? []);
+    const artifactVerification = buildArtifactVerificationSummary(artifacts);
+    const fullOutputPaths = steps.flatMap((step) => getPresentationPaths({
+        primaryPath: step.presentation.fullOutputPath,
+        secondaryPaths: step.presentation.fullOutputPaths,
+    }));
+    const imagePaths = steps.flatMap((step) => getPresentationPaths({
+        primaryPath: step.presentation.imagePath,
+        secondaryPaths: step.presentation.imagePaths,
+    }));
+    const redactedBatchData = steps.map(({ details }) => (details.success
+        ? { command: details.command, result: details.data, success: true }
+        : { command: details.command, error: details.text, success: false }));
+    const stepText = formatBatchStepsText(steps);
+    const batchSummary = batchFailure === undefined
+        ? summary
+        : `Batch failed: ${batchFailure.successCount}/${batchFailure.totalCount} succeeded`;
+    const failureHeader = batchFailure === undefined
+        ? undefined
+        : [
+            batchSummary,
+            `First failing step: ${batchFailure.failedStep.index + 1} — ${batchFailure.failedStep.commandText}`,
+            batchFailure.failureCount > 1 ? `${batchFailure.failureCount} steps failed. See the per-step results below.` : "See the per-step results below.",
+        ].join("\n");
+    const text = failureHeader ? `${failureHeader}\n\n${stepText}` : stepText;
+    const artifactRetentionSummary = currentArtifactManifest ? formatSessionArtifactRetentionSummary(currentArtifactManifest) : undefined;
+    const contentText = artifactRetentionSummary && manifestHasNewNoticeWorthyEntries(options.artifactManifest, currentArtifactManifest)
+        ? `${text}\n\n${artifactRetentionSummary}`
+        : text;
+    const nextActions = batchFailure
+        ? batchFailure.failedStep.nextActions
+        : buildAgentBrowserNextActions({ artifacts, command: "batch", resultCategory: "success" });
+    const changedSteps = steps.map((step) => step.details).filter((details) => details.pageChangeSummary !== undefined);
+    const pageChangeSummary = artifacts.length > 0
+        ? buildPageChangeSummary({
+            artifacts,
+            commandInfo: { command: "batch" },
+            data,
+            nextActions,
+            summary: batchSummary,
+        })
+        : changedSteps.length > 0
+            ? {
+                changeType: "mutation",
+                command: "batch",
+                nextActionIds: nextActions?.map((action) => action.id),
+                summary: `batch → mutation → ${changedSteps.length} changed step${changedSteps.length === 1 ? "" : "s"}`,
+            }
+            : undefined;
+    return {
+        artifactManifest: currentArtifactManifest,
+        artifactRetentionSummary,
+        artifactVerification,
+        artifacts: artifacts.length > 0 ? artifacts : undefined,
+        batchFailure,
+        batchSteps: steps.map((step) => step.details),
+        content: [{ type: "text", text: contentText }, ...images],
+        failureCategory: batchFailure?.failedStep.failureCategory,
+        data: redactedBatchData,
+        fullOutputPath: fullOutputPaths[0],
+        fullOutputPaths: fullOutputPaths.length > 0 ? fullOutputPaths : undefined,
+        imagePath: imagePaths[0],
+        imagePaths: imagePaths.length > 0 ? imagePaths : undefined,
+        nextActions,
+        pageChangeSummary,
+        resultCategory: batchFailure ? "failure" : "success",
+        successCategory: batchFailure ? undefined : classifyPresentationSuccessCategory({ artifactVerification, artifacts }),
+        summary: batchSummary,
+    };
+}

package/dist/extensions/agent-browser/lib/results/presentation/browser-profile-recovery.js

@@ -0,0 +1,55 @@
+/**
+ * Purpose: Build browser profile/config recovery hints for launch/setup failures.
+ * Responsibilities: Recognize local Chrome/profile setup errors, avoid self-loop diagnostics, and emit canonical next actions.
+ * Scope: Error presentation only; upstream profile discovery and doctor execution remain normal agent_browser commands.
+ */
+import { isOpenNavigationCommand } from "../../command-taxonomy.js";
+import { getStartupScopedFlags } from "../../runtime.js";
+import { buildNextToolAction } from "../next-actions.js";
+const BROWSER_PROFILE_CONFIG_HINT = [
+    "Agent-browser profile/config hint: this looks like a local browser profile or Chrome user-data-dir setup problem, not a page-specific failure.",
+    "Do not keep retrying the same open/profile call. Run `profiles` and/or `doctor` through agent_browser, then tell the user whether Chrome/Chromium is installed, which Chrome profile directory names are available, or whether they need to configure a full profile/user-data directory path, a non-default Chromium-compatible `--executable-path`, or remove the profile requirement for public-page browsing.",
+    "Use the top-level `sessionMode: \"fresh\"` field for launch-scoped profile/debug/provider flags; do not pass `--session-mode` inside args.",
+].join(" ");
+function looksLikeBrowserProfileConfigError(errorText) {
+    return /\b(?:No Chrome user data directory found|Cannot resolve profile name|Chrome user data directory|Chrome profile\s+.+?\s+not found|Available profiles|If you meant a directory path)\b/i.test(errorText);
+}
+function isLaunchOrSetupContext(args, commandInfo) {
+    const command = commandInfo.command;
+    if (command === "profiles" || command === "doctor")
+        return true;
+    if (command && isOpenNavigationCommand(command))
+        return true;
+    return (args ? getStartupScopedFlags(args) : []).length > 0;
+}
+function buildBrowserProfileConfigActions(commandInfo) {
+    const actions = [];
+    if (commandInfo.command !== "profiles") {
+        actions.push(buildNextToolAction({
+            args: ["profiles"],
+            id: "inspect-browser-profiles",
+            reason: "List browser profiles/user-data-dir candidates before retrying profile-based launch.",
+            safety: "Read-only local setup inspection; does not open a page or mutate browser state.",
+        }));
+    }
+    if (commandInfo.command !== "doctor") {
+        actions.push(buildNextToolAction({
+            args: ["doctor"],
+            id: "run-agent-browser-doctor",
+            reason: "Inspect local agent-browser browser installation/configuration before retrying.",
+            safety: "Read-only local diagnostics; report findings to the user before changing setup.",
+        }));
+    }
+    return actions;
+}
+export function buildBrowserProfileConfigRecovery(options) {
+    if (!looksLikeBrowserProfileConfigError(options.errorText))
+        return undefined;
+    if (!isLaunchOrSetupContext(options.args, options.commandInfo))
+        return undefined;
+    const actions = buildBrowserProfileConfigActions(options.commandInfo);
+    return {
+        actions: actions.length > 0 ? actions : undefined,
+        hint: BROWSER_PROFILE_CONFIG_HINT,
+    };
+}

package/dist/extensions/agent-browser/lib/results/presentation/common.js

@@ -0,0 +1,46 @@
+/**
+ * Purpose: Share small presentation formatting and redaction helpers across result presentation modules.
+ * Responsibilities: Normalize scalar fields, stringify model-facing values, and apply sensitive-text redaction.
+ * Scope: Leaf helpers only; command-family formatting lives in sibling modules.
+ */
+import { redactSensitiveText, redactSensitiveValue } from "../../runtime.js";
+import { stringifyUnknown, truncateText } from "../text.js";
+export function stringifyModelFacing(value) {
+    return stringifyUnknown(redactSensitiveValue(value));
+}
+export function parseJsonPreviewString(value) {
+    const trimmed = value.trim();
+    if (!trimmed.startsWith("{") && !trimmed.startsWith("["))
+        return value;
+    try {
+        return JSON.parse(trimmed);
+    }
+    catch {
+        return value;
+    }
+}
+export function redactModelFacingText(text) {
+    const parsed = parseJsonPreviewString(text);
+    if (parsed !== text) {
+        return stringifyModelFacing(parsed);
+    }
+    return redactSensitiveText(text);
+}
+export function redactModelFacingTextIfSensitive(text) {
+    return /(?:@|\b(?:access[_-]?key|api[_-]?key|auth|authorization|basic|bearer|connection[_-]?string|cookie|database[_-]?url|db[_-]?url|mongo(?:db)?[_-]?uri|pass(?:word)?|private[_-]?key|redis[_-]?url|secret|session[_-]?id|token)\b)/i.test(text)
+        ? redactModelFacingText(text)
+        : text;
+}
+export function getArrayField(data, key) {
+    return Array.isArray(data[key]) ? data[key] : undefined;
+}
+export function getStringField(data, key) {
+    const value = data[key];
+    return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+}
+export function formatCount(count, singular, plural = `${singular}s`) {
+    return `${count} ${count === 1 ? singular : plural}`;
+}
+export function firstLine(value, maxChars = 160) {
+    return truncateText(value.split("\n", 1)[0] ?? value, maxChars);
+}

package/dist/extensions/agent-browser/lib/results/presentation/content.js

@@ -0,0 +1,24 @@
+/**
+ * Purpose: Share small ToolPresentation content helpers used by batch and compaction code.
+ * Responsibilities: Extract text/image/path fields and format batch step command labels.
+ * Scope: Pure ToolPresentation content helpers only.
+ */
+export function isStringArray(value) {
+    return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+export function getPresentationText(presentation) {
+    return presentation.content
+        .filter((part) => part.type === "text")
+        .map((part) => part.text.trim())
+        .filter((text) => text.length > 0)
+        .join("\n\n");
+}
+export function getPresentationImages(presentation) {
+    return presentation.content.filter((part) => part.type === "image");
+}
+export function getPresentationPaths(options) {
+    return options.secondaryPaths ?? (options.primaryPath ? [options.primaryPath] : []);
+}
+export function formatBatchStepCommand(command, index) {
+    return command && command.length > 0 ? command.join(" ") : `step-${index + 1}`;
+}