pi-agent-browser-native 0.6.17 → 0.7.1

package/dist/extensions/agent-browser/lib/results/presentation/large-output.js

@@ -2,7 +2,7 @@ import { writePersistentSessionArtifactFile, writeSecureTempFile, } from "../../
 import { buildEvictedSessionArtifactEntries } from "../artifact-manifest.js";
 import { countLines, truncateText } from "../text.js";
 import { applyArtifactManifest } from "./artifacts.js";
-import { getPresentationText } from "./content.js";
+import { getPresentationText, projectAgentBrowserObservation } from "./content.js";
 import { redactModelFacingText, stringifyModelFacing } from "./common.js";
 const LARGE_OUTPUT_INLINE_MAX_CHARS = 8_000;
 const LARGE_OUTPUT_INLINE_MAX_LINES = 120;
@@ -88,6 +88,46 @@ function buildSpillArtifactEntries(options) {
         ...buildEvictedSessionArtifactEntries(options.evictedArtifacts, nowMs),
     ];
 }
+export const OBSERVATION_INLINE_MAX_CHARS = 16_000;
+/** Bound the final model-visible result, after recovery/diagnostic assembly. Never truncate executable actions. */
+export async function renderAgentBrowserObservation(options) {
+    const observation = projectAgentBrowserObservation(options.details, options.succeeded);
+    const images = options.content.filter(part => part.type === "image");
+    const prose = options.content.filter(part => part.type === "text").map(part => part.text).join("\n\n");
+    const { data: _data, error: _error, summary: _summary, ...metadata } = observation;
+    let text = options.json ? JSON.stringify(observation, null, 2) : `${prose}\n\nObservation: ${JSON.stringify(metadata)}`;
+    let artifactManifest = options.details.artifactManifest;
+    if (text.length > OBSERVATION_INLINE_MAX_CHARS) {
+        let spill;
+        let spillError;
+        try {
+            spill = await writeLargeOutputSpillFile({ data: { ...observation, ...(!options.json ? { text: prose } : {}) }, persistentArtifactStore: options.persistentArtifactStore, text });
+            artifactManifest = applyArtifactManifest({ content: [], summary: "" }, artifactManifest, buildSpillArtifactEntries({
+                commandInfo: { command: typeof options.details.command === "string" ? options.details.command : undefined },
+                evictedArtifacts: spill.evictedArtifacts, path: spill.path, storageScope: spill.storageScope,
+            })).artifactManifest;
+        }
+        catch (error) {
+            spillError = error instanceof Error ? error.message : String(error);
+        }
+        const compact = {
+            success: options.succeeded, resultCategory: observation.resultCategory,
+            failureCategory: observation.failureCategory, successCategory: observation.successCategory,
+            summary: typeof observation.summary === "string" ? truncateText(observation.summary, 700) : undefined,
+            compacted: true,
+            ...(spill ? { observationPath: spill.path, retrieve: "Read observationPath for the complete redacted observation, including exact recovery actions and requested data." }
+                : { observationUnavailable: truncateText(redactModelFacingText(spillError ?? "Spill could not be written; request a smaller result."), 1_000) }),
+        };
+        for (const key of ["sessionName", "namespace", "codeRun", "error", "failures", "nextActions", "artifactVerification", "imageObservations", "data", "fullOutputPath", "fullOutputPaths"]) {
+            if (observation[key] !== undefined && JSON.stringify({ ...compact, [key]: observation[key] }, null, 2).length <= OBSERVATION_INLINE_MAX_CHARS - 500)
+                compact[key] = observation[key];
+        }
+        text = JSON.stringify(compact, null, 2);
+        if (!options.json)
+            text = `Browser observation compacted.\n${text}`;
+    }
+    return { content: [{ type: "text", text }, ...images], artifactManifest };
+}
 export async function compactLargePresentationOutput(options) {
     const text = getPresentationText(options.presentation);
     if (text.length === 0 || !shouldCompactLargeOutput(text)) {

package/dist/extensions/agent-browser/lib/results/presentation.js

@@ -1,3 +1,4 @@
+import { getScreenshotCapture } from "../orchestration/browser-run/screenshot-observation.js";
 import { isRecord } from "../parsing.js";
 import { buildReadConfirmationNextActions, nextReadConfirmation } from "../read-confirmation.js";
 import { extractUpstreamCommandTokens, parseCommandInfo, redactInvocationArgs } from "../runtime.js";
@@ -56,13 +57,9 @@ export async function buildToolPresentation(options) {
     const recordingBatch = commandInfo.command === "batch" && isAgentBrowserBatchResultArray(envelope?.data)
         && envelope.data.some((row) => row.command?.[0] === "record");
     if (errorText && !recordingCommand && !recordingBatch) {
-        return buildErrorPresentation({
-            args,
-            commandInfo,
-            errorText,
-            presentationCommand: presentationCommandInfo.command,
-            sessionName,
-        });
+        return { ...buildErrorPresentation({ args, commandInfo, errorText, presentationCommand: presentationCommandInfo.command, sessionName }),
+            data: redactPresentationData(commandInfoWithTokens, envelope?.data),
+        };
     }
     let data = enrichStreamStatusData(commandInfoWithTokens, envelope?.data);
     if (commandInfo.command === "session" && commandInfo.subcommand === "info" && isRecord(data)) {
@@ -85,6 +82,7 @@ export async function buildToolPresentation(options) {
             artifactMinUpdatedAtMs: options.artifactMinUpdatedAtMs,
             artifactRequests: options.batchArtifactRequests,
             buildNestedToolPresentation: buildToolPresentation,
+            modelVisible: options.modelVisible,
             cwd,
             data,
             namespace,
@@ -95,14 +93,14 @@ export async function buildToolPresentation(options) {
             summary,
         });
     }
-    else if (commandInfo.command === "snapshot" && isRecord(data)) {
+    else if (options.modelVisible !== false && commandInfo.command === "snapshot" && isRecord(data)) {
         presentation = await buildSnapshotPresentation(data, persistentArtifactStore, artifactManifest);
     }
     else {
         presentation = {
             artifactVerification,
             artifacts: artifacts.length > 0 ? artifacts : undefined,
-            content: [{ type: "text", text: artifactText ?? formatPresentationContentText(commandInfoWithTokens, data, compiledSemanticAction) }],
+            content: options.modelVisible === false ? [] : [{ type: "text", text: artifactText ?? formatPresentationContentText(commandInfoWithTokens, data, compiledSemanticAction) }],
             data: presentationData,
             summary,
         };
@@ -143,8 +141,13 @@ export async function buildToolPresentation(options) {
     }
     const imagePath = artifactRequest?.absolutePath ?? extractImagePath(commandInfo, cwd, data)
         ?? (recordingCommand ? artifacts.find((artifact) => artifact.kind === "image" && artifact.status === "saved")?.absolutePath : undefined);
-    const presentationWithImage = imagePath ? await attachInlineImage(presentation, imagePath) : presentation;
-    const compactedPresentation = await compactLargePresentationOutput({
+    const attachImage = imagePath && !(isRecord(data) && data.changed === false) && !artifacts.some(artifact => artifact.absolutePath === imagePath && ["missing", "stale", "failed"].includes(artifact.status ?? ""));
+    const presentationWithImage = attachImage ? await attachInlineImage(presentation, imagePath, options.modelVisible) : presentation;
+    if (commandInfo.command === "screenshot" && presentationWithImage.imageObservations) {
+        for (const image of presentationWithImage.imageObservations)
+            image.capture = getScreenshotCapture(commandInfoWithTokens.commandTokens ?? []).kind;
+    }
+    const compactedPresentation = options.modelVisible === false ? presentationWithImage : await compactLargePresentationOutput({
         artifactManifest,
         commandInfo,
         data: presentationData,
@@ -241,5 +244,7 @@ export async function buildToolPresentation(options) {
     if (presentationWithManifest.pageChangeSummary?.observed === false && presentationCommandInfo.command !== "batch" && presentationWithManifest.content[0]?.type === "text") {
         presentationWithManifest.content[0] = { ...presentationWithManifest.content[0], text: `${presentationWithManifest.content[0].text}\n\nAction dispatched; application change unverified. Verify the expected URL, text, state, or external receipt before relying on it.` };
     }
+    if (options.modelVisible === false)
+        presentationWithManifest.content = [];
     return sanitizeModelFacingPresentation(presentationWithManifest);
 }

package/dist/extensions/agent-browser/lib/runtime.js

@@ -2,8 +2,10 @@ import { createHash, randomUUID } from "node:crypto";
 import { basename } from "node:path";
 import { extractUpstreamCommandTokens, findCommandStartIndex, parseArgvDescriptor, parseCommandInfo, } from "./argv-descriptor.js";
 import { batchHasSuccessfulCloseAll, getSuccessfulBatchCloseLifecycle } from "./batch-lifecycle.js";
-import { canonicalizeAgentBrowserNamespace, extractExplicitNamespace, extractExplicitSessionName, getAgentBrowserSessionIdentityKey, getBooleanFlagValue, GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES, GLOBAL_VALUE_FLAGS_ALLOWING_DASH_VALUE, isAgentBrowserSessionIdentityKeyInNamespace, isUpstreamEnvFlagEnabled, PREVALIDATED_VALUE_FLAGS, resolveAgentBrowserNamespace, scanUpstreamGlobalFlagOccurrences, stripUpstreamGlobalFlags, } from "./argv-grammar.js";
+import { getBrowserResultMessage } from "./browser-transcript.js";
+import { canonicalizeAgentBrowserNamespace, extractExplicitNamespace, extractExplicitSessionName, getAgentBrowserSessionIdentityKey, getBooleanFlagValue, GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES, GLOBAL_VALUE_FLAGS_ALLOWING_DASH_VALUE, isAgentBrowserSessionIdentityKeyInNamespace, isUpstreamEnvFlagEnabled, PREVALIDATED_VALUE_FLAGS, projectUpstreamGlobalFlags, resolveAgentBrowserNamespace, scanUpstreamGlobalFlagOccurrences, stripUpstreamGlobalFlags, } from "./argv-grammar.js";
 import { needsManagedSession } from "./command-policy.js";
+import { parseBatchCommandArgument } from "./orchestration/batch-stdin.js";
 import { isCloseAllCommand, isCloseCommand, isOpenNavigationCommand } from "./command-taxonomy.js";
 import { hasLaunchScopedFlagToken, LAUNCH_SCOPED_FLAG_DEFINITIONS, LAUNCH_SCOPED_FLAG_LABEL, } from "./launch-scoped-flags.js";
 import { MANAGED_SESSION_NAME_PREFIX, } from "./managed-session-restore.js";
@@ -356,6 +358,19 @@ export function redactInvocationArgs(args) {
             redacted[index] = "[REDACTED]";
         }
     }
+    const batch = projectUpstreamGlobalFlags(args);
+    if (batch.tokens[0] === "batch") {
+        for (let index = 1; index < batch.tokens.length; index++) {
+            if (batch.tokens[index] === "--bail")
+                continue;
+            const step = parseBatchCommandArgument(batch.tokens[index]).step;
+            if (!step)
+                continue;
+            const safe = redactInvocationArgs(step);
+            if (safe.some((token, offset) => token !== step[offset]))
+                redacted[batch.indices[index]] = safe.map(token => `'${token.replaceAll("'", "'\\''")}'`).join(" ");
+        }
+    }
     return redacted;
 }
 export function isPlainTextInspectionArgs(args) {
@@ -478,11 +493,8 @@ export function restoreManagedSessionStateFromBranch(branch, fallbackSessionName
         closedSessionName = sessionName;
     };
     for (const entry of branch) {
-        if (!isRecord(entry) || entry.type !== "message") {
-            continue;
-        }
-        const message = isRecord(entry.message) ? entry.message : undefined;
-        if (!message || message.toolName !== "agent_browser") {
+        const message = getBrowserResultMessage(entry);
+        if (!message) {
             continue;
         }
         const details = isRecord(message.details) ? message.details : undefined;

package/dist/extensions/agent-browser/lib/session-page-state.js

@@ -1,6 +1,7 @@
 import { extractUpstreamCommandTokens } from "./argv-descriptor.js";
 import { getAgentBrowserSessionIdentityKey, isAgentBrowserSessionIdentityKeyInNamespace } from "./argv-grammar.js";
 import { batchHasSuccessfulCloseAll, getSuccessfulBatchCloseLifecycle } from "./batch-lifecycle.js";
+import { getBrowserResultMessage } from "./browser-transcript.js";
 import { isCloseAllCommand, isCloseCommand, isReadOnlyDiagnosticSessionTargetCommand, isRecordPageTransitionCommand, isUnverifiedPageTransitionCommand, isWebMcpPageMutationCommand, isWindowOrDiffPageTransitionCommand } from "./command-taxonomy.js";
 import { isRecord } from "./parsing.js";
 import { findReadConfirmation as findPendingReadConfirmation, parseReadConfirmation } from "./read-confirmation.js";
@@ -351,10 +352,8 @@ export class SessionPageState {
         const state = new SessionPageState();
         let restoredOrder = 0;
         for (const entry of branch) {
-            if (!isRecord(entry) || entry.type !== "message")
-                continue;
-            const message = isRecord(entry.message) ? entry.message : undefined;
-            if (!message || message.toolName !== "agent_browser")
+            const message = getBrowserResultMessage(entry);
+            if (!message)
                 continue;
             const details = isRecord(message.details) ? message.details : undefined;
             if (!details)

package/dist/extensions/agent-browser/lib/tool-surface.js

@@ -0,0 +1,138 @@
+import { Text } from "@earendil-works/pi-tui";
+import { formatAgentBrowserRenderCall } from "./pi-tool-rendering.js";
+import { AGENT_BROWSER_PARAMS, AGENT_BROWSER_CODE_PARAMS, AGENT_BROWSER_ACTION_PARAMS, AGENT_BROWSER_QA_PARAMS, AGENT_BROWSER_ELECTRON_PARAMS, AGENT_BROWSER_SOURCE_PARAMS, AGENT_BROWSER_NETWORK_SOURCE_PARAMS, AGENT_BROWSER_TOOLS_PARAMS, } from "./input-modes/params.js";
+import { ADVANCED_TOOL_PROMPT_GUIDELINES, buildToolPromptGuidelines } from "./playbook.js";
+export const AGENT_BROWSER_TOOL_INVENTORY = {
+    action: { name: "agent_browser_action", description: "Click, check, fill, or select a unique accessible target using stable locators or current refs." },
+    qa: { name: "agent_browser_qa", description: "Check a URL or attached page with visible assertions and a diagnostic pass/fail verdict." },
+    electron: { name: "agent_browser_electron", description: "Discover, launch, probe, and clean up owned Electron desktop applications." },
+    source: { name: "agent_browser_source", description: "Find experimental DOM/React/local-source candidates with a bounded workspace scan." },
+    network: { name: "agent_browser_network_source", description: "Correlate failed requests with initiator and local-source candidates." },
+};
+const advancedNames = new Set(Object.values(AGENT_BROWSER_TOOL_INVENTORY).map(({ name }) => name));
+/** Register once; advanced calls adapt only their input shape and reuse the ordinary executor. */
+export function registerAgentBrowserToolSurface(pi, options) {
+    pi.registerTool({
+        name: "agent_browser",
+        label: "Agent Browser",
+        description: "Browse and interact through native agent-browser commands. One command in args; fixed sequences use batch --bail and JSON-array stdin. Use agent_browser_code for loops/branches and agent_browser_tools for advanced capabilities.",
+        promptSnippet: "Browse pages, inspect current refs, interact, and run native command batches.",
+        promptGuidelines: options.promptGuidelines ?? buildToolPromptGuidelines({ includeWebSearch: false }),
+        parameters: AGENT_BROWSER_PARAMS,
+        renderCall: options.renderCall,
+        renderResult: options.renderResult,
+        execute: options.execute,
+        executionMode: options.executionMode,
+    });
+    pi.registerTool({
+        name: "agent_browser_code",
+        label: "Agent Browser Code",
+        description: "Run fresh JavaScript against a persistent browser. await browser({args,stdin?,timeoutMs?}) returns success/data/error/nextActions and imageObservations; emit(selected JSON) and emitImage(image handle) explicitly choose output. No host APIs or imports. Use native batch for fixed sequences.",
+        promptSnippet: "Branch, loop, and aggregate browser observations with explicit JSON/image output.",
+        parameters: AGENT_BROWSER_CODE_PARAMS,
+        constrainedSampling: { type: "json_schema", strict: "prefer" },
+        execute: options.executeCode,
+        executionMode: options.executionMode,
+        renderCall(args, theme, context) {
+            const text = context.lastComponent instanceof Text ? context.lastComponent : new Text("", 0, 0);
+            text.setText(formatAgentBrowserRenderCall(args, theme, context.expanded));
+            return text;
+        },
+        renderResult: options.renderResult,
+    });
+    pi.registerTool({
+        ...AGENT_BROWSER_TOOL_INVENTORY.action,
+        label: "Browser Action",
+        promptGuidelines: [...ADVANCED_TOOL_PROMPT_GUIDELINES.action],
+        parameters: AGENT_BROWSER_ACTION_PARAMS,
+        renderResult: options.renderResult,
+        executionMode: options.executionMode,
+        execute(id, { outputPath, timeoutMs, ...semanticAction }, signal, onUpdate, ctx) {
+            return options.execute(id, { semanticAction, outputPath, timeoutMs }, signal, onUpdate, ctx);
+        },
+    });
+    pi.registerTool({
+        ...AGENT_BROWSER_TOOL_INVENTORY.qa,
+        label: "Browser QA",
+        promptGuidelines: [...ADVANCED_TOOL_PROMPT_GUIDELINES.qa],
+        parameters: AGENT_BROWSER_QA_PARAMS,
+        renderResult: options.renderResult,
+        executionMode: options.executionMode,
+        execute(id, { outputPath, timeoutMs, sessionMode, ...qa }, signal, onUpdate, ctx) {
+            return options.execute(id, { qa, outputPath, timeoutMs, sessionMode }, signal, onUpdate, ctx);
+        },
+    });
+    pi.registerTool({
+        ...AGENT_BROWSER_TOOL_INVENTORY.electron,
+        label: "Browser Electron",
+        promptGuidelines: [...ADVANCED_TOOL_PROMPT_GUIDELINES.electron],
+        parameters: AGENT_BROWSER_ELECTRON_PARAMS,
+        renderResult: options.renderResult,
+        executionMode: options.executionMode,
+        execute(id, { outputPath, ...electron }, signal, onUpdate, ctx) {
+            return options.execute(id, { electron, outputPath }, signal, onUpdate, ctx);
+        },
+    });
+    pi.registerTool({
+        ...AGENT_BROWSER_TOOL_INVENTORY.source,
+        label: "Browser Source",
+        promptGuidelines: [...ADVANCED_TOOL_PROMPT_GUIDELINES.source],
+        parameters: AGENT_BROWSER_SOURCE_PARAMS,
+        renderResult: options.renderResult,
+        executionMode: options.executionMode,
+        execute(id, { outputPath, timeoutMs, sessionMode, ...sourceLookup }, signal, onUpdate, ctx) {
+            return options.execute(id, { sourceLookup, outputPath, timeoutMs, sessionMode }, signal, onUpdate, ctx);
+        },
+    });
+    pi.registerTool({
+        ...AGENT_BROWSER_TOOL_INVENTORY.network,
+        label: "Browser Network Source",
+        promptGuidelines: [...ADVANCED_TOOL_PROMPT_GUIDELINES.network],
+        parameters: AGENT_BROWSER_NETWORK_SOURCE_PARAMS,
+        renderResult: options.renderResult,
+        executionMode: options.executionMode,
+        execute(id, { outputPath, timeoutMs, sessionMode, ...networkSourceLookup }, signal, onUpdate, ctx) {
+            return options.execute(id, { networkSourceLookup, outputPath, timeoutMs, sessionMode }, signal, onUpdate, ctx);
+        },
+    });
+    pi.registerTool({
+        name: "agent_browser_tools",
+        label: "Browser Tools",
+        description: "List or enable advanced browser tools: action → agent_browser_action; qa → agent_browser_qa; electron → agent_browser_electron; source → agent_browser_source; network → agent_browser_network_source. Omit enable for inventory. Activation only adds tools; it preserves other active tools.",
+        promptSnippet: "Discover and enable specialized browser actions, QA, Electron, and source tools.",
+        parameters: AGENT_BROWSER_TOOLS_PARAMS,
+        async execute(_id, params) {
+            const available = new Set(pi.getAllTools().map(({ name }) => name));
+            const active = new Set(pi.getActiveTools());
+            const added = (params.enable ?? []).map((key) => AGENT_BROWSER_TOOL_INVENTORY[key].name)
+                .filter((name) => available.has(name) && !active.has(name));
+            if (added.length > 0)
+                pi.setActiveTools([...new Set([...active, ...added])]);
+            const current = new Set(pi.getActiveTools());
+            const inventory = Object.entries(AGENT_BROWSER_TOOL_INVENTORY).map(([key, tool]) => ({
+                key, ...tool, available: available.has(tool.name), active: current.has(tool.name),
+            }));
+            return {
+                content: [{ type: "text", text: inventory.map((tool) => `${tool.key}: ${tool.name} (${!tool.available ? "unavailable in this Pi tool selection" : tool.active ? "active" : "inactive"}) — ${tool.description}`).join("\n") }],
+                details: { inventory, added: [...new Set(added)] },
+            };
+        },
+    });
+    pi.on("session_start", async (_event, ctx) => {
+        // Pi owns selected tools and their durable system-message history; keep no parallel registry.
+        const argv = process.argv.slice(2);
+        const delimiter = argv.indexOf("--");
+        if ((delimiter < 0 ? argv : argv.slice(0, delimiter)).some((arg) => arg === "--tools" || arg === "-t"))
+            return;
+        // A host-filtered catalog is an explicit selection, not our default surface.
+        const available = new Set(pi.getAllTools().map(({ name }) => name));
+        if (!["agent_browser", "agent_browser_code", "agent_browser_tools", ...advancedNames].every(name => available.has(name)))
+            return;
+        // Keep Pi's replay helper behind the awaited restoration boundary, not factory registration.
+        const { getCurrentSystemMessage } = await import("@earendil-works/pi-ai");
+        const current = getCurrentSystemMessage(ctx.sessionManager.buildSessionProjection().messages);
+        const restored = new Set(current?.toolsAdded?.map(({ name }) => name));
+        const active = pi.getActiveTools().filter((name) => !advancedNames.has(name) || restored.has(name));
+        pi.setActiveTools([...new Set([...active, ...[...restored].filter((name) => advancedNames.has(name) && available.has(name))])]);
+    });
+}

package/dist/extensions/agent-browser/script-worker.js

@@ -56,11 +56,14 @@ runInContext("const " + stateName + " = (() => {\n" +
     "    });\n" +
     "  };\n" +
     "  const emit = function emit(value) { sendValue({ type: 'emit', value }); };\n" +
+    "  const emitImage = function emitImage(value) { sendValue({ type: 'image', value }); };\n" +
     "  Object.setPrototypeOf(browser, null);\n" +
     "  Object.setPrototypeOf(emit, null);\n" +
+    "  Object.setPrototypeOf(emitImage, null);\n" +
     "  Object.freeze(browser);\n" +
     "  Object.freeze(emit);\n" +
-    "  Object.defineProperties(globalThis, { browser: { value: browser, writable: false, configurable: false }, emit: { value: emit, writable: false, configurable: false } });\n" +
+    "  Object.freeze(emitImage);\n" +
+    "  Object.defineProperties(globalThis, { browser: { value: browser, writable: false, configurable: false }, emit: { value: emit, writable: false, configurable: false }, emitImage: { value: emitImage, writable: false, configurable: false } });\n" +
     "  const complete = (ok, value) => {\n" +
     "    if (ok) {\n" +
     "      try { sendValue(value === undefined ? { type: 'complete', hasValue: false } : { type: 'complete', hasValue: true, value }); }\n" +