pi-agent-browser-native 0.6.16 → 0.7.0

package/dist/extensions/agent-browser/lib/orchestration/script-mode.js

@@ -1,13 +1,12 @@
-import { open } from "node:fs/promises";
-import { AGENT_BROWSER_SCRIPT_FINAL_OUTPUT_MAX_BYTES, AGENT_BROWSER_SCRIPT_IPC_MESSAGE_MAX_BYTES, AGENT_BROWSER_SCRIPT_SPILL_MAX_BYTES, createAgentBrowserScriptCloseArgs, isAgentBrowserScriptSessionName, validateAgentBrowserScriptBrowserParams, } from "../input-modes/script.js";
+import { stat } from "node:fs/promises";
+import { AGENT_BROWSER_SCRIPT_FINAL_OUTPUT_MAX_BYTES, createAgentBrowserScriptCloseArgs, isAgentBrowserScriptSessionName, } from "../input-modes/script.js";
 import { isRecord } from "../parsing.js";
-import { parseCommandInfo, redactSensitiveText } from "../runtime.js";
-import { isSessionArtifactManifest } from "../results/artifact-manifest.js";
+import { redactSensitiveText } from "../runtime.js";
+import { attachInlineImage } from "../results/presentation/artifacts.js";
+import { projectAgentBrowserObservation } from "../results/presentation/content.js";
 import { redactPresentationData } from "../results/presentation/diagnostics.js";
-import { truncateText } from "../results/text.js";
 const SCRIPT_SESSION_ENTRY_TYPE = "agent-browser-script-session";
-const SCRIPT_BROWSER_SUMMARY_MAX_CHARS = 1_024;
-const SCRIPT_BROWSER_TEXT_MAX_CHARS = 8_192;
+// Retire outstanding pre-0.7 isolated-session leases when an old transcript resumes.
 export function getScriptSessionLeasesFromBranch(branch) {
     const leases = new Map();
     for (const entry of branch) {
@@ -16,284 +15,98 @@ export function getScriptSessionLeasesFromBranch(branch) {
         const { cleanup, closeCommandArgs, launchAttempted, sessionName } = entry.data;
         if (!isAgentBrowserScriptSessionName(sessionName))
             continue;
-        const expectedCloseCommandArgs = createAgentBrowserScriptCloseArgs(sessionName);
-        if ((cleanup !== "active" && cleanup !== "closed" && cleanup !== "failed")
-            || launchAttempted !== true
-            || !Array.isArray(closeCommandArgs)
-            || closeCommandArgs.length !== expectedCloseCommandArgs.length
-            || !closeCommandArgs.every((token, index) => token === expectedCloseCommandArgs[index]))
+        const expected = createAgentBrowserScriptCloseArgs(sessionName);
+        if ((cleanup !== "active" && cleanup !== "closed" && cleanup !== "failed") || launchAttempted !== true
+            || !Array.isArray(closeCommandArgs) || closeCommandArgs.length !== expected.length
+            || !closeCommandArgs.every((token, index) => token === expected[index]))
             continue;
-        leases.set(sessionName, { cleanup, closeCommandArgs: expectedCloseCommandArgs, launchAttempted: true, sessionName });
+        leases.set(sessionName, { cleanup, closeCommandArgs: expected, launchAttempted: true, sessionName });
     }
     return leases;
 }
 export function appendScriptSessionLease(pi, sessionName, cleanup) {
-    pi.appendEntry(SCRIPT_SESSION_ENTRY_TYPE, {
-        cleanup,
-        closeCommandArgs: createAgentBrowserScriptCloseArgs(sessionName),
-        launchAttempted: true,
-        sessionName,
-    });
+    pi.appendEntry(SCRIPT_SESSION_ENTRY_TYPE, { cleanup, closeCommandArgs: createAgentBrowserScriptCloseArgs(sessionName), launchAttempted: true, sessionName });
 }
-async function readVerifiedScriptSpill(result) {
-    const details = isRecord(result.details) ? result.details : undefined;
-    const path = typeof details?.fullOutputPath === "string" ? details.fullOutputPath : undefined;
-    const manifest = isSessionArtifactManifest(details?.artifactManifest) ? details.artifactManifest : undefined;
-    if (!path || !manifest?.entries.some((entry) => entry.kind === "spill"
-        && (entry.path === path || entry.absolutePath === path)
-        && (entry.storageScope === "persistent-session" || entry.storageScope === "process-temp")
-        && (entry.retentionState === "live" || entry.retentionState === "ephemeral")))
-        return undefined;
-    let handle;
-    try {
-        handle = await open(path, "r");
-        const stats = await handle.stat();
-        if (!stats.isFile() || stats.size < 0 || stats.size > AGENT_BROWSER_SCRIPT_SPILL_MAX_BYTES)
-            return undefined;
-        const buffer = Buffer.alloc(stats.size);
-        let offset = 0;
-        while (offset < buffer.length) {
-            const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, offset);
-            if (bytesRead === 0)
-                break;
-            offset += bytesRead;
-        }
-        const text = buffer.subarray(0, offset).toString("utf8");
-        try {
-            return JSON.parse(text);
-        }
-        catch {
-            return undefined;
-        }
-    }
-    catch {
-        return undefined;
-    }
-    finally {
-        await handle?.close().catch(() => undefined);
-    }
-}
-function getToolResultText(result) {
-    return result.content
-        .filter((item) => item.type === "text")
-        .map((item) => item.text)
-        .join("\n\n");
-}
-function stripOwnedScriptSessionArgs(args, sessionName) {
-    if (!isAgentBrowserScriptSessionName(sessionName))
-        return args;
-    if (args[0] === "--namespace" && args[1] === "" && args[2] === "--session" && args[3] === sessionName)
-        return args.slice(4);
-    return args[0] === "--session" && args[1] === sessionName ? args.slice(2) : args;
-}
-function getScriptCompatibleNextActions(value, sessionName) {
-    if (!Array.isArray(value))
-        return undefined;
-    const actions = value.flatMap((action) => {
-        if (!isRecord(action))
-            return [];
-        if (!isRecord(action.params))
-            return typeof action.artifactPath === "string" ? [action] : [];
-        if (Object.keys(action.params).some((key) => key !== "args" && key !== "stdin"))
-            return [];
-        if (!Array.isArray(action.params.args) || action.params.args.some((token) => typeof token !== "string"))
-            return [];
-        if (action.params.stdin !== undefined && typeof action.params.stdin !== "string")
-            return [];
-        const normalizedParams = {
-            args: stripOwnedScriptSessionArgs(action.params.args, sessionName),
-            ...(typeof action.params.stdin === "string" ? { stdin: action.params.stdin } : {}),
-        };
-        if (!validateAgentBrowserScriptBrowserParams(normalizedParams).params)
-            return [];
-        return [{ ...action, params: normalizedParams }];
-    });
-    return actions.length > 0 ? actions : undefined;
-}
-function scriptBrowserEnvelopeFitsIpc(envelope) {
-    try {
-        return Buffer.byteLength(JSON.stringify({ envelope, id: Number.MAX_SAFE_INTEGER, type: "response" }), "utf8") + 1 <= AGENT_BROWSER_SCRIPT_IPC_MESSAGE_MAX_BYTES;
-    }
-    catch {
-        return false;
-    }
-}
-function buildOversizedScriptBrowserEnvelope() {
-    const text = "Browser result exceeds the script IPC response limit; narrow the inner browser call output.";
-    return {
-        data: null,
-        details: { failureCategory: "upstream-error", resultCategory: "failure" },
-        error: text,
-        failureCategory: "upstream-error",
-        ok: false,
-        resultCategory: "failure",
-        summary: text,
-        text,
-    };
-}
-export async function buildScriptBrowserEnvelope(result, args, scriptSessionName) {
-    const details = isRecord(result.details) ? result.details : undefined;
-    const fullData = await readVerifiedScriptSpill(result);
-    const commandInfo = parseCommandInfo(args);
-    const data = redactPresentationData(commandInfo, fullData ?? details?.data ?? null);
-    const resultCategory = details?.resultCategory === "failure" || result.isError === true ? "failure" : "success";
-    const text = truncateText(redactSensitiveText(getToolResultText(result)), SCRIPT_BROWSER_TEXT_MAX_CHARS);
-    const summary = truncateText(redactSensitiveText(typeof details?.summary === "string" ? details.summary : text.split("\n", 1)[0] || "Browser call completed."), SCRIPT_BROWSER_SUMMARY_MAX_CHARS);
-    const compatibleNextActions = getScriptCompatibleNextActions(details?.nextActions, scriptSessionName);
-    const redactedNextActions = getScriptCompatibleNextActions(redactPresentationData(commandInfo, compatibleNextActions), undefined);
-    const failureCategory = resultCategory === "failure" && typeof details?.failureCategory === "string"
-        ? details.failureCategory
-        : undefined;
-    const successCategory = resultCategory === "success" && typeof details?.successCategory === "string"
-        ? details.successCategory
-        : undefined;
-    const envelopeDetails = redactPresentationData(commandInfo, {
-        artifactVerification: details?.artifactVerification,
-        artifacts: details?.artifacts,
-        failureCategory,
-        pageChangeSummary: details?.pageChangeSummary,
-        resultCategory,
-        successCategory,
-    });
-    const envelope = {
-        data,
-        details: isRecord(envelopeDetails) ? envelopeDetails : { resultCategory },
-        error: resultCategory === "failure" ? summary : undefined,
-        failureCategory,
-        nextActions: Array.isArray(redactedNextActions) ? redactedNextActions : undefined,
-        ok: resultCategory === "success",
-        resultCategory,
-        successCategory,
-        summary,
-        text,
-    };
-    return scriptBrowserEnvelopeFitsIpc(envelope) ? envelope : buildOversizedScriptBrowserEnvelope();
-}
-function collectUniqueArtifacts(results) {
-    const records = results.flatMap((result) => {
-        const details = isRecord(result.details) ? result.details : undefined;
-        return Array.isArray(details?.artifacts) ? details.artifacts.filter(isRecord) : [];
-    });
-    const unique = new Map(records.map((record) => [String(record.absolutePath ?? record.path ?? JSON.stringify(record)), record]));
-    return unique.size > 0 ? [...unique.values()] : undefined;
-}
-function collectScriptArtifactVerification(results) {
-    const entries = results.flatMap((result) => {
-        const details = isRecord(result.details) ? result.details : undefined;
-        const verification = isRecord(details?.artifactVerification) ? details.artifactVerification : undefined;
-        return Array.isArray(verification?.artifacts) ? verification.artifacts.filter(isRecord) : [];
-    });
-    const unique = [...new Map(entries.map((entry) => [String(entry.absolutePath ?? entry.path ?? JSON.stringify(entry)), entry])).values()];
-    if (unique.length === 0)
-        return undefined;
-    const count = (state) => unique.filter((entry) => entry.state === state).length;
-    return {
-        artifacts: unique,
-        missingCount: count("missing"),
-        pendingCount: count("pending"),
-        unverifiedCount: count("unverified"),
-        verified: unique.every((entry) => entry.state === "verified"),
-        verifiedCount: count("verified"),
-    };
-}
-function getLatestScriptResultDetail(results, key) {
-    for (let index = results.length - 1; index >= 0; index -= 1) {
-        const rawDetails = results[index]?.details;
-        const details = isRecord(rawDetails) ? rawDetails : undefined;
-        if (details?.[key] !== undefined)
-            return details[key];
-    }
-    return undefined;
-}
-function buildScriptCloseNextAction(sessionName) {
+export function createBrowserCodeOutput() {
+    const images = new Map();
+    const selected = new Map();
+    const receipts = new Map();
+    const fileArtifacts = new Map();
+    let selectedBytes = 0;
     return {
-        id: "close-script-session-after-cleanup-failure",
-        params: { args: createAgentBrowserScriptCloseArgs(sessionName) },
-        reason: "Retry closing the isolated wrapper-owned script session after automatic cleanup failed.",
-        safety: "Use these exact args; do not add profile, state, restore, namespace, or connection flags.",
-        tool: "agent_browser",
-    };
-}
-export function buildScriptToolResult(options) {
-    let data;
-    let serializedData;
-    let outputError;
-    try {
-        data = redactPresentationData({ command: "script" }, options.run.data);
-        serializedData = data === undefined ? undefined : JSON.stringify(data);
-        if (data !== undefined && serializedData === undefined)
-            throw new TypeError("Script output is not JSON-serializable.");
-        if (serializedData !== undefined && Buffer.byteLength(serializedData, "utf8") > AGENT_BROWSER_SCRIPT_FINAL_OUTPUT_MAX_BYTES) {
-            throw new RangeError(`Redacted script output exceeds ${AGENT_BROWSER_SCRIPT_FINAL_OUTPUT_MAX_BYTES} bytes.`);
-        }
-    }
-    catch {
-        data = undefined;
-        serializedData = undefined;
-        outputError = "Final script output could not be safely rendered as bounded JSON.";
-    }
-    const steps = options.run.steps.map((step) => ({ ...step, summary: redactSensitiveText(step.summary) }));
-    const rejectedFailureCategory = steps.find((step) => step.failureCategory === "policy-blocked")?.failureCategory
-        ?? (options.run.rejectedCallCount > 0 ? "validation-error" : undefined);
-    const failureCategory = options.cleanupError
-        ? "cleanup-failed"
-        : outputError ? "validation-error" : options.run.failureCategory ?? rejectedFailureCategory;
-    const failed = failureCategory !== undefined || !options.run.ok;
-    const failedStepCount = steps.filter((step) => !step.ok).length;
-    const failedCallCount = Math.max(0, failedStepCount - options.run.rejectedCallCount);
-    const successfulCallCount = steps.filter((step) => step.ok).length;
-    const cleanupSuffix = options.sessionName && !options.cleanupError ? " Isolated script session closed." : "";
-    const summary = `${options.cleanupError
-        ? `Script completed, but isolated session cleanup failed: ${redactSensitiveText(options.cleanupError)}`
-        : outputError
-            ? `Script failed: ${outputError}`
-            : options.run.ok
-                ? rejectedFailureCategory
-                    ? `Script failed: ${options.run.rejectedCallCount} browser call${options.run.rejectedCallCount === 1 ? " was" : "s were"} rejected before dispatch.`
-                    : failedCallCount > 0
-                        ? `Script completed (${options.run.callCount} browser call${options.run.callCount === 1 ? "" : "s"}; ${failedCallCount} returned failure for script handling).`
-                        : `Script completed (${options.run.callCount} browser call${options.run.callCount === 1 ? "" : "s"}).`
-                : `Script failed: ${redactSensitiveText(options.run.error ?? "sandbox execution failed")}`}${cleanupSuffix}`;
-    const dataText = serializedData === undefined ? "" : `\n\n${serializedData}`;
-    const cleanupActionText = options.cleanupError && options.sessionName
-        ? `\n\nNext action: ${JSON.stringify({ args: createAgentBrowserScriptCloseArgs(options.sessionName) })}`
-        : "";
-    const artifactManifest = getLatestScriptResultDetail(options.innerResults, "artifactManifest");
-    const artifactRetentionSummary = getLatestScriptResultDetail(options.innerResults, "artifactRetentionSummary");
-    const artifacts = collectUniqueArtifacts(options.innerResults);
-    const artifactVerification = collectScriptArtifactVerification(options.innerResults);
-    const nextActions = options.cleanupError && options.sessionName ? [buildScriptCloseNextAction(options.sessionName)] : undefined;
-    return {
-        content: [{ type: "text", text: `${summary}${dataText}${cleanupActionText}` }],
-        details: {
-            artifactManifest,
-            artifactRetentionSummary,
-            artifacts,
-            artifactVerification,
-            data,
-            failureCategory,
-            nextActions,
-            resultCategory: failed ? "failure" : "success",
-            scriptRun: {
-                aborted: options.run.aborted,
-                callCount: options.run.callCount,
-                emitCount: options.run.emitCount,
-                failedCallCount,
-                preDispatchRejectedCallCount: options.run.rejectedCallCount,
-                successfulCallCount,
-                timedOut: options.run.timedOut,
-            },
-            scriptSession: options.sessionName ? {
-                cleanup: options.cleanupError ? "failed" : "closed",
-                closeCommandArgs: createAgentBrowserScriptCloseArgs(options.sessionName),
-                ...(options.cleanupError ? { error: redactSensitiveText(options.cleanupError) } : {}),
-                launchAttempted: true,
-                sessionName: options.sessionName,
-            } : undefined,
-            scriptSteps: steps,
-            successCategory: failed ? undefined : "completed",
-            summary,
+        async observe(result) {
+            const details = isRecord(result.details) ? result.details : {};
+            const observation = projectAgentBrowserObservation(details, result.isError !== true);
+            for (const entry of observation.artifactVerification?.artifacts ?? [])
+                receipts.set(entry.absolutePath ?? entry.path, entry);
+            for (const artifact of observation.artifacts ?? [])
+                fileArtifacts.set(artifact.absolutePath, artifact);
+            if (observation.imageObservations) {
+                observation.imageObservations = await Promise.all(observation.imageObservations.map(async (image) => {
+                    const file = await stat(image.path);
+                    const id = `image-${images.size + 1}`;
+                    images.set(id, { observation: image, size: file.size, mtime: file.mtimeMs });
+                    return { ...image, id };
+                }));
+            }
+            return observation;
+        },
+        async emitImage(value) {
+            const id = isRecord(value) && typeof value.id === "string" ? value.id : undefined;
+            const image = id ? images.get(id) : undefined;
+            if (!id || !image)
+                throw new Error("emitImage expects an imageObservations handle returned by browser() in this code call.");
+            if (selected.has(id))
+                return;
+            if (selected.size >= 8 || selectedBytes + image.size > 20 * 1_024 * 1_024)
+                throw new Error("Selected images exceed the code output limit (8 images / 20 MiB). Emit fewer images.");
+            const file = await stat(image.observation.path);
+            if (!file.isFile() || file.size !== image.size || file.mtimeMs !== image.mtime)
+                throw new Error("The selected image changed since capture. Capture it again before emitting it.");
+            const presentation = await attachInlineImage({ content: [], summary: "Selected browser image" }, image.observation.path);
+            if (!presentation.content.some(item => item.type === "image")) {
+                throw new Error(presentation.content.filter(item => item.type === "text").map(item => item.text).join("\n") || "The selected image could not be attached.");
+            }
+            selected.set(id, presentation.content.filter(item => item.type === "image"));
+            selectedBytes += file.size;
+        },
+        async finish(run, sessionName, namespace) {
+            let data;
+            let outputError;
+            try {
+                data = redactPresentationData({ command: "code" }, run.data);
+                if (data !== undefined && Buffer.byteLength(JSON.stringify(data), "utf8") > AGENT_BROWSER_SCRIPT_FINAL_OUTPUT_MAX_BYTES)
+                    throw new Error("oversized");
+            }
+            catch {
+                data = undefined;
+                outputError = "Code output could not be rendered as bounded JSON.";
+            }
+            const failureCategory = outputError ? "validation-error" : run.failureCategory
+                ?? (run.rejectedCallCount > 0 ? "validation-error" : undefined);
+            const success = run.ok && failureCategory === undefined;
+            const artifacts = [...receipts.values()];
+            const count = (state) => artifacts.filter(entry => entry.state === state).length;
+            const artifactVerification = artifacts.length ? {
+                artifacts, missingCount: count("missing"), pendingCount: count("pending"), unverifiedCount: count("unverified"),
+                verifiedCount: count("verified"), verified: artifacts.every(entry => entry.state === "verified"),
+            } : undefined;
+            const observation = {
+                success, resultCategory: success ? "success" : "failure", failureCategory, data,
+                error: outputError ?? (run.error ? redactSensitiveText(run.error) : undefined),
+                sessionName, namespace,
+                codeRun: { callCount: run.callCount, emitCount: run.emitCount, failedCallCount: run.steps.filter(step => !step.ok).length, rejectedCallCount: run.rejectedCallCount, aborted: run.aborted, timedOut: run.timedOut },
+                failures: run.failures?.length ? run.failures : undefined,
+                artifactVerification,
+                artifacts: fileArtifacts.size ? [...fileArtifacts.values()] : undefined,
+                imageObservations: [...selected.keys()].map(id => ({ ...images.get(id).observation, id })),
+            };
+            const summary = success ? `Browser code completed (${run.callCount} calls).` : "Browser code failed.";
+            return {
+                content: [...selected.values()].flat(),
+                details: { ...observation, codeSteps: run.steps, summary },
+                isError: !success,
+            };
         },
-        isError: failed,
     };
 }

package/dist/extensions/agent-browser/lib/pi-tool-rendering.js

@@ -1,6 +1,7 @@
 import { getKeybindings, Text, truncateToWidth } from "@earendil-works/pi-tui";
+import { BROWSER_RESULT_TOOLS } from "./browser-transcript.js";
 import { compileAgentBrowserElectron } from "./input-modes/electron.js";
-import { compileAgentBrowserJob, compileAgentBrowserQaPreset } from "./input-modes/job.js";
+import { compileAgentBrowserQaPreset } from "./input-modes/job.js";
 import { compileAgentBrowserNetworkSourceLookup, compileAgentBrowserSourceLookup } from "./input-modes/lookups.js";
 import { compileAgentBrowserSemanticAction } from "./input-modes/semantic-action.js";
 import { isRecord } from "./parsing.js";
@@ -95,14 +96,12 @@ function formatVisualTruncationNotice(remainingLines, totalLines, theme, width)
     return truncateToWidth(notice, Math.max(0, width));
 }
 function getStructuredModeInvocation(input) {
-    if (typeof input.script === "string")
-        return { mode: "script", rawArgs: [], scriptSource: input.script };
+    if (typeof input.code === "string")
+        return { mode: "code", rawArgs: [], scriptSource: input.code };
     if (Array.isArray(input.args))
         return { rawArgs: input.args.filter((value) => typeof value === "string") };
     if (input.semanticAction !== undefined)
         return { mode: "semanticAction", rawArgs: compileAgentBrowserSemanticAction(input.semanticAction).compiled?.args ?? [] };
-    if (input.job !== undefined)
-        return { mode: "job", rawArgs: compileAgentBrowserJob(input.job).compiled?.args ?? [] };
     if (input.qa !== undefined)
         return { mode: "qa", rawArgs: compileAgentBrowserQaPreset(input.qa).compiled?.args ?? [] };
     if (input.sourceLookup !== undefined)
@@ -137,9 +136,10 @@ export function formatAgentBrowserRenderCall(args, theme, expanded = false) {
     const invocationPreview = scriptSource === undefined
         ? formatInvocationPreview(rawArgs)
         : formatScriptSourceForDisplay(scriptSource, expanded);
-    let text = theme.fg("toolTitle", theme.bold("agent_browser"));
+    let text = theme.fg("toolTitle", theme.bold(mode === "code" ? "agent_browser_code" : "agent_browser"));
     if (mode) {
-        text += ` ${theme.fg("accent", mode)}`;
+        if (mode !== "code")
+            text += ` ${theme.fg("accent", mode)}`;
         if (scriptSource !== undefined && expanded) {
             text += `\n${theme.fg("dim", "Source:")}\n${theme.fg("accent", invocationPreview)}`;
         }
@@ -220,9 +220,9 @@ function appendModelVisibleFailureCategoryNotice(content, notice) {
         : item);
 }
 export function buildAgentBrowserToolResultPatch(event) {
-    if (event.toolName !== "agent_browser")
+    if (!BROWSER_RESULT_TOOLS.has(event.toolName))
         return undefined;
-    const preservesParseableJson = agentBrowserToolResultRequestedJson(event) && agentBrowserToolResultHasParseableJsonContent(event.content);
+    const preservesParseableJson = (event.toolName === "agent_browser_code" || agentBrowserToolResultRequestedJson(event)) && agentBrowserToolResultHasParseableJsonContent(event.content);
     const notice = preservesParseableJson ? undefined : formatModelVisibleFailureCategoryNotice(event.details);
     const content = notice ? appendModelVisibleFailureCategoryNotice(event.content, notice) : undefined;
     const shouldMarkError = isRecord(event.details) && event.details.resultCategory === "failure" && event.isError !== true;