pi-agent-browser-native 0.9.3 → 0.9.5

package/dist/extensions/agent-browser/lib/orchestration/browser-run/electron-observation.js

@@ -0,0 +1,203 @@
+import { setTimeout as sleepMs } from "node:timers/promises";
+import { boundElectronProbeString } from "../../electron/cdp.js";
+import { buildNextToolAction, withOptionalSessionArgs } from "../../results/next-actions.js";
+import { extractRefSnapshotFromData } from "../../session-page-state.js";
+import { extractStringResultField, runSessionCommandData } from "./session-state.js";
+import { diagnosticErrorText } from "./diagnostic-values.js";
+import { presentElectronManagedSessionTarget } from "./final-result-electron.js";
+function sessionResult(data, field) {
+    return extractStringResultField(data, "result") ?? extractStringResultField(data, field);
+}
+async function collectManagedSessionCommandData(options) {
+    try {
+        return { data: await runSessionCommandData({ ...options, pinNamespace: true }) };
+    }
+    catch (error) {
+        return { error: diagnosticErrorText(error) };
+    }
+}
+async function collectElectronManagedSessionUrl(options) {
+    const urlResult = await collectManagedSessionCommandData({ ...options, args: ["get", "url"] });
+    const url = sessionResult(urlResult.data, "url");
+    return urlResult.error !== undefined && urlResult.error !== ""
+        ? { error: urlResult.error }
+        : { url };
+}
+export async function collectElectronManagedSessionTarget(options) {
+    if (options.sessionName === undefined || options.sessionName === "") {
+        return undefined;
+    }
+    const request = { ...options, sessionName: options.sessionName };
+    const urlResult = await collectManagedSessionCommandData({ ...request, args: ["get", "url"] });
+    const url = sessionResult(urlResult.data, "url");
+    if ((urlResult.error ?? "") !== "" || url === undefined || url.trim() === "") {
+        return {
+            error: urlResult.error ?? "get url returned no active page URL.",
+            sessionName: options.sessionName,
+        };
+    }
+    const titleResult = await collectManagedSessionCommandData({
+        ...request,
+        args: ["get", "title"],
+    });
+    const title = boundElectronProbeString(sessionResult(titleResult.data, "title"), 160);
+    return {
+        sessionName: options.sessionName,
+        title,
+        url,
+        ...(titleResult.error !== undefined && titleResult.error !== ""
+            ? { error: titleResult.error }
+            : {}),
+    };
+}
+export async function collectQaAttachedTarget(options) {
+    if (options.sessionName === undefined || options.sessionName === "") {
+        return undefined;
+    }
+    const target = options.currentTarget;
+    if (target && ((target.title ?? "") !== "" || target.url !== "")) {
+        return {
+            sessionName: options.sessionName,
+            title: target.title,
+            url: target.url,
+        };
+    }
+    const observed = await collectElectronManagedSessionTarget({
+        cwd: options.cwd,
+        namespace: options.namespace,
+        sessionName: options.sessionName,
+        signal: options.signal,
+    });
+    return observed ? presentElectronManagedSessionTarget(observed) : undefined;
+}
+export function formatQaAttachedTargetText(target) {
+    return target
+        ? ["QA attached target:", target.sessionName, target.title, target.url]
+            .filter((part) => typeof part === "string" && part.length > 0)
+            .join(" — ")
+        : undefined;
+}
+function buildQaAttachedRecoveryNextActions(sessionName) {
+    return [
+        buildNextToolAction({
+            args: withOptionalSessionArgs(sessionName, ["tab", "list"]),
+            id: "list-tabs-before-qa-attached",
+            reason: "Inspect the connected session tabs before retrying qa.attached.",
+            safety: "Read-only tab listing for the attached session.",
+        }),
+        buildNextToolAction({
+            args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]),
+            id: "snapshot-before-qa-attached",
+            reason: "Capture interactive refs on the active page before retrying qa.attached.",
+            safety: "Read-only snapshot; confirms a renderable page is selected.",
+        }),
+    ];
+}
+export async function validateQaAttachedPrecondition(options) {
+    if (options.sessionName === undefined || options.sessionName === "") {
+        return {
+            error: "qa.attached requires an active attached session with a resolvable session name.",
+            nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
+        };
+    }
+    const urlProbe = await collectElectronManagedSessionUrl({
+        ...options,
+        sessionName: options.sessionName,
+    });
+    if (urlProbe.error !== undefined && urlProbe.error !== "") {
+        return {
+            error: `qa.attached could not read the attached session URL: ${urlProbe.error}. Run tab list or snapshot -i before retrying qa.attached.`,
+            nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
+        };
+    }
+    const url = urlProbe.url?.trim();
+    return url === undefined || url === ""
+        ? {
+            error: "qa.attached requires an attached session with a readable page URL. Run tab list, select a stable tab, then snapshot -i before retrying.",
+            nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
+        }
+        : undefined;
+}
+async function verifyHandoffUrl(options) {
+    const urlData = await runSessionCommandData({
+        args: ["get", "url"],
+        cwd: options.cwd,
+        namespace: options.namespace,
+        sessionName: options.sessionName,
+        signal: options.signal,
+        throwOnFailure: true,
+    });
+    if (options.signal?.aborted ?? false) {
+        throw new Error("Electron handoff was aborted.");
+    }
+    const url = extractStringResultField(urlData, "result") ?? extractStringResultField(urlData, "url");
+    if (url === undefined || url === "") {
+        throw new Error("Electron handoff get url returned no active page URL.");
+    }
+}
+async function collectHandoffTabs(options) {
+    const tabs = await runSessionCommandData({
+        args: ["tab", "list"],
+        cwd: options.cwd,
+        namespace: options.namespace,
+        sessionName: options.sessionName,
+        signal: options.signal,
+        throwOnFailure: true,
+    });
+    if (options.signal?.aborted ?? false) {
+        throw new Error("Electron handoff was aborted.");
+    }
+    return tabs;
+}
+async function collectHandoffSnapshot(options) {
+    let snapshot = await runSessionCommandData({
+        args: ["snapshot", "-i"],
+        cwd: options.cwd,
+        namespace: options.namespace,
+        sessionName: options.sessionName,
+        signal: options.signal,
+        throwOnFailure: true,
+    });
+    let refSnapshot = extractRefSnapshotFromData(snapshot);
+    let snapshotRetryCount = 0;
+    while (needsSnapshotRetry(refSnapshot) && snapshotRetryCount < 2) {
+        if (options.signal?.aborted ?? false) {
+            throw new Error("Electron handoff was aborted.");
+        }
+        snapshotRetryCount += 1;
+        // Renderer startup must settle before retrying the native snapshot.
+        // oxlint-disable-next-line no-await-in-loop
+        await sleepMs(250);
+        if (options.signal?.aborted ?? false) {
+            throw new Error("Electron handoff was aborted.");
+        }
+        // Each retry uses the prior snapshot result and the same session identity.
+        // oxlint-disable-next-line no-await-in-loop
+        snapshot = await runSessionCommandData({
+            args: ["snapshot", "-i"],
+            cwd: options.cwd,
+            namespace: options.namespace,
+            sessionName: options.sessionName,
+            signal: options.signal,
+            throwOnFailure: true,
+        });
+        refSnapshot = extractRefSnapshotFromData(snapshot);
+    }
+    return { refSnapshot, snapshot, ...(snapshotRetryCount > 0 ? { snapshotRetryCount } : {}) };
+}
+function needsSnapshotRetry(snapshot) {
+    return snapshot === undefined || snapshot.refIds.length === 0;
+}
+export async function collectElectronHandoff(options) {
+    if (options.handoff === "connect") {
+        return { handoff: "connect" };
+    }
+    if (options.signal?.aborted ?? false) {
+        throw new Error("Electron handoff was aborted.");
+    }
+    await verifyHandoffUrl(options);
+    const tabs = await collectHandoffTabs(options);
+    return options.handoff === "tabs"
+        ? { handoff: "tabs", tabs }
+        : { handoff: "snapshot", ...(await collectHandoffSnapshot(options)), tabs };
+}

package/dist/extensions/agent-browser/lib/orchestration/browser-run/electron-text-diagnostics.js

@@ -0,0 +1,109 @@
+import { withOptionalSessionArgs } from "../../results/next-actions.js";
+import { findElectronLaunchRecordForSession } from "./session-state.js";
+import { getSuccessfulGetTextSelectors, selectorMayExposeSensitiveLiteral, } from "./text-visibility-diagnostics.js";
+const BROAD_TEXT_SELECTORS = new Set([
+    "body",
+    "html",
+    ":root",
+    "*",
+    "main",
+    "div",
+    "section",
+    "article",
+]);
+function isBroadGetTextSelector(selector) {
+    if (selector === undefined ||
+        selector === "" ||
+        /^@e\d+$/.test(selector) ||
+        selectorMayExposeSensitiveLiteral(selector)) {
+        return false;
+    }
+    const normalized = selector.trim().replace(/\s+/g, " ").toLowerCase();
+    return (BROAD_TEXT_SELECTORS.has(normalized) ||
+        /^\[role=(?:"application"|'application'|application)\]$/i.test(normalized));
+}
+export function getSourceLookupElectronContext(options) {
+    const record = findElectronLaunchRecordForSession(options.sessionName, options.electronLaunchRecords, options.namespace);
+    if (!record) {
+        return undefined;
+    }
+    const url = options.currentTarget?.url ?? options.priorTarget?.url;
+    return {
+        appName: record.appName,
+        appPath: record.appPath,
+        executablePath: record.executablePath,
+        launchId: record.launchId,
+        sessionName: record.sessionName ?? options.sessionName,
+        url,
+    };
+}
+export function buildSourceLookupElectronNextActions(sourceLookup) {
+    if (sourceLookup?.status !== "no-candidates" || !sourceLookup.electronContext) {
+        return [];
+    }
+    const actions = [];
+    const { launchId, sessionName } = sourceLookup.electronContext;
+    const hasSession = sessionName !== undefined && sessionName !== "";
+    if (hasSession) {
+        actions.push({
+            id: "snapshot-electron-session",
+            params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) },
+            reason: "Refresh interactive refs in the attached Electron session before retrying source lookup with a narrower target.",
+            safety: "Read-only snapshot; no app mutation.",
+            tool: "agent_browser",
+        });
+    }
+    if (launchId !== undefined && launchId !== "") {
+        actions.push({
+            id: "probe-electron-launch",
+            params: { action: "probe", launchId },
+            reason: "Collect bounded wrapper/session context for the packaged Electron launch after sourceLookup found no candidates.",
+            safety: "Read-only probe of title, URL, focus, tabs, and compact snapshot metadata.",
+            tool: "agent_browser_electron",
+        });
+    }
+    if (hasSession) {
+        actions.push({
+            id: "list-electron-tabs",
+            params: { args: withOptionalSessionArgs(sessionName, ["tab", "list"]) },
+            reason: "Check current Electron tabs/targets before choosing a narrower selector or @ref.",
+            safety: "Read-only tab listing.",
+            tool: "agent_browser",
+        });
+    }
+    return actions;
+}
+export function collectElectronBroadGetTextScopeDiagnostics(options) {
+    const context = getSourceLookupElectronContext(options);
+    if (!context) {
+        return [];
+    }
+    const electronContext = {
+        launchId: context.launchId,
+        sessionName: context.sessionName,
+        url: context.url,
+    };
+    return getSuccessfulGetTextSelectors(options)
+        .filter(isBroadGetTextSelector)
+        .map((selector) => ({
+        electronContext,
+        selector,
+        summary: `Broad Electron get text selector warning: selector ${JSON.stringify(selector)} may read the entire app shell; prefer snapshot -i and a current @ref or a narrower panel selector.`,
+    }));
+}
+export function formatElectronBroadGetTextScopeText(diagnostics) {
+    return diagnostics.length > 0
+        ? diagnostics.map((diagnostic) => diagnostic.summary).join("\n")
+        : undefined;
+}
+export function buildElectronBroadGetTextScopeNextActions(options) {
+    return options.diagnostics.map((diagnostic, index) => ({
+        id: index === 0
+            ? "snapshot-for-electron-text-scope"
+            : `snapshot-for-electron-text-scope-${index + 1}`,
+        params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) },
+        reason: `Refresh Electron refs before trusting broad get text selector ${JSON.stringify(diagnostic.selector)}.`,
+        safety: "Read-only snapshot; prefer a current @ref or narrower selector before extracting app-shell text.",
+        tool: "agent_browser",
+    }));
+}

package/dist/extensions/agent-browser/lib/orchestration/browser-run/fill-diagnostics.js

@@ -0,0 +1,119 @@
+import { boundElectronProbeString } from "../../electron/cdp.js";
+import { isRecord } from "../../parsing.js";
+import { withOptionalSessionArgs } from "../../results/next-actions.js";
+import { runSessionCommandData } from "./session-state.js";
+const ELECTRON_FILL_VERIFICATION_TIMEOUT_MS = 2_000;
+function getTopLevelFillInvocation(commandTokens) {
+    if (commandTokens[0] !== "fill" || commandTokens.length < 3) {
+        return undefined;
+    }
+    const selector = commandTokens.at(1);
+    const expected = commandTokens.slice(2).join(" ");
+    if (selector === undefined || selector === "" || expected.length === 0) {
+        return undefined;
+    }
+    const refId = selector.match(/^@?(e\d+)$/)?.at(1);
+    return { expected, ...(refId !== undefined && refId !== "" ? { refId } : {}), selector };
+}
+function shouldVerifyContenteditableFill(fill, refSnapshot) {
+    if (fill.refId === undefined || fill.refId === "") {
+        return false;
+    }
+    const ref = refSnapshot?.refs?.[fill.refId];
+    return (ref !== undefined &&
+        ref.isContentEditable === true &&
+        (ref.role === "generic" || ref.role === "unknown" || ref.role === "textbox"));
+}
+export function buildFillVerificationNextActions(diagnostic, sessionName) {
+    return [
+        {
+            id: "inspect-after-fill-verification",
+            params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) },
+            reason: "Refresh the UI after a fill that reported success but did not appear to update the target.",
+            safety: "Read-only snapshot; use current refs before retrying.",
+            tool: "agent_browser",
+        },
+        {
+            id: "verify-filled-value",
+            params: {
+                args: withOptionalSessionArgs(sessionName, ["get", diagnostic.method, diagnostic.selector]),
+            },
+            reason: `Check the target ${diagnostic.method} directly before submitting or creating files.`,
+            safety: "Read-only check; selector may still be stale if the UI rerendered.",
+            tool: "agent_browser",
+        },
+    ];
+}
+function extractFillVerificationValue(data) {
+    if (typeof data === "string") {
+        return data;
+    }
+    if (!isRecord(data)) {
+        return undefined;
+    }
+    if (typeof data.value === "string") {
+        return data.value;
+    }
+    return typeof data.result === "string" ? data.result : undefined;
+}
+function buildFillMismatch(fill, actual, method, sessionName) {
+    const reason = method === "text" ? "contenteditable-fill-mismatch" : "value-fill-mismatch";
+    const actualPreview = actual.length > 0
+        ? `"${boundElectronProbeString(actual, 80) ?? "undefined"}"`
+        : `an empty ${method}`;
+    const diagnostic = {
+        actual: actual.length > 0 ? boundElectronProbeString(actual, 160) : "",
+        expected: boundElectronProbeString(fill.expected, 160) ?? fill.expected,
+        method,
+        nextActionIds: [],
+        reason,
+        selector: fill.selector,
+        status: "mismatch",
+        summary: `Fill verification warning: fill ${fill.selector} reported success, but get ${method} returned ${actualPreview}.`,
+    };
+    return {
+        ...diagnostic,
+        nextActionIds: buildFillVerificationNextActions(diagnostic, sessionName).map((action) => action.id),
+    };
+}
+export async function collectFillVerificationDiagnostic(options) {
+    const fill = getTopLevelFillInvocation(options.commandTokens);
+    if (!fill || options.sessionName === undefined || options.sessionName === "") {
+        return undefined;
+    }
+    const contenteditable = shouldVerifyContenteditableFill(fill, options.refSnapshot);
+    if (!contenteditable && options.forceValueVerification !== true) {
+        return undefined;
+    }
+    const method = contenteditable ? "text" : "value";
+    let valueData;
+    try {
+        valueData = await runSessionCommandData({
+            args: ["get", method, fill.selector],
+            cwd: options.cwd,
+            namespace: options.namespace,
+            sessionName: options.sessionName,
+            signal: options.signal,
+            timeoutMs: ELECTRON_FILL_VERIFICATION_TIMEOUT_MS,
+        });
+    }
+    catch {
+        return undefined;
+    }
+    const actual = extractFillVerificationValue(valueData);
+    return actual === undefined || actual === fill.expected
+        ? undefined
+        : buildFillMismatch(fill, actual, method, options.sessionName);
+}
+export function formatFillVerificationText(diagnostic) {
+    if (!diagnostic) {
+        return undefined;
+    }
+    const actual = diagnostic.actual !== undefined
+        ? `actual "${diagnostic.actual}"`
+        : `actual ${diagnostic.method} unavailable`;
+    const recovery = diagnostic.reason === "contenteditable-fill-mismatch"
+        ? "Contenteditable fill may append or prepend instead of replacing. Re-run snapshot -i, then prefer focus/click plus keyboard shortcut selection or direct keyboard insertion only after verifying the editor state."
+        : "Re-run snapshot -i, then prefer click/focus plus keyboard type for custom quick-input controls before submitting.";
+    return `${diagnostic.summary}\nExpected: "${diagnostic.expected}"; ${actual}.\nNext: ${recovery}`;
+}

package/dist/extensions/agent-browser/lib/orchestration/browser-run/final-result-action-recovery.js

@@ -0,0 +1,147 @@
+import { getCompiledSemanticActionCommandIndex, getCompiledSemanticActionSessionPrefix, isCompiledSemanticActionFindCommand, } from "../../input-modes/semantic-action.js";
+import { appendUniqueAgentBrowserNextActions, isStandaloneSnapshotNextAction, } from "../../results/next-actions.js";
+import { buildConnectedSessionNextActions, buildNoActivePageNextActions, buildPendingWebMcpNextActions, buildSessionAwareStaleRefNextActions, buildSessionTabRecoveryNextActions, } from "../../results/recovery-next-actions.js";
+import { buildRichInputRecoveryNextActions, buildVisibleRefFallbackNextActions, } from "../../results/selector-recovery.js";
+import { redactInvocationArgs } from "../../runtime-redaction.js";
+function tabRecovery(options) {
+    if (options.aboutBlankSessionMismatch) {
+        return [
+            ...buildSessionTabRecoveryNextActions({
+                kind: "about-blank",
+                recoveryApplied: options.aboutBlankSessionMismatch.recoveryApplied,
+                sessionName: options.executionPlan.sessionName,
+                tabCorrection: options.aboutBlankSessionMismatch.recoveryApplied
+                    ? options.sessionTabCorrection
+                    : undefined,
+                target: {
+                    title: options.aboutBlankSessionMismatch.targetTitle,
+                    url: options.aboutBlankSessionMismatch.targetUrl,
+                },
+            }),
+        ];
+    }
+    if (options.categoryDetails.resultCategory === "success" &&
+        (options.sessionTabCorrection || options.openResultTabCorrection)) {
+        return [
+            ...buildSessionTabRecoveryNextActions({
+                kind: "tab-drift",
+                recoveryApplied: true,
+                sessionName: options.executionPlan.sessionName,
+                tabCorrection: options.sessionTabCorrection ?? options.openResultTabCorrection,
+                target: options.currentSessionTabTarget ?? options.priorSessionTabTarget,
+            }),
+        ];
+    }
+    return [];
+}
+function connectionRecovery(options) {
+    const actions = [];
+    if (options.categoryDetails.resultCategory === "success" &&
+        options.executionPlan.commandInfo.command === "connect" &&
+        !options.electronLaunchRecord) {
+        actions.push(...buildConnectedSessionNextActions(options.executionPlan.sessionName));
+    }
+    if (options.noActivePageSnapshotFailure) {
+        actions.push(...buildNoActivePageNextActions(options.executionPlan.sessionName));
+    }
+    return actions;
+}
+export function buildPageRecoveryActions(options) {
+    let actions = [...(options.presentation.nextActions ?? [])];
+    if (options.unsettledWebMcpMutation === true && options.currentSessionTabTargetUnknown === true) {
+        actions = actions.filter((action) => !isStandaloneSnapshotNextAction(action));
+        appendUniqueAgentBrowserNextActions(actions, buildPendingWebMcpNextActions(options.executionPlan.sessionName));
+    }
+    appendUniqueAgentBrowserNextActions(actions, connectionRecovery(options));
+    appendUniqueAgentBrowserNextActions(actions, tabRecovery(options));
+    if (options.aboutBlankSessionMismatch?.recoveryApplied === false) {
+        actions = actions.filter((action) => !isStandaloneSnapshotNextAction(action));
+    }
+    if (options.categoryDetails.failureCategory === "stale-ref") {
+        actions = [...buildSessionAwareStaleRefNextActions(options.executionPlan.sessionName)];
+    }
+    return actions;
+}
+export function buildSelectorRecoveryActions(options) {
+    const actions = [];
+    if (options.visibleRefFallbackDiagnostic) {
+        actions.push(...buildVisibleRefFallbackNextActions({
+            diagnostic: options.visibleRefFallbackDiagnostic,
+            sessionName: options.visibleRefFallbackSessionName,
+        }));
+    }
+    if (options.richInputRecoveryDiagnostic) {
+        actions.push(...buildRichInputRecoveryNextActions({
+            diagnostic: options.richInputRecoveryDiagnostic,
+            sessionName: options.visibleRefFallbackSessionName,
+        }));
+    }
+    return actions;
+}
+export function buildSemanticActionCandidateActions(compiled) {
+    const commandIndex = getCompiledSemanticActionCommandIndex(compiled);
+    if (commandIndex < 0 || compiled.args[commandIndex] !== "find") {
+        return [];
+    }
+    const locator = compiled.args[commandIndex + 1];
+    const value = compiled.args.at(commandIndex + 2);
+    if (value === undefined ||
+        value.length === 0 ||
+        locator !== "text" ||
+        compiled.action !== "click") {
+        return [];
+    }
+    const sessionPrefix = getCompiledSemanticActionSessionPrefix(compiled);
+    return [
+        {
+            id: "try-button-name-candidate",
+            params: {
+                args: redactInvocationArgs([
+                    ...sessionPrefix,
+                    "find",
+                    "role",
+                    "button",
+                    compiled.action,
+                    "--name",
+                    value,
+                ]),
+            },
+            reason: "Retry against a button with the same accessible name when text lookup misses.",
+            safety: "Candidate locator fallback only; inspect the page if multiple elements could match the same accessible name.",
+            tool: "agent_browser",
+        },
+        {
+            id: "try-link-name-candidate",
+            params: {
+                args: redactInvocationArgs([
+                    ...sessionPrefix,
+                    "find",
+                    "role",
+                    "link",
+                    compiled.action,
+                    "--name",
+                    value,
+                ]),
+            },
+            reason: "Retry against a link with the same accessible name when text lookup misses.",
+            safety: "Candidate locator fallback only; inspect the page if multiple elements could match the same accessible name.",
+            tool: "agent_browser",
+        },
+    ];
+}
+export function buildStaleSemanticRetry(options) {
+    if (options.categoryDetails.failureCategory !== "stale-ref" ||
+        !options.redactedCompiledSemanticAction ||
+        !isCompiledSemanticActionFindCommand(options.compiledSemanticAction)) {
+        return [];
+    }
+    return [
+        {
+            id: "retry-semantic-action-after-stale-ref",
+            params: { args: options.redactedCompiledSemanticAction.args },
+            reason: "Retry the same semantic target via its compiled find command after the upstream stale-ref failure proves the prior action did not execute.",
+            safety: "Use only for the same intended target; direct stale @refs still require a fresh snapshot or stable locator before retrying.",
+            tool: "agent_browser",
+        },
+    ];
+}