pi-agent-browser-native 0.9.2 → 0.9.4

package/dist/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.js

@@ -1,46 +1,66 @@
 import { isRecord, parseRefId } from "../../parsing.js";
-import { redactSensitiveText } from "../../runtime.js";
-import { withOptionalSessionArgs } from "../../results/next-actions.js";
+import { redactSensitiveText } from "../../runtime-redaction.js";
+import { withOptionalSessionArgs, } from "../../results/next-actions.js";
 import { runSessionCommandData } from "./session-state.js";
+import { getAgentBrowserProcessTimeoutMs } from "../../process.js";
+import { getAgentBrowserProcessEnvironment } from "../../process-environment.js";
 const CLICK_DISPATCH_MARKER_PREFIX = "__piAgentBrowserClickDispatchProbe_";
 const CLICK_DISPATCH_CLEANUP_TIMEOUT_MS = 2_000;
-const ACCESSIBLE_REF_CLICK_DISPATCH_ROLES = new Set(["button", "checkbox", "menuitem", "radio", "switch", "tab"]);
+const ACCESSIBLE_REF_CLICK_DISPATCH_ROLES = new Set([
+    "button",
+    "checkbox",
+    "menuitem",
+    "radio",
+    "switch",
+    "tab",
+]);
 function normalizeAccessibleName(name) {
     return name.replace(/\s+/g, " ").trim().toLowerCase();
 }
 function getClickDispatchProbeTarget(commandTokens, refSnapshot) {
-    if (commandTokens[0] !== "click" || commandTokens.includes("--new-tab"))
+    if (commandTokens[0] !== "click" || commandTokens.includes("--new-tab")) {
         return undefined;
+    }
     const selector = commandTokens.slice(1).find((token) => token !== "--human");
-    if (!selector || selector.startsWith("-"))
+    if (selector === undefined || selector === "" || selector.startsWith("-")) {
         return undefined;
+    }
     const refId = parseRefId(selector);
-    if (refId) {
-        const ref = refSnapshot?.refs?.[refId];
-        if (!ref || !ACCESSIBLE_REF_CLICK_DISPATCH_ROLES.has(ref.role))
-            return undefined;
-        const matchingRefs = Object.values(refSnapshot?.refs ?? {}).filter((candidate) => candidate.role.toLowerCase() === ref.role.toLowerCase() && normalizeAccessibleName(candidate.name) === normalizeAccessibleName(ref.name));
-        if (matchingRefs.length !== 1)
-            return undefined;
-        return { kind: "accessible", name: ref.name, refId, role: ref.role };
+    if (refId !== undefined && refId !== "") {
+        return getAccessibleProbeTarget(refId, refSnapshot);
     }
-    if (selector.startsWith("xpath="))
+    if (selector.startsWith("xpath=")) {
         return { kind: "xpath", selector: selector.slice("xpath=".length) };
+    }
     return undefined;
 }
+function getAccessibleProbeTarget(refId, refSnapshot) {
+    const refs = refSnapshot?.refs ?? {};
+    const ref = refs[refId];
+    if (!ref || !ACCESSIBLE_REF_CLICK_DISPATCH_ROLES.has(ref.role)) {
+        return undefined;
+    }
+    const matchingRefs = Object.values(refs).filter((candidate) => candidate !== undefined &&
+        candidate.role.toLowerCase() === ref.role.toLowerCase() &&
+        normalizeAccessibleName(candidate.name) === normalizeAccessibleName(ref.name));
+    return matchingRefs.length === 1
+        ? { kind: "accessible", name: ref.name, refId, role: ref.role }
+        : undefined;
+}
 function getEvalResultRecord(data) {
     return isRecord(data) && isRecord(data.result) ? data.result : undefined;
 }
 function getClickDispatchIdentityAttribute(probe) {
     return `data-pi-click-dispatch-${probe.marker.toLowerCase()}`;
 }
-function buildClickDispatchProbeInstallScript(probe) {
-    const target = probe.target;
-    const resolveTarget = target.kind === "selector"
-        ? `(() => { try { return document.querySelector(${JSON.stringify(target.selector)}); } catch { return null; } })()`
-        : target.kind === "xpath"
-            ? `(() => { try { return document.evaluate(${JSON.stringify(target.selector)}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; } catch { return null; } })()`
-            : `(() => {
+function buildClickTargetResolutionScript(target) {
+    if (target.kind === "selector") {
+        return `(() => { try { return document.querySelector(${JSON.stringify(target.selector)}); } catch { return null; } })()`;
+    }
+    if (target.kind === "xpath") {
+        return `(() => { try { return document.evaluate(${JSON.stringify(target.selector)}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; } catch { return null; } })()`;
+    }
+    return `(() => {
   const normalize = (value) => String(value ?? "").replace(/\\s+/g, " ").trim();
   const expectedRole = ${JSON.stringify(target.role)};
   const expectedName = normalize(${JSON.stringify(target.name)});
@@ -68,9 +88,11 @@ function buildClickDispatchProbeInstallScript(probe) {
   const candidates = Array.from(document.querySelectorAll("button,a[href],input,select,textarea,summary,[role],[onclick],[tabindex]")).filter((element) => inferRole(element) === expectedRole && inferName(element) === expectedName && isVisible(element));
   return candidates.length === 1 ? candidates[0] : null;
 })()`;
+}
+function buildClickDispatchProbeInstallScript(probe, timeoutMs) {
     return `(() => {
 const marker = ${JSON.stringify(probe.marker)};
-const element = ${resolveTarget};
+const element = ${buildClickTargetResolutionScript(probe.target)};
 if (!element) return { status: "target-not-found", marker };
 const cssEscape = (value) => {
   if (window.CSS && typeof window.CSS.escape === "function") return window.CSS.escape(value);
@@ -129,10 +151,15 @@ const listeners = eventTypes.map((type) => {
 });
 const identityAttribute = ${JSON.stringify(getClickDispatchIdentityAttribute(probe))};
 state.removeIdentityMarker = () => element.removeAttribute(identityAttribute);
+let cleanupTimer;
 state.cleanup = () => {
+  clearTimeout(cleanupTimer);
   state.removeIdentityMarker();
   listeners.forEach(([type, listener]) => document.removeEventListener(type, listener, true));
+  try { delete window[marker]; } catch {}
 };
+// A native pending decision can forbid cleanup eval; expire without another command.
+cleanupTimer = setTimeout(state.cleanup, ${timeoutMs + CLICK_DISPATCH_CLEANUP_TIMEOUT_MS});
 window[marker] = state;
 element.setAttribute(identityAttribute, marker);
 return { status: "installed", marker, target: state.target };
@@ -172,7 +199,9 @@ export function formatClickDispatchDiagnosticText(diagnostic) {
     return `Click dispatch diagnostic: ${diagnostic.summary}`;
 }
 export function buildClickDispatchNextActions(options) {
-    const retryArgs = options.commandTokens[0] === "click" || options.commandTokens[0] === "find" ? options.commandTokens : ["click", ...options.commandTokens];
+    const retryArgs = options.commandTokens[0] === "click" || options.commandTokens[0] === "find"
+        ? options.commandTokens
+        : ["click", ...options.commandTokens];
     const actions = [
         {
             id: "inspect-click-dispatch-miss",
@@ -185,8 +214,11 @@ export function buildClickDispatchNextActions(options) {
     if (options.diagnostic?.scrollContainer) {
         actions.push({
             id: "scroll-target-into-view-after-dispatch-miss",
-            params: { args: withOptionalSessionArgs(options.sessionName, ["scrollintoview", retryArgs[1]].filter((item) => typeof item === "string")) },
-            reason: options.diagnostic.scrollContainer.selector
+            params: {
+                args: withOptionalSessionArgs(options.sessionName, ["scrollintoview", retryArgs.at(1)].filter((item) => typeof item === "string")),
+            },
+            reason: options.diagnostic.scrollContainer.selector !== undefined &&
+                options.diagnostic.scrollContainer.selector !== ""
                 ? `The target may be outside nested scroll container ${options.diagnostic.scrollContainer.selector}; scroll the target into view before retrying the click.`
                 : "The target may be inside an offscreen nested scroll container; scroll the target into view before retrying the click.",
             safety: "Use only for the same current page and target; run snapshot -i again if the page rerendered.",
@@ -202,67 +234,146 @@ export function buildClickDispatchNextActions(options) {
     });
     return actions;
 }
+function nativeProbeSelector(target) {
+    if (target.kind === "accessible") {
+        return `@${target.refId}`;
+    }
+    return target.kind === "xpath" ? `xpath=${target.selector}` : target.selector;
+}
+async function verifyClickDispatchProbeIdentity(options, probe) {
+    const identity = await runSessionCommandData({
+        args: [
+            "get",
+            "attr",
+            nativeProbeSelector(probe.target),
+            getClickDispatchIdentityAttribute(probe),
+        ],
+        cwd: options.cwd,
+        namespace: options.namespace,
+        sessionName: options.sessionName,
+        signal: options.signal,
+    });
+    if (!isRecord(identity) || identity.value !== probe.marker) {
+        return false;
+    }
+    const removed = await runSessionCommandData({
+        args: ["eval", "--stdin"],
+        cwd: options.cwd,
+        namespace: options.namespace,
+        sessionName: options.sessionName,
+        signal: options.signal,
+        stdin: `(() => {
+const state = window[${JSON.stringify(probe.marker)}];
+if (!state || typeof state.removeIdentityMarker !== "function") return { status: "probe-missing" };
+state.removeIdentityMarker();
+return { status: "identity-marker-removed" };
+})()`,
+    });
+    return getEvalResultRecord(removed)?.status === "identity-marker-removed";
+}
+function canPrepareClickDispatchProbe(options) {
+    if ((options.sessionName ?? "") === "" ||
+        options.commandTokens[0] !== "click" ||
+        options.commandTokens.includes("--new-tab")) {
+        return false;
+    }
+    // This optional diagnostic must not require approving a fresh eval on every click retry.
+    if (getAgentBrowserProcessEnvironment()
+        .AGENT_BROWSER_CONFIRM_ACTIONS?.split(",")
+        .some((action) => action.trim().toLowerCase() === "evaluate") === true) {
+        return false;
+    }
+    return true;
+}
 export async function prepareClickDispatchProbe(options) {
-    if (!options.sessionName || options.commandTokens[0] !== "click" || options.commandTokens.includes("--new-tab"))
+    if (!canPrepareClickDispatchProbe(options)) {
         return undefined;
+    }
     const target = getClickDispatchProbeTarget(options.commandTokens, options.refSnapshot);
-    if (!target)
+    if (!target) {
         return undefined;
-    const probe = { marker: `${CLICK_DISPATCH_MARKER_PREFIX}${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`, target };
+    }
+    const probe = {
+        marker: `${CLICK_DISPATCH_MARKER_PREFIX}${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`,
+        target,
+    };
     let prepared = false;
     try {
-        const installData = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, namespace: options.namespace, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeInstallScript(probe) });
-        if (getEvalResultRecord(installData)?.status !== "installed")
+        const installData = await runSessionCommandData({
+            args: ["eval", "--stdin"],
+            cwd: options.cwd,
+            namespace: options.namespace,
+            sessionName: options.sessionName,
+            signal: options.signal,
+            stdin: buildClickDispatchProbeInstallScript(probe, options.timeoutMs ?? getAgentBrowserProcessTimeoutMs()),
+        });
+        if (getEvalResultRecord(installData)?.status !== "installed") {
             return undefined;
+        }
         // Name matching and in-page XPath only find candidates. Native resolution
         // must prove identity, including frame scope, before missing events mean failure.
-        const selector = target.kind === "accessible" ? `@${target.refId}` : target.kind === "xpath" ? `xpath=${target.selector}` : target.selector;
-        const identity = await runSessionCommandData({ args: ["get", "attr", selector, getClickDispatchIdentityAttribute(probe)], cwd: options.cwd, namespace: options.namespace, sessionName: options.sessionName, signal: options.signal });
-        if (!isRecord(identity) || identity.value !== probe.marker)
+        if (!(await verifyClickDispatchProbeIdentity(options, probe))) {
             return undefined;
-        const removed = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, namespace: options.namespace, sessionName: options.sessionName, signal: options.signal, stdin: `(() => {
-const state = window[${JSON.stringify(probe.marker)}];
-if (!state || typeof state.removeIdentityMarker !== "function") return { status: "probe-missing" };
-state.removeIdentityMarker();
-return { status: "identity-marker-removed" };
-})()` });
-        if (getEvalResultRecord(removed)?.status !== "identity-marker-removed")
-            return undefined;
-        prepared = !options.signal?.aborted;
+        }
+        prepared = options.signal?.aborted !== true;
         return prepared ? probe : undefined;
     }
     finally {
         // Even a lost/aborted install response may have left listeners in-page.
-        if (!prepared)
+        if (!prepared) {
             await cleanupClickDispatchProbe({ ...options, probe });
+        }
     }
 }
+function optionalBoolean(value, key) {
+    return typeof value?.[key] === "boolean" ? value[key] : undefined;
+}
+function nearestScrollContainer(target) {
+    return isRecord(target?.nearestScrollContainer) ? target.nearestScrollContainer : undefined;
+}
 function getClickDispatchScrollContainerDiagnostic(result) {
     const target = isRecord(result.target) ? result.target : undefined;
-    const scrollContainer = isRecord(target?.nearestScrollContainer) ? target.nearestScrollContainer : undefined;
-    const targetOutsideViewport = typeof target?.targetOutsideViewport === "boolean" ? target.targetOutsideViewport : undefined;
-    const targetOutsideContainer = typeof scrollContainer?.targetOutsideContainer === "boolean" ? scrollContainer.targetOutsideContainer : undefined;
-    if (!scrollContainer && !targetOutsideViewport)
+    const scrollContainer = nearestScrollContainer(target);
+    const targetOutsideViewport = optionalBoolean(target, "targetOutsideViewport");
+    const targetOutsideContainer = optionalBoolean(scrollContainer, "targetOutsideContainer");
+    if (!scrollContainer && targetOutsideViewport !== true) {
         return undefined;
-    if (targetOutsideContainer !== true && targetOutsideViewport !== true)
+    }
+    if (targetOutsideContainer !== true && targetOutsideViewport !== true) {
         return undefined;
-    const selector = typeof scrollContainer?.selector === "string" ? redactSensitiveText(scrollContainer.selector) : undefined;
-    const summary = selector
+    }
+    const selector = typeof scrollContainer?.selector === "string"
+        ? redactSensitiveText(scrollContainer.selector)
+        : undefined;
+    const summary = describeClickScrollContainer(selector);
+    return { selector, summary, targetOutsideContainer, targetOutsideViewport };
+}
+function describeClickScrollContainer(selector) {
+    return selector !== undefined && selector !== ""
         ? `Target appears outside nested scroll container ${selector}; use scrollintoview on the target or scroll that container before retrying.`
         : "Target appears outside the viewport or a nested scroll container; use scrollintoview on the target before retrying.";
-    return { selector, summary, targetOutsideContainer, targetOutsideViewport };
 }
 export async function collectClickDispatchDiagnostic(options) {
-    if (!options.probe || !options.sessionName)
+    if (!options.probe || (options.sessionName ?? "") === "") {
         return undefined;
-    const data = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, namespace: options.namespace, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeCheckScript(options.probe) });
+    }
+    const data = await runSessionCommandData({
+        args: ["eval", "--stdin"],
+        cwd: options.cwd,
+        namespace: options.namespace,
+        sessionName: options.sessionName,
+        signal: options.signal,
+        stdin: buildClickDispatchProbeCheckScript(options.probe),
+    });
     const result = getEvalResultRecord(data);
-    if (!result)
+    if (!result) {
         return undefined;
+    }
     options.probe.cleaned = true;
     const status = typeof result.status === "string" ? result.status : undefined;
-    if (status !== "no-native-event-observed")
+    if (status !== "no-native-event-observed") {
         return undefined;
+    }
     const nativeEventCount = typeof result.nativeEventCount === "number" ? result.nativeEventCount : 0;
     const scrollContainer = getClickDispatchScrollContainerDiagnostic(result);
     const targetLabel = "no trusted DOM event reached the selected element";
@@ -279,8 +390,9 @@ export async function collectClickDispatchDiagnostic(options) {
     };
 }
 export async function cleanupClickDispatchProbe(options) {
-    if (!options.probe || options.probe.cleaned || !options.sessionName)
+    if (!options.probe || options.probe.cleaned === true || (options.sessionName ?? "") === "") {
         return;
+    }
     await runSessionCommandData({
         args: ["eval", "--stdin"],
         cwd: options.cwd,
@@ -288,5 +400,7 @@ export async function cleanupClickDispatchProbe(options) {
         sessionName: options.sessionName,
         stdin: buildClickDispatchProbeCleanupScript(options.probe),
         timeoutMs: CLICK_DISPATCH_CLEANUP_TIMEOUT_MS,
-    }).catch(() => undefined);
+    }).catch(() => {
+        // Probe expiry also removes listeners; cleanup failure must not replace the caller's result.
+    });
 }

package/dist/extensions/agent-browser/lib/orchestration/browser-run/diagnostic-values.js

@@ -0,0 +1,31 @@
+import { isRecord } from "../../parsing.js";
+import { stringifyUnknown } from "../../results/text.js";
+export function diagnosticResultRecord(data) {
+    const result = isRecord(data) && isRecord(data.result) ? data.result : data;
+    return isRecord(result) ? result : undefined;
+}
+export function rawStringField(data, field) {
+    const value = data[field];
+    return typeof value === "string" ? value : undefined;
+}
+export function rawNumberField(data, field) {
+    const value = data[field];
+    return typeof value === "number" ? value : undefined;
+}
+export function diagnosticErrorText(error) {
+    if (error instanceof Error) {
+        return error.message;
+    }
+    switch (typeof error) {
+        case "object":
+            return error === null ? "null" : stringifyUnknown(error);
+        case "string":
+        case "number":
+        case "bigint":
+        case "boolean":
+        case "undefined":
+        case "symbol":
+        case "function":
+            return String(error);
+    }
+}