pi-agent-browser-native 0.6.6 → 0.6.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +40 -1
  2. package/README.md +21 -6
  3. package/dist/extensions/agent-browser/index.js +138 -75
  4. package/dist/extensions/agent-browser/lib/command-taxonomy.js +6 -5
  5. package/dist/extensions/agent-browser/lib/electron/cleanup.js +10 -1
  6. package/dist/extensions/agent-browser/lib/input-modes/params.js +20 -7
  7. package/dist/extensions/agent-browser/lib/launch-scoped-flags.js +0 -1
  8. package/dist/extensions/agent-browser/lib/managed-session-restore.js +13 -12
  9. package/dist/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.js +6 -25
  10. package/dist/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.js +2 -2
  11. package/dist/extensions/agent-browser/lib/orchestration/browser-run/final-result.js +2 -3
  12. package/dist/extensions/agent-browser/lib/orchestration/browser-run/index.js +10 -3
  13. package/dist/extensions/agent-browser/lib/orchestration/browser-run/managed-session-daemon-policy.js +3 -0
  14. package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/wait-timeouts.js +1 -1
  15. package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare.js +104 -117
  16. package/dist/extensions/agent-browser/lib/orchestration/browser-run/process-output.js +64 -36
  17. package/dist/extensions/agent-browser/lib/orchestration/browser-run/session-state.js +100 -135
  18. package/dist/extensions/agent-browser/lib/orchestration/electron-host/index.js +3 -1
  19. package/dist/extensions/agent-browser/lib/orchestration/input-plan.js +3 -1
  20. package/dist/extensions/agent-browser/lib/page-target-validation.js +10 -10
  21. package/dist/extensions/agent-browser/lib/parsing.js +7 -0
  22. package/dist/extensions/agent-browser/lib/playbook.js +6 -9
  23. package/dist/extensions/agent-browser/lib/process.js +28 -113
  24. package/dist/extensions/agent-browser/lib/recording-reservations.js +3 -1
  25. package/dist/extensions/agent-browser/lib/results/action-recommendations.js +5 -2
  26. package/dist/extensions/agent-browser/lib/results/envelope.js +9 -4
  27. package/dist/extensions/agent-browser/lib/results/next-actions.js +8 -0
  28. package/dist/extensions/agent-browser/lib/results/presentation/artifacts.js +45 -43
  29. package/dist/extensions/agent-browser/lib/results/presentation/batch.js +2 -1
  30. package/dist/extensions/agent-browser/lib/results/presentation/errors.js +10 -2
  31. package/dist/extensions/agent-browser/lib/results/presentation.js +9 -2
  32. package/dist/extensions/agent-browser/lib/results/recovery-actions.js +4 -4
  33. package/dist/extensions/agent-browser/lib/runtime.js +18 -2
  34. package/dist/extensions/agent-browser/lib/session-page-state.js +29 -10
  35. package/docs/ARCHITECTURE.md +8 -5
  36. package/docs/COMMAND_REFERENCE.md +25 -16
  37. package/docs/ELECTRON.md +6 -6
  38. package/docs/RELEASE.md +14 -5
  39. package/docs/REQUIREMENTS.md +1 -1
  40. package/docs/SUPPORT_MATRIX.md +18 -5
  41. package/docs/TOOL_CONTRACT.md +38 -25
  42. package/package.json +5 -1
@@ -13,7 +13,7 @@ export const QUICK_START_GUIDELINES = [
13
13
  "Common first calls (first-call recipe): { args: [\"open\", \"<url>\"] } → { args: [\"snapshot\", \"-i\"] } → { args: [\"click\", \"@eN\"] } or { args: [\"fill\", \"@eN\", \"<text>\"] } using @refs and visible labels from that snapshot, then { args: [\"snapshot\", \"-i\"] } after navigation or DOM changes. On https://example.com/ the main link label is Learn more (use exact snapshot text, not guessed link copy).",
14
14
  "Locator-first clicks/fills and native select changes without hand-building argv: { semanticAction: { action: \"click\", locator: \"text\", value: \"Close\" } }, { semanticAction: { action: \"fill\", locator: \"label\", value: \"Email\", text: \"user@example.com\" } }, direct current targets such as { semanticAction: { action: \"fill\", selector: \"@e1\", text: \"prompt\" } }, or { semanticAction: { action: \"select\", selector: \"#flavor\", value: \"chocolate\" } }; add semanticAction.session when targeting a named upstream browser session; details.compiledSemanticAction shows the semantic target, while details.effectiveArgs may show a resolved current @ref for active-session role/name click/check/fill actions to avoid hidden duplicate matches; semanticAction does not expose uncheck while upstream find ... uncheck is not runtime-supported, so use raw uncheck with a stable selector or current ref; selector-not-found failures may append bounded click try-*-candidate next actions or, for fill misses with current editable refs, details.richInputRecovery with focus/click actions that do not copy fill text; stale-ref failures can return retry-semantic-action-after-stale-ref for compiled find actions when retry safety is provable.",
15
15
  `Common advanced calls: { args: ["batch", "--bail"], stdin: "[[\"open\",\"https://example.com\"],[\"snapshot\",\"-i\"]]" }, { job: { steps: [{ action: "open", url: "https://example.com" }, { action: "assertText", text: "Example Domain" }, { action: "screenshot", path: ".dogfood/example.png" }] } }, { qa: { url: "https://example.com", expectedText: "Example Domain", screenshotPath: ".dogfood/qa-example.png" } } (example.com smoke only; elsewhere match exact visible text from snapshot -i), { electron: { action: "list", query: "code" } }, { electron: { action: "launch", appName: "Visual Studio Code", handoff: "snapshot" } }, { electron: { action: "probe" } }, { qa: { attached: true, expectedText: "Explorer" } }, { args: ["eval", "--stdin"], stdin: "document.title", outputPath: "logs/page-title.json" }, { args: ["auth", "save", "name", "--password-stdin"], stdin: "<password from user-approved secret source>" }, { args: ["--profile", "Default", "open", "https://example.com/account"], sessionMode: "fresh" }, and { args: ["open", "--enable", "react-devtools", "https://example.com"], sessionMode: "fresh" }. For app pages with a native dropdown, job steps can include { action: "select", selector: "#flavor", value: "chocolate" } before the dependent assertion; for locator-friendly pages, job click/fill steps can use semantic locator fields such as { action: "fill", locator: "role", role: "searchbox", name: "Search", text: "agent browser" }; for human-paced input, job type steps can use { action: "type", selector: "#prompt", text: "hello", delayMs: 20, press: "Enter" }; delayed typing is capped at 200 characters per step, and generated per-character rows are compacted in visible batch prose while full rows remain in details.batchSteps.`,
16
- "Constrained job navigation is explicit only: click (and select/submit flows that may navigate) does not prove the next page loaded; add an assertUrl that does not already match the starting page and/or assertText for new page state after navigation-prone steps before screenshot or later interactions. Keep jobs short around navigation, click, and rerender boundaries on dynamic React/product apps; avoid a whole checkout in one job. If a long job times out and details.timeoutPartialProgress shows a mutating incomplete step, inspect current page state and continue with a shorter job or single action instead of blindly retrying the mutating step. Example: { job: { steps: [{ action: \"open\", url: \"https://shop.example/checkout\" }, { action: \"fill\", selector: \"#email\", text: \"user@example.com\" }, { action: \"click\", selector: \"#continue\" }, { action: \"assertUrl\", url: \"**/shipping\" }, { action: \"assertText\", text: \"Shipping address\" }, { action: \"screenshot\", path: \".dogfood/shipping.png\" }] } }. Top-level click may add pageChangeSummary hints, but job never auto-inserts post-click asserts.",
16
+ "Constrained job navigation is explicit only: click (and select/submit flows that may navigate) does not prove the next page loaded; add an assertUrl that does not already match the starting page and/or assertText for new page state after navigation-prone steps before screenshot or later interactions. assertText takes only text, not selector or locator fields. Clicks can stale subsequent @refs: split the job and re-snapshot before using those refs. Keep jobs short around navigation, click, and rerender boundaries on dynamic React/product apps; avoid a whole checkout in one job. If a long job times out and details.timeoutPartialProgress shows a mutating incomplete step, inspect current page state and continue with a shorter job or single action instead of blindly retrying the mutating step. Example: { job: { steps: [{ action: \"open\", url: \"https://shop.example/checkout\" }, { action: \"fill\", selector: \"#email\", text: \"user@example.com\" }, { action: \"click\", selector: \"#continue\" }, { action: \"assertUrl\", url: \"**/shipping\" }, { action: \"assertText\", text: \"Shipping address\" }, { action: \"screenshot\", path: \".dogfood/shipping.png\" }] } }. Top-level click may add pageChangeSummary hints, but job never auto-inserts post-click asserts.",
17
17
  "High-value command reference: click <selector> --new-tab opens link-like targets in a new tab; select <selector> <value...> changes native dropdown values; wrapper-handled scroll <dir> [px|percent] and scroll to end/top target document scrolling before upstream fallback, while scroll <selector> <dir> [px|percent] targets nested scrollers; download <selector> <path> saves a file triggered by a click; read [url] returns agent-readable text (explicit URLs prefer markdown without requiring a Chrome page; omit the URL for rendered active-tab DOM); get title/url need no selector; get text/html/value/count <selector> and get attr <selector> <name> read elements/page state (use body for whole-page text/html); screenshot [selector] [path] captures a page or element image; pdf <path> saves a PDF; tab list and tab <tab-id-or-label> inspect or recover the active tab; react tree, react inspect <fiberId>, react renders start/stop, and react suspense introspect React after --enable react-devtools; vitals [url] measures Core Web Vitals; pushstate <url> performs SPA navigation; tap <selector> and swipe <direction> [distance] support iOS/provider touch flows.",
18
18
  "For artifact-producing commands, read the visible artifact block and details.artifactVerification before using files: check requested path, absolute path, existence, size bytes, artifact kind, optional mediaType, status, optional limitation, and verified/missing/pending/unverified counts. details.artifacts contains per-file metadata; record start rows are pending/openRecording until record stop writes the target. Upstream record start uses a fresh active page for video capture, so prior in-page DOM and JavaScript state does not carry over; the wrapper blocks prior @e… refs as stale-ref even when the start fails as already-active, and record restart with a URL navigates and invalidates the same way (plain record restart keeps the page), so take a fresh snapshot before continuing. The wrapper creates parent directories for direct artifact paths and can save simple loopback HTTP(S) anchor downloads directly to the requested path before upstream download fallback. Browser close does not delete explicit saved files; if close reports details.artifactCleanup, use host file tools to remove paths listed in explicitArtifactPaths (when non-empty) after inspection. If close fails with details.promptGuard.reason=requested-artifacts-missing-before-close, save the exact required artifact path before closing. A bare inbound image/video path is not a requested output artifact and does not block close. For annotated screenshots inside batch, put --annotate in top-level args (for example { args: [\"--annotate\", \"batch\"], stdin: \"[[\\\"screenshot\\\",\\\"/tmp/page.png\\\"]]\" }) rather than inside the screenshot step; if annotation labels crowd a dense page, use a scoped or non-annotated screenshot plus snapshot refs instead.",
19
19
  "When failure output shows Next actions, prefer those exact native agent_browser follow-up payloads over guessed commands. The same actions are available in details.nextActions to callers that expose structured details; short stdin is shown inline, while long stdin stays details-only.",
@@ -46,14 +46,14 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
46
46
  "For Electron desktop apps, prefer top-level electron for wrapper-owned discovery, isolated launch, status, compact probe, and cleanup: list first, treat likely-sensitive annotations as hints rather than enforcement, launch with the default snapshot handoff unless handoff: \"tabs\" is the safer diagnostic starting point, use electron.probe or snapshot -i/qa.attached for current-session state, and always cleanup the returned launchId when done. electron.launch uses an isolated temporary profile; it does not reuse the app's normal signed-in profile or attach to an already-running authenticated app. For signed-in local app state, host-launch the normal app with --remote-debugging-port when appropriate, then use raw args connect <port|url>; after connect, run get url to verify the active target before page-content reads, inspect tab list, select the stable tab id such as tab t2, verify it again with get url, then run a condition wait or snapshot -i before using refs. close commands (`close`, `quit`, or `exit`) only close the browser/CDP session; leave manually launched app shutdown, profile cleanup, and explicit artifacts to the host owner.",
47
47
  "For provider or specialized app workflows, load version-matched upstream guidance with skills get agentcore|electron|slack|dogfood|vercel-sandbox|derive-client through the native tool; add --full when you need references/templates, and use skills get --all only for broad skill audits. Use derive-client when recording HAR traffic to generate a standalone API client; prefer network har start (text bodies by default) or network har start --content all|none before multi-step capture. For accessibility audits use a11y or a11y --tags wcag2a,wcag2aa (CDP browsers only). Hosted sandbox workflows should use upstream @agent-browser/sandbox helpers outside this wrapper. Provider launches such as -p ios, --provider browserbase/kernel/browseruse/browserless/agentcore, and iOS --device are upstream-owned setup paths; use sessionMode fresh when switching providers and expect external credentials or local Appium/Xcode setup to be required.",
48
48
  "For dialogs and frames, use dialog status/accept/dismiss and frame <selector|main> through native args; dialog commands and eval snippets that look like alert/confirm/prompt/dialog triggers are shorter-bounded than normal browser calls, and timed-out dialog-like interactions may add inspect-dialog-after-timeout, dismiss-dialog-after-timeout, or recover-fresh-session-after-dialog-timeout nextActions. When --confirm-actions produces a pending confirmation, use details.nextActions or exact confirm <id> / deny <id> calls instead of inventing ids.",
49
- "If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For headed demos, put --headed on the first launch with sessionMode=fresh and verify with screenshot/tab/get-url evidence because tool success cannot prove the OS window is visible to the user. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; for disappearance checks, use wait --fn predicates instead of stale upstream-help examples like wait <selector> --state hidden. Use electron.probe/status for wrapper-owned launch health or target mismatch. Fixed waits are a last resort: use explicit --timeout or top-level timeoutMs for legitimately slow waits, and treat a successful payload like \"waited\":\"timeout\" as elapsed time only—verify completion with an observed condition, fresh snapshot, or screenshot.",
49
+ "If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For headed demos, put --headed on the first launch with sessionMode=fresh and verify with screenshot/tab/get-url evidence because tool success cannot prove the OS window is visible to the user. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; for disappearance checks, use wait --fn predicates instead of stale upstream-help examples like wait <selector> --state hidden. Use electron.probe/status for wrapper-owned launch health or target mismatch. Fixed waits are a last resort: their duration is positional (wait <ms>, not wait --time <ms>). Use explicit --timeout or top-level timeoutMs for legitimately slow waits, and treat a successful payload like \"waited\":\"timeout\" as elapsed time only—verify completion with an observed condition, fresh snapshot, or screenshot.",
50
50
  "For feed, timeline, or inbox reading tasks, focus on the main timeline/list region and read the first item there rather than unrelated composer or sidebar content.",
51
51
  "For read-only browsing tasks, use read <url> for documentation or other unstructured text without requiring a Chrome page, or read with no URL for rendered active-tab DOM. Prefer the current snapshot, structured ref labels, getters, or scoped eval --stdin when you need interactive structure or targeted page state. Only click into media viewers, detail routes, or new pages when the current view does not contain the needed information.",
52
52
  "For downloads, prefer download <selector> <path> when an element click should save a file; simple loopback anchor downloads are saved to the requested path when the wrapper can resolve an HTTP(S) href. Do not rely on click alone when you need the downloaded file on disk.",
53
- "On dashboards with nested scroll containers, verify scroll with a screenshot or fresh snapshot -i; if the viewport did not move, details.data.scrolled may be false/noMovement true and you should prefer scrollintoview <@ref> or target the actual scrollable region with scroll <selector> <dir> [px|percent]. For native selects, use select <selector> <value...> (or semanticAction/job select) instead of clicking option refs; for custom comboboxes, a click/semanticAction may only focus the field, so re-snapshot and fall back to type, press Enter/arrow keys, or visible option refs.",
53
+ "On dashboards with nested scroll containers, verify scroll with a screenshot or fresh snapshot -i; if the viewport did not move, details.data.scrolled may be false/noMovement true and you should prefer scrollintoview <@ref> or target the actual scrollable region with scroll <selector> <dir> [px|percent]. For native selects, use select <selector> <value...> (or semanticAction/job select) instead of clicking option refs; for custom comboboxes, a click/semanticAction may only focus the field, so re-snapshot and use keyboard type <text> for focused input, press ArrowDown or press Enter, or visible option refs. Raw type requires both <selector> and <text>.",
54
54
  "When using eval --stdin, scope checks and actions to the target element or route whenever possible instead of relying on broad page-wide text heuristics.",
55
55
  "When using eval --stdin for extraction, pass the JavaScript through the native tool stdin field, not as an extra args token after --stdin, and return the value you want instead of relying on console.log as the primary result channel. Prefer plain expressions like ({ title: document.title }) or explicitly invoked functions like (() => ({ title: document.title }))(); use outputPath when the eval/get/snapshot data should be saved as a durable local file, but never reuse a screenshot, download, recording, or other browser artifact destination as outputPath. If a function-shaped snippet returns {}, details.evalStdinHint may warn that the function was serialized instead of called. Local file pages and caller-selected output paths are supported when upstream allows them. If get text on a broad CSS selector surfaces details.selectorTextVisibility or selectorTextVisibilityAll, prefer a visible @ref, a more specific selector, or the inspect-visible-text-candidates nextAction over hidden tab content.",
56
- "When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If details.clickDispatch reports a click-dispatch miss, refresh/inspect/retry the real click first; for static local fixtures only, an explicit eval --stdin programmatic .click() can exercise app handlers, but treat it as an untrusted scripted workaround and never use it to bypass stop-before-submit/order/purchase boundaries. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.",
56
+ "When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If details.clickDispatch reports a click-dispatch miss, refresh/inspect/retry the real click first; for static local fixtures only, an explicit eval --stdin programmatic .click() can exercise app handlers, but treat it as an untrusted scripted workaround and never use it to bypass stop-before-submit/order/purchase boundaries. If an upstream click failure says the target is covered by another element at the target's click point, use the inspect-overlay-state nextAction to refresh refs and inspect the blocker before deciding whether to retry; do not blindly repeat the blocked click. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.",
57
57
  "When commands save or spill files (screenshots, downloads, PDFs, traces, recordings, HAR, large snapshot spills), use the user's exact requested paths when given and treat paths as provisional until details.artifactVerification shows every row verified: branch on missingCount, pendingCount, unverifiedCount, per-entry state, and optional limitation before downstream file use or PASS/FAIL reporting.",
58
58
  "For evidence-only screenshots, QA captures, or other audit artifacts, save to an explicit path and branch on details.artifactVerification plus details.artifacts before reporting PASS/FAIL; do not require vision review of inline image attachments unless the user asked for visual inspection.",
59
59
  "Respect explicit user stop boundaries yourself. When the surrounding authenticated employee or automation context is explicitly unattended/auto-approved, ordinary non-destructive form submissions within the requested flow may proceed without separate confirmation. Still require explicit authorization for purchases, production-control actions, destructive or irreversible actions, and account, security, or privacy changes. The wrapper does not infer broad business intent from prompt text; details.promptGuard is reserved for concrete artifact-before-close checks.",
@@ -62,10 +62,6 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
62
62
  ];
63
63
  export const TOOL_PROMPT_GUIDELINES_SUFFIX = [
64
64
  "Prefer agent_browser over bash, osascript, AppleScript, or generic browser shell for sites, docs, clicks, fills, screenshots, eval, and batch.",
65
- "Pass exact agent-browser CLI arguments in agent_browser args when you are not using script, semanticAction, job, or qa, excluding the binary name and --json (agent_browser injects --json automatically).",
66
- "Use top-level agent_browser stdin only for eval --stdin, batch, auth save --password-stdin, or wrapper-generated job/qa batches instead of shell heredocs or password args; script puts any inner stdin on browser({ stdin }), and other command/stdin combinations are rejected before launch.",
67
- `Let the agent_browser extension-managed session handle the common path unless you explicitly need a fresh launch for launch-scoped flags (${LAUNCH_SCOPED_FLAG_LABEL}).`,
68
- "Use agent_browser sessionMode=fresh when switching from an existing implicit session to a new profile/browser executable/debug/init-script/provider launch without inventing a fixed explicit session name; later auto calls will follow that new session.",
69
65
  ];
70
66
  export const INSPECTION_TOOL_CALL_EXAMPLES = [
71
67
  '{ "args": ["--help"] }',
@@ -73,7 +69,8 @@ export const INSPECTION_TOOL_CALL_EXAMPLES = [
73
69
  ];
74
70
  export const WRAPPER_TAB_RECOVERY_BEHAVIOR = [
75
71
  "After open/goto/navigate calls with --profile, --restore, --session-name, or --state, agent_browser best-effort re-selects the tab whose URL matches the returned page when restored tabs steal focus during launch or reconnect.",
76
- "After the wrapper observes tab-drift risk for a session (for example open correction, overlapping stale opens, or resumed session state), later active-tab commands best-effort pin that tab inside the same upstream invocation. Routine same-session commands are not preflighted with tab list just because a target tab or ref snapshot is known.",
72
+ "After confirmed shutdown of an automatically restored managed session, the wrapper retains its complete recorded URL, including the fragment, until the first current-page operation (including get url and reload). Non-page calls such as tab list or read <url> may start a daemon without fulfilling that reopen. The wrapper uses native open once, verifies the observed tab, and discards old refs/frame scope; it does not restore unsaved forms, JavaScript memory, or history. Explicit navigation, caller-owned/attached sessions, and restore-disabled sessions are not auto-reopened.",
73
+ "For a still-live browser after tab drift or resume, the wrapper verifies/selects the intended tab before ref/semantic helpers and page commands; failed selection stops the call without navigating. Local commands, read <url>, URL a11y/vitals, diff url, window new, and explicit tab/navigation/connection/state recovery do not require the prior tab. Batch checks follow effective rows past non-page prefixes and stop at explicit context changes, preserving caller argv/stdin and continue-on-error behavior. Same-tab reselection is avoided because it clears refs. Use exact batch --bail for fail-fast, not --bail=<value>. Routine same-session calls skip tab-list preflights.",
77
74
  "For sessions with observed tab-drift risk, after a successful command on a known target tab, agent_browser also best-effort restores that intended tab if a restored/background tab steals focus after the command completes. Routine same-session commands skip this post-command tab-list probe.",
78
75
  "If a known session target unexpectedly reports about:blank, agent_browser best-effort re-selects the prior intended target when it still exists; if recovery fails, it records the observed about:blank target and reports exact recovery guidance instead of treating the prior page as active.",
79
76
  "If upstream reports tab_gone, the pinned bound tab is gone; use details.nextActions (tab list / tab new) instead of assuming another tab is yours.",
@@ -1,11 +1,11 @@
1
1
  import { AsyncLocalStorage } from "node:async_hooks";
2
2
  import { spawn } from "node:child_process";
3
- import { lstat, mkdir, readdir } from "node:fs/promises";
3
+ import { lstat, mkdir, readdir, readlink, stat } from "node:fs/promises";
4
4
  import { dirname, isAbsolute, join } from "node:path";
5
5
  import { env as processEnv, platform as processPlatform } from "node:process";
6
+ import { spawn as crossSpawn } from "cross-spawn";
6
7
  import { parseArgvDescriptor } from "./argv-descriptor.js";
7
- import { isKnownCommandToken } from "./command-taxonomy.js";
8
- import { extractExplicitSessionName, getFlagName, GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES, GLOBAL_VALUE_FLAGS, optionalGlobalValueFlagConsumesNext, resolveAgentBrowserNamespace, } from "./argv-grammar.js";
8
+ import { extractExplicitSessionName, resolveAgentBrowserNamespace } from "./argv-grammar.js";
9
9
  import { commitManagedSessionRestoreSuppression, getManagedSessionRestoreEnv, getManagedSessionRestoreProtectedEnv, getOwnedManagedSessionCompatibilityEnv, getOwnedManagedSessionNamespaceEnv, isOwnedManagedSessionTarget, validateManagedSessionRestoreContextForSpawn, } from "./managed-session-restore.js";
10
10
  import { getPageTargetValidationError, } from "./page-target-validation.js";
11
11
  import { getImplicitSessionIdleTimeoutMs } from "./runtime.js";
@@ -26,114 +26,19 @@ export const SAFE_AGENT_BROWSER_OPERATION_TIMEOUT_MS = 25_000;
26
26
  const DEFAULT_AGENT_BROWSER_PROCESS_TIMEOUT_MS = 35_000;
27
27
  /** Grace period after `exit` before resolving when `close` is delayed by inherited stdio handles. */
28
28
  const EXIT_STDIO_GRACE_MS = 100;
29
- const WINDOWS_AGENT_BROWSER_MISSING_MARKER = "PI_AGENT_BROWSER_COMMAND_NOT_FOUND:agent-browser.cmd";
30
29
  const attachedBrowserSessionContext = new AsyncLocalStorage();
31
- const WINDOWS_COMMANDS_WITH_ADJACENT_SUBCOMMAND = new Set([
32
- "auth", "clipboard", "cookies", "dashboard", "device", "dialog", "diff", "find", "get", "is", "keyboard",
33
- "mouse", "network", "plugin", "profiler", "react", "record", "session", "set", "skills", "state", "storage",
34
- "stream", "tab", "trace", "webmcp", "window",
35
- ]);
36
30
  export function withAttachedBrowserSessionContext(preserve, run) {
37
31
  return attachedBrowserSessionContext.run(preserve || attachedBrowserSessionContext.getStore() === true, run);
38
32
  }
39
- export function getWindowsExplicitDefaultNamespaceEnv(args, parentNamespace, platform = processPlatform) {
40
- return platform === "win32" && resolveAgentBrowserNamespace(args, parentNamespace) === ""
41
- ? { AGENT_BROWSER_NAMESPACE: "" }
42
- : {};
43
- }
44
33
  function appendTail(text, addition, maxChars) {
45
34
  const combined = text + addition;
46
35
  return combined.length <= maxChars ? combined : combined.slice(combined.length - maxChars);
47
36
  }
48
- function quoteWindowsPowerShellArg(value) {
49
- return `'${value.replace(/'/g, "''")}'`;
50
- }
51
- /** Exported for unit tests that lock Windows launcher argv ordering. */
52
- export function reorderWindowsLeadingGlobalArgs(args) {
53
- const leadingGlobals = [];
54
- for (let index = 0; index < args.length; index += 1) {
55
- const token = args[index];
56
- if (isKnownCommandToken(token)) {
57
- if (index === 0)
58
- return args;
59
- const firstPositional = args[index + 1];
60
- return WINDOWS_COMMANDS_WITH_ADJACENT_SUBCOMMAND.has(token) && firstPositional && !firstPositional.startsWith("-")
61
- ? [token, firstPositional, ...leadingGlobals, ...args.slice(index + 2)]
62
- : [token, ...leadingGlobals, ...args.slice(index + 1)];
63
- }
64
- if (!token.startsWith("-"))
65
- return args;
66
- if (token.startsWith("--restore=")) {
67
- leadingGlobals.push(token);
68
- continue;
69
- }
70
- if (token === "--restore") {
71
- const value = args[index + 1];
72
- if (optionalGlobalValueFlagConsumesNext(token, value)) {
73
- leadingGlobals.push(`--restore=${value}`);
74
- index += 1;
75
- }
76
- else {
77
- leadingGlobals.push(token);
78
- }
79
- continue;
80
- }
81
- if (token.includes("="))
82
- return args;
83
- const flag = getFlagName(token);
84
- if (GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES.has(flag)) {
85
- leadingGlobals.push(token);
86
- if (["true", "false"].includes(args[index + 1] ?? "")) {
87
- leadingGlobals.push(args[index + 1]);
88
- index += 1;
89
- }
90
- continue;
91
- }
92
- if (GLOBAL_VALUE_FLAGS.includes(flag)) {
93
- const value = args[index + 1];
94
- if (value === undefined)
95
- return args;
96
- // PowerShell -> .cmd drops empty argv values. Planning rejects empty
97
- // caller --args; keep this defensive skip so an unexpected empty value
98
- // cannot turn the next flag into its accidental value on native Windows.
99
- if (value === "" && (flag === "--args" || flag === "--namespace")) {
100
- index += 1;
101
- continue;
102
- }
103
- leadingGlobals.push(token, value);
104
- index += 1;
105
- continue;
106
- }
107
- return args;
108
- }
109
- return args;
110
- }
111
37
  export function prepareAgentBrowserSpawnArgs(args, wrapperCompatibilityUserAgent, preserveAttachedBrowserSession = false) {
112
38
  if (preserveAttachedBrowserSession || !wrapperCompatibilityUserAgent)
113
39
  return args;
114
40
  return ["--args", `--user-agent=${wrapperCompatibilityUserAgent.replaceAll(/[\r\n,]/g, "")}`, ...args];
115
41
  }
116
- export function buildAgentBrowserSpawnCommand(args, platform = processPlatform) {
117
- if (platform !== "win32") {
118
- return { command: "agent-browser", args };
119
- }
120
- const invocationArgs = reorderWindowsLeadingGlobalArgs(args).map(quoteWindowsPowerShellArg).join(" ");
121
- const commandLine = [
122
- "$agentBrowser = Get-Command agent-browser.cmd -ErrorAction SilentlyContinue;",
123
- `if (-not $agentBrowser) { [Console]::Error.WriteLine('${WINDOWS_AGENT_BROWSER_MISSING_MARKER}'); exit 127 };`,
124
- `& $agentBrowser.Source ${invocationArgs}`.trimEnd(),
125
- ].join(" ");
126
- return { command: "powershell.exe", args: ["-NoLogo", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", commandLine] };
127
- }
128
- export function isWindowsAgentBrowserCommandMissing(stderr) {
129
- const normalized = stderr.toLowerCase();
130
- return normalized.includes(WINDOWS_AGENT_BROWSER_MISSING_MARKER.toLowerCase()) || (normalized.includes("agent-browser.cmd") && (normalized.includes("commandnotfoundexception") ||
131
- normalized.includes("not recognized as the name of a cmdlet") ||
132
- normalized.includes("not recognized as an internal or external command")));
133
- }
134
- export function shouldCommitManagedRestoreAfterWindowsProcess(input) {
135
- return !input.spawnError && !(input.exitCode !== 0 && isWindowsAgentBrowserCommandMissing(input.stderr));
136
- }
137
42
  function terminateSpawnedChild(child, signal) {
138
43
  if (processPlatform === "win32" && child.pid) {
139
44
  const killer = spawn("taskkill.exe", ["/PID", String(child.pid), "/T", "/F"], { stdio: "ignore" });
@@ -252,13 +157,30 @@ export function isTrustedSocketDirAncestor(metadata, uid, platform = processPlat
252
157
  return (mode & 0o022) === 0;
253
158
  return metadata.uid === 0 && ((mode & 0o022) === 0 || (mode & 0o1000) !== 0);
254
159
  }
255
- async function hasTrustedSocketDirAncestry(socketDir, uid) {
160
+ async function hasTrustedSocketDirAncestry(socketDir, uid, visited = new Set()) {
256
161
  for (let current = dirname(socketDir);;) {
162
+ current = current.replace(/\/+$/, "") || "/";
163
+ if (visited.has(current))
164
+ return true;
165
+ visited.add(current);
257
166
  const metadata = await lstat(current);
167
+ // The operating environment supplies /; its reported owner may be unmapped in a user namespace.
168
+ if (current === "/" && metadata.isDirectory() && (metadata.mode & 0o022) === 0)
169
+ return true;
258
170
  if (isTrustedAndroidAppDataRoot(current, metadata, uid))
259
171
  return true;
260
172
  if (!isTrustedSocketDirAncestor(metadata, uid))
261
173
  return false;
174
+ if (metadata.isSymbolicLink()) {
175
+ // Native stat rejects broken/cyclic links before walking their destination ancestry.
176
+ if (!isTrustedSocketDirAncestor(await stat(current), uid))
177
+ return false;
178
+ const target = await readlink(current);
179
+ const targetPath = isAbsolute(target) ? target : `${dirname(current)}/${target}`;
180
+ // Keep '..' after symlinks intact; '/.' includes the target itself in the parent walk.
181
+ if (!await hasTrustedSocketDirAncestry(`${targetPath}/.`, uid, visited))
182
+ return false;
183
+ }
262
184
  const parent = dirname(current);
263
185
  if (parent === current)
264
186
  return true;
@@ -364,7 +286,7 @@ export function buildAgentBrowserProcessEnv(baseEnv = processEnv, overrides = un
364
286
  clampUpstreamDefaultTimeout(childEnv);
365
287
  return childEnv;
366
288
  }
367
- function getManagedPreSpawnPolicyError(options, currentPageUrl, pageUrlUnknown = false, trustedFirstBatchTabSelection = false) {
289
+ function getManagedPreSpawnPolicyError(options, currentPageUrl, pageUrlUnknown = false) {
368
290
  if (!validateManagedSessionRestoreContextForSpawn(options)) {
369
291
  return "Managed session restore policy, storage, or checkout identity changed after planning; refusing to start agent-browser.";
370
292
  }
@@ -373,11 +295,10 @@ function getManagedPreSpawnPolicyError(options, currentPageUrl, pageUrlUnknown =
373
295
  currentPageUrl,
374
296
  pageUrlUnknown,
375
297
  stdin: options.stdin,
376
- trustedFirstBatchTabSelection,
377
298
  });
378
299
  }
379
300
  export async function runAgentBrowserProcess(options) {
380
- const { cwd, env, managedSessionRestoreState, managedStateCurrentPageUrl, managedStatePageUrlUnknown, signal, stdin, trustedFirstBatchTabSelection } = options;
301
+ const { cwd, env, managedSessionRestoreState, managedStateCurrentPageUrl, managedStatePageUrlUnknown, signal, stdin } = options;
381
302
  const preserveAttachedBrowserSession = options.preserveAttachedBrowserSession === true || attachedBrowserSessionContext.getStore() === true;
382
303
  const ownedManagedSession = options.ownedManagedSession === true || isOwnedManagedSessionTarget(options.args);
383
304
  const args = options.args;
@@ -395,7 +316,7 @@ export async function runAgentBrowserProcess(options) {
395
316
  restoreState: managedSessionRestoreState,
396
317
  stdin,
397
318
  };
398
- const planningPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown, trustedFirstBatchTabSelection);
319
+ const planningPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown);
399
320
  if (planningPolicyError) {
400
321
  return {
401
322
  aborted: false,
@@ -415,7 +336,6 @@ export async function runAgentBrowserProcess(options) {
415
336
  ...env,
416
337
  ...getManagedSessionRestoreProtectedEnv(managedSessionRestoreOptions, managedSessionRestoreEnv),
417
338
  ...getOwnedManagedSessionNamespaceEnv(managedSessionRestoreOptions),
418
- ...getWindowsExplicitDefaultNamespaceEnv(args, parentEnv.AGENT_BROWSER_NAMESPACE),
419
339
  ...ownedManagedSessionCompatibilityEnv,
420
340
  };
421
341
  const explicitSocketDir = processOverrides[AGENT_BROWSER_SOCKET_DIR_ENV];
@@ -520,13 +440,8 @@ export async function runAgentBrowserProcess(options) {
520
440
  if (stdoutSpillHandle) {
521
441
  await stdoutSpillHandle.close().catch(() => undefined);
522
442
  }
523
- const windowsMissingBinary = processPlatform === "win32" && exitCode !== 0 && isWindowsAgentBrowserCommandMissing(stderr);
524
- if (processPlatform === "win32" && !windowsMissingBinary && !spawnError)
443
+ if (processPlatform === "win32" && !spawnError) {
525
444
  agentBrowserStarted = true;
526
- if (windowsMissingBinary && !spawnError) {
527
- spawnError = Object.assign(new Error("spawn agent-browser ENOENT"), { code: "ENOENT" });
528
- }
529
- else if (processPlatform === "win32" && shouldCommitManagedRestoreAfterWindowsProcess({ exitCode, spawnError, stderr })) {
530
445
  commitManagedSessionRestoreSuppression(managedSessionRestoreOptions);
531
446
  }
532
447
  if (!spawnError && stdoutSpillError) {
@@ -548,13 +463,13 @@ export async function runAgentBrowserProcess(options) {
548
463
  });
549
464
  };
550
465
  const childEnv = buildAgentBrowserProcessEnv(parentEnv, effectiveEnv);
551
- const spawnPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown, trustedFirstBatchTabSelection);
466
+ const spawnPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown);
552
467
  if (spawnPolicyError) {
553
468
  resolve({ aborted: false, agentBrowserStarted: false, exitCode: 1, spawnError: new Error(spawnPolicyError), stderr: "", stdout: "", timedOut: false });
554
469
  return;
555
470
  }
556
- const spawnCommand = buildAgentBrowserSpawnCommand(prepareAgentBrowserSpawnArgs(args, ownedManagedSessionCompatibilityEnv.AGENT_BROWSER_USER_AGENT, preserveAttachedBrowserSession));
557
- const child = spawn(spawnCommand.command, spawnCommand.args, {
471
+ const spawnBrowser = processPlatform === "win32" ? crossSpawn : spawn;
472
+ const child = spawnBrowser("agent-browser", prepareAgentBrowserSpawnArgs(args, ownedManagedSessionCompatibilityEnv.AGENT_BROWSER_USER_AGENT, preserveAttachedBrowserSession), {
558
473
  cwd,
559
474
  env: childEnv,
560
475
  stdio: ["pipe", "pipe", "pipe"],
@@ -1,3 +1,4 @@
1
+ import { isAbsolute } from "node:path";
1
2
  import { getAgentBrowserSessionIdentityKey } from "./argv-grammar.js";
2
3
  import { isRecord } from "./parsing.js";
3
4
  import { isPendingRecordingArtifact } from "./results/artifact-manifest.js";
@@ -80,7 +81,8 @@ function parseReservationTransition(data) {
80
81
  state: "closed",
81
82
  };
82
83
  }
83
- if (typeof data.absolutePath !== "string" || typeof data.cwd !== "string" || typeof data.path !== "string")
84
+ if (typeof data.absolutePath !== "string" || !isAbsolute(data.absolutePath)
85
+ || typeof data.cwd !== "string" || !isAbsolute(data.cwd) || typeof data.path !== "string")
84
86
  return undefined;
85
87
  return {
86
88
  reservation: {
@@ -1,6 +1,6 @@
1
1
  import { isOpenNavigationCommand, isPageMutationCommand } from "../command-taxonomy.js";
2
2
  import { isPendingRecordingArtifact } from "./artifact-manifest.js";
3
- import { applySessionToNextActions, buildNextToolAction } from "./next-actions.js";
3
+ import { applySessionToNextActions, buildInspectOverlayStateAction, buildNextToolAction } from "./next-actions.js";
4
4
  import { AGENT_BROWSER_RECOVERY_NEXT_ACTION_IDS, buildRecoveryNextActions, } from "./recovery-actions.js";
5
5
  function buildArtifactAction(path) {
6
6
  return {
@@ -223,7 +223,10 @@ export function buildAgentBrowserNextActions(options) {
223
223
  }
224
224
  break;
225
225
  case "upstream-error":
226
- if (isOpenNavigationCommand(options.command)) {
226
+ if (options.overlayBlockedClick) {
227
+ actions.push(buildInspectOverlayStateAction(options.sessionName));
228
+ }
229
+ else if (isOpenNavigationCommand(options.command)) {
227
230
  actions.push(buildNextToolAction({
228
231
  args: ["get", "url"],
229
232
  id: "inspect-page-after-navigation-error",
@@ -17,7 +17,7 @@ async function readEnvelopeSource(options) {
17
17
  throw new Error(`agent-browser output spill file could not be read: ${message}`);
18
18
  }
19
19
  }
20
- function extractEnvelopeErrorText(error) {
20
+ export function extractEnvelopeErrorText(error) {
21
21
  if (typeof error === "string") {
22
22
  return error.trim() || undefined;
23
23
  }
@@ -48,7 +48,8 @@ export async function parseAgentBrowserEnvelope(options) {
48
48
  return { parseError: error instanceof Error ? error.message : String(error) };
49
49
  }
50
50
  const trimmed = stdout.trim();
51
- if (trimmed.length === 0) {
51
+ const plainText = typeof options !== "string" && options.plainText === true;
52
+ if (trimmed.length === 0 && !plainText) {
52
53
  return { parseError: "agent-browser returned no JSON output." };
53
54
  }
54
55
  try {
@@ -81,6 +82,8 @@ export async function parseAgentBrowserEnvelope(options) {
81
82
  return { envelope: parsed };
82
83
  }
83
84
  catch (error) {
85
+ if (plainText)
86
+ return { envelope: { success: true, data: trimmed } };
84
87
  const message = error instanceof Error ? error.message : String(error);
85
88
  return { parseError: `agent-browser returned invalid JSON: ${message}` };
86
89
  }
@@ -149,10 +152,12 @@ export function getAgentBrowserErrorText(options) {
149
152
  if (parseError)
150
153
  return parseError;
151
154
  if (envelope?.success === false) {
152
- if ((hasStructuredBatchStepFailure(envelope.data) || detectConfirmationRequired(envelope.data)) && envelope.error === undefined) {
155
+ const explicitErrorText = extractEnvelopeErrorText(envelope.error);
156
+ if ((hasStructuredBatchStepFailure(envelope.data) || detectConfirmationRequired(envelope.data)) && explicitErrorText === undefined) {
153
157
  return undefined;
154
158
  }
155
- const envelopeErrorText = extractEnvelopeErrorText(envelope.error);
159
+ const envelopeErrorText = explicitErrorText
160
+ ?? extractEnvelopeErrorText(typeof envelope.data === "string" ? envelope.data : isRecord(envelope.data) ? envelope.data.error : undefined);
156
161
  if (envelopeErrorText && isUpstreamIpcReadTimeoutMessage(envelopeErrorText)) {
157
162
  return buildUpstreamIpcReadTimeoutMessage();
158
163
  }
@@ -44,6 +44,14 @@ export function buildNextToolAction(options) {
44
44
  tool: "agent_browser",
45
45
  };
46
46
  }
47
+ export function buildInspectOverlayStateAction(sessionName) {
48
+ return buildNextToolAction({
49
+ args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]),
50
+ id: "inspect-overlay-state",
51
+ reason: "Refresh interactive refs and inspect whether an overlay, banner, modal, or dialog is blocking the intended click.",
52
+ safety: "Read-only inspection; do not blindly retry the blocked click, and use current refs from this snapshot before interacting.",
53
+ });
54
+ }
47
55
  export function appendUniqueAgentBrowserNextActions(target, additions) {
48
56
  if (!additions || additions.length === 0)
49
57
  return target;
@@ -1,22 +1,43 @@
1
- import { readFile, stat } from "node:fs/promises";
1
+ import { open, readFile, stat } from "node:fs/promises";
2
2
  import { extname, resolve } from "node:path";
3
3
  import { getAgentBrowserSessionIdentityKey } from "../../argv-grammar.js";
4
+ import { getExplicitArtifactDestination } from "../../orchestration/browser-run/artifact-paths.js";
4
5
  import { isRecord, parsePositiveInteger } from "../../parsing.js";
6
+ import { extractUpstreamCommandTokens } from "../../runtime.js";
5
7
  import { formatSessionArtifactRetentionSummary, getSessionArtifactManifestEntryKey, isPendingRecordingArtifact, isPendingRecordingCommand, mergeSessionArtifactManifest, } from "../artifact-manifest.js";
6
8
  import { classifyAgentBrowserSuccessCategory } from "../categories.js";
7
- const IMAGE_EXTENSION_TO_MIME_TYPE = {
8
- ".gif": "image/gif",
9
- ".jpeg": "image/jpeg",
10
- ".jpg": "image/jpeg",
11
- ".png": "image/png",
12
- ".webp": "image/webp",
13
- };
9
+ const PNG_HEADER = Buffer.from("89504e470d0a1a0a0000000d49484452", "hex");
14
10
  const INLINE_IMAGE_MAX_BYTES_ENV = "PI_AGENT_BROWSER_INLINE_IMAGE_MAX_BYTES";
15
11
  const DEFAULT_INLINE_IMAGE_MAX_BYTES = 5 * 1_024 * 1_024;
16
12
  const ARTIFACT_MTIME_TOLERANCE_MS = 2_000;
17
- function getImageMimeType(filePath) {
18
- const extension = extname(filePath).toLowerCase();
19
- return IMAGE_EXTENSION_TO_MIME_TYPE[extension];
13
+ function getImageMimeType(bytes) {
14
+ if (bytes.length < 16)
15
+ return undefined;
16
+ if (bytes.subarray(0, 16).equals(PNG_HEADER))
17
+ return "image/png";
18
+ if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff && bytes[3] !== 0xf7)
19
+ return "image/jpeg";
20
+ if (["GIF87a", "GIF89a"].includes(bytes.toString("utf8", 0, 6)))
21
+ return "image/gif";
22
+ if (bytes.toString("utf8", 0, 4) === "RIFF" && bytes.toString("utf8", 8, 12) === "WEBP")
23
+ return "image/webp";
24
+ return undefined;
25
+ }
26
+ async function getFileImageMimeType(path) {
27
+ try {
28
+ const file = await open(path, "r");
29
+ try {
30
+ const bytes = Buffer.alloc(16);
31
+ const { bytesRead } = await file.read(bytes, 0, bytes.length, 0);
32
+ return getImageMimeType(bytes.subarray(0, bytesRead));
33
+ }
34
+ finally {
35
+ await file.close();
36
+ }
37
+ }
38
+ catch {
39
+ return undefined;
40
+ }
20
41
  }
21
42
  function getInlineImageMaxBytes(env = process.env) {
22
43
  return parsePositiveInteger(env[INLINE_IMAGE_MAX_BYTES_ENV]) ?? DEFAULT_INLINE_IMAGE_MAX_BYTES;
@@ -74,17 +95,6 @@ const PATH_FIELD_CANDIDATES = [
74
95
  "profilePath",
75
96
  "videoPath",
76
97
  ];
77
- const ARTIFACT_EXTENSION_TO_MEDIA_TYPE = {
78
- ".cpuprofile": "application/json",
79
- ".har": "application/json",
80
- ".html": "text/html",
81
- ".json": "application/json",
82
- ".pdf": "application/pdf",
83
- ".txt": "text/plain",
84
- ".webm": "video/webm",
85
- ".zip": "application/zip",
86
- ...IMAGE_EXTENSION_TO_MIME_TYPE,
87
- };
88
98
  function isDownloadWaitSubcommand(subcommand) {
89
99
  return subcommand === "--download" || subcommand === "-d";
90
100
  }
@@ -153,6 +163,7 @@ async function buildFileArtifactMetadata(options) {
153
163
  const pendingRecording = isPendingRecordingCommand(options.commandInfo.command, options.commandInfo.subcommand, kind);
154
164
  let exists;
155
165
  let sizeBytes;
166
+ let mediaType;
156
167
  let stale = false;
157
168
  let updatedAtMs;
158
169
  if (!pendingRecording) {
@@ -161,6 +172,7 @@ async function buildFileArtifactMetadata(options) {
161
172
  exists = true;
162
173
  sizeBytes = fileStats.size;
163
174
  updatedAtMs = fileStats.mtimeMs;
175
+ mediaType = fileStats.isFile() ? await getFileImageMimeType(absolutePath) : undefined;
164
176
  const commandCreatesArtifact = !(options.commandInfo.command === "wait" && isDownloadWaitSubcommand(options.commandInfo.subcommand));
165
177
  stale = commandCreatesArtifact && artifactMtimeIsOutsideCommandWindow(updatedAtMs, options.artifactMinUpdatedAtMs, options.artifactMaxUpdatedAtMs);
166
178
  }
@@ -176,11 +188,11 @@ async function buildFileArtifactMetadata(options) {
176
188
  exists,
177
189
  extension,
178
190
  kind,
179
- mediaType: extension ? ARTIFACT_EXTENSION_TO_MEDIA_TYPE[extension] : undefined,
191
+ mediaType,
180
192
  namespace: options.namespace,
181
193
  path: displayPath,
182
194
  recordingState: pendingRecording ? "openRecording" : undefined,
183
- requestedPath: options.artifactRequest?.path,
195
+ requestedPath: options.artifactRequest?.path ?? getExplicitArtifactDestination(extractUpstreamCommandTokens(options.commandInfo.commandTokens ?? [])),
184
196
  session: options.sessionName,
185
197
  sizeBytes,
186
198
  status: pendingRecording ? "pending" : exists === false ? "missing" : stale ? "stale" : options.artifactRequest?.status ?? "saved",
@@ -212,7 +224,7 @@ async function buildPreviousRestartRecordingArtifact(options) {
212
224
  exists: true,
213
225
  extension: previousRecording.extension ?? (extname(absolutePath).toLowerCase() || undefined),
214
226
  kind: "video",
215
- mediaType: previousRecording.mediaType,
227
+ mediaType: fileStats.isFile() ? await getFileImageMimeType(absolutePath) : undefined,
216
228
  namespace: previousRecording.namespace ?? options.namespace,
217
229
  path: previousRecording.path,
218
230
  requestedPath: previousRecording.requestedPath,
@@ -232,7 +244,6 @@ async function buildPreviousRestartRecordingArtifact(options) {
232
244
  exists: false,
233
245
  extension: previousRecording.extension ?? (extname(absolutePath).toLowerCase() || undefined),
234
246
  kind: "video",
235
- mediaType: previousRecording.mediaType,
236
247
  namespace: previousRecording.namespace ?? options.namespace,
237
248
  path: previousRecording.path,
238
249
  requestedPath: previousRecording.requestedPath,
@@ -449,13 +460,12 @@ export function formatArtifactMetadataLines(artifacts) {
449
460
  return [
450
461
  `${formatArtifactLabel(artifact)}: ${artifact.path}`,
451
462
  `Artifact type: ${artifact.kind}`,
452
- `Requested path: ${artifact.requestedPath ?? artifact.path}`,
463
+ artifact.requestedPath ? `Requested path: ${artifact.requestedPath}` : undefined,
453
464
  `Absolute path: ${artifact.absolutePath}`,
454
465
  "Exists: pending until record stop",
455
466
  `Status: ${artifact.status ?? "pending"}`,
456
467
  `Recording state: ${artifact.recordingState ?? "openRecording"}`,
457
468
  `Will exist on stop: ${artifact.willExistOnStop !== false}`,
458
- artifact.subcommand === "start" ? "Page state: record start uses a fresh active page for video capture; prior in-page DOM and JavaScript state does not carry over. Take a fresh snapshot before continuing." : undefined,
459
469
  artifact.session ? `Session: ${artifact.session}` : undefined,
460
470
  artifact.cwd ? `CWD: ${artifact.cwd}` : undefined,
461
471
  `Machine data: details.artifacts[${index}]`,
@@ -464,14 +474,14 @@ export function formatArtifactMetadataLines(artifacts) {
464
474
  return [
465
475
  `${formatArtifactLabel(artifact)}: ${artifact.path}`,
466
476
  `Artifact type: ${artifact.kind}`,
467
- `Requested path: ${artifact.requestedPath ?? artifact.path}`,
477
+ artifact.requestedPath ? `Requested path: ${artifact.requestedPath}` : undefined,
468
478
  `Absolute path: ${artifact.absolutePath}`,
469
479
  `Exists: ${artifact.exists === true}`,
470
480
  artifact.exists === false ? "not found on disk" : undefined,
471
481
  typeof artifact.sizeBytes === "number" ? `Size: ${formatByteCount(artifact.sizeBytes)}` : undefined,
472
482
  typeof artifact.sizeBytes === "number" ? `Size bytes: ${artifact.sizeBytes}` : undefined,
473
483
  `Status: ${artifact.status ?? (artifact.exists === false ? "missing" : "saved")}`,
474
- artifact.tempPath ? `Temp path: ${artifact.tempPath}` : undefined,
484
+ artifact.tempPath ? `Reported path: ${artifact.tempPath}` : undefined,
475
485
  artifact.mediaType ? `Media type: ${artifact.mediaType}` : undefined,
476
486
  artifact.session ? `Session: ${artifact.session}` : undefined,
477
487
  artifact.cwd ? `CWD: ${artifact.cwd}` : undefined,
@@ -519,21 +529,10 @@ export function extractImagePath(commandInfo, cwd, data) {
519
529
  if (!isTrustedScreenshotOutput(commandInfo)) {
520
530
  return undefined;
521
531
  }
522
- if (typeof data === "string") {
523
- const mimeType = getImageMimeType(data);
524
- return mimeType ? resolve(cwd, data) : undefined;
525
- }
526
- if (!isRecord(data) || typeof data.path !== "string") {
527
- return undefined;
528
- }
529
- const mimeType = getImageMimeType(data.path);
530
- return mimeType ? resolve(cwd, data.path) : undefined;
532
+ const path = typeof data === "string" ? data : isRecord(data) && typeof data.path === "string" ? data.path : undefined;
533
+ return path?.trim() && !isNonFileArtifactPathCandidate(path) ? resolve(cwd, path) : undefined;
531
534
  }
532
535
  export async function attachInlineImage(presentation, imagePath) {
533
- const mimeType = getImageMimeType(imagePath);
534
- if (!mimeType) {
535
- return presentation;
536
- }
537
536
  try {
538
537
  const fileStats = await stat(imagePath);
539
538
  const inlineImageMaxBytes = getInlineImageMaxBytes();
@@ -543,6 +542,9 @@ export async function attachInlineImage(presentation, imagePath) {
543
542
  return presentation;
544
543
  }
545
544
  const file = await readFile(imagePath);
545
+ const mimeType = getImageMimeType(file);
546
+ if (!mimeType)
547
+ return presentation;
546
548
  presentation.content.push({ type: "image", data: file.toString("base64"), mimeType });
547
549
  presentation.imagePath = imagePath;
548
550
  return presentation;
@@ -13,7 +13,7 @@ import { extractAgentBrowserLifecycle, stringifyModelFacing } from "./common.js"
13
13
  import { buildArtifactVerificationSummary, classifyPresentationSuccessCategory, manifestHasNewNoticeWorthyEntries } from "./artifacts.js";
14
14
  import { formatBatchStepCommand, getPresentationImages, getPresentationPaths, getPresentationText, isStringArray } from "./content.js";
15
15
  import { buildPageChangeSummary } from "./navigation.js";
16
- import { appendSelectorRecoveryHint, getClipboardWritePayloadCandidates, redactClipboardPermissionErrorValue } from "./errors.js";
16
+ import { appendSelectorRecoveryHint, getClipboardWritePayloadCandidates, isOverlayBlockedClickError, redactClipboardPermissionErrorValue } from "./errors.js";
17
17
  export function isAgentBrowserBatchResultArray(value) {
18
18
  return Array.isArray(value) && value.every(isRecord);
19
19
  }
@@ -208,6 +208,7 @@ async function buildBatchStepPresentation(options) {
208
208
  command: command?.[0],
209
209
  confirmationId: confirmationRequired?.id,
210
210
  failureCategory,
211
+ overlayBlockedClick: isOverlayBlockedClickError(command?.[0], errorText, command),
211
212
  resultCategory: "failure",
212
213
  sessionName,
213
214
  subcommand: command?.[1],