pi-agent-browser-native 0.2.47 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -19
- package/README.md +52 -19
- package/dist/extensions/agent-browser/index.js +785 -0
- package/dist/extensions/agent-browser/lib/argv-descriptor.js +71 -0
- package/dist/extensions/agent-browser/lib/argv-grammar.js +121 -0
- package/dist/extensions/agent-browser/lib/bash-guard.js +190 -0
- package/dist/extensions/agent-browser/lib/command-policy.js +85 -0
- package/dist/extensions/agent-browser/lib/command-taxonomy.js +302 -0
- package/dist/extensions/agent-browser/lib/config-policy.js +686 -0
- package/dist/extensions/agent-browser/lib/config.js +122 -0
- package/dist/extensions/agent-browser/lib/electron/cdp.js +51 -0
- package/dist/extensions/agent-browser/lib/electron/cleanup.js +212 -0
- package/dist/extensions/agent-browser/lib/electron/discovery.js +633 -0
- package/dist/extensions/agent-browser/lib/electron/launch.js +351 -0
- package/{extensions/agent-browser/lib/electron/text.ts → dist/extensions/agent-browser/lib/electron/text.js} +5 -5
- package/dist/extensions/agent-browser/lib/executable-path.js +20 -0
- package/dist/extensions/agent-browser/lib/fs-utils.js +18 -0
- package/dist/extensions/agent-browser/lib/input-modes/electron.js +165 -0
- package/dist/extensions/agent-browser/lib/input-modes/job.js +519 -0
- package/dist/extensions/agent-browser/lib/input-modes/lookups.js +440 -0
- package/dist/extensions/agent-browser/lib/input-modes/params.js +164 -0
- package/dist/extensions/agent-browser/lib/input-modes/semantic-action.js +119 -0
- package/dist/extensions/agent-browser/lib/input-modes/shared.js +42 -0
- package/dist/extensions/agent-browser/lib/input-modes/types.js +21 -0
- package/dist/extensions/agent-browser/lib/input-modes.js +10 -0
- package/dist/extensions/agent-browser/lib/json-schema.js +58 -0
- package/dist/extensions/agent-browser/lib/launch-scoped-flags.js +59 -0
- package/dist/extensions/agent-browser/lib/navigation-policy.js +83 -0
- package/dist/extensions/agent-browser/lib/orchestration/batch-stdin.js +62 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/artifact-paths.js +39 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.js +276 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.js +909 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/final-result.js +443 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/index.js +47 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/direct-anchor-download.js +141 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/network-page-filter.js +108 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/scroll-shims.js +112 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/snapshot-filter.js +158 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/wait-timeouts.js +54 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare.js +762 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/process-output.js +491 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.js +40 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/session-artifacts.js +5 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/session-state.js +731 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/types.js +1 -0
- package/dist/extensions/agent-browser/lib/orchestration/electron-host/index.js +718 -0
- package/dist/extensions/agent-browser/lib/orchestration/input-plan.js +247 -0
- package/dist/extensions/agent-browser/lib/orchestration/output-file.js +68 -0
- package/{extensions/agent-browser/lib/parsing.ts → dist/extensions/agent-browser/lib/parsing.js} +12 -11
- package/dist/extensions/agent-browser/lib/pi-tool-rendering.js +241 -0
- package/dist/extensions/agent-browser/lib/playbook.js +121 -0
- package/dist/extensions/agent-browser/lib/process.js +448 -0
- package/dist/extensions/agent-browser/lib/prompt-policy.js +91 -0
- package/dist/extensions/agent-browser/lib/results/action-recommendations.js +220 -0
- package/dist/extensions/agent-browser/lib/results/artifact-manifest.js +111 -0
- package/{extensions/agent-browser/lib/results/artifact-state.ts → dist/extensions/agent-browser/lib/results/artifact-state.js} +4 -8
- package/dist/extensions/agent-browser/lib/results/categories.js +76 -0
- package/dist/extensions/agent-browser/lib/results/confirmation.js +63 -0
- package/dist/extensions/agent-browser/lib/results/contracts.js +8 -0
- package/dist/extensions/agent-browser/lib/results/editable-ref-evidence.js +74 -0
- package/dist/extensions/agent-browser/lib/results/envelope.js +166 -0
- package/dist/extensions/agent-browser/lib/results/network-routes.js +92 -0
- package/dist/extensions/agent-browser/lib/results/network.js +73 -0
- package/dist/extensions/agent-browser/lib/results/next-actions.js +72 -0
- package/dist/extensions/agent-browser/lib/results/presentation/artifacts.js +515 -0
- package/dist/extensions/agent-browser/lib/results/presentation/batch.js +397 -0
- package/dist/extensions/agent-browser/lib/results/presentation/browser-profile-recovery.js +55 -0
- package/dist/extensions/agent-browser/lib/results/presentation/common.js +46 -0
- package/dist/extensions/agent-browser/lib/results/presentation/content.js +24 -0
- package/dist/extensions/agent-browser/lib/results/presentation/diagnostics.js +960 -0
- package/dist/extensions/agent-browser/lib/results/presentation/errors.js +205 -0
- package/dist/extensions/agent-browser/lib/results/presentation/large-output.js +134 -0
- package/dist/extensions/agent-browser/lib/results/presentation/navigation.js +159 -0
- package/dist/extensions/agent-browser/lib/results/presentation/registry.js +216 -0
- package/dist/extensions/agent-browser/lib/results/presentation/semantic-action.js +104 -0
- package/dist/extensions/agent-browser/lib/results/presentation/skills.js +152 -0
- package/dist/extensions/agent-browser/lib/results/presentation.js +177 -0
- package/dist/extensions/agent-browser/lib/results/recovery-actions.js +107 -0
- package/dist/extensions/agent-browser/lib/results/recovery-next-actions.js +50 -0
- package/dist/extensions/agent-browser/lib/results/selector-recovery.js +225 -0
- package/{extensions/agent-browser/lib/results/shared.ts → dist/extensions/agent-browser/lib/results/shared.js} +0 -1
- package/dist/extensions/agent-browser/lib/results/snapshot-high-value-controls.js +208 -0
- package/dist/extensions/agent-browser/lib/results/snapshot-refs.js +78 -0
- package/dist/extensions/agent-browser/lib/results/snapshot-segments.js +331 -0
- package/dist/extensions/agent-browser/lib/results/snapshot-spill.js +40 -0
- package/dist/extensions/agent-browser/lib/results/snapshot.js +264 -0
- package/dist/extensions/agent-browser/lib/results/text.js +40 -0
- package/{extensions/agent-browser/lib/results.ts → dist/extensions/agent-browser/lib/results.js} +2 -32
- package/dist/extensions/agent-browser/lib/runtime.js +816 -0
- package/dist/extensions/agent-browser/lib/session-page-state.js +411 -0
- package/dist/extensions/agent-browser/lib/string-enum-schema.js +13 -0
- package/dist/extensions/agent-browser/lib/temp.js +498 -0
- package/dist/extensions/agent-browser/lib/web-search.js +562 -0
- package/docs/ARCHITECTURE.md +10 -10
- package/docs/COMMAND_REFERENCE.md +35 -21
- package/docs/ELECTRON.md +3 -3
- package/docs/RELEASE.md +46 -26
- package/docs/REQUIREMENTS.md +1 -1
- package/docs/SUPPORT_MATRIX.md +35 -106
- package/docs/TOOL_CONTRACT.md +23 -21
- package/package.json +12 -8
- package/scripts/agent-browser-capability-baseline.mjs +6 -3
- package/scripts/config.mjs +8 -2
- package/scripts/doctor.mjs +19 -17
- package/scripts/platform-smoke.mjs +1 -1
- package/extensions/agent-browser/index.ts +0 -952
- package/extensions/agent-browser/lib/argv-descriptor.ts +0 -90
- package/extensions/agent-browser/lib/argv-grammar.ts +0 -128
- package/extensions/agent-browser/lib/bash-guard.ts +0 -205
- package/extensions/agent-browser/lib/command-policy.ts +0 -71
- package/extensions/agent-browser/lib/command-taxonomy.ts +0 -336
- package/extensions/agent-browser/lib/config-policy.js +0 -690
- package/extensions/agent-browser/lib/config.ts +0 -209
- package/extensions/agent-browser/lib/electron/cdp.ts +0 -69
- package/extensions/agent-browser/lib/electron/cleanup.ts +0 -235
- package/extensions/agent-browser/lib/electron/discovery.ts +0 -710
- package/extensions/agent-browser/lib/electron/launch.ts +0 -499
- package/extensions/agent-browser/lib/executable-path.ts +0 -19
- package/extensions/agent-browser/lib/fs-utils.ts +0 -18
- package/extensions/agent-browser/lib/input-modes/electron.ts +0 -170
- package/extensions/agent-browser/lib/input-modes/job.ts +0 -451
- package/extensions/agent-browser/lib/input-modes/lookups.ts +0 -447
- package/extensions/agent-browser/lib/input-modes/params.ts +0 -205
- package/extensions/agent-browser/lib/input-modes/semantic-action.ts +0 -127
- package/extensions/agent-browser/lib/input-modes/shared.ts +0 -46
- package/extensions/agent-browser/lib/input-modes/types.ts +0 -225
- package/extensions/agent-browser/lib/input-modes.ts +0 -45
- package/extensions/agent-browser/lib/json-schema.ts +0 -73
- package/extensions/agent-browser/lib/launch-scoped-flags.ts +0 -67
- package/extensions/agent-browser/lib/navigation-policy.ts +0 -95
- package/extensions/agent-browser/lib/orchestration/batch-stdin.ts +0 -65
- package/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.ts +0 -257
- package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts +0 -912
- package/extensions/agent-browser/lib/orchestration/browser-run/final-result.ts +0 -512
- package/extensions/agent-browser/lib/orchestration/browser-run/index.ts +0 -53
- package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts +0 -1481
- package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts +0 -564
- package/extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.ts +0 -47
- package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts +0 -868
- package/extensions/agent-browser/lib/orchestration/browser-run/types.ts +0 -564
- package/extensions/agent-browser/lib/orchestration/electron-host/index.ts +0 -855
- package/extensions/agent-browser/lib/orchestration/input-plan.ts +0 -375
- package/extensions/agent-browser/lib/orchestration/output-file.ts +0 -86
- package/extensions/agent-browser/lib/pi-tool-rendering.ts +0 -252
- package/extensions/agent-browser/lib/playbook.ts +0 -142
- package/extensions/agent-browser/lib/process.ts +0 -516
- package/extensions/agent-browser/lib/prompt-policy.ts +0 -105
- package/extensions/agent-browser/lib/results/action-recommendations.ts +0 -264
- package/extensions/agent-browser/lib/results/artifact-manifest.ts +0 -111
- package/extensions/agent-browser/lib/results/categories.ts +0 -106
- package/extensions/agent-browser/lib/results/confirmation.ts +0 -76
- package/extensions/agent-browser/lib/results/contracts.ts +0 -241
- package/extensions/agent-browser/lib/results/editable-ref-evidence.ts +0 -72
- package/extensions/agent-browser/lib/results/envelope.ts +0 -195
- package/extensions/agent-browser/lib/results/network-routes.ts +0 -83
- package/extensions/agent-browser/lib/results/network.ts +0 -78
- package/extensions/agent-browser/lib/results/next-actions.ts +0 -117
- package/extensions/agent-browser/lib/results/presentation/artifacts.ts +0 -588
- package/extensions/agent-browser/lib/results/presentation/batch.ts +0 -450
- package/extensions/agent-browser/lib/results/presentation/browser-profile-recovery.ts +0 -67
- package/extensions/agent-browser/lib/results/presentation/common.ts +0 -53
- package/extensions/agent-browser/lib/results/presentation/content.ts +0 -36
- package/extensions/agent-browser/lib/results/presentation/diagnostics.ts +0 -923
- package/extensions/agent-browser/lib/results/presentation/errors.ts +0 -227
- package/extensions/agent-browser/lib/results/presentation/large-output.ts +0 -182
- package/extensions/agent-browser/lib/results/presentation/navigation.ts +0 -184
- package/extensions/agent-browser/lib/results/presentation/registry.ts +0 -242
- package/extensions/agent-browser/lib/results/presentation/semantic-action.ts +0 -131
- package/extensions/agent-browser/lib/results/presentation/skills.ts +0 -143
- package/extensions/agent-browser/lib/results/presentation.ts +0 -257
- package/extensions/agent-browser/lib/results/recovery-actions.ts +0 -139
- package/extensions/agent-browser/lib/results/recovery-next-actions.ts +0 -71
- package/extensions/agent-browser/lib/results/selector-recovery.ts +0 -320
- package/extensions/agent-browser/lib/results/snapshot-high-value-controls.ts +0 -273
- package/extensions/agent-browser/lib/results/snapshot-refs.ts +0 -100
- package/extensions/agent-browser/lib/results/snapshot-segments.ts +0 -366
- package/extensions/agent-browser/lib/results/snapshot-spill.ts +0 -63
- package/extensions/agent-browser/lib/results/snapshot.ts +0 -329
- package/extensions/agent-browser/lib/results/text.ts +0 -40
- package/extensions/agent-browser/lib/runtime.ts +0 -988
- package/extensions/agent-browser/lib/session-page-state.ts +0 -512
- package/extensions/agent-browser/lib/string-enum-schema.ts +0 -20
- package/extensions/agent-browser/lib/temp.ts +0 -577
- package/extensions/agent-browser/lib/web-search.ts +0 -721
- /package/{extensions/agent-browser/lib/orchestration/browser-run.ts → dist/extensions/agent-browser/lib/orchestration/browser-run.js} +0 -0
package/docs/TOOL_CONTRACT.md
CHANGED
|
@@ -36,7 +36,7 @@ Agent-facing efficiency claims are measured with `npm run benchmark:agent-browse
|
|
|
36
36
|
|
|
37
37
|
## Optional companion web search
|
|
38
38
|
|
|
39
|
-
`agent_browser_web_search` is a separate custom tool, not an `agent_browser` input mode. It is registered only when the extension can see at least one configured/resolvable Exa or Brave credential source from `~/.pi/config/pi-agent-browser-native/config.json`, `.pi/config/pi-agent-browser-native/config.json`, `PI_AGENT_BROWSER_CONFIG`, or the `EXA_API_KEY` / `BRAVE_API_KEY` environment fallbacks, and only when the final available merged config does not set `webSearch.enabled` to `false`. Config layers merge global → project → `PI_AGENT_BROWSER_CONFIG` override; under Pi 0.79+, globally installed and CLI-loaded copies read `.pi/config/...` by default because extensions are developer-trusted code, and they skip the project layer
|
|
39
|
+
`agent_browser_web_search` is a separate custom tool, not an `agent_browser` input mode. It is registered only when the extension can see at least one configured/resolvable Exa or Brave credential source from `~/.pi/config/pi-agent-browser-native/config.json`, `.pi/config/pi-agent-browser-native/config.json`, `PI_AGENT_BROWSER_CONFIG`, or the `EXA_API_KEY` / `BRAVE_API_KEY` environment fallbacks, and only when the final available merged config does not set `webSearch.enabled` to `false`. Config layers merge global → project → `PI_AGENT_BROWSER_CONFIG` override; under Pi 0.79+, globally installed and CLI-loaded copies read `.pi/config/...` by default because extensions are developer-trusted code, and they skip the project layer when Pi reports the project is untrusted or when launched with `--no-approve`. Disable scope is explicit: a global disable is a normal user default, a project disable applies to one repo, and an override file with `webSearch.enabled: false` is the highest-priority hard disable for that run. Command credential sources such as `"!op read 'op://Private/Exa/API Key'"` are allowed only from trusted global or explicit-override config; they make the tool available without running the command at startup, and the key is resolved when the tool executes. Project-local config may use only matching provider env refs (`$EXA_API_KEY` / `${EXA_API_KEY}` for Exa and `$BRAVE_API_KEY` / `${BRAVE_API_KEY}` for Brave); custom env aliases, interpolation literals, and malformed `$` values are rejected. Browser profile/executable config uses the same paths but only trusted global or explicit override values are emitted as host launch prompt guidance; project-local browser config is loaded by default but is not trusted to steer local profiles or executable paths.
|
|
40
40
|
|
|
41
41
|
Use it when live/current external web information would help answer a task, find current docs/news, or discover candidate URLs. Use `agent_browser` when the task needs browser interaction, screenshots, authenticated/profile content, page inspection, or DOM work. The search tool is namespaced to avoid colliding with generic `web_search`, chooses Exa or Brave automatically from available credentials, defaults to Exa when both are available (unless `webSearch.preferredProvider` is set), and must not expose resolved API keys in content, details, errors, status output, docs examples, logs, or PR artifacts.
|
|
42
42
|
|
|
@@ -158,14 +158,14 @@ The extension always plans normal browser commands with `--json` prepended in `e
|
|
|
158
158
|
- For Electron desktop apps, prefer top-level electron for wrapper-owned discovery, isolated launch, status, compact probe, and cleanup: list first, treat likely-sensitive annotations as hints rather than enforcement, launch with the default snapshot handoff unless handoff: "tabs" is the safer diagnostic starting point, use electron.probe or snapshot -i/qa.attached for current-session state, and always cleanup the returned launchId when done. electron.launch uses an isolated temporary profile; it does not reuse the app's normal signed-in profile or attach to an already-running authenticated app. For signed-in local app state, host-launch the normal app with --remote-debugging-port when appropriate, then use raw args connect <port|url>; after connect, inspect tab list, select the stable tab id such as tab t2, then run a condition wait or snapshot -i before using refs. close commands (`close`, `quit`, or `exit`) only close the browser/CDP session; leave manually launched app shutdown, profile cleanup, and explicit artifacts to the host owner.
|
|
159
159
|
- For provider or specialized app workflows, load version-matched upstream guidance with skills get agentcore|electron|slack|dogfood|vercel-sandbox through the native tool; add --full when you need references/templates, and use skills get --all only for broad skill audits. Provider launches such as -p ios, --provider browserbase/kernel/browseruse/browserless/agentcore, and iOS --device are upstream-owned setup paths; use sessionMode fresh when switching providers and expect external credentials or local Appium/Xcode setup to be required.
|
|
160
160
|
- For dialogs and frames, use dialog status/accept/dismiss and frame <selector|main> through native args; dialog commands and eval snippets that look like alert/confirm/prompt/dialog triggers are shorter-bounded than normal browser calls, and timed-out dialog-like interactions may add inspect-dialog-after-timeout, dismiss-dialog-after-timeout, or recover-fresh-session-after-dialog-timeout nextActions. When --confirm-actions produces a pending confirmation, use details.nextActions or exact confirm <id> / deny <id> calls instead of inventing ids.
|
|
161
|
-
- If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For headed demos, put --headed on the first launch with sessionMode=fresh and verify with screenshot/tab/get-url evidence because tool success cannot prove the OS window is visible to the user. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; for disappearance checks in agent-browser 0.27.
|
|
161
|
+
- If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For headed demos, put --headed on the first launch with sessionMode=fresh and verify with screenshot/tab/get-url evidence because tool success cannot prove the OS window is visible to the user. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; for disappearance checks in agent-browser 0.27.2, use wait --fn predicates instead of stale upstream-help examples like wait <selector> --state hidden. Use electron.probe/status for wrapper-owned launch health or target mismatch. Fixed waits are a last resort: use explicit --timeout or top-level timeoutMs for legitimately slow waits, and treat a successful payload like "waited":"timeout" as elapsed time only—verify completion with an observed condition, fresh snapshot, or screenshot.
|
|
162
162
|
- For feed, timeline, or inbox reading tasks, focus on the main timeline/list region and read the first item there rather than unrelated composer or sidebar content.
|
|
163
163
|
- For read-only browsing tasks, prefer extracting the answer from the current snapshot, structured ref labels, or eval --stdin on the current page before navigating away. Only click into media viewers, detail routes, or new pages when the current view does not contain the needed information.
|
|
164
164
|
- For downloads, prefer download <selector> <path> when an element click should save a file; simple loopback anchor downloads are saved to the requested path when the wrapper can resolve an HTTP(S) href. Do not rely on click alone when you need the downloaded file on disk.
|
|
165
165
|
- On dashboards with nested scroll containers, verify scroll with a screenshot or fresh snapshot -i; if the viewport did not move, details.data.scrolled may be false/noMovement true and you should prefer scrollintoview <@ref> or target the actual scrollable region with scroll <selector> <dir> [px|percent]. For native selects, use select <selector> <value...> (or semanticAction/job select) instead of clicking option refs; for custom comboboxes, a click/semanticAction may only focus the field, so re-snapshot and fall back to type, press Enter/arrow keys, or visible option refs.
|
|
166
166
|
- When using eval --stdin, scope checks and actions to the target element or route whenever possible instead of relying on broad page-wide text heuristics.
|
|
167
167
|
- When using eval --stdin for extraction, pass the JavaScript through the native tool stdin field, not as an extra args token after --stdin, and return the value you want instead of relying on console.log as the primary result channel. Prefer plain expressions like ({ title: document.title }) or explicitly invoked functions like (() => ({ title: document.title }))(); use outputPath when the eval/get/snapshot data should be saved as a durable local file. If a function-shaped snippet returns {}, details.evalStdinHint may warn that the function was serialized instead of called. On file:// pages, when upstream JSON returns result: null for non-trivial stdin, details.evalResultWarning may append Eval result warning without failing the tool—treat that as inconclusive DOM verification. If get text on a CSS selector surfaces details.selectorTextVisibility or selectorTextVisibilityAll, prefer a visible @ref, a more specific selector, or the inspect-visible-text-candidates nextAction over hidden tab content.
|
|
168
|
-
- When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If details.clickDispatch reports
|
|
168
|
+
- When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If details.clickDispatch reports a click-dispatch miss, refresh/inspect/retry the real click first; for static local fixtures only, an explicit eval --stdin programmatic .click() can exercise app handlers, but treat it as an untrusted scripted workaround and never use it to bypass stop-before-submit/order/purchase boundaries. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.
|
|
169
169
|
- When commands save or spill files (screenshots, downloads, PDFs, traces, recordings, HAR, large snapshot spills), use the user's exact requested paths when given and treat paths as provisional until details.artifactVerification shows every row verified: branch on missingCount, pendingCount, unverifiedCount, per-entry state, and optional limitation before downstream file use or PASS/FAIL reporting.
|
|
170
170
|
- For evidence-only screenshots, QA captures, or other audit artifacts, save to an explicit path and branch on details.artifactVerification plus details.artifacts before reporting PASS/FAIL; do not require vision review of inline image attachments unless the user asked for visual inspection.
|
|
171
171
|
- Respect explicit user stop boundaries yourself: if the user says to stop before order/post/purchase/submit, do not click that final action. The wrapper does not infer broad business intent from prompt text; details.promptGuard is reserved for concrete artifact-before-close checks.
|
|
@@ -274,20 +274,20 @@ Examples:
|
|
|
274
274
|
- constrained orchestration only: every step compiles to existing upstream `batch` argv and the compiled plan is echoed as `details.compiledJob`
|
|
275
275
|
- optional `failFast` boolean; defaults to `true`, compiling to upstream `batch --bail` so later mutating job steps do not run after an earlier required step fails. Set `failFast: false` only when you explicitly want upstream batch's continue-after-error behavior.
|
|
276
276
|
- there is no separate reusable named “browser recipe” extension surface above `job`, `qa`, and raw `batch` yet; the closed `RQ-0068` decision, evidence bar, and revisit criteria are in [`ARCHITECTURE.md`](ARCHITECTURE.md#no-reusable-recipe-layer-yet) and [`SUPPORT_MATRIX.md`](SUPPORT_MATRIX.md)
|
|
277
|
-
- supported steps (`open.loadState` can emit an extra readiness row immediately after the `open`; otherwise each row becomes one upstream `batch` step except paced `type`, which expands to existing focus/keyboard/wait/press rows; `click` / `fill`
|
|
277
|
+
- supported steps (`open.loadState` can emit an extra readiness row immediately after the `open`; otherwise each row becomes one upstream `batch` step except paced `type`, which expands to existing focus/keyboard/wait/press rows; `click` / `fill` accept either `selector` or semantic locator fields, while `type` / `select` pass `selector` through as the same argv token shape standalone upstream commands would use, including `@refs`):
|
|
278
278
|
- `open` with `url`; optional `loadState` (`domcontentloaded`, `load`, or `networkidle`) inserts `wait --load <state>` immediately after the open when the next step needs page-readiness evidence
|
|
279
|
-
- `click` with `selector`
|
|
280
|
-
- `fill` with `selector` and `text`
|
|
279
|
+
- `click` with `selector` or semantic locator fields
|
|
280
|
+
- `fill` with `selector` and `text`, or semantic locator fields plus `text`
|
|
281
281
|
- `type` with `text`; optional `selector` focuses the target first, optional `delayMs` emits per-character `keyboard type` plus `wait` rows, and optional `press` emits a final `press <key>` row such as `Enter`. Delayed typing is capped at 200 characters per step. Model-visible batch prose compacts generated per-character rows; `details.batchSteps` and `details.compiledJob.steps` retain the full bounded row list.
|
|
282
282
|
- `select` with `selector` plus either `value` or `values` (one or more option values; compiled as `select <selector> <value...>`)
|
|
283
283
|
- `wait` with positive integer `milliseconds`
|
|
284
284
|
- `assertText` with `text` (compiled as passive `wait --text <text>`)
|
|
285
|
-
- `assertUrl` with `url` pattern (
|
|
285
|
+
- `assertUrl` with exact URL or `*` / `**` glob-style `url` pattern (exact URLs compile as `wait --url <url>`; glob-style values compile as `wait --fn <location.href predicate>`)
|
|
286
286
|
- `waitForDownload` with `path` (compiled as `wait --download <path>`)
|
|
287
287
|
- `snapshot` (compiled as `snapshot -i`; useful between mutation-prone steps before reusing current refs)
|
|
288
288
|
- `screenshot` with `path`
|
|
289
289
|
|
|
290
|
-
**Navigation assertions are explicit only.** `job` never treats a successful `click` (or a `select` / submit-style interaction that may navigate) as proof that the expected next page loaded. Top-level `click` may still surface optional `details.navigationSummary` or `pageChangeSummary` hints for operators, but compiled `job` / `batch` steps do **not** auto-insert `assertUrl` or `assertText` after clicks—there is no deterministic expected URL source without caller intent. Use `open.loadState` to wait for initial page readiness after an `open`; after any later navigation-prone step (link/submit clicks, checkout or form flows, tab-sensitive UI), add an explicit `assertUrl` with the destination pattern you expect, `assertText` for on-page copy, or both, **before** screenshots or steps that assume the new page state.
|
|
290
|
+
**Navigation assertions are explicit only.** `job` never treats a successful `click` (or a `select` / submit-style interaction that may navigate) as proof that the expected next page loaded. Top-level `click` may still surface optional `details.navigationSummary` or `pageChangeSummary` hints for operators, but compiled `job` / `batch` steps do **not** auto-insert `assertUrl` or `assertText` after clicks—there is no deterministic expected URL source without caller intent. Use `open.loadState` to wait for initial page readiness after an `open`; after any later navigation-prone step (link/submit clicks, checkout or form flows, tab-sensitive UI), add an explicit `assertUrl` with the exact destination URL or `*` / `**` glob-style pattern you expect, `assertText` for on-page copy, or both, **before** screenshots or steps that assume the new page state. Glob-style `assertUrl` values compile to a JavaScript URL predicate so wrapper jobs do not depend on upstream `wait --url` glob behavior.
|
|
291
291
|
|
|
292
292
|
Example (static landing page):
|
|
293
293
|
|
|
@@ -325,15 +325,17 @@ Compiled shape for the navigation example:
|
|
|
325
325
|
```json
|
|
326
326
|
{
|
|
327
327
|
"args": ["batch", "--bail"],
|
|
328
|
-
"stdin": "[[\"open\",\"https://shop.example/checkout\"],[\"fill\",\"#email\",\"user@example.com\"],[\"click\",\"#continue\"],[\"wait\",\"--
|
|
328
|
+
"stdin": "[[\"open\",\"https://shop.example/checkout\"],[\"fill\",\"#email\",\"user@example.com\"],[\"click\",\"#continue\"],[\"wait\",\"--fn\",\"new RegExp(\\\"^.*/shipping$\\\").test(location.href)\"],[\"wait\",\"--text\",\"Shipping address\"],[\"screenshot\",\".dogfood/shipping.png\"]]"
|
|
329
329
|
}
|
|
330
330
|
```
|
|
331
331
|
|
|
332
332
|
On app pages that expose a native dropdown, add a `select` step such as `{ "action": "select", "selector": "#flavor", "value": "chocolate" }` before the assertion that depends on it. `click` and `fill` steps may use either `selector` or semantic locator fields. A semantic locator step mirrors the top-level `semanticAction` compiler and must include `locator` plus the locator value (`role`/`value` and optional `name` for role locators); it compiles to upstream `find`, for example `{ "action": "fill", "locator": "role", "role": "searchbox", "name": "Search", "text": "agent browser" }` → `find role searchbox fill "agent browser" --name Search` and `{ "action": "click", "locator": "role", "role": "button", "name": "Search" }` → `find role button click --name Search`. Supplying both `selector` and semantic locator fields is rejected.
|
|
333
333
|
|
|
334
|
+
`assertUrl` with no `*` compiles to exact `wait --url <url>`, preserving query strings and literal `?`. Glob-style `assertUrl` values compile to a `wait --fn` regex predicate: single `*` matches any characters except `/`, while `**` or longer star runs collapse to one `.*` that can cross `/`. Other regex metacharacters are escaped and stay literal. Literal `*` exact URL assertions are not supported by this constrained step.
|
|
335
|
+
|
|
334
336
|
Use raw `args` plus `stdin` for upstream `batch` when a flow needs commands, flags, stdin forms, or failure policies outside this constrained schema. `job.failFast: false` keeps the constrained schema but removes `--bail` when collecting later diagnostic artifacts is more important than stopping on the first failure.
|
|
335
337
|
|
|
336
|
-
Because `job` still executes as upstream `batch` with generated stdin, the same wrapper page-scoped `@e…` preflight applies: if you pass `@refs` in `click`/`fill`/`select` selectors after an `open`, non-form `click`, or another step that can navigate or mutate the page, split the work across tool calls or switch to raw `batch` and insert your own `snapshot -i` rows between steps—the constrained `job` vocabulary emits a `snapshot` step only when you include `{ action: "snapshot" }` explicitly. Multiple same-snapshot `fill @e…` rows may run before the first click/submit-style step. Raw `args:["batch"]` stdin can also batch native form-control rows (`check`/`uncheck` checkbox or radio refs and `select` combobox refs) before that click.
|
|
338
|
+
Because `job` still executes as upstream `batch` with generated stdin, the same wrapper page-scoped `@e…` preflight applies: if you pass `@refs` in `click`/`fill`/`select` selectors after an `open`, non-form `click`, or another step that can navigate or mutate the page, split the work across tool calls or switch to raw `batch` and insert your own `snapshot -i` rows between steps—the constrained `job` vocabulary emits a `snapshot` step only when you include `{ action: "snapshot" }` explicitly. Multiple same-snapshot `fill @e…` rows may run before the first click/submit-style step. Raw `args:["batch"]` stdin can also batch native form-control rows (`check`/`uncheck` checkbox or radio refs, checkbox/radio `click`/`tap` refs, and `select` combobox refs) before that click.
|
|
337
339
|
|
|
338
340
|
### `qa`
|
|
339
341
|
|
|
@@ -584,7 +586,7 @@ Example:
|
|
|
584
586
|
- type: positive integer milliseconds
|
|
585
587
|
- optional per-call wrapper subprocess watchdog for browser CLI calls (`args`, `job`, `qa`, `sourceLookup`, and `networkSourceLookup`); Electron actions use nested `electron.timeoutMs` instead
|
|
586
588
|
- use for long opens, large snapshots, paced `job` typing, or captures that legitimately need more than the default watchdog
|
|
587
|
-
-
|
|
589
|
+
- explicit long `wait` steps are forwarded to upstream; top-level `timeoutMs` only controls the wrapper subprocess watchdog and should be at least the wait duration plus a small grace window when supplied manually
|
|
588
590
|
- when the watchdog fires, `details.timeoutMs`, `details.timedOut`, and possibly `details.timeoutPartialProgress` explain what was recovered
|
|
589
591
|
|
|
590
592
|
Example:
|
|
@@ -645,7 +647,7 @@ Primary content should be:
|
|
|
645
647
|
- browser-aware compacting for oversized snapshots so the model gets a concise actionable view before raw page noise
|
|
646
648
|
- compact snapshots should be main-content-first: prefer the primary content block and nearby sections over top-of-page chrome, ads, or unrelated sidebars when those can be distinguished from the snapshot tree. They are DOM/signal-prioritized, not guaranteed viewport-first after scroll; compact output may include `details.snapshotCompaction.viewportOrdering: "dom-signal-prioritized"` and a visible viewport note when viewport context matters.
|
|
647
649
|
- when compacting hides actionable controls, snapshot output should add an `Omitted high-value controls` section for bounded editable/searchbox/textbox/combobox controls, named tab/surface controls, primary action buttons, high-signal named links such as repository search results, and other useful controls such as checkboxes, radios, options, and menuitems that were not already shown in key refs
|
|
648
|
-
- wrapper-side `snapshot -i --search <text>` and `snapshot -i --filter role=<role>` filters should strip those wrapper-only flags before upstream spawn, preserve the full latest ref map in `details.refSnapshot`, and render
|
|
650
|
+
- wrapper-side `snapshot -i --search <text>` and `snapshot -i --filter role=<role>` filters should strip those wrapper-only flags before upstream spawn, preserve the full latest ref map in `details.refSnapshot`, and render matching direct refs plus surrounding snapshot context with `details.snapshotFilter` counts so dense-page agents can find controls without opening raw spill files; the visible summary should distinguish direct ref matches from contextual lines to avoid apparent count mismatches; wrapper-side `--viewport` should also strip before upstream spawn, run one read-only viewport/scroll probe, and report `details.snapshotViewport`; wrapper-side `--diff` should strip before upstream spawn and report `details.snapshotDiff` against the previous wrapper-tracked ref map for that session
|
|
649
651
|
|
|
650
652
|
Examples:
|
|
651
653
|
- small `snapshot` results should include the actual snapshot text
|
|
@@ -689,7 +691,7 @@ These categories are intentionally bounded and stable so agents can branch on th
|
|
|
689
691
|
|
|
690
692
|
For argv-supplied `--allowed-domains`, the wrapper preserves the allowlist for the managed session and checks the final observed `http(s)` page URL after successful-looking browser commands. If a click, redirect, or other post-load navigation ends outside the allowlist, the wrapper reclassifies the result as `failureCategory: "policy-blocked"` and includes the requested allowlist plus observed URL/host in the failure text. Direct upstream blocks may still surface as upstream errors when upstream rejects navigation before a page target is available.
|
|
691
693
|
|
|
692
|
-
Real Pi custom tools only mark a tool result failed when the tool throws during `execute`; returned `isError` fields are not authoritative. The extension therefore also registers a `tool_result` handler that treats any `agent_browser` result with `details.resultCategory: "failure"` as a real Pi tool error. For normal prose output, it appends `Result category: failure; failureCategory: …; Pi tool isError: true.` to model-visible text. For caller-requested `--json` output, it only patches `isError` and preserves
|
|
694
|
+
Real Pi custom tools only mark a tool result failed when the tool throws during `execute`; returned `isError` fields are not authoritative. The extension therefore also registers a `tool_result` handler that treats any `agent_browser` result with `details.resultCategory: "failure"` as a real Pi tool error. For normal prose output, it appends `Result category: failure; failureCategory: …; Pi tool isError: true.` to model-visible text. For caller-requested `--json` output, it only patches `isError` and preserves visible parseable JSON content unchanged. The TUI renderer also repeats that category line at the top of failed rendered results so collapsed failed rows keep the outcome visible. The hook treats `--json` as requested when echoed `details.args` or the original tool `input.args` includes that flag; it skips appending the prose notice when any non-empty text content item is parseable JSON, even if other text items are not parseable. Invalid or non-JSON text still gets the visible prose notice. Implementation: `buildAgentBrowserToolResultPatch` in `extensions/agent-browser/lib/pi-tool-rendering.ts`; `extensions/agent-browser/index.ts` registers the handler. This keeps Pi transcript semantics aligned with the machine-readable result contract, including wrapper-side reclassifications such as `qa-failure` after an upstream-successful batch and `artifact-missing` after an upstream-successful artifact command whose requested file is absent.
|
|
693
695
|
|
|
694
696
|
For `batch`, top-level `details` still carries `resultCategory` plus `successCategory` or `failureCategory` for the **aggregate** tool outcome: if any step fails, the overall result is a failure (`resultCategory: "failure"`) even when later steps succeed—inspect `batchSteps[]` for per-step outcomes. Each `batchSteps[]` entry includes its own `resultCategory` and either `successCategory` or `failureCategory` for that step. `batchFailure.failedStep` duplicates the first failing step’s details, including its `failureCategory` and any `nextActions`.
|
|
695
697
|
|
|
@@ -704,7 +706,7 @@ Ref preflight details (command taxonomy in `extensions/agent-browser/lib/command
|
|
|
704
706
|
|
|
705
707
|
**Presentation redaction (implementation map):** Successful non-`batch` tool calls and each successful `batchSteps[]` row run upstream `data` through `redactPresentationData` in `extensions/agent-browser/lib/results/presentation/diagnostics.ts`: `cookies` still walk objects/arrays and replace case-insensitive `value` keys with `"[REDACTED]"`; `storage` redacts values when the key or value looks credential-like (token, cookie, auth, secret, JWT, bearer/basic credential, high-entropy token-like string, or nested sensitive JSON) but keeps low-risk primitive QA values such as booleans, numbers, and short strings visible. Redacted storage entries add `valueRedacted` plus `valueRedactionReason` in `details.data`; diagnostic formatters mirror the same decision. Every other command’s payload is recursively scrubbed with `redactStructuredPresentationValue`, which redacts known sensitive key names and applies string-level sensitivity heuristics so network, diff, trace/profiler, stream, dashboard, chat, and other structured results do not echo bearer tokens, proxy credentials, or similar fields verbatim into `details.data`. Echoed `command` arrays in `details` and in batch roll-ups use `redactInvocationArgs` from `extensions/agent-browser/lib/runtime.ts` to mask trailing values for sensitive global flags (including `--body`, `--headers`, `--password`, and `--proxy`), preserve the special positional rules for `cookies set`, `storage local|session set`, and `set credentials`, and scrub other argv tokens for URLs and inline secrets. Failed batch steps additionally run `redactExactValues` on structured step errors so literals taken from that step’s argv (cookie value, storage set value, `--password` / `--password=` tokens) cannot reappear inside formatted error blobs.
|
|
706
708
|
|
|
707
|
-
`nextActions` is an optional machine-readable list of exact native `agent_browser` follow-ups. Each entry includes `tool: "agent_browser"`, an `id`, a short `reason`, optional `safety`, and either `params` (`args`, optional `stdin`, optional `sessionMode`, optional `networkSourceLookup`, optional `electron`) or an `artifactPath` for saved-file workflows. Agents should prefer these payloads over prose when present. Tab/session recovery id strings are centralized in `AGENT_BROWSER_RECOVERY_NEXT_ACTION_IDS`, while rich-input focus/click recovery ids are centralized in `AGENT_BROWSER_RICH_INPUT_RECOVERY_NEXT_ACTION_IDS` plus `getAgentBrowserRichInputRecoveryNextActionId(s)` in `extensions/agent-browser/lib/results/recovery-actions.ts` (both registries are also re-exported from `shared.ts`); docs and tests mirror those registries/helpers rather than inventing recovery ids in prose. Current recommendations include: raw `connect` success → session-scoped `list-connected-session-tabs` only, then the agent should inspect/select a stable `tab t<N>` target and run `snapshot -i` explicitly; `snapshot` failures whose upstream error says `No active page` and whose wrapper result has a known session → `list-tabs-after-no-active-page` only, because this path has no wrapper-observed safe tab id to select atomically; browser profile/user-data-dir resolution failures → `inspect-browser-profiles` (`profiles`) and `run-agent-browser-doctor` (`doctor`) before retrying opens; Electron launches → wrapper-tracked `electron.status` / `electron.probe` / `electron.cleanup` actions plus session-scoped tab/snapshot inspection when attached; Electron status/probe mismatch diagnostics → `reattach-electron-launch` plus fresh tab/snapshot inspection; Electron post-command health failures → status/probe/cleanup for the same `launchId`; Electron or contenteditable fill verification mismatches → `inspect-after-fill-verification` and `verify-filled-value`; Electron same-URL ref freshness warnings → `refresh-electron-refs-after-rerender`; packaged-Electron `sourceLookup` no-candidate diagnostics → session snapshot, launch probe, and tab list; Electron cleanup partial failures → status plus retry-cleanup for the same wrapper-owned `launchId`; `open` success → `snapshot -i`; mutating/navigation commands (see `buildAgentBrowserNextActions` in source for the exact command set) → `snapshot -i`; stale refs and selector failures → `snapshot -i` via `refresh-interactive-refs` (prefixed with `--session <name>` when the failed call ran in a named or managed session); selector misses with exact current snapshot role/name matches → direct ref retries via `try-current-visible-ref` or bounded `try-current-visible-ref-N` for non-fill targets; semantic `fill` selector misses with exact current editable refs → `focus-current-editable-ref` / `click-current-editable-ref` or numbered variants that do not include fill text or submit; unknown getter shortcuts such as `title` / `url` → exact read-only retries like `get title` / `get url` with ids `use-get-title` / `use-get-url`; compact `network requests` results with safe request IDs → bounded read-only request detail, `networkSourceLookup`, path filter, or HAR-capture follow-ups; semantic `selector-not-found` failures that compiled from `semanticAction` may append `try-button-name-candidate` or `try-link-name-candidate` after presentation `nextActions` only for the bounded click pair enumerated under `semanticAction`; semantic `stale-ref` failures that compiled from `semanticAction` `find` argv may also include `retry-semantic-action-after-stale-ref` after that snapshot step; successful snapshots or qualifying same-URL non-Electron top-level clicks (see `overlayBlockers` below) with snapshot evidence of likely overlay/banner/dialog close controls may append `inspect-overlay-state` and bounded `try-overlay-blocker-candidate-*` entries; successful top-level `scroll` calls whose pre/post viewport and sampled scroll-container positions do not change may append `inspect-after-noop-scroll` and `verify-noop-scroll-visually`; explicit combobox-targeted actions that focus a combobox without visible options may append `inspect-focused-combobox`, `try-open-combobox-with-arrow`, and `try-open-combobox-with-enter`; `get text <selector>` calls with hidden/multiple CSS matches may append `inspect-visible-text-candidates` with a read-only `eval --stdin` probe (each prefixed with `--session <name>` when `details.sessionName` is set, same `sessionPrefixArgs` rule as other session-scoped follow-ups); confirmations → exact `confirm <id>` and `deny <id>` choices; generic tab drift → `list-tabs-for-recovery` with `tab list` first, then select or confirm the stable target before running `snapshot -i`; about:blank or tab-drift recovery with a wrapper-known target → `list-tabs-for-about-blank-recovery` or `list-tabs-for-tab-drift-recovery`, plus `select-intended-tab-after-drift` and `snapshot-after-tab-recovery` when the wrapper already observed the stable `t<N>` tab id; `wait --text` assertion failures → `inspect-after-text-assertion-failure` with a read-only snapshot; download verification failures or missing successful download artifacts → `wait --download [path]`; saved artifacts → the artifact path to inspect/consume after checking `artifactVerification`/metadata; missing non-download artifacts → `verify-artifact-path` so agents do not trust an absent file. When nothing applies, the field is omitted.
|
|
709
|
+
`nextActions` is an optional machine-readable list of exact native `agent_browser` follow-ups. Each entry includes `tool: "agent_browser"`, an `id`, a short `reason`, optional `safety`, and either `params` (`args`, optional `stdin`, optional `sessionMode`, optional `networkSourceLookup`, optional `electron`) or an `artifactPath` for saved-file workflows. Agents should prefer these payloads over prose when present. Tab/session recovery id strings are centralized in `AGENT_BROWSER_RECOVERY_NEXT_ACTION_IDS`, while rich-input focus/click recovery ids are centralized in `AGENT_BROWSER_RICH_INPUT_RECOVERY_NEXT_ACTION_IDS` plus `getAgentBrowserRichInputRecoveryNextActionId(s)` in `extensions/agent-browser/lib/results/recovery-actions.ts` (both registries are also re-exported from `shared.ts`); docs and tests mirror those registries/helpers rather than inventing recovery ids in prose. Current recommendations include: timed-out jobs/batches with a retryable read-only/idempotent first incomplete step → `retry-timeout-step`, while timed-out flows whose first incomplete step may be mutating → `inspect-current-page-after-timeout` (`snapshot -i`) before splitting the remaining work into shorter batches; raw `connect` success → session-scoped `list-connected-session-tabs` only, then the agent should inspect/select a stable `tab t<N>` target and run `snapshot -i` explicitly; `snapshot` failures whose upstream error says `No active page` and whose wrapper result has a known session → `list-tabs-after-no-active-page` only, because this path has no wrapper-observed safe tab id to select atomically; browser profile/user-data-dir resolution failures → `inspect-browser-profiles` (`profiles`) and `run-agent-browser-doctor` (`doctor`) before retrying opens; Electron launches → wrapper-tracked `electron.status` / `electron.probe` / `electron.cleanup` actions plus session-scoped tab/snapshot inspection when attached; Electron status/probe mismatch diagnostics → `reattach-electron-launch` plus fresh tab/snapshot inspection; Electron post-command health failures → status/probe/cleanup for the same `launchId`; Electron or contenteditable fill verification mismatches → `inspect-after-fill-verification` and `verify-filled-value`; Electron same-URL ref freshness warnings → `refresh-electron-refs-after-rerender`; packaged-Electron `sourceLookup` no-candidate diagnostics → session snapshot, launch probe, and tab list; Electron cleanup partial failures → status plus retry-cleanup for the same wrapper-owned `launchId`; `open` success → `snapshot -i`; mutating/navigation commands (see `buildAgentBrowserNextActions` in source for the exact command set) → `snapshot -i`; stale refs and selector failures → `snapshot -i` via `refresh-interactive-refs` (prefixed with `--session <name>` when the failed call ran in a named or managed session); selector misses with exact current snapshot role/name matches → direct ref retries via `try-current-visible-ref` or bounded `try-current-visible-ref-N` for non-fill targets; semantic `fill` selector misses with exact current editable refs → `focus-current-editable-ref` / `click-current-editable-ref` or numbered variants that do not include fill text or submit; unknown getter shortcuts such as `title` / `url` → exact read-only retries like `get title` / `get url` with ids `use-get-title` / `use-get-url`; compact `network requests` results with safe request IDs → bounded read-only request detail, `networkSourceLookup`, path filter, or HAR-capture follow-ups; semantic `selector-not-found` failures that compiled from `semanticAction` may append `try-button-name-candidate` or `try-link-name-candidate` after presentation `nextActions` only for the bounded click pair enumerated under `semanticAction`; semantic `stale-ref` failures that compiled from `semanticAction` `find` argv may also include `retry-semantic-action-after-stale-ref` after that snapshot step; successful snapshots or qualifying same-URL non-Electron top-level clicks (see `overlayBlockers` below) with snapshot evidence of likely overlay/banner/dialog close controls may append `inspect-overlay-state` and bounded `try-overlay-blocker-candidate-*` entries; successful top-level `scroll` calls whose pre/post viewport and sampled scroll-container positions do not change may append `inspect-after-noop-scroll` and `verify-noop-scroll-visually`; explicit combobox-targeted actions that focus a combobox without visible options may append `inspect-focused-combobox`, `try-open-combobox-with-arrow`, and `try-open-combobox-with-enter`; `get text <selector>` calls with hidden/multiple CSS matches may append `inspect-visible-text-candidates` with a read-only `eval --stdin` probe (each prefixed with `--session <name>` when `details.sessionName` is set, same `sessionPrefixArgs` rule as other session-scoped follow-ups); confirmations → exact `confirm <id>` and `deny <id>` choices; generic tab drift → `list-tabs-for-recovery` with `tab list` first, then select or confirm the stable target before running `snapshot -i`; about:blank or tab-drift recovery with a wrapper-known target → `list-tabs-for-about-blank-recovery` or `list-tabs-for-tab-drift-recovery`, plus `select-intended-tab-after-drift` and `snapshot-after-tab-recovery` when the wrapper already observed the stable `t<N>` tab id; `wait --text` assertion failures → `inspect-after-text-assertion-failure` with a read-only snapshot; download verification failures or missing successful download artifacts → `wait --download [path]`; saved artifacts → the artifact path to inspect/consume after checking `artifactVerification`/metadata; missing non-download artifacts → `verify-artifact-path` so agents do not trust an absent file. When nothing applies, the field is omitted.
|
|
708
710
|
|
|
709
711
|
**Unknown-command getter hints (failure presentation):** `buildErrorPresentation` in `extensions/agent-browser/lib/results/presentation/errors.ts` only runs this path when upstream error text (after model-facing redaction) matches `unknown command`, `unknown subcommand`, or `unrecognized command` (case-insensitive) **and** the failed invocation’s primary command token is one of `attr`, `count`, `html`, `text`, `title`, `url`, or `value`. Visible text then includes a grouped-`get` hint line plus per-token guidance (`get text <selector>`, `get html …`, `get attr …`, `get count …`, `get value …`, `get title`, `get url`). Machine `nextActions` with ids `use-get-title` / `use-get-url` are emitted only for `title` / `url`, with `params.args` optionally prefixed by `--session <name>` when the failed call targeted a named session. If the error string already contains `Agent-browser hint:` from selector recovery (stale-ref or unsupported selector dialect appendages), the getter block is skipped so two stacked `Agent-browser hint:` headers are not emitted.
|
|
710
712
|
|
|
@@ -714,7 +716,7 @@ For `batch`, each `batchSteps[]` entry can carry its own `nextActions` for that
|
|
|
714
716
|
|
|
715
717
|
`pageChangeSummary` is an optional compact summary for mutation-prone and artifact-producing commands. It includes `changeType` (`"navigation"`, `"mutation"`, `"artifact"`, or `"confirmation"`), `command`, a readable `summary`, optional `title`/`url`, optional `artifactCount` or `savedFilePath`, and `nextActionIds` that link the observed change to `nextActions` without repeating full payloads. The wrapper maintains an explicit `eligibleForPageChangeSummary` command capability through `isPageChangeSummaryCommand` in `extensions/agent-browser/lib/command-taxonomy.ts`: those commands still emit a `mutation`-typed summary when upstream JSON lacks navigation metadata, as long as no stronger signal (artifact, saved path, navigation fields, or pending confirmation) applies. That capability is independent from `invalidatesBatchRefs` and `triggersPostMutationSnapshot`, so artifact summaries like `download` / `screenshot` and guarded-but-non-invalidating `fill` are documented directly in the capability table instead of implied by broad set spreading. Commands outside that set omit `pageChangeSummary` unless the parsed payload shows navigation, a confirmation prompt, saved files, or artifacts—including read-only inspection commands, which normally have no summary unless one of those signals appears. For `batch`, the top-level summary favors artifact rollups when any step produced artifacts; otherwise it may synthesize a `mutation` summary from steps that carried their own `pageChangeSummary`. Treat mutation summaries as "upstream attempted the action" evidence, not proof the application handled it; agents should verify URL/text/state for important mutations before continuing.
|
|
716
718
|
|
|
717
|
-
`clickDispatch` may appear after a **top-level non-Electron** `click` when the wrapper installed a
|
|
719
|
+
`clickDispatch` may appear after a **top-level non-Electron** direct `click` when the wrapper installed a target-specific DOM-event probe, upstream reported success, and the post-click probe found no trusted DOM event reached the resolved target. Target-specific probes cover CSS selectors, `xpath=` targets, and role-gated `@e…` refs when the latest wrapper-tracked snapshot has role/name metadata; eligible ref roles are `button`, `checkbox`, `menuitem`, `radio`, `switch`, and `tab`, and duplicate-name refs use the ref's snapshot-order duplicate index rather than requiring a unique accessible name. Raw `find … click` locator calls, including compiled `semanticAction` clicks that still execute as upstream `find`, are not probed because the wrapper has no concrete element before upstream resolves the locator, and document-level probes can falsely fail frame-scoped clicks. It does **not** take a fresh pre-click snapshot because that could recycle upstream refs before the intended click. The wrapper does **not** replay clicks in-page. On a miss it marks the tool failed, appends `Click dispatch diagnostic: …`, and sets `clickDispatch.status` to `"no-native-event-observed"` with `reason: "native-click-produced-no-target-dom-event"`, `nativeEventCount`, and a redacted `target` descriptor (`kind: "selector" | "xpath"` plus `selector`, or `kind: "accessible"` plus `refId`, `role`, optional `duplicateIndex`, and redacted `name`). `details.nextActions` gains `inspect-click-dispatch-miss` (`snapshot -i`) and `retry-click-after-dispatch-miss` (same upstream click argv, session-prefixed when applicable). If a local static fixture must be exercised despite this diagnostic, a caller may explicitly run a programmatic activation via `eval --stdin` such as `document.querySelector(...).click()`, but that emits an untrusted scripted event and is only a debugging/workaround path; it must not be used as proof that real user-like clicking works or to bypass prompt stop boundaries. This diagnostic is only for standalone top-level direct click calls; `find` locator clicks and `batch`/`job`/`qa` click steps remain upstream-owned behavior.
|
|
718
720
|
|
|
719
721
|
`promptGuard` may appear on wrapper-blocked calls only for concrete, machine-checkable prompt requirements. `reason: "requested-artifacts-missing-before-close"` blocks `close` / `quit` / `exit` when the prompt named exact required screenshot paths and the session artifact manifest has not verified those paths; optional recording paths are only required when recording appears available. The wrapper does **not** parse broad user/business intent such as “do not place the order” or “do not post anything” into click/key blocks; agents must follow those instructions themselves. Prompt guards return `failureCategory: "policy-blocked"` and `validationError` text instead of invoking upstream.
|
|
720
722
|
|
|
@@ -776,7 +778,7 @@ Implementation and precedence:
|
|
|
776
778
|
|
|
777
779
|
Additional structured fields can appear when relevant:
|
|
778
780
|
- `compiledSemanticAction` when the call used `semanticAction` and the result includes the unified `details` merge: `{ action, locator, args }` for `find` actions or `{ action: "select", selector, values, args }` for `select`, with the same redaction rules as `args` / `effectiveArgs`; omitted for plain `args`/`job` calls and omitted on some early error returns that omit this field (see the `semanticAction` section above)
|
|
779
|
-
- `compiledJob` when the call used `job` or the job-backed `qa` preset: `{ args: ["batch"], stdin, steps: [{ action, args }] }
|
|
781
|
+
- `compiledJob` when the call used `job` or the job-backed `qa` preset: by default `{ args: ["batch", "--bail"], failFast: true, stdin, steps: [{ action, args }] }`; with `failFast: false`, `{ args: ["batch"], failFast: false, stdin, steps: [{ action, args }] }`. Step args are redacted the same way as other invocation details. Semantic `job` click/fill steps appear here as their compiled upstream `find … click|fill …` argv, not as the input object.
|
|
780
782
|
- `compiledQaPreset` when the call used `qa`: the compiled job fields plus the QA `checks` object. `args` is `batch --bail` and `failFast` is `true` for QA presets. `checks.attached` is `true` for current-session QA, `checks.url` is present only for URL-opening QA, and `checks.diagnosticsResetAtStart` is `true` only for URL-opening QA because `qa.attached` preserves existing session diagnostics.
|
|
781
783
|
- `compiledSourceLookup` when the call used `sourceLookup`: `{ args: ["batch"], stdin, steps, query }` with the generated local-evidence plan and original query fields (`selector?`, `reactFiberId?`, `componentName?`, `includeDomHints?`, `maxWorkspaceFiles?`).
|
|
782
784
|
- `sourceLookup` when the call used `sourceLookup`: `{ status, candidates, limitations, summary, workspaceRoot?, electronContext? }`; wrapper-tracked packaged Electron no-candidate diagnostics may carry `workspaceRoot` plus `electronContext` and live Electron nextActions without marking the successful batch as a tool failure.
|
|
@@ -788,7 +790,7 @@ Additional structured fields can appear when relevant:
|
|
|
788
790
|
- `batchFailure` and `batchSteps` for `batch` rendering, including mixed-success runs
|
|
789
791
|
- `navigationSummary` for navigation-style commands like `click`, `back`, `forward`, and `reload`
|
|
790
792
|
- `pageChangeSummary` for compact mutation/artifact/navigation summaries on commands that can change browser state
|
|
791
|
-
- `clickDispatch` when a top-level non-Electron `click` reported upstream success but the
|
|
793
|
+
- `clickDispatch` when a top-level non-Electron direct `click` reported upstream success but the target-specific probe found no trusted event reached the resolved target; shape follows `ClickDispatchDiagnostic` in `extensions/agent-browser/lib/orchestration/browser-run/types.ts`
|
|
792
794
|
- `promptGuard` when the requested-artifact-before-close guard blocks browser close before required prompt artifact paths are verified; implementation lives in `extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.ts`
|
|
793
795
|
- `overlayBlockers` for conservative overlay/banner/dialog blocker candidates when a successful snapshot itself contains strong modal evidence, or after a direct click stays on the same URL, no `clickDispatch` diagnostic fired, and a fresh snapshot provides evidence (`candidates`, `summary`, and `snapshot` per `OverlayBlockerDiagnostic` in `extensions/agent-browser/index.ts`)
|
|
794
796
|
- `visibleRefFallback` after a raw `find` or compiled `semanticAction` fails with `selector-not-found` and a fresh snapshot finds exact role/name `@ref` matches. Shape follows `VisibleRefFallbackDiagnostic` in `extensions/agent-browser/lib/results/selector-recovery.ts`: `{ candidates, snapshot, summary, target }`, where each candidate has `ref`, `role`, `name`, optional direct ref `args`, and `reason`; visible text appends `Current snapshot ref fallback`. Non-fill candidates with direct args add `try-current-visible-ref` or numbered `try-current-visible-ref-N` actions. Fill candidates omit direct args and target text so recovery details do not repeat potentially sensitive fill text.
|
|
@@ -805,7 +807,7 @@ Additional structured fields can appear when relevant:
|
|
|
805
807
|
- `electronGetTextScopeWarning` after a successful wrapper-tracked attached Electron `get text <selector>` (standalone or successful `batch`) when a broad non-ref CSS selector such as `body`, `html`, `main`, `div`, or `[role=application]` may read the whole app shell. Ordinary browser pages, including `file://` fixtures, do not qualify without wrapper-owned Electron launch provenance. Shape: `{ selector, summary, electronContext: { launchId?, sessionName?, url? } }`; multiple batched diagnostics use `electronGetTextScopeWarnings`. Visible text appends `Broad Electron get text selector warning`, and next actions use `snapshot-for-electron-text-scope` ids with session-scoped `snapshot -i` payloads.
|
|
806
808
|
- `evalStdinHint` after a successful `eval --stdin` when caller stdin (trimmed) looks function-shaped to the wrapper’s lightweight detector (in `extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts`: leading `function` / `async function`, parenthesized arrow `(…) =>`, or a concise `name =>` / `async name =>` form) **and** upstream JSON `data` is an object whose `result` field is a plain empty object (`{}`). Arrays such as `[]` do not qualify. It includes `reason` and `suggestion`; visible output appends `Eval stdin hint` with the same guidance. This is a heuristic for the common mistake of returning a function object instead of invoking it or passing a plain expression, not a JavaScript parser or proof that the page returned no useful data. Before this diagnostic path runs, the wrapper also recovers the common malformed native-tool call `args: ["eval", "--stdin", "..."]` with no top-level `stdin` by moving trailing `args` tokens after `--stdin` into the process stdin stream.
|
|
807
809
|
- `evalResultWarning` after a successful `eval --stdin` when the current or prior page URL is `file:` (from navigation summary, session tab target, or persisted session page state), upstream JSON `data.result` is strictly `null`, and stdin is non-empty and not a trivial literal `null`/`undefined`. Fields: `reason`, `suggestion`. Visible output appends `Eval result warning` without failing the tool. Use snapshot -i, ref-based getters, screenshots, or http(s) fixtures when file:// null results are inconclusive.
|
|
808
|
-
- `timeoutPartialProgress` after `runAgentBrowserProcess` reports `timedOut` (wrapper child-process watchdog) when best-effort recovery finds useful context. `summary` is a short sentence counting recovered planned-step state and declared artifact paths, plus whether page context came from live session reads or only from a planned URL (when nothing in the plan declares an artifact path, the fraction may read `0/0` while `currentPage` can still carry session or planned URL context). `steps` lists planned argv from the compiled `job` or `qa` batch plan (`compiledJob` in `extensions/agent-browser/index.ts`, which is only populated for those top-level modes) or, when that object is absent, from the same JSON-array `batch` stdin the tool sends upstream—whether caller-authored or wrapper-generated for `sourceLookup` / `networkSourceLookup` (1-based indices; only JSON-array stdin whose elements are string[] argv arrays is parsed). Generated rows such as `open.loadState` waits may include `generatedFrom`. Each step includes `status` (`completed`, `failed`, `pending`, or `unknown`) and optional `reason`; the first incomplete step becomes `retryStep`, but `retry.args` and top-level `retry-timeout-step` are emitted only for read-only or idempotent commands such as waits, snapshots, screenshots, navigation, and diagnostics. Mutating steps such as clicks, fills, keyboard typing, presses, selects, or checks are still identified as the first incomplete step but omit executable retry args because they may already have run. When a retryable step timed out during `sessionMode: "fresh"` and no live URL was recovered, `retry-timeout-step` uses top-level `sessionMode: "fresh"` instead of prefixing the abandoned generated session name. `currentPage` comes from session-scoped `get url` / `get title` when the session answers, otherwise a fallback URL may be inferred from the last `open` / `navigate` / `pushstate` step in the plan; `liveUrlRecovered` is true only when the wrapper recovered a live URL, so planned URLs are not treated as proof that the page actually opened. `openedButPostOpenTimedOut` is set when a live opened page was recovered and a later step appears to have timed out. `artifacts` covers declared output paths on `screenshot`, `pdf`, `download`, and `wait --download` steps (absolute path, existence, `state`, optional `sizeBytes`, `stepIndex`). Visible text repeats the same block under `Timeout partial progress`, applying URL and path-segment redaction; the prose `Planned steps` list shows at most six steps, then an omitted-count line when the plan is longer. This is recovery evidence only; missing entries do not prove the upstream step never ran or that no other side effects occurred.
|
|
810
|
+
- `timeoutPartialProgress` after `runAgentBrowserProcess` reports `timedOut` (wrapper child-process watchdog) when best-effort recovery finds useful context. `summary` is a short sentence counting recovered planned-step state and declared artifact paths, plus whether page context came from live session reads or only from a planned URL (when nothing in the plan declares an artifact path, the fraction may read `0/0` while `currentPage` can still carry session or planned URL context). `steps` lists planned argv from the compiled `job` or `qa` batch plan (`compiledJob` in `extensions/agent-browser/index.ts`, which is only populated for those top-level modes) or, when that object is absent, from the same JSON-array `batch` stdin the tool sends upstream—whether caller-authored or wrapper-generated for `sourceLookup` / `networkSourceLookup` (1-based indices; only JSON-array stdin whose elements are string[] argv arrays is parsed). Generated rows such as `open.loadState` waits may include `generatedFrom`. Each step includes `status` (`completed`, `failed`, `pending`, or `unknown`) and optional `reason`; the first incomplete step becomes `retryStep`, but `retry.args` and top-level `retry-timeout-step` are emitted only for read-only or idempotent commands such as waits, snapshots, screenshots, navigation, and diagnostics. Mutating steps such as clicks, fills, keyboard typing, presses, selects, or checks are still identified as the first incomplete step but omit executable retry args because they may already have run; when the timed-out session is still usable, `details.nextActions` can instead include `inspect-current-page-after-timeout` (`snapshot -i`) so the agent verifies current state before continuing with a shorter split flow. When a retryable step timed out during `sessionMode: "fresh"` and no live URL was recovered, `retry-timeout-step` uses top-level `sessionMode: "fresh"` instead of prefixing the abandoned generated session name. `currentPage` comes from session-scoped `get url` / `get title` when the session answers, otherwise a fallback URL may be inferred from the last `open` / `navigate` / `pushstate` step in the plan; `liveUrlRecovered` is true only when the wrapper recovered a live URL, so planned URLs are not treated as proof that the page actually opened. `openedButPostOpenTimedOut` is set when a live opened page was recovered and a later step appears to have timed out. `artifacts` covers declared output paths on `screenshot`, `pdf`, `download`, and `wait --download` steps (absolute path, existence, `state`, optional `sizeBytes`, `stepIndex`). Visible text repeats the same block under `Timeout partial progress`, applying URL and path-segment redaction; the prose `Planned steps` list shows at most six steps, then an omitted-count line when the plan is longer. This is recovery evidence only; missing entries do not prove the upstream step never ran or that no other side effects occurred.
|
|
809
811
|
- `managedSessionOutcome` after a managed-session plan reaches process execution (`buildManagedSessionOutcome` / `formatManagedSessionOutcomeText` in `extensions/agent-browser/lib/orchestration/browser-run/session-state.ts`). Populated when `buildExecutionPlan` injects an extension-managed implicit or fresh `--session`, and also when a successful explicit `--session <current-wrapper-managed-session> close` closes the current managed session. It remains omitted for unrelated explicit user-managed sessions and for sessionless inspection/local paths that skip injection. Fields: `status` (`created`, `replaced`, `unchanged`, `closed`, `preserved`, or `abandoned`), `sessionMode`, `attemptedSessionName`, `previousSessionName`, `currentSessionName`, optional `replacedSessionName`, `activeBefore`, `activeAfter`, `succeeded`, and `summary` (machine-oriented; may include generated session names). Model-visible echo: only when `sessionMode` is `"fresh"` **and** `succeeded` is false, the wrapper appends action-oriented `Managed session outcome` and `Recovery` lines without repeating generated session ids in visible prose; session names remain in `details.managedSessionOutcome`. Failed fresh launches may also append `details.nextActions` such as `run-agent-browser-doctor`, `verify-current-managed-session`, `snapshot-current-managed-session`, or `retry-fresh-managed-session`. When other trailing diagnostic prose is also emitted in the same result, that block is concatenated **after** semantic-action candidate lines, overlay/selector-visibility tails, eval hints/warnings, and `Timeout partial progress` (see `rawAppendedDiagnosticText` in `extensions/agent-browser/lib/orchestration/browser-run/final-result.ts`). For `"auto"` failures the same struct may appear on `details` without that extra line. When post-upstream analysis (for example **`qa`** preset failure) flips the overall tool result after a successful batch, or a fresh `job`/batch opens the requested page and then a later step fails, the managed-session transition still reflects that the fresh browser became current. The visible recovery says the fresh launch became current and points to `failureCategory` / `qaPreset` / `batchFailure` for the post-launch failure instead of telling the agent that the old session was preserved.
|
|
810
812
|
- `imagePath` / `imagePaths` for Pi inline image attachments from the **`screenshot`** command (including batched screenshot steps). **`diff screenshot`** still records the diff output as an `image`-kind entry in `details.artifacts`, but it does **not** populate `imagePath` / `imagePaths` or attach an inline image: only plain `screenshot` is treated as a trusted live-capture path for automatic inlining (`isTrustedScreenshotOutput` in `extensions/agent-browser/lib/results/presentation/artifacts.ts`).
|
|
811
813
|
- `artifacts` for saved files such as screenshots, `state save` outputs, `diff screenshot` diff images, PDFs, downloads, `wait --download` files, traces, CPU profiles, completed WebM recordings, path-bearing HAR captures, and future recording output paths reported by `record start` / `record restart`. Non-file URL payloads such as `data:` / `blob:` / `http(s):` values are not treated as verified local artifacts. For direct artifact commands and batch artifact steps, the wrapper creates parent directories for requested paths before spawning upstream. Each artifact includes the original saved or requested `path`, resolved `absolutePath`, `kind`/`artifactType`, optional `mediaType`, optional `extension`, best-effort disk metadata such as `exists` and `sizeBytes`, plus `requestedPath`, `status`, `cwd`, `session`, and `tempPath` when applicable. Pending `record start` / `record restart` artifacts use `status: "pending"`, omit `exists` rather than reporting false, and include `recordingState: "openRecording"` / `willExistOnStop: true`.
|
|
@@ -815,7 +817,7 @@ Additional structured fields can appear when relevant:
|
|
|
815
817
|
- `fullOutputPath` / `fullOutputPaths` when large snapshot output or other oversized tool output is compacted and spilled to a private file; persisted sessions keep that path under a private session-scoped artifact directory with a bounded per-session budget so it survives reload/resume without unbounded growth
|
|
816
818
|
- `artifactManifest` for a bounded, metadata-only inventory of recent session artifacts. Entries include path metadata, artifact `kind`, source `command`/`subcommand` when safe, `storageScope` (`persistent-session`, `process-temp`, or `explicit-path`), and `retentionState` (`live`, `ephemeral`, `missing`, or `evicted`). The default recent window is 100 entries and can be configured with `PI_AGENT_BROWSER_SESSION_ARTIFACT_MANIFEST_MAX_ENTRIES`. The manifest must not store command args, output contents, headers, DOM snapshots, or downloaded file contents.
|
|
817
819
|
- `artifactRetentionSummary` with a concise count of live, evicted, ephemeral, and missing artifacts from the current manifest; results append this summary to model-facing text only when retention state affects recovery, such as spill files, ephemeral files, or evictions. Routine explicit saved files keep the summary in details to avoid noisy browsing transcripts.
|
|
818
|
-
- `artifactCleanup` after a successful close command (`close`, `quit`, or `exit`) when `artifactManifest` exists and `entries` is non-empty. Fields: `owner: "host-file-tools"`, `summary` (same retention summary string as `artifactRetentionSummary` for that manifest), `note` explaining that browser close commands do not delete explicit screenshots/downloads/PDFs/traces/HAR/recordings, and `explicitArtifactPaths`: up to ten **distinct existing** paths taken from manifest rows with `storageScope: "explicit-path"` in encounter order (de-duplicated after checking the filesystem); deleted/stale explicit paths are skipped. When the recent window has no existing explicit rows—for example only spill/ephemeral inventory or explicit paths already deleted—the array is empty but `summary` / `note` still surface so agents know close is not file deletion. The native browser tool intentionally does not expose a delete operation for arbitrary user-chosen artifact paths; agents should inspect `artifactVerification` / manifest metadata, then remove files with normal host file tools when cleanup is required.
|
|
820
|
+
- `artifactCleanup` after a successful close command (`close`, `quit`, or `exit`) when `artifactManifest` exists and `entries` is non-empty. Fields: `owner: "host-file-tools"`, `summary` (same retention summary string as `artifactRetentionSummary` for that manifest), `note` explaining that browser close commands do not delete explicit screenshots/downloads/PDFs/traces/HAR/recordings, and `explicitArtifactPaths`: up to ten **distinct existing** paths taken from manifest rows with `storageScope: "explicit-path"` in encounter order (de-duplicated after checking the filesystem); deleted/stale explicit paths are skipped. When the recent window has no existing explicit rows—for example only spill/ephemeral inventory or explicit paths already deleted—the array is empty but `summary` / `note` still surface so agents know close is not file deletion. The visible close text stays compact and points operators to `details.artifactCleanup.explicitArtifactPaths` instead of listing paths inline. The native browser tool intentionally does not expose a delete operation for arbitrary user-chosen artifact paths; agents should inspect `artifactVerification` / manifest metadata, then remove files with normal host file tools when cleanup is required.
|
|
819
821
|
- compact **snapshot** metadata on successful presentation when `details.data.compacted` is true (oversized trees): `previewMode` (`"structured"` vs outline `"outline"`), `structuredPreviewUsed`, `previewRefIds`, `previewSections` (per-section `linesShown` / `omittedLines` / root `role` / `title`), `additionalSectionsOmitted`, counts such as `refCount`, `snapshotLineCount`, and `roleCounts`, optional `highValueControlRefIds` aligned with the visible bounded `Omitted high-value controls` lines, and optional `spillError` when the wrapper could not write the raw spill file; the model text still ends with `Full raw snapshot path:` or an explicit unavailable reason plus `details.fullOutputPath` when a path exists
|
|
820
822
|
- `sessionRecoveryHint` when startup-scoped flags need `sessionMode: "fresh"` while an implicit session is already active: includes `reason`, `recommendedSessionMode` (`"fresh"`), redacted `exampleArgs`, and `exampleParams` where `sessionMode` is `"fresh"` and `args` is the same redacted argv as `exampleArgs` (from `buildExecutionPlan` in `extensions/agent-browser/lib/runtime.ts`, merged through `redactRecoveryHint` in `extensions/agent-browser/index.ts`)
|
|
821
823
|
- `inspection: true` plus `stdout` for successful plain-text inspection commands like `--help` and `--version`
|
|
@@ -836,7 +838,7 @@ Worth doing in v1:
|
|
|
836
838
|
- `diff screenshot` → same file-artifact pattern as above for the **diff** image path only (summary text uses “Saved diff image” only when the diff output exists; missing output says “Diff image reported; file not verified” and fails as `artifact-missing`); baseline paths and other fields stay in the structured payload but are not echoed as separate saved artifacts in the visible artifact block, and there is no Pi inline image attachment for the diff output
|
|
837
839
|
- `state load` → completion text may mention the loaded path, but the wrapper does **not** treat that path as a new saved artifact (`artifacts` / `artifactManifest` stay unset) the way `state save` does
|
|
838
840
|
- auth, cookies, storage, clipboard, dialog, frame, state, network, debug, diff, stream, dashboard, chat, and other structured results → concise summaries that avoid expanding secret-bearing payloads; credential-like keys, values, URLs, body snippets, bearer/basic credentials, clipboard write text, cookie values, and likely secret storage values are redacted before model-facing output and `details.data`, while benign primitive storage values may remain visible for local QA
|
|
839
|
-
- TUI display → custom `agent_browser` call/result rendering with colorized command/output text and a built-in-style collapsed view for long visible output; `ctrl+o` expansion reveals the full rendered tool result without changing the model-facing content
|
|
841
|
+
- TUI display → custom `agent_browser` call/result rendering with colorized command/output text and a built-in-style collapsed view for long visible output; top-level native modes render as `agent_browser qa → batch --bail`, `agent_browser job → batch --bail` by default (`agent_browser job → batch` when `failFast:false`), or `agent_browser semanticAction → find …` so reviewers can see both the native input mode and compiled upstream command; failed results keep `resultCategory` / `failureCategory` visible before truncated output; `ctrl+o` expansion reveals the full rendered tool result without changing the model-facing content
|
|
840
842
|
- snapshots → origin + ref count + main-content-first compact preview, with the raw snapshot spill path printed directly in content and kept in `details.fullOutputPath` plus `details.artifactManifest` when the inline result would otherwise be too large
|
|
841
843
|
- oversized generic outputs such as large `eval --stdin` payloads → compact preview plus the actual spill file path instead of dumping the whole payload into model context
|
|
842
844
|
- extraction-style commands like `eval --stdin` and `get title` → scalar-first text with lightweight origin context when available
|
|
@@ -880,7 +882,7 @@ If `agent-browser` is not on `PATH`, fail with a message that:
|
|
|
880
882
|
- If a known session target unexpectedly reports about:blank, agent_browser best-effort re-selects the prior intended target when it still exists; if recovery fails, it records the observed about:blank target and reports exact recovery guidance instead of treating the prior page as active.
|
|
881
883
|
<!-- agent-browser-playbook:end wrapper-tab-recovery -->
|
|
882
884
|
- on local Unix launches, set a short private socket directory for wrapper-spawned `agent-browser` processes so extension-generated session names do not fail the upstream Unix socket-path length limit in longer cwd/session-name combinations
|
|
883
|
-
- keep wrapper-spawned commands bounded by clamping `AGENT_BROWSER_DEFAULT_TIMEOUT` to 25 seconds while the default wrapper child-process watchdog is 35 seconds (`PI_AGENT_BROWSER_PROCESS_TIMEOUT_MS` overrides it, and top-level `timeoutMs` overrides it per call for browser CLI subprocesses).
|
|
885
|
+
- keep wrapper-spawned commands bounded by clamping `AGENT_BROWSER_DEFAULT_TIMEOUT` to the upstream documented default of 25 seconds while the default wrapper child-process watchdog is 35 seconds (`PI_AGENT_BROWSER_PROCESS_TIMEOUT_MS` overrides it, and top-level `timeoutMs` overrides it per call for browser CLI subprocesses). Explicit `wait <ms>` or `wait --timeout <ms>` calls can exceed that default; when top-level `timeoutMs` is omitted, the wrapper derives a per-call subprocess watchdog from the requested wait duration plus a small grace window. Dialog commands use `PI_AGENT_BROWSER_DIALOG_PROCESS_TIMEOUT_MS` (default 5000 ms), and click/tap/find refs or tokens plus `eval --stdin` snippets whose text looks like alert/confirm/prompt/dialog triggers use `PI_AGENT_BROWSER_DIALOG_TRIGGER_PROCESS_TIMEOUT_MS` (default 8000 ms). Timed-out compiled `job` / `qa` or caller `batch` calls may add `details.timeoutPartialProgress` and visible `Timeout partial progress` evidence with per-step status, retry payloads, current page title/URL, and declared artifact path checks; timed-out dialog-like commands may add dialog status/dismiss/fresh-session recovery next actions
|
|
884
886
|
- interactive or long-running upstream families such as `chat` without a prompt, `dashboard start`, `stream enable`, `trace start`, `profiler start`, `record start`, `inspect`, `install`, `upgrade`, `doctor --fix`, and `confirm-interactive` are passed through thinly but remain bounded by the same wrapper timeout/session planning rules; prefer explicit arguments, single-shot `chat <message>`, non-interactive flags like `doctor --offline --quick` or `doctor --json`, and cleanup pairs such as `dashboard stop`, `stream disable`, `trace stop`, `profiler stop`, and `record stop`
|
|
885
887
|
- treat successful plain-text inspection commands like `--help` and `--version` as stateless: do not inject the implicit managed session and do not let those calls claim the managed-session slot
|
|
886
888
|
- if startup-scoped flags like `--profile`, `--executable-path`, `--session-name`, `--cdp`, `--state`, `--auto-connect`, `--init-script`, `--enable`, `-p` / `--provider`, or iOS `--device` are supplied after the implicit session is already active while `sessionMode` is `"auto"`, return a validation error with a structured recovery hint that recommends `sessionMode: "fresh"`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-agent-browser-native",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.49",
|
|
4
4
|
"description": "pi extension that exposes agent-browser as a native tool for browser automation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "Mitch Fultz (https://github.com/fitchmultz)",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"pi-agent-browser-doctor": "scripts/doctor.mjs"
|
|
32
32
|
},
|
|
33
33
|
"files": [
|
|
34
|
-
"
|
|
34
|
+
"dist",
|
|
35
35
|
"platform-smoke.config.mjs",
|
|
36
36
|
"scripts/config.mjs",
|
|
37
37
|
"scripts/doctor.mjs",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
],
|
|
53
53
|
"pi": {
|
|
54
54
|
"extensions": [
|
|
55
|
-
"./extensions/agent-browser/index.
|
|
55
|
+
"./dist/extensions/agent-browser/index.js"
|
|
56
56
|
]
|
|
57
57
|
},
|
|
58
58
|
"peerDependencies": {
|
|
@@ -62,9 +62,9 @@
|
|
|
62
62
|
"typebox": "*"
|
|
63
63
|
},
|
|
64
64
|
"devDependencies": {
|
|
65
|
-
"@earendil-works/pi-ai": "
|
|
66
|
-
"@earendil-works/pi-coding-agent": "
|
|
67
|
-
"@earendil-works/pi-tui": "
|
|
65
|
+
"@earendil-works/pi-ai": "0.79.1",
|
|
66
|
+
"@earendil-works/pi-coding-agent": "0.79.1",
|
|
67
|
+
"@earendil-works/pi-tui": "0.79.1",
|
|
68
68
|
"@types/node": "^25.6.1",
|
|
69
69
|
"tsx": "^4.21.0",
|
|
70
70
|
"typebox": "^1.1.38",
|
|
@@ -86,9 +86,13 @@
|
|
|
86
86
|
"smoke:platform:windows-native": "node scripts/platform-smoke.mjs run --target windows-native",
|
|
87
87
|
"smoke:platform:all": "npm run smoke:platform:doctor && node scripts/platform-smoke.mjs run --target macos,ubuntu,windows-native",
|
|
88
88
|
"typecheck": "node ./scripts/project.mjs verify typecheck",
|
|
89
|
-
"test": "tsx --test test/**/*.test.ts",
|
|
89
|
+
"test": "node ./scripts/build.mjs && tsx --test test/**/*.test.ts",
|
|
90
90
|
"verify": "node ./scripts/project.mjs verify",
|
|
91
|
-
"prepublishOnly": "npm run verify -- release && npm pack --dry-run"
|
|
91
|
+
"prepublishOnly": "npm run verify -- release && npm pack --dry-run",
|
|
92
|
+
"build": "node ./scripts/build.mjs",
|
|
93
|
+
"startup-profile": "node ./scripts/profile-startup.mjs",
|
|
94
|
+
"prepack": "npm run build",
|
|
95
|
+
"prepare": "npm run build"
|
|
92
96
|
},
|
|
93
97
|
"packageManager": "npm@11.14.0"
|
|
94
98
|
}
|
|
@@ -14,8 +14,8 @@ export const COMMAND_REFERENCE_BASELINE_BLOCK_IDS = Object.freeze(["upstream-bas
|
|
|
14
14
|
|
|
15
15
|
const sourceEvidence = Object.freeze({
|
|
16
16
|
repository: "vercel-labs/agent-browser",
|
|
17
|
-
upstreamHead: "
|
|
18
|
-
upstreamPackageVersion: "0.27.
|
|
17
|
+
upstreamHead: "5185339ca3fdab9848e11b8ec676eecfdec3733f",
|
|
18
|
+
upstreamPackageVersion: "0.27.2",
|
|
19
19
|
inspectedSources: Object.freeze([
|
|
20
20
|
"agent-browser --version",
|
|
21
21
|
"agent-browser --help",
|
|
@@ -609,6 +609,9 @@ const inventorySections = Object.freeze([
|
|
|
609
609
|
"AWS_PROFILE",
|
|
610
610
|
"AWS_ACCESS_KEY_ID",
|
|
611
611
|
"AWS_SECRET_ACCESS_KEY",
|
|
612
|
+
"AWS_SESSION_TOKEN",
|
|
613
|
+
"AWS_REGION",
|
|
614
|
+
"AWS_DEFAULT_REGION",
|
|
612
615
|
],
|
|
613
616
|
[
|
|
614
617
|
root("--profile <name|path>"),
|
|
@@ -706,7 +709,7 @@ const inventorySections = Object.freeze([
|
|
|
706
709
|
]);
|
|
707
710
|
|
|
708
711
|
export const CAPABILITY_BASELINE = Object.freeze({
|
|
709
|
-
targetVersion: "0.27.
|
|
712
|
+
targetVersion: "0.27.2",
|
|
710
713
|
sourceEvidence,
|
|
711
714
|
helpCommands,
|
|
712
715
|
inventorySections,
|
package/scripts/config.mjs
CHANGED
|
@@ -9,7 +9,13 @@ import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "n
|
|
|
9
9
|
import { dirname } from "node:path";
|
|
10
10
|
import process from "node:process";
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
async function loadConfigPolicyModule() {
|
|
13
|
+
const sourcePolicyUrl = new URL("../extensions/agent-browser/lib/config-policy.js", import.meta.url);
|
|
14
|
+
if (existsSync(sourcePolicyUrl)) return import(sourcePolicyUrl.href);
|
|
15
|
+
return import("../dist/extensions/agent-browser/lib/config-policy.js");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const {
|
|
13
19
|
AGENT_BROWSER_CONFIG_ENV,
|
|
14
20
|
BRAVE_API_KEY_ENV,
|
|
15
21
|
DEFAULT_WEB_SEARCH_PROVIDER,
|
|
@@ -26,7 +32,7 @@ import {
|
|
|
26
32
|
isWebSearchProvider,
|
|
27
33
|
loadAgentBrowserConfigStateSync,
|
|
28
34
|
summarizeConfigFiles,
|
|
29
|
-
}
|
|
35
|
+
} = await loadConfigPolicyModule();
|
|
30
36
|
|
|
31
37
|
const DEFAULT_CONFIG = { version: 1 };
|
|
32
38
|
|
package/scripts/doctor.mjs
CHANGED
|
@@ -20,9 +20,12 @@ import { CAPABILITY_BASELINE, CAPABILITY_BASELINE_SOURCE } from "./agent-browser
|
|
|
20
20
|
const execFile = promisify(execFileCallback);
|
|
21
21
|
const PACKAGE_NAME = "pi-agent-browser-native";
|
|
22
22
|
const REPO_URL_FRAGMENT = "github.com/fitchmultz/pi-agent-browser-native";
|
|
23
|
-
const
|
|
23
|
+
const EXTENSION_ENTRYPOINTS = Object.freeze([
|
|
24
|
+
"extensions/agent-browser/index.ts",
|
|
25
|
+
"dist/extensions/agent-browser/index.js",
|
|
26
|
+
]);
|
|
24
27
|
const EXPECTED_VERSION = CAPABILITY_BASELINE.targetVersion;
|
|
25
|
-
const
|
|
28
|
+
const MINIMUM_PI_VERSION = "0.79.0";
|
|
26
29
|
const DEFAULT_AGENT_DIR = resolve(homedir(), ".pi/agent");
|
|
27
30
|
const THIS_PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
28
31
|
|
|
@@ -67,7 +70,7 @@ Options:
|
|
|
67
70
|
Checks:
|
|
68
71
|
1. agent-browser is installed on PATH.
|
|
69
72
|
2. agent-browser --version matches the package capability baseline.
|
|
70
|
-
3. pi --version is at least the
|
|
73
|
+
3. pi --version is at least the minimum Pi runtime version for this release.
|
|
71
74
|
4. Pi settings and repo-local autoload locations do not point at multiple active pi-agent-browser-native sources.
|
|
72
75
|
|
|
73
76
|
Examples:
|
|
@@ -163,15 +166,13 @@ function sourceLooksLikeThisPackage(source, cwd, sourceBaseDir = cwd) {
|
|
|
163
166
|
|
|
164
167
|
if (!isPathLikeSource(text)) return false;
|
|
165
168
|
const resolvedSource = resolve(sourceBaseDir, expandUserPath(text));
|
|
166
|
-
const
|
|
167
|
-
const
|
|
169
|
+
const cwdEntrypoints = EXTENSION_ENTRYPOINTS.map((entrypoint) => resolve(cwd, entrypoint));
|
|
170
|
+
const packageEntrypoints = EXTENSION_ENTRYPOINTS.map((entrypoint) => resolve(THIS_PACKAGE_ROOT, entrypoint));
|
|
168
171
|
return (
|
|
169
172
|
resolvedSource === cwd ||
|
|
170
|
-
resolvedSource === cwdEntrypoint ||
|
|
171
173
|
resolvedSource === THIS_PACKAGE_ROOT ||
|
|
172
|
-
resolvedSource ===
|
|
173
|
-
isInsidePath(
|
|
174
|
-
isInsidePath(packageEntrypoint, resolvedSource)
|
|
174
|
+
cwdEntrypoints.some((entrypoint) => resolvedSource === entrypoint || isInsidePath(entrypoint, resolvedSource)) ||
|
|
175
|
+
packageEntrypoints.some((entrypoint) => resolvedSource === entrypoint || isInsidePath(entrypoint, resolvedSource))
|
|
175
176
|
);
|
|
176
177
|
}
|
|
177
178
|
|
|
@@ -302,14 +303,14 @@ async function checkPiVersion({ runPi }) {
|
|
|
302
303
|
try {
|
|
303
304
|
const rawOutput = await runPi(["--version"]);
|
|
304
305
|
const version = normalizePiVersion(rawOutput);
|
|
305
|
-
const supported = versionAtLeast(version,
|
|
306
|
+
const supported = versionAtLeast(version, MINIMUM_PI_VERSION);
|
|
306
307
|
if (supported === false) {
|
|
307
308
|
return {
|
|
308
|
-
status: "
|
|
309
|
-
title: `Pi ${
|
|
309
|
+
status: "fail",
|
|
310
|
+
title: `Pi ${MINIMUM_PI_VERSION} or newer is required; found ${version || "<empty>"}.`,
|
|
310
311
|
lines: [
|
|
311
|
-
"This
|
|
312
|
-
"Update Pi before
|
|
312
|
+
"This release enforces the Pi 0.79.0 runtime floor through the read-only doctor and release/package validation because it depends on Project Trust, package loading, session lifecycle, TUI rendering, and tool_result patch behavior from that baseline.",
|
|
313
|
+
"Update Pi before using this package or running lifecycle/package validation.",
|
|
313
314
|
],
|
|
314
315
|
};
|
|
315
316
|
}
|
|
@@ -317,17 +318,17 @@ async function checkPiVersion({ runPi }) {
|
|
|
317
318
|
return {
|
|
318
319
|
status: "warn",
|
|
319
320
|
title: `Could not parse pi --version output: ${version || "<empty>"}.`,
|
|
320
|
-
lines: [`Pi ${
|
|
321
|
+
lines: [`Pi ${MINIMUM_PI_VERSION} or newer is required for this release; run this doctor from the same shell that launches Pi so the setup gate can verify the host runtime.`],
|
|
321
322
|
};
|
|
322
323
|
}
|
|
323
|
-
return { status: "pass", title: `Pi version
|
|
324
|
+
return { status: "pass", title: `Pi version satisfies the minimum runtime floor: ${version}`, lines: [] };
|
|
324
325
|
} catch (error) {
|
|
325
326
|
const code = error && typeof error === "object" ? error.code : undefined;
|
|
326
327
|
return {
|
|
327
328
|
status: "warn",
|
|
328
329
|
title: "Could not inspect pi --version.",
|
|
329
330
|
lines: [
|
|
330
|
-
`Pi ${
|
|
331
|
+
`Pi ${MINIMUM_PI_VERSION} or newer is required for this release; run this doctor from the same shell that launches Pi so the setup gate can verify the host runtime.`,
|
|
331
332
|
"Make sure the same shell that launches pi can run `pi --version` when debugging lifecycle or package-install behavior.",
|
|
332
333
|
code && code !== "ENOENT" ? `Spawn error: ${String(code)}` : undefined,
|
|
333
334
|
].filter(Boolean),
|
|
@@ -434,6 +435,7 @@ export async function evaluateDoctor(options = {}) {
|
|
|
434
435
|
|
|
435
436
|
const piVersionCheck = await checkPiVersion({ runPi });
|
|
436
437
|
checks.push(piVersionCheck);
|
|
438
|
+
if (piVersionCheck.status === "fail") failures.push(piVersionCheck);
|
|
437
439
|
|
|
438
440
|
if (!options.skipSourceCheck) {
|
|
439
441
|
const sourceCheck = await checkPiSources({ cwd, agentDir, settingsPaths, readText, pathExists });
|
|
@@ -62,7 +62,7 @@ Environment:
|
|
|
62
62
|
PLATFORM_SMOKE_MAC_USER macOS SSH user; default $USER
|
|
63
63
|
PLATFORM_SMOKE_MAC_WORK_ROOT macOS Crabbox work root
|
|
64
64
|
PLATFORM_SMOKE_MAC_PORT macOS SSH port; default 22
|
|
65
|
-
PLATFORM_SMOKE_UBUNTU_IMAGE Ubuntu local-container image; default pi-agent-browser-native-platform:node24-agent-browser0.27.
|
|
65
|
+
PLATFORM_SMOKE_UBUNTU_IMAGE Ubuntu local-container image; default pi-agent-browser-native-platform:node24-agent-browser0.27.2
|
|
66
66
|
PLATFORM_SMOKE_WINDOWS_VM Parallels Windows template VM
|
|
67
67
|
PLATFORM_SMOKE_WINDOWS_SNAPSHOT Parallels snapshot name
|
|
68
68
|
PLATFORM_SMOKE_WINDOWS_USER Windows SSH user
|