pi-agent-browser-native 0.2.33 → 0.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +34 -4
- package/docs/ARCHITECTURE.md +6 -0
- package/docs/COMMAND_REFERENCE.md +28 -5
- package/docs/RELEASE.md +11 -3
- package/docs/SUPPORT_MATRIX.md +8 -6
- package/docs/TOOL_CONTRACT.md +61 -7
- package/extensions/agent-browser/index.ts +2 -1
- package/extensions/agent-browser/lib/input-modes/job.ts +62 -0
- package/extensions/agent-browser/lib/input-modes/params.ts +4 -4
- package/extensions/agent-browser/lib/input-modes.ts +3 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts +67 -1
- package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts +26 -1
- package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts +34 -7
- package/extensions/agent-browser/lib/orchestration/browser-run/types.ts +6 -0
- package/extensions/agent-browser/lib/playbook.ts +17 -16
- package/extensions/agent-browser/lib/results/categories.ts +1 -1
- package/extensions/agent-browser/lib/results/presentation/registry.ts +34 -6
- package/extensions/agent-browser/lib/results/presentation/semantic-action.ts +133 -0
- package/extensions/agent-browser/lib/results/presentation.ts +11 -6
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,25 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.2.34 - 2026-05-24
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Deterministic maintainer dogfood mode: `npm run verify -- dogfood` runs a model-free live-browser smoke through the native wrapper against public `example.com`, covering top-level `qa`, `semanticAction`, `qa.attached`, constrained `job`, screenshot artifact verification, and session close.
|
|
10
|
+
- Opt-in efficiency-benchmark JSONL sampling via `--sample-jsonl`, so maintainers can measure real transcript model-visible byte output without changing deterministic scenario metrics.
|
|
11
|
+
- Architecture note for the Tier A/Tier B prompt-guidance budget, keeping always-on `promptGuidelines` short while preserving detailed browser playbook guidance in docs.
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
|
|
15
|
+
- Always-on `agent_browser` prompt guidance is smaller and focused on Tier A rules: input-mode choice, refs/session/artifacts/nextActions, extraction basics, and explicit stop-before-order/post/purchase/submit boundaries.
|
|
16
|
+
- `semanticAction` success output now better mirrors raw browser-action navigation and page-change summaries, while docs make the input-mode chooser clearer.
|
|
17
|
+
- QA preset pass output is more compact, and `qa.attached` preflight now treats URL-only current-page checks as valid attached-session evidence.
|
|
18
|
+
- Constrained `job` docs now make post-click navigation assertions explicit with `assertUrl` / `assertText` instead of implying hidden automatic navigation checks.
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- Stabilized concurrency-sensitive fake-upstream tests by waiting for the older explicit-session open to reach the fake binary before launching the newer one, and by covering the documented planned-URL fallback separately from strict live current-page recovery assertions.
|
|
23
|
+
|
|
5
24
|
## 0.2.33 - 2026-05-23
|
|
6
25
|
|
|
7
26
|
### Added
|
package/README.md
CHANGED
|
@@ -150,13 +150,15 @@ It does **not** edit Pi settings and does **not** run upstream `agent-browser do
|
|
|
150
150
|
|
|
151
151
|
You usually prompt the agent in natural language. These JSON snippets show the exact native tool shape the agent should use.
|
|
152
152
|
|
|
153
|
-
Open a page and inspect it:
|
|
153
|
+
Open a page and inspect it (first-call recipe: open → snapshot -i → interact with current `@refs` → snapshot -i after changes). Do not pass `--json` in `args`; the wrapper injects it.
|
|
154
154
|
|
|
155
155
|
```json
|
|
156
156
|
{ "args": ["open", "https://example.com"] }
|
|
157
157
|
{ "args": ["snapshot", "-i"] }
|
|
158
158
|
```
|
|
159
159
|
|
|
160
|
+
On `https://example.com/`, the main link label is **Learn more**—use exact visible text from your snapshot, not guessed copy such as `More information...`.
|
|
161
|
+
|
|
160
162
|
Click a visible ref, then refresh refs after navigation or a DOM update:
|
|
161
163
|
|
|
162
164
|
```json
|
|
@@ -211,7 +213,8 @@ For supported upstream `find` flows and native dropdown selection you can omit h
|
|
|
211
213
|
|
|
212
214
|
Typical pitfalls:
|
|
213
215
|
|
|
214
|
-
- Supply **exactly one** of `args`, `semanticAction`, `job`, `qa`, `sourceLookup`, `networkSourceLookup`, or `electron` per call (not more, not none).
|
|
216
|
+
- Supply **exactly one** of `args`, `semanticAction`, `job`, `qa`, `sourceLookup`, `networkSourceLookup`, or `electron` per call (not more, not none). Prefer `args` for routine browse; `semanticAction` for stable locators; `job`/`qa` for multi-step checks; `electron` for desktop apps; treat `sourceLookup` / `networkSourceLookup` as experimental candidates-only.
|
|
217
|
+
- Do not pass `--json` in `args`; the wrapper injects it automatically.
|
|
215
218
|
- `semanticAction` and `job` are **not** valid inside `batch` stdin; batch steps stay upstream argv string arrays (spell a `find` step as tokens there if you need it in a batch).
|
|
216
219
|
- Commands or locators outside the supported shorthand still require explicit `args`. Common page getters are grouped under `get`: use `get title`, `get url`, or `get text <selector>` rather than shortcut commands such as `title` or `url`; unknown getter shortcuts can return read-only `details.nextActions` like `use-get-title`.
|
|
217
220
|
- For `locator: "role"`, pass either `value: "button"` or `role: "button"`; if both are present they must match.
|
|
@@ -228,6 +231,8 @@ Typical pitfalls:
|
|
|
228
231
|
|
|
229
232
|
For short repeatable workflows, pass a top-level `job` instead of hand-writing `batch` stdin. The wrapper only supports constrained steps (`open`, `click`, `fill`, `select`, `wait`, `assertText`, `assertUrl`, `waitForDownload`, and `screenshot`), compiles them to existing upstream `batch` commands, and echoes the compiled commands as `details.compiledJob` for auditability. The same compile path backs top-level `qa`, so long `qa` runs surface the same timeout evidence shape. If a long `job`, `qa`, or `batch` hits the wrapper watchdog, `details.timeoutPartialProgress` may recover planned steps, current page title/URL, and declared artifact paths that already exist on disk (see [`docs/TOOL_CONTRACT.md#details`](docs/TOOL_CONTRACT.md#details)). There is no separate catalog of reusable named browser recipes above `job`, `qa`, and raw `batch`; see [`docs/ARCHITECTURE.md#no-reusable-recipe-layer-yet`](docs/ARCHITECTURE.md#no-reusable-recipe-layer-yet) for the closed `RQ-0068` decision and when to revisit it.
|
|
230
233
|
|
|
234
|
+
**Navigation inside `job` is explicit.** A successful `click` does not prove the next page loaded; add `assertUrl` and/or `assertText` after navigation-prone clicks (forms, checkout, tabs, submit buttons) before screenshots or steps that assume the new page.
|
|
235
|
+
|
|
231
236
|
```json
|
|
232
237
|
{
|
|
233
238
|
"job": {
|
|
@@ -240,6 +245,21 @@ For short repeatable workflows, pass a top-level `job` instead of hand-writing `
|
|
|
240
245
|
}
|
|
241
246
|
```
|
|
242
247
|
|
|
248
|
+
```json
|
|
249
|
+
{
|
|
250
|
+
"job": {
|
|
251
|
+
"steps": [
|
|
252
|
+
{ "action": "open", "url": "https://shop.example/checkout" },
|
|
253
|
+
{ "action": "fill", "selector": "#email", "text": "user@example.com" },
|
|
254
|
+
{ "action": "click", "selector": "#continue" },
|
|
255
|
+
{ "action": "assertUrl", "url": "**/shipping" },
|
|
256
|
+
{ "action": "assertText", "text": "Shipping address" },
|
|
257
|
+
{ "action": "screenshot", "path": ".dogfood/shipping.png" }
|
|
258
|
+
]
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
```
|
|
262
|
+
|
|
243
263
|
On app pages that expose a native dropdown, add a `select` step such as `{ "action": "select", "selector": "#flavor", "value": "chocolate" }` before the assertion that depends on it.
|
|
244
264
|
|
|
245
265
|
Use raw `args`/`stdin` when you need full upstream `batch` power, custom flags, or commands outside the constrained job schema. Do not pass `stdin` with `job`, `qa`, `sourceLookup`, `networkSourceLookup`, or `electron`; those modes generate or manage their own input.
|
|
@@ -315,6 +335,8 @@ For asynchronous exports, click first and then wait for the download:
|
|
|
315
335
|
|
|
316
336
|
When a user gives exact artifact paths for screenshots, recordings, downloads, PDFs, traces, or HAR files, use those paths or explicitly report why the artifact was unavailable; do not silently substitute a different path in the final report. With upstream `agent-browser 0.27.0`, treat `details.savedFilePath` as upstream-reported metadata and confirm `details.artifacts[].exists` before relying on the requested `wait --download <path>` file being present on disk.
|
|
317
337
|
|
|
338
|
+
For evidence-only screenshots or QA captures, branch on `details.artifactVerification` and `details.artifacts` before reporting PASS/FAIL; inline image attachments are optional when size limits allow—do not require vision review unless the user asked for visual inspection.
|
|
339
|
+
|
|
318
340
|
Artifact cleanup is host-owned, not a browser command. `close` shuts down the browser session but does **not** delete explicit screenshots, downloads, PDFs, traces, HAR files, or recordings saved to paths you chose. When the session’s non-empty `details.artifactManifest` is in scope, a successful `close` appends an `Artifact lifecycle` note and sets `details.artifactCleanup` with the same retention summary as `details.artifactRetentionSummary`, a fixed `note` about host-owned cleanup, and `explicitArtifactPaths`: up to ten distinct paths from manifest rows whose `storageScope` is `explicit-path` (this list can be empty if the recent window only holds spills or other non-explicit inventory). Remove any listed paths with normal file tools after inspection.
|
|
319
341
|
|
|
320
342
|
Start a fresh profiled browser after the implicit public-browsing session already exists:
|
|
@@ -381,7 +403,7 @@ Use SPA and Web Vitals helpers as normal command tokens:
|
|
|
381
403
|
|
|
382
404
|
```json
|
|
383
405
|
{ "args": ["pushstate", "/dashboard"] }
|
|
384
|
-
{ "args": ["vitals", "https://example.com"
|
|
406
|
+
{ "args": ["vitals", "https://example.com"] }
|
|
385
407
|
```
|
|
386
408
|
|
|
387
409
|
For setup that must happen before first navigation, open a blank fresh page, stage routes/cookies/scripts, then navigate:
|
|
@@ -417,7 +439,7 @@ The full `npm run verify` gate runs:
|
|
|
417
439
|
- command-reference baseline checks
|
|
418
440
|
- live command-reference verification against the targeted installed upstream `agent-browser`
|
|
419
441
|
|
|
420
|
-
Step order and which subprocesses run live in [`scripts/project.mjs`](scripts/project.mjs); [`test/project-verify.test.ts`](test/project-verify.test.ts) locks default, `release`, `real-upstream`, `package-pi`, and combined-docs orchestration so a gate cannot disappear accidentally. Run `npm run verify -- --help` for opt-in modes and supported passthrough flags.
|
|
442
|
+
Step order and which subprocesses run live in [`scripts/project.mjs`](scripts/project.mjs); [`test/project-verify.test.ts`](test/project-verify.test.ts) locks default, `release`, `real-upstream`, `dogfood`, `package-pi`, and combined-docs orchestration so a gate cannot disappear accidentally. Run `npm run verify -- --help` for opt-in modes and supported passthrough flags.
|
|
421
443
|
|
|
422
444
|
The deterministic agent-efficiency benchmark’s **standalone JSON/Markdown accounting run** is not part of default `npm run verify` (only `npm run verify -- benchmark` or `npm run benchmark:agent-browser` invokes the script). The full unit suite still exercises `test/agent-browser.efficiency-benchmark.test.ts`. Use the script before and after agent-facing abstractions to prove call-count, output-size, stale-ref, artifact, failure-category coverage, success-rate, and elapsed-time effects before changing the wrapper UX:
|
|
423
445
|
|
|
@@ -438,6 +460,14 @@ npm run verify -- real-upstream
|
|
|
438
460
|
|
|
439
461
|
That mode sets `PI_AGENT_BROWSER_REAL_UPSTREAM=1` and runs `test/agent-browser.real-upstream-contract.test.ts` against the real `agent-browser` on `PATH` (version must match the capability baseline). It covers inspection, skills, a broad core interaction and navigation matrix on localhost fixtures (including `batch` stdin and `pushstate`), plus `vitals`, network route/requests/HAR, diff snapshot/screenshot/url, trace/profiler, console/errors/highlight, stream enable/status/disable, `cookies set --curl`, a `react tree` missing-renderer path, and `wait --download` with the on-disk caveat documented in release notes. The harness uses a throwaway temp `HOME` and dedicated socket/screenshot directories so the run does not touch your normal browser profile paths. Browser-opening or credential-dependent families such as `inspect`, `dashboard`, `chat`, provider clouds, and OS clipboard flows stay in fake-upstream or manual validation unless a safe deterministic fixture is added. For prerequisites, isolation details, and troubleshooting, see [`docs/RELEASE.md`](docs/RELEASE.md#real-upstream-contract-validation).
|
|
440
462
|
|
|
463
|
+
A deterministic live-browser wrapper smoke is available without an LLM choosing tool calls:
|
|
464
|
+
|
|
465
|
+
```bash
|
|
466
|
+
npm run verify -- dogfood
|
|
467
|
+
```
|
|
468
|
+
|
|
469
|
+
That mode drives the native wrapper through top-level `qa`, `semanticAction`, `qa.attached`, constrained `job`, screenshot artifact verification, and session close against public `example.com`. It complements, but does not replace, the interactive Pi/tmux release dogfood in [`docs/RELEASE.md`](docs/RELEASE.md#pre-release-checks).
|
|
470
|
+
|
|
441
471
|
For package release confidence, follow [`docs/RELEASE.md`](docs/RELEASE.md). The release gate is:
|
|
442
472
|
|
|
443
473
|
```bash
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -53,6 +53,12 @@ That means:
|
|
|
53
53
|
- no manual user orchestration as the main workflow
|
|
54
54
|
- any future slash commands should be minimal and secondary
|
|
55
55
|
|
|
56
|
+
### Prompt guidance budget
|
|
57
|
+
|
|
58
|
+
Runtime `promptGuidelines` are a Tier A budget, not a full manual. They stay short enough to load on every `agent_browser`-aware turn and carry only high-impact rules: input-mode choice, the open → snapshot → ref loop, launch-scoped session handling, artifact verification, structured `nextActions`, extraction basics, and hard safety boundaries such as “stop before order/post/purchase/submit.”
|
|
59
|
+
|
|
60
|
+
Tier B guidance lives in `SHARED_BROWSER_PLAYBOOK_GUIDELINES`, generated README/command-reference fragments, and targeted docs. When a workflow needs examples, caveats, or long command-family coverage, add it there instead of expanding always-on prompt text. If a Tier B rule prevents a repeated real failure, promote only the smallest durable sentence into Tier A and keep the generated-doc mirrors aligned.
|
|
61
|
+
|
|
56
62
|
### No reusable recipe layer yet
|
|
57
63
|
|
|
58
64
|
Do **not** add reusable browser recipes as a first-class runtime surface yet.
|
|
@@ -27,6 +27,8 @@ Use `npm run benchmark:agent-browser` or `npm run verify -- benchmark` before an
|
|
|
27
27
|
|
|
28
28
|
## Core mental model
|
|
29
29
|
|
|
30
|
+
Input mode chooser (one per call): **`args`** for the default open → snapshot -i → click/fill `@refs` flow; **`semanticAction`** for stable role/text/label targets; **`job`** / **`qa`** for multi-step checks; **`electron`** for desktop apps only; **`sourceLookup`** / **`networkSourceLookup`** are **experimental candidates-only** helpers (not authoritative mappings). Do not pass `--json` in `args`—the wrapper injects it. Match link and button text to the latest snapshot (on `https://example.com/` the main link is `Learn more`, not legacy `More information...` copy). See [`TOOL_CONTRACT.md`](TOOL_CONTRACT.md#input-mode-chooser) for snapshot variants (`-i` vs `--compact` vs full) and batching three or more getters.
|
|
31
|
+
|
|
30
32
|
Tool parameters (use exactly one of `args`, `semanticAction`, `job`, `qa`, `sourceLookup`, `networkSourceLookup`, or `electron`):
|
|
31
33
|
|
|
32
34
|
```json
|
|
@@ -63,8 +65,8 @@ Tool parameters (use exactly one of `args`, `semanticAction`, `job`, `qa`, `sour
|
|
|
63
65
|
- `semanticAction`: optional shorthand for common `find` flows and native dropdown `select`; compiles to upstream argv and is rejected together with `args`, `job`, `qa`, `sourceLookup`, `networkSourceLookup`, or `electron` on the same call.
|
|
64
66
|
- `job`: optional constrained short-workflow schema; compiles to existing upstream `batch` args/stdin and reports the compiled plan in `details.compiledJob`.
|
|
65
67
|
- `qa`: optional lightweight QA preset; compiles to the same batch path and reports `details.compiledQaPreset` plus `details.qaPreset` pass/fail evidence.
|
|
66
|
-
- `sourceLookup`:
|
|
67
|
-
- `networkSourceLookup`:
|
|
68
|
+
- `sourceLookup`: **EXPERIMENTAL — candidates only** for local UI-to-source hints; compiles to the same `batch` path, reports `details.compiledSourceLookup` and `details.sourceLookup`, and never reclassifies a fully successful upstream batch as failed the way `qa` can (see [`TOOL_CONTRACT.md`](TOOL_CONTRACT.md#sourcelookup) and the longer notes below).
|
|
69
|
+
- `networkSourceLookup`: **EXPERIMENTAL — candidates only** for failed request-to-source hints; compiles to generated `batch`, reports `details.compiledNetworkSourceLookup` and `details.networkSourceLookup`, and never assigns blame or edits files.
|
|
68
70
|
- `electron`: optional Electron desktop-app shorthand. `list`, `status`, `cleanup`, and `probe` are wrapper-owned host/session helpers; `launch` starts a wrapper-owned isolated Electron profile and attaches through upstream `connect`.
|
|
69
71
|
- `stdin`: only for `batch`, `eval --stdin`, and `auth save --password-stdin`; other command/stdin combinations are rejected before `agent-browser` is launched. `job`, `qa`, `sourceLookup`, `networkSourceLookup`, and `electron` generate or manage their own input.
|
|
70
72
|
- `sessionMode`:
|
|
@@ -105,7 +107,7 @@ React introspection requires the React DevTools init hook to be installed before
|
|
|
105
107
|
Use `vitals [url]` for Core Web Vitals plus React hydration timing when available, and `pushstate <url>` for client-side SPA navigation without a full reload:
|
|
106
108
|
|
|
107
109
|
```json
|
|
108
|
-
{ "args": ["vitals", "https://example.com"
|
|
110
|
+
{ "args": ["vitals", "https://example.com"] }
|
|
109
111
|
{ "args": ["pushstate", "/dashboard?tab=settings"] }
|
|
110
112
|
```
|
|
111
113
|
|
|
@@ -186,6 +188,8 @@ Use `batch --bail` when later steps should stop after the first failed command.
|
|
|
186
188
|
|
|
187
189
|
For short constrained flows, use top-level `job` instead of hand-writing `batch` stdin. Supported job steps are `open`, `click`, `fill`, `select`, `wait`, `assertText`, `assertUrl`, `waitForDownload`, and `screenshot`; `select` requires `selector` plus `value` or `values`, and compiles to upstream `select <selector> <value...>`. The wrapper compiles steps to upstream `batch` and records `details.compiledJob.steps[]`. There is still no separate first-class catalog of reusable named browser recipes above `job`, the `qa` preset, and raw `batch`; see [`ARCHITECTURE.md`](ARCHITECTURE.md#no-reusable-recipe-layer-yet) for the closed `RQ-0068` decision and revisit bar.
|
|
188
190
|
|
|
191
|
+
**Job navigation is explicit.** A `click` step (or other navigation-prone interaction) does not prove the next page loaded. The wrapper does not auto-insert `assertUrl` or `assertText` after clicks inside `job`; add those steps yourself with the URL pattern or on-page text you expect, especially after forms, checkout, tabs, or submit buttons, before screenshots or later steps.
|
|
192
|
+
|
|
189
193
|
```json
|
|
190
194
|
{
|
|
191
195
|
"job": {
|
|
@@ -198,11 +202,28 @@ For short constrained flows, use top-level `job` instead of hand-writing `batch`
|
|
|
198
202
|
}
|
|
199
203
|
```
|
|
200
204
|
|
|
205
|
+
Navigation-prone flow (open → fill → click → assert destination → screenshot):
|
|
206
|
+
|
|
207
|
+
```json
|
|
208
|
+
{
|
|
209
|
+
"job": {
|
|
210
|
+
"steps": [
|
|
211
|
+
{ "action": "open", "url": "https://shop.example/checkout" },
|
|
212
|
+
{ "action": "fill", "selector": "#email", "text": "user@example.com" },
|
|
213
|
+
{ "action": "click", "selector": "#continue" },
|
|
214
|
+
{ "action": "assertUrl", "url": "**/shipping" },
|
|
215
|
+
{ "action": "assertText", "text": "Shipping address" },
|
|
216
|
+
{ "action": "screenshot", "path": ".dogfood/shipping.png" }
|
|
217
|
+
]
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
```
|
|
221
|
+
|
|
201
222
|
On app pages that expose a native dropdown, add a `select` step such as `{ "action": "select", "selector": "#flavor", "value": "chocolate" }` before the assertion that depends on it.
|
|
202
223
|
|
|
203
224
|
Use raw `args: ["batch"]` with `stdin` when you need arbitrary upstream commands, flags, or batch failure policies outside the constrained schema. Do not pass `stdin` with `job`, `qa`, `sourceLookup`, `networkSourceLookup`, or `electron`; those modes generate or manage their own input.
|
|
204
225
|
|
|
205
|
-
For quick smoke/QA checks, use top-level `qa`. It clears enabled network/console/page-error buffers before opening the target URL, waits for page readiness, checks expected text/selector, inspects fresh network requests, console messages, and page errors, and can capture an evidence screenshot. The readiness wait defaults to `loadState: "domcontentloaded"`; set `loadState` to `"load"` or `"networkidle"` only when that stricter state is useful and the site is not expected to keep background requests alive. QA network diagnostics classify failed requests by likely impact and list failed rows first in the network preview: actionable document/script/API-style failures fail the preset, while common low-impact browser icon misses such as `favicon.ico` are surfaced as warnings (`qaPreset.warnings`) so they do not fail an otherwise healthy page. Failed QA presets report `details.resultCategory: "failure"`, `failureCategory: "qa-failure"`, and real Pi sessions treat the diagnostic as a failed tool result. Prose output also gets a model-visible result-category line including `Pi tool isError: true`; caller-requested `--json` output keeps the JSON string parseable and relies on the patched `isError` plus `details` fields.
|
|
226
|
+
For quick smoke/QA checks, use top-level `qa`. It clears enabled network/console/page-error buffers before opening the target URL, waits for page readiness, checks expected text/selector, inspects fresh network requests, console messages, and page errors, and can capture an evidence screenshot. The readiness wait defaults to `loadState: "domcontentloaded"`; set `loadState` to `"load"` or `"networkidle"` only when that stricter state is useful and the site is not expected to keep background requests alive. QA network diagnostics classify failed requests by likely impact and list failed rows first in the network preview: actionable document/script/API-style failures fail the preset, while common low-impact browser icon misses such as `favicon.ico` are surfaced as warnings (`qaPreset.warnings`) so they do not fail an otherwise healthy page. Successful QA with no failed checks returns compact model-visible prose (page URL/title when known, checks run, optional screenshot verification) while keeping the full step matrix in `details.qaPreset` and `details.batchSteps`. Failed QA presets report `details.resultCategory: "failure"`, `failureCategory: "qa-failure"`, keep verbose per-step batch output, and real Pi sessions treat the diagnostic as a failed tool result. Prose output also gets a model-visible result-category line including `Pi tool isError: true`; caller-requested `--json` output keeps the JSON string parseable and relies on the patched `isError` plus `details` fields.
|
|
206
227
|
|
|
207
228
|
The same classification drives plain `network requests` presentation: when any row counts as failed (HTTP status ≥ 400, `failed: true`, or a string `error`), model-facing text starts with a line like `Network failure summary: 0 actionable, 1 benign low-impact (1 total).`, and each preview line can end with an impact tag such as `[benign: low-impact browser icon asset]` or `[actionable: document, script, API, or non-benign request failure]`. When safe request IDs are present, `details.nextActions` adds bounded read-only follow-ups such as `network request <id>`, `networkSourceLookup` for actionable failed rows, `network requests --filter <path>`, and `network har start`; prefer those payloads over rebuilding request-id commands from prose. Rules live in `classifyNetworkRequestFailure` / `summarizeNetworkFailures` in `extensions/agent-browser/lib/results/network.ts`; QA aggregation is `analyzeQaPresetResults` in `extensions/agent-browser/index.ts`.
|
|
208
229
|
|
|
@@ -212,7 +233,7 @@ The same classification drives plain `network requests` presentation: when any r
|
|
|
212
233
|
|
|
213
234
|
Optional `loadState`, `checkNetwork`, `checkConsole`, and `checkErrors` default to `"domcontentloaded"`, `true`, `true`, and `true`; set a check to `false` to skip that diagnostic. Omit `expectedText` and `expectedSelector` when you only need load plus diagnostics.
|
|
214
235
|
|
|
215
|
-
For attached Electron or manually connected CDP sessions, use `qa.attached` after the session exists. It does not open a URL and rejects `sessionMode: "fresh"` because it checks the current managed session.
|
|
236
|
+
For attached Electron or manually connected CDP sessions, use `qa.attached` after the session exists. It does not open a URL and rejects `sessionMode: "fresh"` because it checks the current managed session. Before running diagnostics, the wrapper requires a readable `http:` or `https:` page URL on the attached session; missing URLs, read failures, and non-http(s) surfaces fail fast with recovery `nextActions` such as `tab list` and `snapshot -i` instead of running the full QA batch.
|
|
216
237
|
|
|
217
238
|
```json
|
|
218
239
|
{ "qa": { "attached": true, "expectedText": "Explorer", "screenshotPath": ".dogfood/electron.png" } }
|
|
@@ -310,6 +331,8 @@ The upstream screenshot aliases are `screenshot --full` for full-page capture an
|
|
|
310
331
|
|
|
311
332
|
Prefer `download <selector> <path>` when the target element itself is the downloadable link/control. Use `click` plus `wait --download [path]` when a previous action starts the download indirectly.
|
|
312
333
|
|
|
334
|
+
For evidence-only screenshots, QA captures, or audit artifacts, save to an explicit path and branch on `details.artifactVerification` plus `details.artifacts` before reporting PASS/FAIL. Inline image attachments are optional convenience when size limits allow; do not require vision review unless the user asked for visual inspection.
|
|
335
|
+
|
|
313
336
|
Wrapper result rendering is metadata-first for saved files:
|
|
314
337
|
- screenshots return a saved-path summary, visible artifact metadata, structured `details.artifacts` metadata, and an inline image attachment when safe; the visible block includes artifact type, requested path, absolute path, existence, size, cwd, session, and repair/copy status when applicable
|
|
315
338
|
- downloads, PDFs, `wait --download` files, `state save` state files, diff screenshot output images, traces, CPU profiles, completed WebM recordings from `record stop`, and path-bearing HAR captures return concise saved-path summaries plus structured `details.artifacts` metadata without inlining large files
|
package/docs/RELEASE.md
CHANGED
|
@@ -35,9 +35,17 @@ npm run verify -- release
|
|
|
35
35
|
|
|
36
36
|
`npm publish` runs npm’s `prepublishOnly` script from `package.json`, which executes the same `npm run verify -- release` gate and then `npm pack --dry-run`. That concatenated gate is everything in the default `npm run verify` step (generated playbook drift, TypeScript, the unit/fake suite, generated command-reference blocks, and live upstream command-reference sampling against the targeted `agent-browser` on `PATH`) plus the packaged Pi smoke in `package-pi`. Using `npm publish --ignore-scripts` skips that contract intentionally.
|
|
37
37
|
|
|
38
|
-
`prepublishOnly` intentionally does **not** run `npm run verify -- lifecycle`, `npm run verify -- real-upstream`, or `npm run verify -- benchmark`; those are separate `npm run verify` modes in [`scripts/project.mjs`](../scripts/project.mjs). Treat the bullets below as the full pre-publish contract even though only the `release` slice is automated at publish time.
|
|
38
|
+
`prepublishOnly` intentionally does **not** run `npm run verify -- lifecycle`, `npm run verify -- real-upstream`, `npm run verify -- dogfood`, or `npm run verify -- benchmark`; those are separate `npm run verify` modes in [`scripts/project.mjs`](../scripts/project.mjs). Treat the bullets below as the full pre-publish contract even though only the `release` slice is automated at publish time.
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
For a deterministic real-browser wrapper smoke without model choice in the loop, run:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
npm run verify -- dogfood
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
This mode uses the extension harness and the real `agent-browser` on `PATH` against public `example.com`, then verifies top-level `qa`, `semanticAction`, `qa.attached`, constrained `job`, screenshot artifact verification, and session close. Use `npm run verify -- dogfood --keep-artifacts` or `--artifact-dir <path>` only while debugging, then delete retained screenshots. This smoke complements, but does not replace, human-readable interactive transcript evidence.
|
|
47
|
+
|
|
48
|
+
Every release also requires interactive `tmux`-driven Pi dogfood with the native `agent_browser` tool against real sites. For extension-focused release smokes, use `pi --no-extensions --no-skills -e .` from the checkout before publish so auto-loaded dogfood/QA skills cannot replace the bounded smoke workflow; run separate skill-enabled dogfood only when validating skill routing or report-generation behavior. Drive prompts with `tmux send-keys`, exercise at least one simple static site and one real documentation/product site, include the higher-level `qa` or `job`/`batch` surfaces when they changed, close every opened browser session, remove screenshots/temp artifacts, and record the outcome in the release notes or support-matrix evidence. Automated localhost, fake-upstream, and deterministic dogfood gates do not replace this human-readable live-site transcript evidence. When `electron.*` surfaces, attached-session diagnostics, or `qa.attached` changed, add a local Electron pass: `electron.list` → `electron.launch` (expect isolated profile behavior) → `snapshot -i` or `electron.probe` / `qa.attached` → `electron.cleanup` with the returned `launchId`, verifying status/mismatch guidance if you simulate a dead renderer or stale refs. For dense-dashboard stress coverage, use the [public Grafana stress checklist](#public-grafana-stress-checklist) below; it is a maintainer workflow, not bundled product skill or recipe runtime.
|
|
41
49
|
|
|
42
50
|
When reviewing saved session JSONL after a failed smoke or a `qa` preset that reclassified an upstream-successful batch, expect `agent_browser` tool rows to carry `isError: true` whenever `details.resultCategory` is `failure`. For normal prose output, model-visible text should end with a `Pi tool isError: true` category line; for caller-requested `--json` output, the hook preserves parseable JSON and only patches `isError`. The extension applies that patch on the `tool_result` path so Pi’s transcript matches the wrapper contract ([`TOOL_CONTRACT.md`](TOOL_CONTRACT.md#details)). Preserve a normal Pi session directory for those checks; avoiding `--no-session` keeps this evidence intact ([`AGENTS.md`](../AGENTS.md) preferred validation workflow).
|
|
43
51
|
|
|
@@ -117,7 +125,7 @@ Evaluator expectations after the queued Sauce Demo fixes: the agent should indep
|
|
|
117
125
|
|
|
118
126
|
[`scripts/agent-browser-efficiency-benchmark.mjs`](../scripts/agent-browser-efficiency-benchmark.mjs) is an accounting-only benchmark: it does not shell out to `agent-browser`, launch a browser, or read or write Pi sessions. It models representative `agent_browser` call shapes (including optional `stdin` for `batch` and top-level `job`, `qa`, or experimental `sourceLookup` / `networkSourceLookup` objects that compile to batch) and aggregates success rate, tool-call counts, UTF-8 size of model-visible strings, stale-ref failure and recovery counts, artifact success, distinct failure-category coverage, and summed elapsed-time estimates. When extending scenarios, keep them aligned with the closed `RQ-0068` “no reusable recipe layer” rationale in [`ARCHITECTURE.md`](ARCHITECTURE.md#no-reusable-recipe-layer-yet) (benchmark ids cited there are the canonical inventory for that evidence bar).
|
|
119
127
|
|
|
120
|
-
- **During development:** `npm run benchmark:agent-browser` prints a Markdown report; `npm run benchmark:agent-browser -- --json` saves machine-readable metrics; `npm run benchmark:agent-browser -- --compare path/to/prior.json` fails with exit code `1` on regressions (see the script’s `--help` for exit codes).
|
|
128
|
+
- **During development:** `npm run benchmark:agent-browser` prints a Markdown report; `npm run benchmark:agent-browser -- --json` saves machine-readable metrics; `npm run benchmark:agent-browser -- --compare path/to/prior.json` fails with exit code `1` on regressions (see the script’s `--help` for exit codes). Optional `--sample-jsonl path/to/session.jsonl` adds a `jsonlSample` section with real UTF-8 byte totals and per-workflow/overall p95 sizes for model-visible `agent_browser` tool-result text without changing deterministic scenario metrics; comparison ignores `jsonlSample` blocks.
|
|
121
129
|
- **Default gate:** `npm run verify` checks generated playbook drift, runs `tsc --noEmit`, runs the full unit/fake suite under `test/**/*.test.ts` (including [`test/agent-browser.efficiency-benchmark.test.ts`](../test/agent-browser.efficiency-benchmark.test.ts) for scenario coverage and comparison behavior), verifies generated command-reference baseline blocks, and samples live upstream command-reference tokens. It does not spawn the standalone benchmark script’s JSON/Markdown run; that is what the opt-in slice below adds.
|
|
122
130
|
- **Opt-in slice:** `npm run verify -- benchmark` runs the benchmark script once with `--json` and then that same test module alone. It is intentionally **not** part of `npm run verify -- release`, so routine publish gates stay decoupled from benchmark churn while still allowing a focused check after editing scenarios or `CURRENT_BENCHMARK_VERSION`.
|
|
123
131
|
|
package/docs/SUPPORT_MATRIX.md
CHANGED
|
@@ -37,12 +37,14 @@ Re-run the gates below before each release; this table records what the closure
|
|
|
37
37
|
|
|
38
38
|
| Gate | Evidence | Status |
|
|
39
39
|
| --- | --- | --- |
|
|
40
|
-
| Default local gate | `npm run verify` checks generated playbook drift, `tsc --noEmit`, unit/fake tests, generated command-reference blocks, and live command-reference sampling. | Pass on 2026-05-
|
|
41
|
-
| Real upstream contract | `npm run verify -- real-upstream` runs the localhost fixture matrix against the real installed `agent-browser` matching the baseline. | Pass on 2026-05-
|
|
42
|
-
| Packaged Pi smoke | `npm run verify -- package-pi` validates package contents, loads exactly one packaged `agent_browser` tool, and executes fake-upstream `--version`. | Pass on 2026-05-
|
|
43
|
-
|
|
|
44
|
-
|
|
|
45
|
-
|
|
|
40
|
+
| Default local gate | `npm run verify` checks generated playbook drift, `tsc --noEmit`, unit/fake tests, generated command-reference blocks, and live command-reference sampling. | Pass on 2026-05-24 (`npm run verify`, `agent-browser 0.27.0` on `PATH`). |
|
|
41
|
+
| Real upstream contract | `npm run verify -- real-upstream` runs the localhost fixture matrix against the real installed `agent-browser` matching the baseline. | Pass on 2026-05-24 (`npm run verify -- real-upstream`, `agent-browser 0.27.0` on `PATH`). |
|
|
42
|
+
| Packaged Pi smoke | `npm run verify -- package-pi` validates package contents, loads exactly one packaged `agent_browser` tool, and executes fake-upstream `--version`. | Pass on 2026-05-24 as part of `npm run verify -- release`. |
|
|
43
|
+
| Deterministic dogfood smoke | `npm run verify -- dogfood` (`scripts/verify-agent-browser-dogfood.ts`) drives the native wrapper against public `example.com` through top-level `qa`, `semanticAction`, `qa.attached`, constrained `job`, screenshot artifact verification, and session close with the real `agent-browser` on `PATH`. | Pass on 2026-05-24 (`npm run verify -- dogfood --artifact-dir /tmp/pi-agent-browser-release-dogfood --json`; artifacts removed). |
|
|
44
|
+
| Efficiency benchmark | `npm run verify -- benchmark` runs deterministic browser workflow accounting plus focused benchmark tests, including JSONL sampling fixtures and job/qa/sourceLookup/networkSourceLookup/Electron scenario coverage. | Pass on 2026-05-24 (`npm run verify -- benchmark`). |
|
|
45
|
+
| `verify -- release` / `prepublishOnly` | `npm run verify -- release` chains the default gate with packaged Pi smoke (`verifySteps` `release` in [`scripts/project.mjs`](../scripts/project.mjs)). `package.json` `prepublishOnly` runs that compose before `npm pack --dry-run` during `npm publish`. It intentionally omits lifecycle, real-upstream, dogfood, and benchmark modes—see [`RELEASE.md`](RELEASE.md#pre-release-checks). | Pass on 2026-05-24 (`npm run verify -- release`). `prepublishOnly` still needs a fresh run during actual publish. |
|
|
46
|
+
| Configured-source lifecycle | `npm run verify -- lifecycle` (`scripts/verify-lifecycle.mjs`) drives `/reload`, restart, `/resume`, session continuity, slash-command sentinel tokens (`v1` then `v2` after rewriting the packaged extension to simulate pickup), and persisted spill reachability with a fake upstream on `PATH`. Default Pi model is `zai/glm-5.1`; default per-step wait is **180000 ms** (`DEFAULT_TIMEOUT_MS`); override model with `--model <id>` and waits with `--timeout-ms <ms>`. Passthrough flags in [`scripts/project.mjs`](../scripts/project.mjs): `--keep-artifacts`, `--model`, `--verbose`, and `--timeout-ms` plus a value (for example `npm run verify -- lifecycle --model openai-codex/gpt-5.5:minimal --keep-artifacts --verbose --timeout-ms 600000`). | Pass on 2026-05-24 (`npm run verify -- lifecycle`). Treat any future unexplained red lifecycle gate as a release blocker. |
|
|
47
|
+
| Quick isolated Pi smoke | `pi --no-extensions --no-skills -e . --tools agent_browser` from repo root; native `agent_browser` only. | Pass on 2026-05-24 for a fresh interactive tmux smoke: the agent opened `https://example.com`, verified `Example Domain`, clicked `Learn more` to `https://www.iana.org/help/example-domains`, opened `https://github.com/fitchmultz/pi-agent-browser-native`, verified the project page, saved `/tmp/piab-release-artifacts-TL5YWe/release-smoke.png` with verified `image/png` artifact metadata, closed the browser session, and reported PASS; screenshot, session dir, and tmux session were removed after recording evidence. Broader historical coverage also includes version/help/skills, open/snapshot/click, eval stdin, batch stdin, screenshot, explicit session, `sessionMode: "fresh"`, network requests, console/errors, diff snapshot, stream status/disable, dashboard start/stop, and chat credential-failure pass-through during RQ-0055. |
|
|
46
48
|
|
|
47
49
|
## Baseline checklist by inventory section
|
|
48
50
|
|
package/docs/TOOL_CONTRACT.md
CHANGED
|
@@ -33,11 +33,38 @@ The native command reference in `docs/COMMAND_REFERENCE.md` is driven by the sam
|
|
|
33
33
|
|
|
34
34
|
Agent-facing efficiency claims are measured with `npm run benchmark:agent-browser` or `npm run verify -- benchmark`. The benchmark is deterministic and does not launch a browser; it tracks representative workflow success, tool calls, model-visible output size, stale-ref failures and recoveries, artifact success, failure-category coverage, and elapsed-time estimates so future abstractions can prove they reduce agent work before replacing raw tool use.
|
|
35
35
|
|
|
36
|
+
## Input mode chooser
|
|
37
|
+
|
|
38
|
+
Use exactly one top-level input per call:
|
|
39
|
+
|
|
40
|
+
| When you need | Use | Notes |
|
|
41
|
+
| --- | --- | --- |
|
|
42
|
+
| Routine browse, click, fill, screenshots, upstream commands | `args` | Default path: `open` → `snapshot -i` → `click`/`fill` `@eN` → `snapshot -i` after navigation or DOM changes. Do not pass `--json`; the wrapper injects it (see [Wrapper `--json`](#wrapper-json)). |
|
|
43
|
+
| Stable visible role/text/label/placeholder targets | `semanticAction` | Compiles to upstream `find` or `select`; optional `session` for a named upstream browser. |
|
|
44
|
+
| Short multi-step smoke or evidence flows | `job` or `qa` | Both compile to `batch`; `qa` may reclassify diagnostics as failure. |
|
|
45
|
+
| Desktop Electron apps (list/launch/probe/cleanup) | `electron` | Wrapper-owned lifecycle; not for ordinary websites. |
|
|
46
|
+
| Local UI source hints (experimental) | `sourceLookup` | **Candidates only** with confidence/evidence; not guaranteed DOM-to-file mappings. |
|
|
47
|
+
| Failed fetch/API source hints (experimental) | `networkSourceLookup` | **Candidates only** from initiator metadata and bounded workspace URL literals; not definitive blame. |
|
|
48
|
+
|
|
49
|
+
For link and button text, use the **exact** visible label from the latest `snapshot -i` (or `semanticAction` locators), not guessed copy. The `https://example.com/` smoke page uses heading `Example Domain` and link `Learn more`; do not assume older example.com strings such as `More information...`.
|
|
50
|
+
|
|
51
|
+
## Snapshot and getter batching
|
|
52
|
+
|
|
53
|
+
- **`snapshot -i`**: default for interaction—interactive `@eN` refs, main-content-first trimming, and the usual click/fill workflow.
|
|
54
|
+
- **`snapshot --compact`**: denser same-page tree when you still need refs but want less output than full interactive snapshot.
|
|
55
|
+
- **Full `snapshot`** (no `-i`): use only when you need the complete accessibility tree; expect larger output and possible spill files.
|
|
56
|
+
- Re-run `snapshot -i` after navigation, scrolling, rerendering, or other major DOM changes; refs are page-scoped.
|
|
57
|
+
- When you need **three or more** `get title` / `get url` / `get text` / similar reads for known refs or selectors on the same page, prefer one `batch` stdin array (for example `[["get","text","@e1"],["get","text","@e2"]]`) instead of serial tool calls.
|
|
58
|
+
|
|
59
|
+
## Wrapper `--json`
|
|
60
|
+
|
|
61
|
+
The extension always plans normal browser commands with `--json` prepended in `effectiveArgs` so upstream returns structured JSON for presentation and `details`. **Do not** include `--json` in caller `args`; it is unnecessary and can confuse planning or transcript hooks that treat caller-requested JSON differently. Plain-text inspection (`--help`, `--version`) and read-only `skills list` / `skills get` / `skills path` keep their own output shapes and skip implicit session injection as documented under `sessionMode`.
|
|
62
|
+
|
|
36
63
|
<!-- agent-browser-playbook:start shared-guidelines -->
|
|
37
64
|
<!-- Generated from extensions/agent-browser/lib/playbook.ts. Run `npm run docs -- playbook write` to update. -->
|
|
38
65
|
- Standard workflow: open the page, snapshot -i, interact using current @refs from that snapshot, and re-snapshot after navigation, scrolling, rerendering, or other major DOM changes because refs are page-scoped; the wrapper fails mutation-prone stale/recycled refs before upstream can silently target a different current-page element.
|
|
39
66
|
- For ordinary forms from one snapshot, batch multiple fill @refs before the submit/click step to avoid serial tool calls; if a fill may autosubmit, navigate, or rerender later fields, split the flow and refresh refs first.
|
|
40
|
-
- When snapshot -i compacts because the tree is oversized, scan visible output for Omitted high-value controls and optional details.data.highValueControlRefIds before opening the spill file: those list bounded searchboxes, textboxes, comboboxes, buttons, tabs, checkboxes, radios, options, and menuitems that did not fit the key/other ref previews.
|
|
67
|
+
- Snapshot choice: prefer snapshot -i for routine clicks/fills (interactive @refs, main-content-first). Use snapshot --compact when you need a denser same-page tree without full spill; use full snapshot (no -i) only when you need the complete accessibility tree. Re-snapshot after navigation or major DOM changes. When snapshot -i compacts because the tree is oversized, scan visible output for Omitted high-value controls and optional details.data.highValueControlRefIds before opening the spill file: those list bounded searchboxes, textboxes, comboboxes, buttons, tabs, checkboxes, radios, options, and menuitems that did not fit the key/other ref previews.
|
|
41
68
|
- When a visible text or accessible-name target should survive ref churn, prefer find locators such as role, text, label, placeholder, alt, title, or testid with the intended action instead of guessing a CSS selector.
|
|
42
69
|
- For desktop or host-controlled rich inputs, if semanticAction fill misses, refresh refs and prefer a current editable @ref from details.richInputRecovery or the latest snapshot; focus or click that ref, then use keyboard inserttext or keyboard type with the intended text. Do not auto-submit with Enter or a submit button unless the user flow explicitly calls for it.
|
|
43
70
|
- Do not assume Playwright selector dialects such as text=Close or button:has-text('Close') are supported wrapper syntax unless current upstream agent-browser behavior has been verified.
|
|
@@ -62,6 +89,9 @@ Agent-facing efficiency claims are measured with `npm run benchmark:agent-browse
|
|
|
62
89
|
- When using eval --stdin for extraction, return the value you want instead of relying on console.log as the primary result channel. Prefer plain expressions like ({ title: document.title }) or explicitly invoked functions like (() => ({ title: document.title }))(); if a function-shaped snippet returns {}, details.evalStdinHint may warn that the function was serialized instead of called. If get text on a CSS selector surfaces details.selectorTextVisibility or selectorTextVisibilityAll, prefer a visible @ref, a more specific selector, or the inspect-visible-text-candidates nextAction over hidden tab content.
|
|
63
90
|
- When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.
|
|
64
91
|
- When commands save or spill files (screenshots, downloads, PDFs, traces, recordings, HAR, large snapshot spills), use the user's exact requested paths when given and treat paths as provisional until details.artifactVerification shows every row verified: branch on missingCount, pendingCount, unverifiedCount, per-entry state, and optional limitation before downstream file use or PASS/FAIL reporting.
|
|
92
|
+
- For evidence-only screenshots, QA captures, or other audit artifacts, save to an explicit path and branch on details.artifactVerification plus details.artifacts before reporting PASS/FAIL; do not require vision review of inline image attachments unless the user asked for visual inspection.
|
|
93
|
+
- Respect explicit user stop boundaries: if the user says to stop before order/post/purchase/submit, do not click that final action.
|
|
94
|
+
- Successful record stop needs ffmpeg on PATH; the wrapper may warn after record start when ffmpeg is missing.
|
|
65
95
|
- Do not call --help or other exploratory inspection commands unless the user explicitly asks for them or debugging the browser integration is necessary.
|
|
66
96
|
<!-- agent-browser-playbook:end shared-guidelines -->
|
|
67
97
|
|
|
@@ -90,6 +120,8 @@ Illustrative shapes (each real call uses exactly one of `args`, `semanticAction`
|
|
|
90
120
|
- exact CLI args passed after `agent-browser`
|
|
91
121
|
- no shell operators
|
|
92
122
|
- do not include the binary name
|
|
123
|
+
- do not include `--json`; the wrapper injects it (see [Wrapper `--json`](#wrapper-json))
|
|
124
|
+
- first-call recipe: `open` → `snapshot -i` → `click` / `fill` with current `@eN` refs from that snapshot → `snapshot -i` again after navigation or DOM changes
|
|
93
125
|
|
|
94
126
|
Examples:
|
|
95
127
|
|
|
@@ -167,7 +199,9 @@ Examples:
|
|
|
167
199
|
- `waitForDownload` with `path` (compiled as `wait --download <path>`)
|
|
168
200
|
- `screenshot` with `path`
|
|
169
201
|
|
|
170
|
-
|
|
202
|
+
**Navigation assertions are explicit only.** `job` never treats a successful `click` (or a `select` / submit-style interaction that may navigate) as proof that the expected next page loaded. Top-level `click` may still surface optional `details.navigationSummary` or `pageChangeSummary` hints for operators, but compiled `job` / `batch` steps do **not** auto-insert `assertUrl` or `assertText` after clicks—there is no deterministic expected URL source without caller intent. After any navigation-prone step (link/submit clicks, checkout or form flows, tab-sensitive UI), add an explicit `assertUrl` with the destination pattern you expect, `assertText` for on-page copy, or both, **before** screenshots or steps that assume the new page state.
|
|
203
|
+
|
|
204
|
+
Example (static landing page):
|
|
171
205
|
|
|
172
206
|
```json
|
|
173
207
|
{
|
|
@@ -181,12 +215,29 @@ Example:
|
|
|
181
215
|
}
|
|
182
216
|
```
|
|
183
217
|
|
|
184
|
-
|
|
218
|
+
Example (open → fill → click → assert destination → screenshot):
|
|
219
|
+
|
|
220
|
+
```json
|
|
221
|
+
{
|
|
222
|
+
"job": {
|
|
223
|
+
"steps": [
|
|
224
|
+
{ "action": "open", "url": "https://shop.example/checkout" },
|
|
225
|
+
{ "action": "fill", "selector": "#email", "text": "user@example.com" },
|
|
226
|
+
{ "action": "click", "selector": "#continue" },
|
|
227
|
+
{ "action": "assertUrl", "url": "**/shipping" },
|
|
228
|
+
{ "action": "assertText", "text": "Shipping address" },
|
|
229
|
+
{ "action": "screenshot", "path": ".dogfood/shipping.png" }
|
|
230
|
+
]
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Compiled shape for the navigation example:
|
|
185
236
|
|
|
186
237
|
```json
|
|
187
238
|
{
|
|
188
239
|
"args": ["batch"],
|
|
189
|
-
"stdin": "[[\"open\",\"https://example.com\"],[\"wait\",\"--text\",\"
|
|
240
|
+
"stdin": "[[\"open\",\"https://shop.example/checkout\"],[\"fill\",\"#email\",\"user@example.com\"],[\"click\",\"#continue\"],[\"wait\",\"--url\",\"**/shipping\"],[\"wait\",\"--text\",\"Shipping address\"],[\"screenshot\",\".dogfood/shipping.png\"]]"
|
|
190
241
|
}
|
|
191
242
|
```
|
|
192
243
|
|
|
@@ -202,11 +253,12 @@ Because `job` still executes as upstream `batch` with generated stdin, the same
|
|
|
202
253
|
- optional; mutually exclusive with `args`, `semanticAction`, `job`, `sourceLookup`, `networkSourceLookup`, and `electron`
|
|
203
254
|
- lightweight preset built on the same batch compiler path as `job`
|
|
204
255
|
- URL form: clears enabled diagnostic buffers first (`network requests --clear`, `console --clear`, `errors --clear`), then opens `url`, waits with `wait --load <state>` using the resolved `loadState`, optionally asserts `expectedText` (string or string array) and/or `expectedSelector` (each may be omitted for a load-plus-diagnostics-only smoke), then runs enabled diagnostics: `network requests`, `console`, and `errors`
|
|
205
|
-
- attached form: `qa: { attached: true, expectedText?, expectedSelector?, screenshotPath?, checkNetwork?, checkConsole?, checkErrors?, loadState? }` runs the same waits, optional assertions, diagnostics, and screenshot against the current attached managed session without opening a URL. It rejects `url` and cannot be used with `sessionMode: "fresh"`; attach first with `electron.launch` or raw `args: ["connect", "<port-or-url>"]`, then run `qa.attached`.
|
|
256
|
+
- attached form: `qa: { attached: true, expectedText?, expectedSelector?, screenshotPath?, checkNetwork?, checkConsole?, checkErrors?, loadState? }` runs the same waits, optional assertions, diagnostics, and screenshot against the current attached managed session without opening a URL. It rejects `url` and cannot be used with `sessionMode: "fresh"`; attach first with `electron.launch` or raw `args: ["connect", "<port-or-url>"]`, then run `qa.attached`. Before spawning the diagnostic batch, the wrapper preflights the attached session: `get url` must succeed and return an `http:` or `https:` page URL. Missing URLs, read failures, and non-http(s) surfaces fail fast with `failureCategory: "validation-error"`, `details.validationError`, and recovery `nextActions` such as `list-tabs-before-qa-attached` and `snapshot-before-qa-attached` instead of running the full QA batch.
|
|
206
257
|
- `loadState` is optional and must be `domcontentloaded`, `load`, or `networkidle`; it defaults to `domcontentloaded` so analytics-heavy or long-polling pages do not hang routine QA. Use `networkidle` only when the site is expected to go fully quiet.
|
|
207
258
|
- `checkNetwork`, `checkConsole`, and `checkErrors` default to `true`; set a field to `false` to omit that diagnostic
|
|
208
259
|
- optional `screenshotPath` adds an evidence screenshot step
|
|
209
260
|
- reports `details.compiledQaPreset` with the compiled batch plan and resolved `loadState`, plus `details.qaPreset` with `{ passed, failedChecks, warnings, summary }`
|
|
261
|
+
- on success with no failed checks, model-visible prose collapses to a compact pass summary (current page URL/title when known, checks run, optional screenshot path plus artifact verification, pointer to `details.qaPreset` and `details.batchSteps` for the full matrix). Failed QA and QA reclassified to `qa-failure` keep the verbose per-step batch output.
|
|
210
262
|
- fails the native tool result with `failureCategory: "qa-failure"` when diagnostics report page errors, console error messages, actionable failed network requests, or any batch step failure. Benign classification (implementation: `classifyNetworkRequestFailure` → `isBenignAssetFailure` in `extensions/agent-browser/lib/results/network.ts`) applies only when the row is already treated as failed (`status >= 400`, `failed: true`, or a string `error`—see `isFailedNetworkRequest`), the URL path’s last segment matches the icon basename heuristic (`favicon` plus `.ico`/`.png`/`.svg`, or `apple-touch-icon` plus `.png`, each allowing an optional `[-.\w]*` stem suffix before the extension), **and** at least one of `status === 404`, `failed === true`, or `typeof request.error === "string"` holds (so a **status-only** failure such as `500` on that path with neither `failed` nor a string `error` stays actionable). It also requires the upstream `resourceType` / `mimeType` (whichever is present) to be absent or look image-like: `image`, `img`, `other`, or a value starting with `image/`. Those rows are counted in `qaPreset.warnings` (for example `N benign network request failure(s) ignored`) and omitted from the actionable failed-network tally; every other failed request stays actionable.
|
|
211
263
|
|
|
212
264
|
Example:
|
|
@@ -358,7 +410,7 @@ For an app you launched manually with remote debugging enabled, skip `electron.c
|
|
|
358
410
|
|
|
359
411
|
- type: object with at least one of `selector`, `reactFiberId`, or `componentName`
|
|
360
412
|
- optional; mutually exclusive with `args`, `semanticAction`, `job`, `qa`, `networkSourceLookup`, and `electron`
|
|
361
|
-
-
|
|
413
|
+
- **EXPERIMENTAL — candidates only:** opt-in helper for local app debugging; it reports candidate source locations with confidence and evidence instead of claiming a guaranteed DOM-to-file mapping. Do not treat output as authoritative file ownership or edit targets without verification.
|
|
362
414
|
- compiles to existing upstream `batch` commands only:
|
|
363
415
|
- `selector` adds `is visible <selector>` and, unless `includeDomHints: false`, adds `get html <selector>` for source-like DOM attributes (`data-source-file`, `data-file`, `data-component-file`, `data-source`, plus optional `data-source-line` / `data-line` and `data-source-column` / `data-column`) and for `.ts`/`.tsx`/`.js`/`.jsx` paths embedded in HTML text
|
|
364
416
|
- `reactFiberId` runs `react inspect <id>`; this requires the page to have been launched with `--enable react-devtools` before first navigation and for the app build to expose source information
|
|
@@ -383,7 +435,7 @@ Use raw `args` for direct upstream React inspection when you already know the ex
|
|
|
383
435
|
|
|
384
436
|
- type: object with at least one of `requestId`, `filter`, or `url`, plus optional `maxWorkspaceFiles`
|
|
385
437
|
- optional; mutually exclusive with `args`, `semanticAction`, `job`, `qa`, `sourceLookup`, and `electron`
|
|
386
|
-
-
|
|
438
|
+
- **EXPERIMENTAL — candidates only:** failed-request source-hint helper; it reports failed network requests and candidate source hints with evidence instead of assigning blame or proving root cause
|
|
387
439
|
- compiles to existing upstream `batch` commands only: `network request <requestId>` when provided plus `network requests` with `--filter <filter-or-url>` when a filter or URL is provided (if both are set, `filter` wins; when only `url` is set, it becomes the `--filter` argument); optional `session` prepends `--session <name>` before that generated `batch`
|
|
388
440
|
- detects failed requests from `status >= 400`, `failed: true`, or an `error` field
|
|
389
441
|
- candidate sources come from source-like initiator/stack metadata in upstream network results and bounded local workspace search for URL/path literals under the Pi session cwd
|
|
@@ -442,6 +494,8 @@ Recommended use:
|
|
|
442
494
|
|
|
443
495
|
## Wrapper behavior
|
|
444
496
|
|
|
497
|
+
Caller `args` should omit `--json`; the wrapper prepends it for normal execution so `details` and presentation stay structured. See [Wrapper `--json`](#wrapper-json).
|
|
498
|
+
|
|
445
499
|
The extension should:
|
|
446
500
|
- inject `--json`
|
|
447
501
|
- invoke `agent-browser` directly, not through a shell
|
|
@@ -717,6 +717,7 @@ interface ElectronBroadGetTextScopeDiagnostic {
|
|
|
717
717
|
}
|
|
718
718
|
|
|
719
719
|
interface QaAttachedTarget {
|
|
720
|
+
error?: string;
|
|
720
721
|
sessionName: string;
|
|
721
722
|
title?: string;
|
|
722
723
|
url?: string;
|
|
@@ -3062,7 +3063,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
3062
3063
|
name: "agent_browser",
|
|
3063
3064
|
label: "Agent Browser",
|
|
3064
3065
|
description:
|
|
3065
|
-
"Browse and interact with websites using agent-browser. Use this for web research, reading live docs, opening pages, taking snapshots or screenshots, clicking links, filling forms, extracting page content, and authenticated/profile-based browser work.",
|
|
3066
|
+
"Browse and interact with websites using agent-browser. Use this for web research, reading live docs, opening pages, taking snapshots or screenshots, clicking links, filling forms, extracting page content, and authenticated/profile-based browser work. Input choice: default `args` for open → snapshot -i → click/fill @refs; `semanticAction` for stable role/text/label targets; `job` or `qa` for multi-step checks; `electron` only for desktop apps; experimental `sourceLookup` / `networkSourceLookup` for candidates only.",
|
|
3066
3067
|
promptSnippet:
|
|
3067
3068
|
"Browse websites, read live docs, click and fill pages, extract browser content, take screenshots, and automate real web workflows.",
|
|
3068
3069
|
promptGuidelines: toolPromptGuidelines,
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Scope: Job and QA modes only.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
import type { ArtifactVerificationSummary } from "../results/contracts.js";
|
|
7
8
|
import { isRecord } from "../parsing.js";
|
|
8
9
|
import { summarizeNetworkFailures } from "../results/network.js";
|
|
9
10
|
import { getBatchResultItems, getCommandNameFromBatchItem, getSelectValues } from "./shared.js";
|
|
@@ -93,6 +94,67 @@ export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAge
|
|
|
93
94
|
return { compiled: { args: ["batch"], stdin: JSON.stringify(steps.map((step) => step.args)), steps } };
|
|
94
95
|
}
|
|
95
96
|
|
|
97
|
+
export function isHttpOrHttpsUrl(url: string): boolean {
|
|
98
|
+
try {
|
|
99
|
+
const protocol = new URL(url).protocol;
|
|
100
|
+
return protocol === "http:" || protocol === "https:";
|
|
101
|
+
} catch {
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function describeQaChecksRun(checks: CompiledAgentBrowserQaPreset["checks"]): string {
|
|
107
|
+
const parts = [`load:${checks.loadState}`];
|
|
108
|
+
if (checks.expectedText.length > 0) parts.push(`text×${checks.expectedText.length}`);
|
|
109
|
+
if (checks.expectedSelector) parts.push("selector");
|
|
110
|
+
if (checks.checkNetwork) parts.push("network");
|
|
111
|
+
if (checks.checkConsole) parts.push("console");
|
|
112
|
+
if (checks.checkErrors) parts.push("errors");
|
|
113
|
+
if (checks.screenshotPath) parts.push("screenshot");
|
|
114
|
+
return parts.join(", ");
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function extractQaPageContext(options: {
|
|
118
|
+
attachedTarget?: { title?: string; url?: string };
|
|
119
|
+
batchData?: unknown;
|
|
120
|
+
compiled?: CompiledAgentBrowserQaPreset;
|
|
121
|
+
}): { title?: string; url?: string } {
|
|
122
|
+
if (options.attachedTarget?.title || options.attachedTarget?.url) {
|
|
123
|
+
return { title: options.attachedTarget.title, url: options.attachedTarget.url };
|
|
124
|
+
}
|
|
125
|
+
for (const item of getBatchResultItems(options.batchData)) {
|
|
126
|
+
if (getCommandNameFromBatchItem(item) !== "open" || !isRecord(item.result)) continue;
|
|
127
|
+
const url = typeof item.result.url === "string" ? item.result.url : undefined;
|
|
128
|
+
const title = typeof item.result.title === "string" ? item.result.title : undefined;
|
|
129
|
+
if (url || title) return { title, url };
|
|
130
|
+
}
|
|
131
|
+
if (options.compiled?.checks.url) {
|
|
132
|
+
return { url: options.compiled.checks.url };
|
|
133
|
+
}
|
|
134
|
+
return {};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function buildQaCompactPassText(options: {
|
|
138
|
+
artifactVerification?: ArtifactVerificationSummary;
|
|
139
|
+
batchStepCount: number;
|
|
140
|
+
checks: CompiledAgentBrowserQaPreset["checks"];
|
|
141
|
+
page?: { title?: string; url?: string };
|
|
142
|
+
qaPreset: AgentBrowserQaPresetAnalysis;
|
|
143
|
+
}): string {
|
|
144
|
+
const lines = [options.qaPreset.summary];
|
|
145
|
+
const pageParts = [options.page?.title, options.page?.url].filter((part): part is string => typeof part === "string" && part.length > 0);
|
|
146
|
+
if (pageParts.length > 0) lines.push(`Page: ${pageParts.join(" — ")}`);
|
|
147
|
+
lines.push(`Checks run: ${describeQaChecksRun(options.checks)} (${options.batchStepCount} batch step${options.batchStepCount === 1 ? "" : "s"})`);
|
|
148
|
+
if (options.checks.screenshotPath) {
|
|
149
|
+
const verification = options.artifactVerification;
|
|
150
|
+
lines.push(verification
|
|
151
|
+
? `Screenshot: ${options.checks.screenshotPath} (${verification.verifiedCount}/${verification.artifacts.length} verified on disk)`
|
|
152
|
+
: `Screenshot: ${options.checks.screenshotPath}`);
|
|
153
|
+
}
|
|
154
|
+
lines.push("Full diagnostic matrix: see details.qaPreset and details.batchSteps.");
|
|
155
|
+
return lines.join("\n");
|
|
156
|
+
}
|
|
157
|
+
|
|
96
158
|
export function analyzeQaPresetResults(data: unknown): AgentBrowserQaPresetAnalysis | undefined {
|
|
97
159
|
const items = getBatchResultItems(data);
|
|
98
160
|
if (items.length === 0) return undefined;
|
|
@@ -25,8 +25,8 @@ import {
|
|
|
25
25
|
export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
26
26
|
|
|
27
27
|
args: Type.Optional(
|
|
28
|
-
Type.Array(Type.String({ description: "Exact agent-browser CLI arguments, excluding the binary name." }), {
|
|
29
|
-
description: "Exact agent-browser CLI arguments, excluding the binary name and any shell operators. Required unless semanticAction, job, qa, sourceLookup, networkSourceLookup, or electron is provided.",
|
|
28
|
+
Type.Array(Type.String({ description: "Exact agent-browser CLI arguments, excluding the binary name. Do not pass --json; the wrapper injects it. First-call recipe: open → snapshot -i → click/fill @eN → snapshot -i." }), {
|
|
29
|
+
description: "Exact agent-browser CLI arguments, excluding the binary name and any shell operators. Required unless semanticAction, job, qa, sourceLookup, networkSourceLookup, or electron is provided. Do not include --json (wrapper injects it). Typical first calls: open, snapshot -i, click/fill current @refs, then snapshot -i again after navigation or DOM changes.",
|
|
30
30
|
minItems: 1,
|
|
31
31
|
}),
|
|
32
32
|
),
|
|
@@ -79,7 +79,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
79
79
|
componentName: Type.Optional(Type.String({ description: "Component name to correlate with react tree output and bounded local workspace search." })),
|
|
80
80
|
includeDomHints: Type.Optional(Type.Boolean({ description: "Whether selector lookups should inspect DOM HTML attributes for source-like metadata. Defaults to true." })),
|
|
81
81
|
maxWorkspaceFiles: Type.Optional(Type.Number({ description: "Maximum local source files to scan when componentName is provided. Defaults to 2000 and cannot exceed 5000.", minimum: 1, maximum: SOURCE_LOOKUP_MAX_WORKSPACE_FILES })),
|
|
82
|
-
}),
|
|
82
|
+
}, { description: "EXPERIMENTAL: local UI-to-source candidates only (confidence/evidence, not guaranteed mappings). Compiles to batch; mutually exclusive with other input modes." }),
|
|
83
83
|
),
|
|
84
84
|
networkSourceLookup: Type.Optional(
|
|
85
85
|
Type.Object({
|
|
@@ -88,7 +88,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
88
88
|
session: Type.Optional(Type.String({ description: "Optional upstream session name; prepends --session <name> before the generated batch." })),
|
|
89
89
|
url: Type.Optional(Type.String({ description: "Optional failed request URL or URL fragment to correlate with local source." })),
|
|
90
90
|
maxWorkspaceFiles: Type.Optional(Type.Number({ description: "Maximum local source files to scan for URL literals. Defaults to 2000 and cannot exceed 5000.", minimum: 1, maximum: SOURCE_LOOKUP_MAX_WORKSPACE_FILES })),
|
|
91
|
-
}),
|
|
91
|
+
}, { description: "EXPERIMENTAL: failed-request-to-source candidates only (initiator metadata and bounded workspace URL literals; not definitive blame). Compiles to batch; mutually exclusive with other input modes." }),
|
|
92
92
|
),
|
|
93
93
|
electron: Type.Optional(
|
|
94
94
|
Type.Union([
|