pi-agent-browser-native 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +89 -2
- package/docs/ARCHITECTURE.md +7 -3
- package/docs/TOOL_CONTRACT.md +18 -10
- package/extensions/agent-browser/index.ts +123 -26
- package/extensions/agent-browser/lib/results/presentation.ts +233 -21
- package/extensions/agent-browser/lib/results/shared.ts +17 -0
- package/extensions/agent-browser/lib/results/snapshot.ts +32 -16
- package/extensions/agent-browser/lib/runtime.ts +22 -12
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.0 - 2026-04-12
|
|
4
|
+
|
|
5
|
+
### Changed
|
|
6
|
+
- `batch` now reuses the richer standalone renderers, so batched snapshots keep the compact main-content-first view and batched screenshots keep inline image attachments instead of degrading to raw JSON-ish text
|
|
7
|
+
- the tool schema now uses `sessionMode: "auto" | "fresh"` instead of the old implicit-session boolean so agents have a first-class way to request a fresh profiled/debug launch, and blocked startup-scoped reuse errors now include structured recovery hints
|
|
8
|
+
- plain-text inspection commands like `agent_browser --help` and `--version` are now always allowed, removing the old prompt-dependent inspection gate and making the inspection contract local and predictable
|
|
9
|
+
- navigation actions like `click`, `dblclick`, `back`, `forward`, and `reload` now include lightweight post-action title/url summaries when the wrapper can address the active session, reducing guess-and-check follow-up snapshots
|
|
10
|
+
- compact snapshot rendering is leaner by default: fewer additional sections, fewer refs, smaller role summaries, and the raw spill path now stays in `details.fullOutputPath` instead of dominating the visible snapshot body
|
|
11
|
+
- README and injected tool guidance now include a compact agent quick start with the core call shapes for `open` + `snapshot`, `click` + re-snapshot, `batch`, `eval --stdin`, and fresh profiled launches
|
|
12
|
+
|
|
13
|
+
### Migration notes
|
|
14
|
+
- replace any use of `useActiveSession` with `sessionMode`
|
|
15
|
+
- use `sessionMode: "fresh"` when you need a new `--profile`, `--session-name`, or `--cdp` launch after the implicit session is already active
|
|
16
|
+
|
|
3
17
|
## 0.1.6 - 2026-04-12
|
|
4
18
|
|
|
5
19
|
### Changed
|
package/README.md
CHANGED
|
@@ -87,6 +87,67 @@ This avoids duplicate `agent_browser` registrations if you also have the publish
|
|
|
87
87
|
|
|
88
88
|
The native tool exposed to the agent is named `agent_browser`.
|
|
89
89
|
|
|
90
|
+
The primary session control parameter is `sessionMode`:
|
|
91
|
+
|
|
92
|
+
- `"auto"` (default) reuses the implicit `pi`-scoped session when possible
|
|
93
|
+
- `"fresh"` skips that implicit session so startup-scoped flags like `--profile`, `--session-name`, and `--cdp` can launch a fresh upstream session
|
|
94
|
+
|
|
95
|
+
## Agent quick start
|
|
96
|
+
|
|
97
|
+
### Mental model
|
|
98
|
+
|
|
99
|
+
- `args` — exact CLI args after `agent-browser`
|
|
100
|
+
- `stdin` — raw stdin only for `batch` and `eval --stdin`
|
|
101
|
+
- `sessionMode`
|
|
102
|
+
- `"auto"` — default, reuse the implicit `pi`-scoped session
|
|
103
|
+
- `"fresh"` — skip the implicit session for a new profile/debug launch
|
|
104
|
+
|
|
105
|
+
### Common call shapes
|
|
106
|
+
|
|
107
|
+
Open a page, then take an interactive snapshot:
|
|
108
|
+
|
|
109
|
+
```json
|
|
110
|
+
{ "args": ["open", "https://example.com"] }
|
|
111
|
+
{ "args": ["snapshot", "-i"] }
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Click a ref, then re-snapshot after navigation or a major DOM change:
|
|
115
|
+
|
|
116
|
+
```json
|
|
117
|
+
{ "args": ["click", "@e2"] }
|
|
118
|
+
{ "args": ["snapshot", "-i"] }
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Run a multi-step browser flow in one tool call:
|
|
122
|
+
|
|
123
|
+
```json
|
|
124
|
+
{ "args": ["batch"], "stdin": "[[\"open\",\"https://example.com\"],[\"snapshot\",\"-i\"]]" }
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Evaluate page JavaScript via stdin:
|
|
128
|
+
|
|
129
|
+
```json
|
|
130
|
+
{ "args": ["eval", "--stdin"], "stdin": "document.title" }
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Start a fresh profiled launch after you already used the implicit session:
|
|
134
|
+
|
|
135
|
+
```json
|
|
136
|
+
{ "args": ["--profile", "Default", "open", "https://example.com/account"], "sessionMode": "fresh" }
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Name a new upstream session explicitly when you want to keep reusing it:
|
|
140
|
+
|
|
141
|
+
```json
|
|
142
|
+
{ "args": ["--session", "auth-flow", "open", "https://example.com"] }
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### First useful prompt in a fresh `pi` session
|
|
146
|
+
|
|
147
|
+
```text
|
|
148
|
+
Use the agent_browser tool to open https://react.dev and then take an interactive snapshot.
|
|
149
|
+
```
|
|
150
|
+
|
|
90
151
|
## Local development
|
|
91
152
|
|
|
92
153
|
Do not track or rely on a repo-local `.pi/extensions/agent-browser.ts` autoload shim for this package. When the package is also installed globally, that creates a duplicate `agent_browser` registration and blocks `pi` startup from this working directory.
|
|
@@ -116,14 +177,40 @@ Validated workflow examples:
|
|
|
116
177
|
- run `batch` with JSON via `stdin`
|
|
117
178
|
- run `eval --stdin`
|
|
118
179
|
- take a screenshot with inline attachment support
|
|
119
|
-
- inspect `agent_browser --help` and `--version`
|
|
180
|
+
- inspect `agent_browser --help` and `--version` via the tool's plain-text inspection fallback
|
|
181
|
+
|
|
182
|
+
Inspection commands like `agent_browser --help` and `--version` are always supported. They return plain text and are useful for debugging or capability checks, but they are not required for normal browsing workflows.
|
|
120
183
|
|
|
121
184
|
Current cautions:
|
|
122
185
|
- passing `--profile` is an explicit upstream choice; this extension does not add its own profile-cloning or isolation layer
|
|
123
|
-
- startup-scoped flags like `--profile`, `--session-name`, and `--cdp` are for the first command that launches a session; if the implicit session is already active,
|
|
186
|
+
- startup-scoped flags like `--profile`, `--session-name`, and `--cdp` are for the first command that launches a session; if the implicit session is already active, retry that call with `sessionMode: "fresh"` or provide an explicit `--session ...` for the new launch
|
|
124
187
|
- implicit `piab-*` sessions are extension-managed convenience sessions; they are best-effort closed on `pi` shutdown, get an idle timeout to reduce stale background daemons, and clean up private temp spill artifacts on shutdown
|
|
125
188
|
- explicit upstream sessions like `--session`, `--profile`, `--session-name`, and `--cdp` are treated as user-managed and are not auto-closed by the extension
|
|
126
189
|
|
|
190
|
+
### Switching from public browsing to a fresh profile/debug launch
|
|
191
|
+
|
|
192
|
+
A common agent workflow is:
|
|
193
|
+
|
|
194
|
+
1. browse a public page with the default implicit session
|
|
195
|
+
2. then switch to a fresh authenticated/profile/debug launch
|
|
196
|
+
|
|
197
|
+
Use `sessionMode: "fresh"` for that transition instead of relying on the implicit session:
|
|
198
|
+
|
|
199
|
+
```json
|
|
200
|
+
{
|
|
201
|
+
"args": ["--profile", "Default", "open", "https://example.com/account"],
|
|
202
|
+
"sessionMode": "fresh"
|
|
203
|
+
}
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
If you want to name the new upstream session yourself, pass an explicit session instead:
|
|
207
|
+
|
|
208
|
+
```json
|
|
209
|
+
{
|
|
210
|
+
"args": ["--session", "auth-flow", "--profile", "Default", "open", "https://example.com/account"]
|
|
211
|
+
}
|
|
212
|
+
```
|
|
213
|
+
|
|
127
214
|
## Docs
|
|
128
215
|
|
|
129
216
|
- [`docs/REQUIREMENTS.md`](docs/REQUIREMENTS.md) — product requirements and constraints
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -59,17 +59,19 @@ The published package should exclude agent-only and superseded repo materials su
|
|
|
59
59
|
|
|
60
60
|
### Default
|
|
61
61
|
|
|
62
|
-
If the caller does not provide `--session`, the extension should use an implicit session name derived from the current `pi` session id.
|
|
62
|
+
If the caller does not provide `--session`, the extension should default to `sessionMode: "auto"` and use an implicit session name derived from the current `pi` session id.
|
|
63
63
|
|
|
64
64
|
Why:
|
|
65
65
|
- works out of the box
|
|
66
66
|
- gives continuity across calls
|
|
67
67
|
- avoids forcing the agent to invent session names for basic browsing
|
|
68
68
|
|
|
69
|
-
### Explicit upstream sessions
|
|
69
|
+
### Explicit upstream sessions and fresh launches
|
|
70
70
|
|
|
71
71
|
If the caller provides `--session`, `--profile`, `--cdp`, or similar upstream flags, the extension should respect them with minimal interference.
|
|
72
72
|
|
|
73
|
+
The tool should also expose a first-class `sessionMode: "fresh"` escape hatch so agents can intentionally skip the implicit session and launch a fresh upstream session without inventing a fixed explicit session name.
|
|
74
|
+
|
|
73
75
|
### Ownership
|
|
74
76
|
|
|
75
77
|
V1 ownership rule:
|
|
@@ -92,7 +94,9 @@ The extension should surface that clearly and avoid hidden restart behavior in v
|
|
|
92
94
|
|
|
93
95
|
That means explicit startup-scoping flags like `--profile`, `--session-name`, and `--cdp` should remain explicit upstream choices instead of being wrapped in extra hidden restart or cloning logic.
|
|
94
96
|
|
|
95
|
-
If the implicit session is already active and one of those startup-scoped flags appears again
|
|
97
|
+
If the implicit session is already active and one of those startup-scoped flags appears again while `sessionMode` is still `"auto"`, the extension should fail clearly instead of silently sending a command shape that upstream would ignore.
|
|
98
|
+
|
|
99
|
+
That failure should include a structured recovery hint pointing to `sessionMode: "fresh"` as the first-line fix, while still allowing an explicit `--session` when the caller wants to name the new upstream session.
|
|
96
100
|
|
|
97
101
|
## Preferring the native tool
|
|
98
102
|
|
package/docs/TOOL_CONTRACT.md
CHANGED
|
@@ -32,7 +32,7 @@ The tool also needs an operating playbook, not just a capability list. The model
|
|
|
32
32
|
{
|
|
33
33
|
"args": ["open", "https://example.com"],
|
|
34
34
|
"stdin": "optional raw stdin content",
|
|
35
|
-
"
|
|
35
|
+
"sessionMode": "auto"
|
|
36
36
|
}
|
|
37
37
|
```
|
|
38
38
|
|
|
@@ -69,15 +69,20 @@ Examples:
|
|
|
69
69
|
{ "args": ["batch"], "stdin": "[[\"open\",\"https://example.com\"],[\"snapshot\",\"-i\"]]" }
|
|
70
70
|
```
|
|
71
71
|
|
|
72
|
-
### `
|
|
72
|
+
### `sessionMode`
|
|
73
73
|
|
|
74
|
-
- type: `
|
|
74
|
+
- type: `"auto" | "fresh"`
|
|
75
75
|
- optional
|
|
76
|
-
- default: `
|
|
76
|
+
- default: `"auto"`
|
|
77
77
|
|
|
78
78
|
Behavior:
|
|
79
79
|
- if `args` already include `--session`, upstream session choice wins
|
|
80
|
-
-
|
|
80
|
+
- `"auto"` prepends the implicit active session when appropriate
|
|
81
|
+
- `"fresh"` skips the implicit session so startup-scoped flags like `--profile`, `--session-name`, or `--cdp` can launch a fresh upstream session
|
|
82
|
+
|
|
83
|
+
Recommended use:
|
|
84
|
+
- use `"auto"` for the common browse/snapshot/click flow inside one `pi` session
|
|
85
|
+
- use `"fresh"` when switching from an already-active implicit session to a new profile/debug/auth launch without inventing a fixed explicit session name
|
|
81
86
|
|
|
82
87
|
## Wrapper behavior
|
|
83
88
|
|
|
@@ -87,8 +92,8 @@ The extension should:
|
|
|
87
92
|
- parse JSON output into tool details
|
|
88
93
|
- handle observed JSON result shapes, including the array returned by `batch --json`
|
|
89
94
|
- allow plain-text fallback for inspection commands like `--help` and `--version`
|
|
90
|
-
-
|
|
91
|
-
-
|
|
95
|
+
- support those inspection commands unconditionally so the tool contract stays local and predictable
|
|
96
|
+
- still describe normal browser workflows in guidance so models do not overuse inspection for routine tasks
|
|
92
97
|
- surface stderr and non-zero exits clearly
|
|
93
98
|
- attach images when the result points to a screenshot-like artifact
|
|
94
99
|
|
|
@@ -104,7 +109,8 @@ Primary content should be:
|
|
|
104
109
|
|
|
105
110
|
Examples:
|
|
106
111
|
- small `snapshot` results should include the actual snapshot text
|
|
107
|
-
- oversized `snapshot` results should switch to a compact view that preserves the primary content, nearby sections, high-value refs,
|
|
112
|
+
- oversized `snapshot` results should switch to a compact view that preserves the primary content, nearby sections, and a trimmed set of high-value refs, while exposing the full raw snapshot path via `details.fullOutputPath`
|
|
113
|
+
- successful navigation actions like `click`, `back`, `forward`, and `reload` should include a lightweight post-action title/url summary when the wrapper can address the active session
|
|
108
114
|
- `tab list` should include a readable tab summary
|
|
109
115
|
- `screenshot` should include the saved-path summary plus the inline image attachment when available
|
|
110
116
|
|
|
@@ -116,6 +122,7 @@ Recommended details:
|
|
|
116
122
|
{
|
|
117
123
|
"args": ["snapshot", "-i"],
|
|
118
124
|
"effectiveArgs": ["--session", "pi-abc123", "--json", "snapshot", "-i"],
|
|
125
|
+
"sessionMode": "auto",
|
|
119
126
|
"sessionName": "pi-abc123",
|
|
120
127
|
"usedImplicitSession": true,
|
|
121
128
|
"data": {
|
|
@@ -136,7 +143,8 @@ For oversized snapshots, details should switch to a compact metadata object and
|
|
|
136
143
|
|
|
137
144
|
Worth doing in v1:
|
|
138
145
|
- screenshots → inline image attachment
|
|
139
|
-
- snapshots → origin + ref count + main-content-first compact preview, with
|
|
146
|
+
- snapshots → origin + ref count + main-content-first compact preview, with the raw snapshot spill path kept in `details.fullOutputPath` when the inline result would otherwise be too large
|
|
147
|
+
- navigation actions like `click`, `back`, `forward`, and `reload` → lightweight post-action title/url summary when available
|
|
140
148
|
- tab lists → compact summary/table
|
|
141
149
|
- stream status → enabled/connected/port summary
|
|
142
150
|
|
|
@@ -158,7 +166,7 @@ If `agent-browser` is not on `PATH`, fail with a message that:
|
|
|
158
166
|
- clean up private temp spill artifacts owned by the implicit session on shutdown
|
|
159
167
|
- treat explicit upstream session choices like `--session`, `--profile`, `--session-name`, and `--cdp` as user-managed
|
|
160
168
|
- pass explicit `--profile` straight through to upstream `agent-browser`; no profile-cloning or isolation layer is added in v1
|
|
161
|
-
- if startup-scoped flags like `--profile`, `--session-name`, or `--cdp` are supplied after the implicit session is already active
|
|
169
|
+
- if startup-scoped flags like `--profile`, `--session-name`, or `--cdp` are supplied after the implicit session is already active while `sessionMode` is `"auto"`, return a validation error with a structured recovery hint that recommends `sessionMode: "fresh"`
|
|
162
170
|
|
|
163
171
|
## Non-goals
|
|
164
172
|
|
|
@@ -27,21 +27,29 @@ import {
|
|
|
27
27
|
} from "./lib/runtime.js";
|
|
28
28
|
import { cleanupSecureTempArtifacts } from "./lib/temp.js";
|
|
29
29
|
|
|
30
|
+
const DEFAULT_SESSION_MODE = "auto" as const;
|
|
31
|
+
|
|
30
32
|
const AGENT_BROWSER_PARAMS = Type.Object({
|
|
31
33
|
args: Type.Array(Type.String({ description: "Exact agent-browser CLI arguments, excluding the binary name." }), {
|
|
32
34
|
description: "Exact agent-browser CLI arguments, excluding the binary name and any shell operators.",
|
|
33
35
|
minItems: 1,
|
|
34
36
|
}),
|
|
35
37
|
stdin: Type.Optional(Type.String({ description: "Optional raw stdin content for commands like eval --stdin or batch." })),
|
|
36
|
-
|
|
37
|
-
Type.
|
|
38
|
-
description:
|
|
39
|
-
|
|
38
|
+
sessionMode: Type.Optional(
|
|
39
|
+
Type.Union([Type.Literal("auto"), Type.Literal("fresh")], {
|
|
40
|
+
description:
|
|
41
|
+
"Session handling mode. `auto` reuses the implicit pi-scoped session when possible. `fresh` skips the implicit session so startup-scoped flags like --profile, --session-name, or --cdp can launch a fresh upstream session.",
|
|
42
|
+
default: DEFAULT_SESSION_MODE,
|
|
40
43
|
}),
|
|
41
44
|
),
|
|
42
45
|
});
|
|
43
46
|
const PROJECT_RULE_PROMPT =
|
|
44
47
|
"Project rule: when browser automation is needed, prefer the native `agent_browser` tool. Do not run direct `agent-browser` bash commands unless the user explicitly asks for a bash-oriented workflow or browser-integration debugging.";
|
|
48
|
+
const QUICK_START_GUIDELINES = [
|
|
49
|
+
"Quick start mental model: args are the exact agent-browser CLI args after the binary; stdin is only for batch and eval --stdin; sessionMode=fresh starts a fresh upstream launch when you need new --profile, --session-name, or --cdp state.",
|
|
50
|
+
"Common first calls: { args: [\"open\", \"https://example.com\"] } then { args: [\"snapshot\", \"-i\"] }; after navigation, use { args: [\"click\", \"@e2\"] } then { args: [\"snapshot\", \"-i\"] }.",
|
|
51
|
+
"Common advanced calls: { args: [\"batch\"], stdin: \"[[\\\"open\\\",\\\"https://example.com\\\"],[\\\"snapshot\\\",\\\"-i\\\"]]\" }, { args: [\"eval\", \"--stdin\"], stdin: \"document.title\" }, and { args: [\"--profile\", \"Default\", \"open\", \"https://example.com/account\"], sessionMode: \"fresh\" }.",
|
|
52
|
+
] as const;
|
|
45
53
|
const BRAVE_SEARCH_PROMPT_GUIDELINE =
|
|
46
54
|
"When a non-empty BRAVE_API_KEY is available in the current environment, prefer the Brave Search API via bash/curl to discover specific destination URLs, then open the chosen URL with agent_browser instead of browsing a search engine results page just to find the target.";
|
|
47
55
|
const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
|
|
@@ -49,6 +57,7 @@ const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
|
|
|
49
57
|
"For authenticated or user-specific content like feeds, inboxes, dashboards, and accounts, prefer --profile Default on the first browser call and let the implicit session carry continuity. Use --auto-connect only if profile-based reuse is unavailable or the task is specifically about attaching to a running debug-enabled browser.",
|
|
50
58
|
"Do not invent fixed explicit session names for routine tasks. Use the implicit session unless you truly need multiple isolated browser sessions in the same conversation.",
|
|
51
59
|
"When using --profile, --session-name, or --cdp, put them on the first command for that session. If you intentionally use an explicit --session, keep using that same explicit session for follow-ups.",
|
|
60
|
+
"If you already used the implicit session and now need startup-scoped flags like --profile, --session-name, or --cdp, retry with sessionMode set to fresh or pass an explicit --session for the new launch.",
|
|
52
61
|
"If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <n> / snapshot -i to recover state before retrying different URLs or fallback strategies. Only use wait with an explicit argument like milliseconds, --load, --url, --fn, or --text.",
|
|
53
62
|
"For feed, timeline, or inbox reading tasks, focus on the main timeline/list region and read the first item there rather than unrelated composer or sidebar content.",
|
|
54
63
|
"For read-only browsing tasks, prefer extracting the answer from the current snapshot, structured ref labels, or eval --stdin on the current page before navigating away. Only click into media viewers, detail routes, or new pages when the current view does not contain the needed information.",
|
|
@@ -62,7 +71,8 @@ const TOOL_PROMPT_GUIDELINES_SUFFIX = [
|
|
|
62
71
|
"Do not fall back to osascript, AppleScript, or generic browser-driving bash commands when this tool can do the job.",
|
|
63
72
|
"Pass exact agent-browser CLI arguments in args, excluding the binary name.",
|
|
64
73
|
"Use stdin for commands like eval --stdin and batch instead of shell heredocs.",
|
|
65
|
-
"Let the implicit session handle the common path unless you explicitly need upstream flags like --
|
|
74
|
+
"Let the implicit session handle the common path unless you explicitly need a fresh launch for upstream flags like --profile, --session-name, or --cdp.",
|
|
75
|
+
"Use sessionMode=fresh when switching from an existing implicit session to a new profile/debug launch without inventing a fixed explicit session name.",
|
|
66
76
|
] as const;
|
|
67
77
|
|
|
68
78
|
function buildMissingBinaryMessage(): string {
|
|
@@ -92,15 +102,81 @@ function isPlainTextInspectionArgs(args: string[]): boolean {
|
|
|
92
102
|
return args.includes("--help") || args.includes("-h") || args.includes("--version") || args.includes("-V");
|
|
93
103
|
}
|
|
94
104
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
105
|
+
const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
|
|
106
|
+
|
|
107
|
+
interface NavigationSummary {
|
|
108
|
+
title?: string;
|
|
109
|
+
url?: string;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
113
|
+
return typeof value === "object" && value !== null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function shouldCaptureNavigationSummary(command: string | undefined, data: unknown): boolean {
|
|
117
|
+
return (
|
|
118
|
+
command !== undefined &&
|
|
119
|
+
NAVIGATION_SUMMARY_COMMANDS.has(command) &&
|
|
120
|
+
(!isRecord(data) || (typeof data.title !== "string" && typeof data.url !== "string"))
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function extractStringResultField(data: unknown, fieldName: "title" | "url"): string | undefined {
|
|
125
|
+
if (typeof data === "string") {
|
|
126
|
+
const text = data.trim();
|
|
127
|
+
return text.length > 0 ? text : undefined;
|
|
128
|
+
}
|
|
129
|
+
if (!isRecord(data) || typeof data[fieldName] !== "string") {
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
const text = data[fieldName].trim();
|
|
133
|
+
return text.length > 0 ? text : undefined;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async function collectNavigationSummary(options: {
|
|
137
|
+
cwd: string;
|
|
138
|
+
sessionName?: string;
|
|
139
|
+
signal?: AbortSignal;
|
|
140
|
+
}): Promise<NavigationSummary | undefined> {
|
|
141
|
+
const { cwd, sessionName, signal } = options;
|
|
142
|
+
if (!sessionName) return undefined;
|
|
143
|
+
|
|
144
|
+
const readField = async (fieldName: "title" | "url"): Promise<string | undefined> => {
|
|
145
|
+
const processResult = await runAgentBrowserProcess({
|
|
146
|
+
args: ["--json", "--session", sessionName, "get", fieldName],
|
|
147
|
+
cwd,
|
|
148
|
+
signal,
|
|
149
|
+
});
|
|
150
|
+
if (processResult.aborted || processResult.spawnError || processResult.exitCode !== 0) {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
const parsed = await parseAgentBrowserEnvelope({
|
|
154
|
+
stdout: processResult.stdout,
|
|
155
|
+
stdoutPath: processResult.stdoutSpillPath,
|
|
156
|
+
});
|
|
157
|
+
try {
|
|
158
|
+
if (parsed.parseError || parsed.envelope?.success === false) {
|
|
159
|
+
return undefined;
|
|
160
|
+
}
|
|
161
|
+
return extractStringResultField(parsed.envelope?.data, fieldName);
|
|
162
|
+
} finally {
|
|
163
|
+
if (processResult.stdoutSpillPath) {
|
|
164
|
+
await rm(processResult.stdoutSpillPath, { force: true }).catch(() => undefined);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
const title = await readField("title");
|
|
170
|
+
const url = await readField("url");
|
|
171
|
+
if (!title && !url) return undefined;
|
|
172
|
+
return { title, url };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function mergeNavigationSummaryIntoData(data: unknown, navigationSummary: NavigationSummary): unknown {
|
|
176
|
+
if (isRecord(data)) {
|
|
177
|
+
return { ...data, navigationSummary };
|
|
178
|
+
}
|
|
179
|
+
return { navigationSummary, result: data };
|
|
104
180
|
}
|
|
105
181
|
|
|
106
182
|
function buildSharedBrowserPlaybookGuidelines(hasBraveApiKey: boolean): string[] {
|
|
@@ -115,6 +191,9 @@ function buildBrowserSystemPromptAppendix(hasBraveApiKey: boolean): string {
|
|
|
115
191
|
return [
|
|
116
192
|
PROJECT_RULE_PROMPT,
|
|
117
193
|
"",
|
|
194
|
+
"Quick start:",
|
|
195
|
+
...QUICK_START_GUIDELINES.map((guideline) => `- ${guideline}`),
|
|
196
|
+
"",
|
|
118
197
|
"Browser operating playbook:",
|
|
119
198
|
...buildSharedBrowserPlaybookGuidelines(hasBraveApiKey).map((guideline) => `- ${guideline}`),
|
|
120
199
|
].join("\n");
|
|
@@ -123,6 +202,7 @@ function buildBrowserSystemPromptAppendix(hasBraveApiKey: boolean): string {
|
|
|
123
202
|
function buildToolPromptGuidelines(hasBraveApiKey: boolean): string[] {
|
|
124
203
|
return [
|
|
125
204
|
...TOOL_PROMPT_GUIDELINES_PREFIX,
|
|
205
|
+
...QUICK_START_GUIDELINES,
|
|
126
206
|
...buildSharedBrowserPlaybookGuidelines(hasBraveApiKey),
|
|
127
207
|
...TOOL_PROMPT_GUIDELINES_SUFFIX,
|
|
128
208
|
];
|
|
@@ -194,16 +274,6 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
194
274
|
promptGuidelines: toolPromptGuidelines,
|
|
195
275
|
parameters: AGENT_BROWSER_PARAMS,
|
|
196
276
|
async execute(_toolCallId, params, signal, onUpdate, ctx) {
|
|
197
|
-
const promptPolicy = buildPromptPolicy(getLatestUserPrompt(ctx.sessionManager.getBranch()));
|
|
198
|
-
if (!promptPolicy.allowAgentBrowserInspection && isPlainTextInspectionArgs(params.args)) {
|
|
199
|
-
const errorText = buildInspectionDeflectionMessage();
|
|
200
|
-
return {
|
|
201
|
-
content: [{ type: "text", text: errorText }],
|
|
202
|
-
details: { args: params.args, inspectionBlocked: true },
|
|
203
|
-
isError: true,
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
|
|
207
277
|
const validationError = validateToolArgs(params.args);
|
|
208
278
|
if (validationError) {
|
|
209
279
|
return {
|
|
@@ -213,10 +283,11 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
213
283
|
};
|
|
214
284
|
}
|
|
215
285
|
|
|
286
|
+
const sessionMode = params.sessionMode ?? DEFAULT_SESSION_MODE;
|
|
216
287
|
const executionPlan = buildExecutionPlan(params.args, {
|
|
217
288
|
implicitSessionActive,
|
|
218
289
|
implicitSessionName,
|
|
219
|
-
|
|
290
|
+
sessionMode,
|
|
220
291
|
});
|
|
221
292
|
|
|
222
293
|
if (executionPlan.validationError) {
|
|
@@ -224,6 +295,8 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
224
295
|
content: [{ type: "text", text: executionPlan.validationError }],
|
|
225
296
|
details: {
|
|
226
297
|
args: params.args,
|
|
298
|
+
sessionMode,
|
|
299
|
+
sessionRecoveryHint: executionPlan.recoveryHint,
|
|
227
300
|
startupScopedFlags: executionPlan.startupScopedFlags,
|
|
228
301
|
validationError: executionPlan.validationError,
|
|
229
302
|
},
|
|
@@ -235,6 +308,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
235
308
|
content: [{ type: "text", text: `Running agent-browser ${buildInvocationPreview(executionPlan.effectiveArgs)}` }],
|
|
236
309
|
details: {
|
|
237
310
|
effectiveArgs: executionPlan.effectiveArgs,
|
|
311
|
+
sessionMode,
|
|
238
312
|
sessionName: executionPlan.sessionName,
|
|
239
313
|
usedImplicitSession: executionPlan.usedImplicitSession,
|
|
240
314
|
},
|
|
@@ -257,6 +331,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
257
331
|
details: {
|
|
258
332
|
args: params.args,
|
|
259
333
|
effectiveArgs: executionPlan.effectiveArgs,
|
|
334
|
+
sessionMode,
|
|
260
335
|
spawnError: processResult.spawnError.message,
|
|
261
336
|
},
|
|
262
337
|
isError: true,
|
|
@@ -268,12 +343,28 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
268
343
|
stdout: processResult.stdout,
|
|
269
344
|
stdoutPath: processResult.stdoutSpillPath,
|
|
270
345
|
});
|
|
346
|
+
let presentationEnvelope = parsed.envelope;
|
|
271
347
|
const processSucceeded = !processResult.aborted && !processResult.spawnError && processResult.exitCode === 0;
|
|
272
348
|
const plainTextInspection = isPlainTextInspectionArgs(params.args) && processSucceeded && parsed.parseError !== undefined;
|
|
273
349
|
const envelopeSuccess = plainTextInspection ? true : parsed.envelope?.success !== false;
|
|
274
350
|
const parseSucceeded = plainTextInspection || parsed.parseError === undefined;
|
|
275
351
|
const succeeded = processSucceeded && parseSucceeded && envelopeSuccess;
|
|
276
352
|
|
|
353
|
+
let navigationSummary: NavigationSummary | undefined;
|
|
354
|
+
if (succeeded && shouldCaptureNavigationSummary(executionPlan.commandInfo.command, parsed.envelope?.data)) {
|
|
355
|
+
navigationSummary = await collectNavigationSummary({
|
|
356
|
+
cwd: ctx.cwd,
|
|
357
|
+
sessionName: executionPlan.sessionName,
|
|
358
|
+
signal,
|
|
359
|
+
});
|
|
360
|
+
if (navigationSummary && presentationEnvelope) {
|
|
361
|
+
presentationEnvelope = {
|
|
362
|
+
...presentationEnvelope,
|
|
363
|
+
data: mergeNavigationSummaryIntoData(presentationEnvelope.data, navigationSummary),
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
277
368
|
implicitSessionActive = resolveImplicitSessionActiveState({
|
|
278
369
|
command: executionPlan.commandInfo.command,
|
|
279
370
|
priorActive: implicitSessionActive,
|
|
@@ -300,7 +391,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
300
391
|
: await buildToolPresentation({
|
|
301
392
|
commandInfo: executionPlan.commandInfo,
|
|
302
393
|
cwd: ctx.cwd,
|
|
303
|
-
envelope:
|
|
394
|
+
envelope: presentationEnvelope,
|
|
304
395
|
errorText,
|
|
305
396
|
});
|
|
306
397
|
|
|
@@ -308,16 +399,22 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
308
399
|
content: presentation.content,
|
|
309
400
|
details: {
|
|
310
401
|
args: params.args,
|
|
402
|
+
batchSteps: presentation.batchSteps,
|
|
311
403
|
command: executionPlan.commandInfo.command,
|
|
312
404
|
subcommand: executionPlan.commandInfo.subcommand,
|
|
313
405
|
data: presentation.data,
|
|
314
406
|
error: parsed.envelope?.error,
|
|
407
|
+
navigationSummary,
|
|
315
408
|
effectiveArgs: executionPlan.effectiveArgs,
|
|
316
409
|
exitCode: processResult.exitCode,
|
|
317
410
|
fullOutputPath: presentation.fullOutputPath,
|
|
411
|
+
fullOutputPaths: presentation.fullOutputPaths,
|
|
318
412
|
imagePath: presentation.imagePath,
|
|
413
|
+
imagePaths: presentation.imagePaths,
|
|
319
414
|
parseError: parsed.parseError,
|
|
415
|
+
sessionMode,
|
|
320
416
|
sessionName: executionPlan.sessionName,
|
|
417
|
+
sessionRecoveryHint: executionPlan.recoveryHint,
|
|
321
418
|
startupScopedFlags: executionPlan.startupScopedFlags,
|
|
322
419
|
stderr: processResult.stderr || undefined,
|
|
323
420
|
stdout: parseSucceeded ? undefined : processResult.stdout,
|
|
@@ -9,9 +9,17 @@
|
|
|
9
9
|
import { readFile, stat } from "node:fs/promises";
|
|
10
10
|
import { resolve } from "node:path";
|
|
11
11
|
|
|
12
|
-
import type
|
|
12
|
+
import { parseCommandInfo, type CommandInfo } from "../runtime.js";
|
|
13
13
|
import { buildSnapshotPresentation, formatRawSnapshotText, formatSnapshotSummary } from "./snapshot.js";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
type AgentBrowserBatchResult,
|
|
16
|
+
type AgentBrowserEnvelope,
|
|
17
|
+
type BatchStepPresentationDetails,
|
|
18
|
+
type ToolPresentation,
|
|
19
|
+
isRecord,
|
|
20
|
+
parsePositiveInteger,
|
|
21
|
+
stringifyUnknown,
|
|
22
|
+
} from "./shared.js";
|
|
15
23
|
|
|
16
24
|
const IMAGE_EXTENSION_TO_MIME_TYPE: Record<string, string> = {
|
|
17
25
|
".gif": "image/gif",
|
|
@@ -23,6 +31,13 @@ const IMAGE_EXTENSION_TO_MIME_TYPE: Record<string, string> = {
|
|
|
23
31
|
|
|
24
32
|
const INLINE_IMAGE_MAX_BYTES_ENV = "PI_AGENT_BROWSER_INLINE_IMAGE_MAX_BYTES";
|
|
25
33
|
const DEFAULT_INLINE_IMAGE_MAX_BYTES = 5 * 1_024 * 1_024;
|
|
34
|
+
const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
|
|
35
|
+
const NAVIGATION_SUMMARY_FIELD = "navigationSummary";
|
|
36
|
+
|
|
37
|
+
interface NavigationSummary {
|
|
38
|
+
title?: string;
|
|
39
|
+
url?: string;
|
|
40
|
+
}
|
|
26
41
|
|
|
27
42
|
function getImageMimeType(filePath: string): string | undefined {
|
|
28
43
|
const extension = filePath.toLowerCase().slice(filePath.lastIndexOf("."));
|
|
@@ -90,24 +105,213 @@ function getScreenshotSummary(data: Record<string, unknown>): string | undefined
|
|
|
90
105
|
return typeof data.path === "string" ? `Saved image: ${data.path}` : undefined;
|
|
91
106
|
}
|
|
92
107
|
|
|
93
|
-
function
|
|
94
|
-
return
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
108
|
+
function isNavigationObservableCommand(command: string | undefined): boolean {
|
|
109
|
+
return command !== undefined && NAVIGATION_SUMMARY_COMMANDS.has(command);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function isNavigationSummary(value: unknown): value is NavigationSummary {
|
|
113
|
+
return isRecord(value) && (typeof value.title === "string" || typeof value.url === "string");
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function getNavigationSummary(data: Record<string, unknown>): NavigationSummary | undefined {
|
|
117
|
+
const candidate = data[NAVIGATION_SUMMARY_FIELD];
|
|
118
|
+
return isNavigationSummary(candidate) ? candidate : undefined;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function formatNavigationSummary(summary: NavigationSummary): string | undefined {
|
|
122
|
+
const title = typeof summary.title === "string" && summary.title.trim().length > 0 ? summary.title.trim() : undefined;
|
|
123
|
+
const url = typeof summary.url === "string" && summary.url.trim().length > 0 ? summary.url.trim() : undefined;
|
|
124
|
+
if (!title && !url) return undefined;
|
|
125
|
+
if (title && url) return `${title}\n${url}`;
|
|
126
|
+
return title ?? url;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function stripNavigationSummary(data: Record<string, unknown>): Record<string, unknown> {
|
|
130
|
+
const { [NAVIGATION_SUMMARY_FIELD]: _navigationSummary, ...rest } = data;
|
|
131
|
+
return rest;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function formatNavigationActionResult(data: Record<string, unknown>): string | undefined {
|
|
135
|
+
const actionData = stripNavigationSummary(data);
|
|
136
|
+
const lines: string[] = [];
|
|
137
|
+
if (typeof actionData.clicked === "string" || typeof actionData.clicked === "boolean") {
|
|
138
|
+
lines.push(`Clicked: ${String(actionData.clicked)}`);
|
|
139
|
+
}
|
|
140
|
+
if (typeof actionData.href === "string") {
|
|
141
|
+
lines.push(`Href: ${actionData.href}`);
|
|
142
|
+
}
|
|
143
|
+
if (typeof actionData.navigated === "boolean") {
|
|
144
|
+
lines.push(`Navigated: ${actionData.navigated}`);
|
|
145
|
+
}
|
|
146
|
+
if (lines.length > 0) {
|
|
147
|
+
return lines.join("\n");
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const actionText = stringifyUnknown(actionData).trim();
|
|
151
|
+
if (actionText.length === 0 || actionText === "{}") {
|
|
152
|
+
return undefined;
|
|
153
|
+
}
|
|
154
|
+
return actionText;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function isStringArray(value: unknown): value is string[] {
|
|
158
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function getPresentationText(presentation: ToolPresentation): string {
|
|
162
|
+
return presentation.content
|
|
163
|
+
.filter((part): part is Extract<ToolPresentation["content"][number], { type: "text" }> => part.type === "text")
|
|
164
|
+
.map((part) => part.text.trim())
|
|
165
|
+
.filter((text) => text.length > 0)
|
|
102
166
|
.join("\n\n");
|
|
103
167
|
}
|
|
104
168
|
|
|
169
|
+
function getPresentationImages(presentation: ToolPresentation): Array<Extract<ToolPresentation["content"][number], { type: "image" }>> {
|
|
170
|
+
return presentation.content.filter(
|
|
171
|
+
(part): part is Extract<ToolPresentation["content"][number], { type: "image" }> => part.type === "image",
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function getPresentationPaths(options: {
|
|
176
|
+
primaryPath?: string;
|
|
177
|
+
secondaryPaths?: string[];
|
|
178
|
+
}): string[] {
|
|
179
|
+
return options.secondaryPaths ?? (options.primaryPath ? [options.primaryPath] : []);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function formatBatchStepCommand(command: string[] | undefined, index: number): string {
|
|
183
|
+
return command && command.length > 0 ? command.join(" ") : `step-${index + 1}`;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function formatBatchStepError(error: unknown): string {
|
|
187
|
+
const errorText = stringifyUnknown(error).trim();
|
|
188
|
+
return errorText.length > 0 ? `Error: ${errorText}` : "Error: batch step failed.";
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async function buildBatchStepPresentation(options: {
|
|
192
|
+
cwd: string;
|
|
193
|
+
index: number;
|
|
194
|
+
item: AgentBrowserBatchResult;
|
|
195
|
+
}): Promise<{ details: BatchStepPresentationDetails; presentation: ToolPresentation }> {
|
|
196
|
+
const { cwd, index, item } = options;
|
|
197
|
+
const command = isStringArray(item.command) ? item.command : undefined;
|
|
198
|
+
const commandText = formatBatchStepCommand(command, index);
|
|
199
|
+
|
|
200
|
+
if (item.success === false) {
|
|
201
|
+
const errorText = formatBatchStepError(item.error);
|
|
202
|
+
const presentation: ToolPresentation = {
|
|
203
|
+
content: [{ type: "text", text: errorText }],
|
|
204
|
+
summary: errorText,
|
|
205
|
+
};
|
|
206
|
+
return {
|
|
207
|
+
details: {
|
|
208
|
+
command,
|
|
209
|
+
commandText,
|
|
210
|
+
data: item.error,
|
|
211
|
+
index,
|
|
212
|
+
success: false,
|
|
213
|
+
summary: errorText,
|
|
214
|
+
text: errorText,
|
|
215
|
+
},
|
|
216
|
+
presentation,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const presentation = await buildToolPresentation({
|
|
221
|
+
commandInfo: parseCommandInfo(command ?? []),
|
|
222
|
+
cwd,
|
|
223
|
+
envelope: { data: item.result, success: true },
|
|
224
|
+
});
|
|
225
|
+
const fullOutputPaths = getPresentationPaths({
|
|
226
|
+
primaryPath: presentation.fullOutputPath,
|
|
227
|
+
secondaryPaths: presentation.fullOutputPaths,
|
|
228
|
+
});
|
|
229
|
+
const imagePaths = getPresentationPaths({
|
|
230
|
+
primaryPath: presentation.imagePath,
|
|
231
|
+
secondaryPaths: presentation.imagePaths,
|
|
232
|
+
});
|
|
233
|
+
const text = getPresentationText(presentation) || presentation.summary;
|
|
234
|
+
|
|
235
|
+
return {
|
|
236
|
+
details: {
|
|
237
|
+
command,
|
|
238
|
+
commandText,
|
|
239
|
+
data: presentation.data,
|
|
240
|
+
fullOutputPath: fullOutputPaths[0],
|
|
241
|
+
fullOutputPaths: fullOutputPaths.length > 0 ? fullOutputPaths : undefined,
|
|
242
|
+
imagePath: imagePaths[0],
|
|
243
|
+
imagePaths: imagePaths.length > 0 ? imagePaths : undefined,
|
|
244
|
+
index,
|
|
245
|
+
success: true,
|
|
246
|
+
summary: presentation.summary,
|
|
247
|
+
text,
|
|
248
|
+
},
|
|
249
|
+
presentation,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async function buildBatchPresentation(options: {
|
|
254
|
+
cwd: string;
|
|
255
|
+
data: AgentBrowserBatchResult[];
|
|
256
|
+
summary: string;
|
|
257
|
+
}): Promise<ToolPresentation> {
|
|
258
|
+
const { cwd, data, summary } = options;
|
|
259
|
+
const steps: Array<{ details: BatchStepPresentationDetails; presentation: ToolPresentation }> = [];
|
|
260
|
+
for (const [index, item] of data.entries()) {
|
|
261
|
+
steps.push(await buildBatchStepPresentation({ cwd, index, item }));
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const images = steps.flatMap((step) => getPresentationImages(step.presentation));
|
|
265
|
+
const fullOutputPaths = steps.flatMap((step) => getPresentationPaths({
|
|
266
|
+
primaryPath: step.presentation.fullOutputPath,
|
|
267
|
+
secondaryPaths: step.presentation.fullOutputPaths,
|
|
268
|
+
}));
|
|
269
|
+
const imagePaths = steps.flatMap((step) => getPresentationPaths({
|
|
270
|
+
primaryPath: step.presentation.imagePath,
|
|
271
|
+
secondaryPaths: step.presentation.imagePaths,
|
|
272
|
+
}));
|
|
273
|
+
const text =
|
|
274
|
+
steps.length === 0
|
|
275
|
+
? "(no batch steps)"
|
|
276
|
+
: steps
|
|
277
|
+
.map(({ details, presentation }) => {
|
|
278
|
+
const inlineImageCount = getPresentationImages(presentation).length;
|
|
279
|
+
const lines = [`Step ${details.index + 1} — ${details.commandText}`];
|
|
280
|
+
if (details.text.length > 0) {
|
|
281
|
+
lines.push(details.text);
|
|
282
|
+
}
|
|
283
|
+
if (inlineImageCount > 0) {
|
|
284
|
+
lines.push(`(${inlineImageCount} inline image attachment${inlineImageCount === 1 ? "" : "s"} below)`);
|
|
285
|
+
}
|
|
286
|
+
return lines.join("\n");
|
|
287
|
+
})
|
|
288
|
+
.join("\n\n");
|
|
289
|
+
|
|
290
|
+
return {
|
|
291
|
+
batchSteps: steps.map((step) => step.details),
|
|
292
|
+
content: [{ type: "text", text }, ...images],
|
|
293
|
+
data,
|
|
294
|
+
fullOutputPath: fullOutputPaths[0],
|
|
295
|
+
fullOutputPaths: fullOutputPaths.length > 0 ? fullOutputPaths : undefined,
|
|
296
|
+
imagePath: imagePaths[0],
|
|
297
|
+
imagePaths: imagePaths.length > 0 ? imagePaths : undefined,
|
|
298
|
+
summary,
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
|
|
105
302
|
function formatSummary(commandInfo: CommandInfo, data: unknown): string {
|
|
106
303
|
if (Array.isArray(data) && commandInfo.command === "batch") {
|
|
107
304
|
const successCount = data.filter((item) => isRecord(item) && item.success !== false).length;
|
|
108
305
|
return `Batch: ${successCount}/${data.length} succeeded`;
|
|
109
306
|
}
|
|
110
307
|
if (isRecord(data)) {
|
|
308
|
+
const navigationSummary = getNavigationSummary(data);
|
|
309
|
+
if (navigationSummary && isNavigationObservableCommand(commandInfo.command)) {
|
|
310
|
+
const navigationText = formatNavigationSummary(navigationSummary);
|
|
311
|
+
if (navigationText) {
|
|
312
|
+
return `${commandInfo.command ?? "navigation"} → ${navigationText.split("\n", 1)[0] ?? navigationText}`;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
111
315
|
if (commandInfo.command === "snapshot") {
|
|
112
316
|
return formatSnapshotSummary(data);
|
|
113
317
|
}
|
|
@@ -136,9 +340,6 @@ function formatSummary(commandInfo: CommandInfo, data: unknown): string {
|
|
|
136
340
|
}
|
|
137
341
|
|
|
138
342
|
function formatContentText(commandInfo: CommandInfo, data: unknown): string {
|
|
139
|
-
if (Array.isArray(data) && commandInfo.command === "batch") {
|
|
140
|
-
return formatBatchContent(data as AgentBrowserBatchResult[]);
|
|
141
|
-
}
|
|
142
343
|
if (typeof data === "string") {
|
|
143
344
|
return data;
|
|
144
345
|
}
|
|
@@ -149,6 +350,15 @@ function formatContentText(commandInfo: CommandInfo, data: unknown): string {
|
|
|
149
350
|
return stringifyUnknown(data);
|
|
150
351
|
}
|
|
151
352
|
|
|
353
|
+
const navigationSummary = getNavigationSummary(data);
|
|
354
|
+
if (navigationSummary && isNavigationObservableCommand(commandInfo.command)) {
|
|
355
|
+
const navigationText = formatNavigationSummary(navigationSummary);
|
|
356
|
+
if (navigationText) {
|
|
357
|
+
const actionText = formatNavigationActionResult(data);
|
|
358
|
+
return actionText ? `${actionText}\n\nCurrent page:\n${navigationText}` : `Current page:\n${navigationText}`;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
152
362
|
if (commandInfo.command === "snapshot") {
|
|
153
363
|
return formatRawSnapshotText(data);
|
|
154
364
|
}
|
|
@@ -232,13 +442,15 @@ export async function buildToolPresentation(options: {
|
|
|
232
442
|
const data = envelope?.data;
|
|
233
443
|
const summary = formatSummary(commandInfo, data);
|
|
234
444
|
const presentation =
|
|
235
|
-
commandInfo.command === "
|
|
236
|
-
? await
|
|
237
|
-
:
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
445
|
+
commandInfo.command === "batch" && Array.isArray(data)
|
|
446
|
+
? await buildBatchPresentation({ cwd, data: data as AgentBrowserBatchResult[], summary })
|
|
447
|
+
: commandInfo.command === "snapshot" && isRecord(data)
|
|
448
|
+
? await buildSnapshotPresentation(data)
|
|
449
|
+
: {
|
|
450
|
+
content: [{ type: "text" as const, text: formatContentText(commandInfo, data) }],
|
|
451
|
+
data,
|
|
452
|
+
summary,
|
|
453
|
+
};
|
|
242
454
|
|
|
243
455
|
const imagePath = extractImagePath(cwd, data);
|
|
244
456
|
if (!imagePath) {
|
|
@@ -19,11 +19,28 @@ export interface AgentBrowserBatchResult {
|
|
|
19
19
|
success?: boolean;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
export interface BatchStepPresentationDetails {
|
|
23
|
+
command?: string[];
|
|
24
|
+
commandText: string;
|
|
25
|
+
data?: unknown;
|
|
26
|
+
fullOutputPath?: string;
|
|
27
|
+
fullOutputPaths?: string[];
|
|
28
|
+
imagePath?: string;
|
|
29
|
+
imagePaths?: string[];
|
|
30
|
+
index: number;
|
|
31
|
+
success: boolean;
|
|
32
|
+
summary: string;
|
|
33
|
+
text: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
22
36
|
export interface ToolPresentation {
|
|
37
|
+
batchSteps?: BatchStepPresentationDetails[];
|
|
23
38
|
content: Array<{ text: string; type: "text" } | { data: string; mimeType: string; type: "image" }>;
|
|
24
39
|
data?: unknown;
|
|
25
40
|
fullOutputPath?: string;
|
|
41
|
+
fullOutputPaths?: string[];
|
|
26
42
|
imagePath?: string;
|
|
43
|
+
imagePaths?: string[];
|
|
27
44
|
summary: string;
|
|
28
45
|
}
|
|
29
46
|
|
|
@@ -12,11 +12,13 @@ import { type ToolPresentation, compareRefIds, countLines, isRecord, normalizeWh
|
|
|
12
12
|
const SNAPSHOT_INLINE_MAX_CHARS = 6_000;
|
|
13
13
|
const SNAPSHOT_INLINE_MAX_LINES = 80;
|
|
14
14
|
const SNAPSHOT_INLINE_MAX_REFS = 60;
|
|
15
|
-
const SNAPSHOT_PRIMARY_PREVIEW_LINES =
|
|
16
|
-
const SNAPSHOT_SECTION_PREVIEW_LINES =
|
|
17
|
-
const SNAPSHOT_MAX_ADDITIONAL_SECTIONS =
|
|
18
|
-
const SNAPSHOT_KEY_REF_MAX_LINES =
|
|
19
|
-
const SNAPSHOT_OTHER_REF_MAX_LINES =
|
|
15
|
+
const SNAPSHOT_PRIMARY_PREVIEW_LINES = 8;
|
|
16
|
+
const SNAPSHOT_SECTION_PREVIEW_LINES = 2;
|
|
17
|
+
const SNAPSHOT_MAX_ADDITIONAL_SECTIONS = 2;
|
|
18
|
+
const SNAPSHOT_KEY_REF_MAX_LINES = 8;
|
|
19
|
+
const SNAPSHOT_OTHER_REF_MAX_LINES = 4;
|
|
20
|
+
const SNAPSHOT_ROLE_COUNT_MAX_ENTRIES = 4;
|
|
21
|
+
const SNAPSHOT_FALLBACK_PREVIEW_MAX_LINES = 12;
|
|
20
22
|
const SNAPSHOT_NAME_MAX_CHARS = 96;
|
|
21
23
|
const SNAPSHOT_LINE_MAX_CHARS = 140;
|
|
22
24
|
const SNAPSHOT_SPILL_FILE_PREFIX = "pi-agent-browser-snapshot";
|
|
@@ -165,7 +167,12 @@ function formatRoleCounts(roleCounts: Record<string, number>): string | undefine
|
|
|
165
167
|
if (right[1] !== left[1]) return right[1] - left[1];
|
|
166
168
|
return getRolePriority(left[0]) - getRolePriority(right[0]);
|
|
167
169
|
});
|
|
168
|
-
|
|
170
|
+
const visibleEntries = ordered.slice(0, SNAPSHOT_ROLE_COUNT_MAX_ENTRIES).map(([role, count]) => `${role} ${count}`);
|
|
171
|
+
const omittedEntries = Math.max(0, ordered.length - visibleEntries.length);
|
|
172
|
+
if (omittedEntries > 0) {
|
|
173
|
+
visibleEntries.push(`+${omittedEntries} more`);
|
|
174
|
+
}
|
|
175
|
+
return visibleEntries.join(", ");
|
|
169
176
|
}
|
|
170
177
|
|
|
171
178
|
function parseSnapshotLines(snapshot: string): SnapshotLine[] {
|
|
@@ -373,10 +380,10 @@ function buildSegmentPreview(segment: SnapshotSegment, maxLines: number): Snapsh
|
|
|
373
380
|
|
|
374
381
|
function buildFallbackSnapshotOutline(snapshotLines: SnapshotLine[]): SnapshotPreview {
|
|
375
382
|
const selected = new Set<number>();
|
|
376
|
-
for (let index = 0; index < snapshotLines.length && selected.size <
|
|
383
|
+
for (let index = 0; index < snapshotLines.length && selected.size < 4; index += 1) {
|
|
377
384
|
if (!isNoiseSnapshotLine(snapshotLines[index])) selected.add(index);
|
|
378
385
|
}
|
|
379
|
-
for (let index = 0; index < snapshotLines.length && selected.size <
|
|
386
|
+
for (let index = 0; index < snapshotLines.length && selected.size < SNAPSHOT_FALLBACK_PREVIEW_MAX_LINES; index += 1) {
|
|
380
387
|
const line = snapshotLines[index];
|
|
381
388
|
if (isNoiseSnapshotLine(line)) continue;
|
|
382
389
|
if (SNAPSHOT_SIGNAL_ROLES.has(line.role) || line.ref || line.name.length > 0) {
|
|
@@ -385,7 +392,7 @@ function buildFallbackSnapshotOutline(snapshotLines: SnapshotLine[]): SnapshotPr
|
|
|
385
392
|
}
|
|
386
393
|
const chosenLines = [...selected]
|
|
387
394
|
.sort((left, right) => left - right)
|
|
388
|
-
.slice(0,
|
|
395
|
+
.slice(0, SNAPSHOT_FALLBACK_PREVIEW_MAX_LINES)
|
|
389
396
|
.map((index) => snapshotLines[index]);
|
|
390
397
|
return {
|
|
391
398
|
omittedCount: Math.max(0, snapshotLines.length - chosenLines.length),
|
|
@@ -508,6 +515,8 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
|
|
|
508
515
|
const snapshotSegments = useStructuredPreview ? buildSnapshotSegments(snapshotLines) : [];
|
|
509
516
|
const primarySegment = useStructuredPreview ? choosePrimarySegment(snapshotSegments) : undefined;
|
|
510
517
|
const additionalSegments = useStructuredPreview ? chooseAdditionalSegments(snapshotSegments, primarySegment) : [];
|
|
518
|
+
const additionalSegmentCount = useStructuredPreview && primarySegment ? Math.max(0, snapshotSegments.length - 1) : 0;
|
|
519
|
+
const omittedAdditionalSectionCount = Math.max(0, additionalSegmentCount - additionalSegments.length);
|
|
511
520
|
const primaryPreview = primarySegment ? buildSegmentPreview(primarySegment, SNAPSHOT_PRIMARY_PREVIEW_LINES) : undefined;
|
|
512
521
|
const additionalPreviews = additionalSegments
|
|
513
522
|
.map((segment) => ({
|
|
@@ -548,11 +557,9 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
|
|
|
548
557
|
const lines: string[] = [
|
|
549
558
|
`Origin: ${origin}`,
|
|
550
559
|
`Refs: ${refEntries.length}`,
|
|
551
|
-
...(roleCountsText ? [`
|
|
560
|
+
...(roleCountsText ? [`Top roles: ${roleCountsText}`] : []),
|
|
552
561
|
"",
|
|
553
|
-
|
|
554
|
-
? `Compact snapshot view. Full raw snapshot: ${fullOutputPath}`
|
|
555
|
-
: `Compact snapshot view. Full raw snapshot unavailable: ${spillErrorText ?? "temp spill file could not be created."}`,
|
|
562
|
+
"Compact snapshot view.",
|
|
556
563
|
];
|
|
557
564
|
|
|
558
565
|
if (fallbackPreview) {
|
|
@@ -581,6 +588,9 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
|
|
|
581
588
|
lines.push(`- ... (${preview.omittedCount} more lines in this section)`);
|
|
582
589
|
}
|
|
583
590
|
});
|
|
591
|
+
if (omittedAdditionalSectionCount > 0) {
|
|
592
|
+
lines.push(`- ... (${omittedAdditionalSectionCount} more sections omitted)`);
|
|
593
|
+
}
|
|
584
594
|
}
|
|
585
595
|
}
|
|
586
596
|
|
|
@@ -589,11 +599,16 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
|
|
|
589
599
|
lines.push("", "Other refs:", ...otherRefEntries.map(formatCompactRef));
|
|
590
600
|
}
|
|
591
601
|
if (omittedOtherRefs > 0) {
|
|
592
|
-
lines.push(
|
|
593
|
-
`- ... (${omittedOtherRefs} additional refs ${fullOutputPath ? "in the full snapshot file" : "were omitted with the full raw snapshot"})`,
|
|
594
|
-
);
|
|
602
|
+
lines.push(`- ... (${omittedOtherRefs} additional refs omitted)`);
|
|
595
603
|
}
|
|
596
604
|
|
|
605
|
+
lines.push(
|
|
606
|
+
"",
|
|
607
|
+
fullOutputPath
|
|
608
|
+
? "Full raw snapshot path is available in details.fullOutputPath."
|
|
609
|
+
: `Full raw snapshot unavailable: ${spillErrorText ?? "temp spill file could not be created."}`,
|
|
610
|
+
);
|
|
611
|
+
|
|
597
612
|
return {
|
|
598
613
|
content: [{ type: "text", text: lines.join("\n") }],
|
|
599
614
|
data: {
|
|
@@ -603,6 +618,7 @@ export async function buildSnapshotPresentation(data: Record<string, unknown>):
|
|
|
603
618
|
previewMode: fallbackPreview ? "outline" : "structured",
|
|
604
619
|
spillError: spillErrorText,
|
|
605
620
|
previewRefIds: [...previewRefIds],
|
|
621
|
+
additionalSectionsOmitted: omittedAdditionalSectionCount,
|
|
606
622
|
previewSections: [
|
|
607
623
|
...(primarySegment
|
|
608
624
|
? [
|
|
@@ -16,13 +16,6 @@ const IMPLICIT_SESSION_IDLE_TIMEOUT_ENV = "PI_AGENT_BROWSER_IMPLICIT_SESSION_IDL
|
|
|
16
16
|
const IMPLICIT_SESSION_CLOSE_TIMEOUT_ENV = "PI_AGENT_BROWSER_IMPLICIT_SESSION_CLOSE_TIMEOUT_MS";
|
|
17
17
|
const DEFAULT_IMPLICIT_SESSION_IDLE_TIMEOUT_MS = 15 * 60 * 1000;
|
|
18
18
|
const DEFAULT_IMPLICIT_SESSION_CLOSE_TIMEOUT_MS = 5_000;
|
|
19
|
-
const INSPECTION_ALLOW_PATTERNS = [
|
|
20
|
-
/\bagent[_ -]?browser\s+--(?:help|version)\b/i,
|
|
21
|
-
/\bagent[_ -]?browser\b.*\b(?:help|version|docs?|documentation|tool contract|tool guidance|tool description)\b/i,
|
|
22
|
-
/\b(?:help|version|docs?|documentation|tool contract|tool guidance|tool description)\b.*\bagent[_ -]?browser\b/i,
|
|
23
|
-
/\bdebug(?:ging)?\b.*\b(?:agent[_ -]?browser|agent_browser|browser integration)\b/i,
|
|
24
|
-
/\bwhy\s+(?:isn't|is not|doesn't|does not)\b.*\b(?:agent[_ -]?browser|agent_browser)\b/i,
|
|
25
|
-
];
|
|
26
19
|
const LEGACY_BASH_ALLOW_PATTERNS = [
|
|
27
20
|
/\b(?:bash-oriented workflow|bash workflow)\b/i,
|
|
28
21
|
/\b(?:use|via|through|with)\s+bash\b/i,
|
|
@@ -62,9 +55,19 @@ export interface CommandInfo {
|
|
|
62
55
|
subcommand?: string;
|
|
63
56
|
}
|
|
64
57
|
|
|
58
|
+
export type SessionMode = "auto" | "fresh";
|
|
59
|
+
|
|
60
|
+
export interface SessionRecoveryHint {
|
|
61
|
+
exampleArgs: string[];
|
|
62
|
+
exampleParams: { args: string[]; sessionMode: "fresh" };
|
|
63
|
+
reason: string;
|
|
64
|
+
recommendedSessionMode: "fresh";
|
|
65
|
+
}
|
|
66
|
+
|
|
65
67
|
export interface ExecutionPlan {
|
|
66
68
|
commandInfo: CommandInfo;
|
|
67
69
|
effectiveArgs: string[];
|
|
70
|
+
recoveryHint?: SessionRecoveryHint;
|
|
68
71
|
sessionName?: string;
|
|
69
72
|
startupScopedFlags: string[];
|
|
70
73
|
usedImplicitSession: boolean;
|
|
@@ -72,7 +75,6 @@ export interface ExecutionPlan {
|
|
|
72
75
|
}
|
|
73
76
|
|
|
74
77
|
export interface PromptPolicy {
|
|
75
|
-
allowAgentBrowserInspection: boolean;
|
|
76
78
|
allowLegacyAgentBrowserBash: boolean;
|
|
77
79
|
}
|
|
78
80
|
|
|
@@ -177,7 +179,6 @@ export function getStartupScopedFlags(args: string[]): string[] {
|
|
|
177
179
|
|
|
178
180
|
export function buildPromptPolicy(prompt: string): PromptPolicy {
|
|
179
181
|
return {
|
|
180
|
-
allowAgentBrowserInspection: INSPECTION_ALLOW_PATTERNS.some((pattern) => pattern.test(prompt)),
|
|
181
182
|
allowLegacyAgentBrowserBash: LEGACY_BASH_ALLOW_PATTERNS.some((pattern) => pattern.test(prompt)),
|
|
182
183
|
};
|
|
183
184
|
}
|
|
@@ -212,21 +213,29 @@ export function getLatestUserPrompt(branch: unknown[]): string {
|
|
|
212
213
|
|
|
213
214
|
export function buildExecutionPlan(
|
|
214
215
|
args: string[],
|
|
215
|
-
options: { implicitSessionActive: boolean; implicitSessionName: string;
|
|
216
|
+
options: { implicitSessionActive: boolean; implicitSessionName: string; sessionMode: SessionMode },
|
|
216
217
|
): ExecutionPlan {
|
|
217
218
|
const commandInfo = parseCommandInfo(args);
|
|
218
219
|
const explicitSessionName = extractExplicitSessionName(args);
|
|
219
220
|
const startupScopedFlags = getStartupScopedFlags(args);
|
|
220
221
|
const effectiveArgs = args.includes("--json") ? [] : ["--json"];
|
|
222
|
+
let recoveryHint: SessionRecoveryHint | undefined;
|
|
221
223
|
let sessionName = explicitSessionName;
|
|
222
224
|
let usedImplicitSession = false;
|
|
223
225
|
let validationError: string | undefined;
|
|
224
226
|
|
|
225
|
-
if (!explicitSessionName && options.
|
|
227
|
+
if (!explicitSessionName && options.sessionMode === "auto") {
|
|
226
228
|
if (options.implicitSessionActive && startupScopedFlags.length > 0) {
|
|
229
|
+
recoveryHint = {
|
|
230
|
+
exampleArgs: args,
|
|
231
|
+
exampleParams: { args, sessionMode: "fresh" },
|
|
232
|
+
reason:
|
|
233
|
+
"Startup-scoped flags like --profile, --session-name, and --cdp need a fresh upstream launch once the implicit session is already active.",
|
|
234
|
+
recommendedSessionMode: "fresh",
|
|
235
|
+
};
|
|
227
236
|
validationError = [
|
|
228
237
|
`The current implicit agent-browser session is already running, so startup-scoped flags ${startupScopedFlags.join(", ")} would be ignored by upstream agent-browser.`,
|
|
229
|
-
"
|
|
238
|
+
"Retry this call with `sessionMode: \"fresh\"` to force a fresh upstream launch, or pass an explicit `--session ...` if you want to name the new session yourself.",
|
|
230
239
|
].join(" ");
|
|
231
240
|
} else {
|
|
232
241
|
effectiveArgs.push("--session", options.implicitSessionName);
|
|
@@ -240,6 +249,7 @@ export function buildExecutionPlan(
|
|
|
240
249
|
return {
|
|
241
250
|
commandInfo,
|
|
242
251
|
effectiveArgs,
|
|
252
|
+
recoveryHint,
|
|
243
253
|
sessionName,
|
|
244
254
|
startupScopedFlags,
|
|
245
255
|
usedImplicitSession,
|
package/package.json
CHANGED