pi-browser-harness 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +16 -1
- package/package.json +2 -2
- package/skills/pi-browser-harness/SKILL.md +29 -0
- package/src/cdp/commands.ts +4 -0
- package/src/cdp/discovery.ts +15 -12
- package/src/daemon/bridge.ts +262 -99
- package/src/daemon/spawn.ts +72 -14
- package/src/daemon/transport.ts +7 -2
- package/src/domains/click.ts +0 -5
- package/src/domains/console.ts +0 -2
- package/src/domains/files.ts +53 -4
- package/src/domains/form-batch.ts +0 -1
- package/src/domains/form.ts +1 -9
- package/src/domains/headers.ts +89 -0
- package/src/domains/history.ts +3 -3
- package/src/domains/js.ts +3 -9
- package/src/domains/keyboard.ts +2 -7
- package/src/domains/navigate.ts +0 -3
- package/src/domains/network.ts +1 -3
- package/src/domains/page.ts +1 -7
- package/src/domains/readpage/read-page.ts +0 -1
- package/src/domains/screenshot.ts +0 -4
- package/src/domains/scroll.ts +0 -1
- package/src/domains/search/web-search.ts +0 -1
- package/src/domains/setup.ts +0 -2
- package/src/domains/snapshot.ts +1 -4
- package/src/domains/tabs.ts +1 -9
- package/src/domains/viewport.ts +0 -1
- package/src/index.ts +16 -12
- package/src/prompt.ts +22 -122
- package/src/registry.ts +3 -0
- package/src/state.ts +32 -2
- package/src/util/paths.ts +11 -1
package/src/domains/files.ts
CHANGED
|
@@ -104,7 +104,7 @@ export const uploadFileTool = defineBrowserTool({
|
|
|
104
104
|
description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.",
|
|
105
105
|
promptSnippet: "Upload a file to a file input by ref (preferred) or selector",
|
|
106
106
|
promptGuidelines: [
|
|
107
|
-
"
|
|
107
|
+
"File inputs are often hidden; browser_snapshot still surfaces them.",
|
|
108
108
|
"File path must be absolute and readable.",
|
|
109
109
|
],
|
|
110
110
|
parameters: UploadArgs,
|
|
@@ -145,22 +145,70 @@ export const uploadFileTool = defineBrowserTool({
|
|
|
145
145
|
},
|
|
146
146
|
});
|
|
147
147
|
|
|
148
|
+
// Chrome stores the override in its default browser context, so it outlives this client and
|
|
149
|
+
// silently redirects the user's own downloads. Track it, so session teardown can undo it.
|
|
150
|
+
// The flag only records what THIS process set: the daemon is shared by every pi process on the
|
|
151
|
+
// machine and the CDP primitive is last-writer-wins, so another session may have reset or
|
|
152
|
+
// re-pointed it since. A crashed or killed session never reaches teardown, so the override can
|
|
153
|
+
// also outlive the process that set it.
|
|
154
|
+
let activeDownloadOverride: string | undefined;
|
|
155
|
+
|
|
156
|
+
// Shutdown must not block on a browser that is already gone: a pending restore against a
|
|
157
|
+
// disconnected daemon would otherwise trigger the one-shot reconnect (and Chrome's consent
|
|
158
|
+
// prompt) that 0.11.1 confined to explicit tool calls, and wait up to the connect timeout.
|
|
159
|
+
const SHUTDOWN_RESTORE_TIMEOUT_MS = 2_000;
|
|
160
|
+
|
|
161
|
+
export const restoreDownloadBehavior = async (client: BrowserClient): Promise<Result<void, ToolErr>> => {
|
|
162
|
+
const pending = activeDownloadOverride;
|
|
163
|
+
if (pending === undefined) return ok(undefined);
|
|
164
|
+
if (!client.status().alive) {
|
|
165
|
+
return err({ kind: "not_connected", message: `browser not connected; downloads may keep going to ${pending} until Chrome restarts` });
|
|
166
|
+
}
|
|
167
|
+
const r = await cdpCallBrowser(client, "Browser.setDownloadBehavior", { behavior: "default" }, { timeoutMs: SHUTDOWN_RESTORE_TIMEOUT_MS });
|
|
168
|
+
if (!r.success) {
|
|
169
|
+
return err({ ...r.error, message: `${r.error.message} (downloads may keep going to ${pending})` });
|
|
170
|
+
}
|
|
171
|
+
activeDownloadOverride = undefined;
|
|
172
|
+
return ok(undefined);
|
|
173
|
+
};
|
|
174
|
+
|
|
148
175
|
const DownloadArgs = Type.Object({
|
|
149
|
-
downloadPath: Type.
|
|
176
|
+
downloadPath: Type.Optional(
|
|
177
|
+
Type.String({ description: "Absolute path to a writable directory where downloads should be saved" }),
|
|
178
|
+
),
|
|
179
|
+
restore: Type.Optional(
|
|
180
|
+
Type.Boolean({
|
|
181
|
+
description:
|
|
182
|
+
"Undo a previous browser_download: reset Chrome's download behavior back to the user's own default.",
|
|
183
|
+
}),
|
|
184
|
+
),
|
|
150
185
|
});
|
|
151
186
|
|
|
152
187
|
export const downloadTool = defineBrowserTool({
|
|
153
188
|
name: "browser_download",
|
|
154
189
|
label: "Browser Download",
|
|
155
|
-
description:
|
|
190
|
+
description:
|
|
191
|
+
"Configure Chrome's download behavior: set the save directory and disable the save-as prompt, or restore the user's default. Restored automatically when this session shuts down cleanly.",
|
|
156
192
|
promptSnippet: "Configure download directory",
|
|
157
193
|
promptGuidelines: [
|
|
158
194
|
"Pass an absolute path to an existing writable directory.",
|
|
159
|
-
"
|
|
195
|
+
"The override applies to the user's own downloads too, not just yours, and only a clean session shutdown undoes it. Call browser_download({restore:true}) as soon as you are done.",
|
|
160
196
|
],
|
|
161
197
|
parameters: DownloadArgs,
|
|
162
198
|
concurrency: "serialized",
|
|
163
199
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
200
|
+
if (args.restore === true) {
|
|
201
|
+
if (args.downloadPath !== undefined) {
|
|
202
|
+
return err({ kind: "invalid_state", message: "Pass either downloadPath or restore, not both" });
|
|
203
|
+
}
|
|
204
|
+
const r = await cdpCallBrowser(client, "Browser.setDownloadBehavior", { behavior: "default" });
|
|
205
|
+
if (!r.success) return r;
|
|
206
|
+
activeDownloadOverride = undefined;
|
|
207
|
+
return ok({ text: "Downloads restored to Chrome's default location", details: { restored: true } });
|
|
208
|
+
}
|
|
209
|
+
if (args.downloadPath === undefined) {
|
|
210
|
+
return err({ kind: "invalid_state", message: "Provide downloadPath to set a directory, or restore:true to undo" });
|
|
211
|
+
}
|
|
164
212
|
try {
|
|
165
213
|
await mkdir(args.downloadPath, { recursive: true });
|
|
166
214
|
} catch (e) {
|
|
@@ -187,6 +235,7 @@ export const downloadTool = defineBrowserTool({
|
|
|
187
235
|
eventsEnabled: true,
|
|
188
236
|
});
|
|
189
237
|
if (!r.success) return r;
|
|
238
|
+
activeDownloadOverride = args.downloadPath;
|
|
190
239
|
return ok({
|
|
191
240
|
text: `Downloads will save to: ${args.downloadPath}`,
|
|
192
241
|
details: { downloadPath: args.downloadPath },
|
|
@@ -28,7 +28,6 @@ export const fillFormTool = defineBrowserTool({
|
|
|
28
28
|
promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming",
|
|
29
29
|
promptGuidelines: [
|
|
30
30
|
"Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.",
|
|
31
|
-
"Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.",
|
|
32
31
|
"The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.",
|
|
33
32
|
"Re-run browser_snapshot afterwards to verify the form state if needed.",
|
|
34
33
|
],
|
package/src/domains/form.ts
CHANGED
|
@@ -27,15 +27,12 @@ export const fillTool = defineBrowserTool({
|
|
|
27
27
|
"Fill a form field (input/textarea/contenteditable), or tick a checkbox/radio by passing 'true'/'false'. PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Refuses a disabled field rather than writing a value the page will ignore. Returns the field's value after writing, plus a compact diff of page changes.",
|
|
28
28
|
promptSnippet: "Fill a form field by ref (preferred) or selector (works with React/Vue controlled inputs)",
|
|
29
29
|
promptGuidelines: [
|
|
30
|
-
"PREFER `ref` from browser_snapshot (the '[eN]' handle) over a guessed CSS selector — refs survive re-renders and don't require you to invent a selector.",
|
|
31
30
|
"Pass the desired value; no browser_click is needed first.",
|
|
32
31
|
"For a checkbox or radio pass 'true' or 'false' — it toggles `checked`, not the submit value. browser_set_checked does the same thing with a boolean.",
|
|
33
|
-
"A disabled field is refused: the error says so, rather than reporting a write the page never accepted.",
|
|
34
|
-
"A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
|
|
35
32
|
"Use browser_type instead only for keystroke-sensitive widgets (autocomplete, masked/segmented inputs).",
|
|
36
33
|
"Fill sets the value but does not focus or submit. To submit a tag/autocomplete input, follow with browser_dispatch_key({ ref, key: 'Enter' }) — not browser_press_key, which targets the focused element and may miss this field.",
|
|
37
34
|
"For a custom (div-based) dropdown, browser_fill won't validate against its option list — open it, run browser_snapshot, then browser_click the option's ref.",
|
|
38
|
-
"
|
|
35
|
+
"Confirm the field value browser_fill returns matches what you intended.",
|
|
39
36
|
],
|
|
40
37
|
parameters: FillArgs,
|
|
41
38
|
concurrency: "serialized",
|
|
@@ -118,8 +115,6 @@ export const focusTool = defineBrowserTool({
|
|
|
118
115
|
"Focus an element via the DOM .focus() method — deterministic, no coordinate accuracy needed. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use before browser_type when a click might miss the field.",
|
|
119
116
|
promptSnippet: "Focus an element by ref (preferred) or selector",
|
|
120
117
|
promptGuidelines: [
|
|
121
|
-
"PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.",
|
|
122
|
-
"Use before browser_type to guarantee the right field is focused without relying on click coordinates.",
|
|
123
118
|
"For simply setting a value, prefer browser_fill (it doesn't need a separate focus step).",
|
|
124
119
|
],
|
|
125
120
|
parameters: FocusArgs,
|
|
@@ -196,10 +191,8 @@ export const selectOptionTool = defineBrowserTool({
|
|
|
196
191
|
"Select an option in a native <select> element by value, visible label, or index. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Sets the selection and fires bubbling 'input'/'change' events so framework listeners update. On no match, returns the available options.",
|
|
197
192
|
promptSnippet: "Select an option in a native <select> by ref (preferred) or selector",
|
|
198
193
|
promptGuidelines: [
|
|
199
|
-
"PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.",
|
|
200
194
|
"Provide exactly one of value, label, or index to choose the option.",
|
|
201
195
|
"For custom (non-native) dropdowns built from divs, click to open then browser_click the option instead.",
|
|
202
|
-
"If no option matches, the error details list the available options so you can retry.",
|
|
203
196
|
],
|
|
204
197
|
parameters: SelectOptionArgs,
|
|
205
198
|
concurrency: "serialized",
|
|
@@ -263,7 +256,6 @@ export const setCheckedTool = defineBrowserTool({
|
|
|
263
256
|
"Set a checkbox or radio to a desired checked state by ref. Only acts if the current state differs, fires change, and reports the final state. Get the ref from browser_snapshot's [eN].",
|
|
264
257
|
promptSnippet: "Set a checkbox/radio checked state by ref",
|
|
265
258
|
promptGuidelines: [
|
|
266
|
-
"checked:true ticks the box, checked:false unticks it. Idempotent — no-op if already in the desired state.",
|
|
267
259
|
"For radios, set checked:true on the option you want; the group deselects the others.",
|
|
268
260
|
],
|
|
269
261
|
parameters: SetCheckedArgs,
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import type { BrowserClient } from "../client";
|
|
3
|
+
import { type Result, err, ok } from "../util/result";
|
|
4
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
5
|
+
import { cdpCall } from "./cdp-call";
|
|
6
|
+
|
|
7
|
+
const HeaderValue = Type.String({
|
|
8
|
+
maxLength: 8192,
|
|
9
|
+
pattern: "^[^\\r\\n]*$",
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
const SetHeadersArgs = Type.Object({
|
|
13
|
+
headers: Type.Record(Type.String(), HeaderValue, {
|
|
14
|
+
minProperties: 1,
|
|
15
|
+
maxProperties: 64,
|
|
16
|
+
description: "Extra HTTP request headers for the current owned tab. Names must be valid HTTP tokens.",
|
|
17
|
+
}),
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
const HEADER_NAME = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/;
|
|
21
|
+
|
|
22
|
+
const currentOwnedTarget = (
|
|
23
|
+
client: BrowserClient,
|
|
24
|
+
): Result<{ readonly targetId: string }, ToolErr> => {
|
|
25
|
+
const current = client.current();
|
|
26
|
+
if (current === undefined || current === null) {
|
|
27
|
+
return err({ kind: "invalid_state", message: "No tab attached." });
|
|
28
|
+
}
|
|
29
|
+
if (!client.owns(current.targetId)) {
|
|
30
|
+
return err({
|
|
31
|
+
kind: "invalid_state",
|
|
32
|
+
message: `Current tab ${current.targetId} is not owned by this harness session.`,
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
return ok({ targetId: current.targetId });
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export const setHeadersTool = defineBrowserTool({
|
|
39
|
+
name: "browser_set_headers",
|
|
40
|
+
label: "Browser Set Headers",
|
|
41
|
+
description:
|
|
42
|
+
"Set extra HTTP request headers on the current owned tab. Headers persist for subsequent document, subresource, fetch, and XHR requests until cleared or the tab closes.",
|
|
43
|
+
promptSnippet: "Set extra request headers on the current owned tab",
|
|
44
|
+
promptGuidelines: [
|
|
45
|
+
"If no tab is attached yet, call browser_new_tab without a URL first.",
|
|
46
|
+
"Call before navigation when a site requires routing or environment headers.",
|
|
47
|
+
"Call browser_clear_headers before switching tabs or navigating this tab to another origin.",
|
|
48
|
+
"Header values are never echoed in tool output.",
|
|
49
|
+
],
|
|
50
|
+
parameters: SetHeadersArgs,
|
|
51
|
+
concurrency: "serialized",
|
|
52
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
53
|
+
const target = currentOwnedTarget(client);
|
|
54
|
+
if (!target.success) return target;
|
|
55
|
+
const invalidName = Object.keys(args.headers).find((name) => !HEADER_NAME.test(name));
|
|
56
|
+
if (invalidName !== undefined) {
|
|
57
|
+
return err({ kind: "invalid_state", message: `Invalid HTTP header name: ${JSON.stringify(invalidName)}` });
|
|
58
|
+
}
|
|
59
|
+
const result = await cdpCall(client, "Network.setExtraHTTPHeaders", { headers: args.headers });
|
|
60
|
+
if (!result.success) return result;
|
|
61
|
+
const headerNames = Object.keys(args.headers).sort();
|
|
62
|
+
return ok({
|
|
63
|
+
text: `Set ${headerNames.length} extra HTTP header(s) on tab ${target.data.targetId}: ${headerNames.join(", ")}. Values are hidden. Clear them before switching tabs or navigating to another origin.`,
|
|
64
|
+
details: { targetId: target.data.targetId, headerNames, count: headerNames.length },
|
|
65
|
+
});
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
export const clearHeadersTool = defineBrowserTool({
|
|
70
|
+
name: "browser_clear_headers",
|
|
71
|
+
label: "Browser Clear Headers",
|
|
72
|
+
description: "Clear all extra HTTP request headers previously set on the current owned tab.",
|
|
73
|
+
promptSnippet: "Clear extra request headers from the current owned tab",
|
|
74
|
+
promptGuidelines: [
|
|
75
|
+
"Call after finishing an environment-specific workflow, before switching tabs, or before navigating to another origin.",
|
|
76
|
+
],
|
|
77
|
+
parameters: Type.Object({}),
|
|
78
|
+
concurrency: "serialized",
|
|
79
|
+
async handler(_args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
80
|
+
const target = currentOwnedTarget(client);
|
|
81
|
+
if (!target.success) return target;
|
|
82
|
+
const result = await cdpCall(client, "Network.setExtraHTTPHeaders", { headers: {} });
|
|
83
|
+
if (!result.success) return result;
|
|
84
|
+
return ok({
|
|
85
|
+
text: `Cleared extra HTTP headers from tab ${target.data.targetId}.`,
|
|
86
|
+
details: { targetId: target.data.targetId },
|
|
87
|
+
});
|
|
88
|
+
},
|
|
89
|
+
});
|
package/src/domains/history.ts
CHANGED
|
@@ -18,7 +18,7 @@ export const goBackTool = defineBrowserTool({
|
|
|
18
18
|
label: "Browser Go Back",
|
|
19
19
|
description: "Navigate back one page in history.",
|
|
20
20
|
promptSnippet: "Go back one page",
|
|
21
|
-
promptGuidelines: [
|
|
21
|
+
promptGuidelines: [],
|
|
22
22
|
parameters: Type.Object({}),
|
|
23
23
|
concurrency: "serialized",
|
|
24
24
|
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
@@ -38,7 +38,7 @@ export const goForwardTool = defineBrowserTool({
|
|
|
38
38
|
label: "Browser Go Forward",
|
|
39
39
|
description: "Navigate forward one page in history.",
|
|
40
40
|
promptSnippet: "Go forward one page",
|
|
41
|
-
promptGuidelines: [
|
|
41
|
+
promptGuidelines: [],
|
|
42
42
|
parameters: Type.Object({}),
|
|
43
43
|
concurrency: "serialized",
|
|
44
44
|
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
@@ -58,7 +58,7 @@ export const reloadTool = defineBrowserTool({
|
|
|
58
58
|
label: "Browser Reload",
|
|
59
59
|
description: "Reload the current page.",
|
|
60
60
|
promptSnippet: "Reload the page",
|
|
61
|
-
promptGuidelines: [
|
|
61
|
+
promptGuidelines: [],
|
|
62
62
|
parameters: Type.Object({}),
|
|
63
63
|
concurrency: "serialized",
|
|
64
64
|
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
package/src/domains/js.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
2
|
import { resolve, isAbsolute } from "node:path";
|
|
3
|
+
import { isPathWithin } from "../util/paths";
|
|
3
4
|
import { tmpdir } from "node:os";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import { Type } from "typebox";
|
|
@@ -35,8 +36,6 @@ export const executeJsTool = defineBrowserTool({
|
|
|
35
36
|
"DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.",
|
|
36
37
|
"For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.",
|
|
37
38
|
"For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.",
|
|
38
|
-
"DO NOT call browser_screenshot to read a value or check element state. Use this tool.",
|
|
39
|
-
"`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.",
|
|
40
39
|
"For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.",
|
|
41
40
|
"Result must be JSON-serializable (Runtime.evaluate returnByValue=true).",
|
|
42
41
|
],
|
|
@@ -121,10 +120,7 @@ const allowedRoots = (): ReadonlyArray<string> => {
|
|
|
121
120
|
return [tmpdir(), process.cwd(), ...(env !== undefined ? [env] : [])].map((d) => resolve(d));
|
|
122
121
|
};
|
|
123
122
|
|
|
124
|
-
const isPathAllowed = (p: string): boolean =>
|
|
125
|
-
const abs = resolve(p);
|
|
126
|
-
return allowedRoots().some((root) => abs === root || abs.startsWith(`${root}/`));
|
|
127
|
-
};
|
|
123
|
+
const isPathAllowed = (p: string): boolean => allowedRoots().some((root) => isPathWithin(root, p));
|
|
128
124
|
|
|
129
125
|
const MAX_SOURCE_BYTES = 1_000_000;
|
|
130
126
|
|
|
@@ -140,9 +136,7 @@ export const runScriptTool = defineBrowserTool({
|
|
|
140
136
|
"Execute a temporary JavaScript script with full Node.js + browser-daemon access. Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR. Mandatory timeout. The script is full RCE on the harness's process — only invoke scripts you wrote and reviewed.",
|
|
141
137
|
promptSnippet: "Run a temporary script with daemon + Node access (security-bounded)",
|
|
142
138
|
promptGuidelines: [
|
|
143
|
-
"Write the script with the write tool first; pass its absolute path.",
|
|
144
|
-
"Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR — otherwise rejected.",
|
|
145
|
-
"Default timeout is 60s; pass timeoutMs to extend (max 600s).",
|
|
139
|
+
"Write the script with the write tool first; pass its absolute path. Use the OS temp directory rather than hardcoding /tmp.",
|
|
146
140
|
"Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.",
|
|
147
141
|
"Script MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }. Throw on errors.",
|
|
148
142
|
],
|
package/src/domains/keyboard.ts
CHANGED
|
@@ -31,7 +31,6 @@ export const typeTool = defineBrowserTool({
|
|
|
31
31
|
description: "Type text into the currently focused element. Use browser_click first to focus an input field.",
|
|
32
32
|
promptSnippet: "Type text into the focused element",
|
|
33
33
|
promptGuidelines: [
|
|
34
|
-
"Use browser_type to enter text. Click on an input field with browser_click first to focus it.",
|
|
35
34
|
"For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.",
|
|
36
35
|
],
|
|
37
36
|
parameters: TypeArgs,
|
|
@@ -84,9 +83,7 @@ export const pressKeyTool = defineBrowserTool({
|
|
|
84
83
|
promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)",
|
|
85
84
|
promptGuidelines: [
|
|
86
85
|
"Use browser_press_key for keyboard shortcuts and navigation keys.",
|
|
87
|
-
"
|
|
88
|
-
"Use Space as ' ' (a single space character).",
|
|
89
|
-
"Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
|
|
86
|
+
"browser_press_key modifiers combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
|
|
90
87
|
],
|
|
91
88
|
parameters: PressKeyArgs,
|
|
92
89
|
concurrency: "serialized",
|
|
@@ -138,11 +135,9 @@ export const dispatchKeyTool = defineBrowserTool({
|
|
|
138
135
|
"Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
|
|
139
136
|
promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector",
|
|
140
137
|
promptGuidelines: [
|
|
141
|
-
"
|
|
142
|
-
"Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).",
|
|
138
|
+
"Does NOT insert text (use browser_type or browser_press_key for actual typing).",
|
|
143
139
|
"Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
|
|
144
140
|
"The event carries keyCode/which (e.g. 13 for Enter) for legacy handlers, but is untrusted (isTrusted === false) — a few libraries may still ignore it.",
|
|
145
|
-
"eventType defaults to 'keydown'.",
|
|
146
141
|
],
|
|
147
142
|
parameters: DispatchKeyArgs,
|
|
148
143
|
concurrency: "serialized",
|
package/src/domains/navigate.ts
CHANGED
|
@@ -22,8 +22,6 @@ export const navigateTool = defineBrowserTool({
|
|
|
22
22
|
"Navigate to a URL. Creates a new tab on first call (when no real tabs exist). Otherwise navigates the current tab in place.",
|
|
23
23
|
promptSnippet: "Navigate to a URL",
|
|
24
24
|
promptGuidelines: [
|
|
25
|
-
"Use browser_navigate to go to URLs.",
|
|
26
|
-
"Use browser_wait_for_load after browser_navigate to wait for the page to finish loading.",
|
|
27
25
|
"For extracting data from a page you already navigated to, use browser_execute_js or browser_http_get (faster for APIs).",
|
|
28
26
|
"Note: Google and some sites with strict anti-bot detection may reject CDP navigation. Use browser_http_get for search results.",
|
|
29
27
|
],
|
|
@@ -80,7 +78,6 @@ export const openUrlsTool = defineBrowserTool({
|
|
|
80
78
|
promptGuidelines: [
|
|
81
79
|
"Use after browser_web_search to open citations in parallel.",
|
|
82
80
|
"After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.",
|
|
83
|
-
"Use browser_wait_for_load on a tab before extracting data from SPAs.",
|
|
84
81
|
],
|
|
85
82
|
parameters: OpenUrlsArgs,
|
|
86
83
|
concurrency: "serialized",
|
package/src/domains/network.ts
CHANGED
|
@@ -24,7 +24,6 @@ export const httpGetTool = defineBrowserTool({
|
|
|
24
24
|
"Fetch a URL outside the browser (faster than browser_navigate for APIs and static pages). Timeout covers headers AND body read.",
|
|
25
25
|
promptSnippet: "HTTP GET (outside browser; for APIs/static pages)",
|
|
26
26
|
promptGuidelines: [
|
|
27
|
-
"Faster than navigate+execute_js for JSON/HTML APIs.",
|
|
28
27
|
"No JS rendering — for SPAs use browser_navigate.",
|
|
29
28
|
],
|
|
30
29
|
parameters: HttpGetArgs,
|
|
@@ -224,8 +223,7 @@ export const networkRequestsTool = defineBrowserTool({
|
|
|
224
223
|
promptSnippet: "Inspect recent network requests on the current page",
|
|
225
224
|
promptGuidelines: [
|
|
226
225
|
"Buffer is reset on tab switch — only requests on the current tab since attach are visible.",
|
|
227
|
-
|
|
228
|
-
"includeResponseBodies:true costs an extra CDP call per matched request — use only when you need payloads.",
|
|
226
|
+
"Pass includeResponseBodies:true to browser_network_requests only when you need payloads.",
|
|
229
227
|
"If bufferOverflowed:true in the result, older events were dropped (capacity 500 records/tab).",
|
|
230
228
|
],
|
|
231
229
|
parameters: NetworkRequestsArgs,
|
package/src/domains/page.ts
CHANGED
|
@@ -28,10 +28,7 @@ export const pageInfoTool = defineBrowserTool({
|
|
|
28
28
|
"Get current page state: URL, title, viewport size, scroll position, page dimensions. If a JS dialog is open, returns dialog info instead.",
|
|
29
29
|
promptSnippet: "Get current page URL, title, viewport, and scroll position",
|
|
30
30
|
promptGuidelines: [
|
|
31
|
-
"
|
|
32
|
-
"If browser_page_info returns a dialog, use browser_handle_dialog before any other browser actions.",
|
|
33
|
-
"JS dialogs freeze the page's JS thread, so no other interaction works until the dialog is handled.",
|
|
34
|
-
"browser_page_info auto-detects alert, confirm, prompt, and beforeunload dialogs.",
|
|
31
|
+
"If browser_page_info returns a dialog, call browser_handle_dialog before any other browser actions: dialogs freeze the page's JS thread.",
|
|
35
32
|
],
|
|
36
33
|
parameters: Type.Object({}),
|
|
37
34
|
concurrency: "parallel",
|
|
@@ -87,7 +84,6 @@ export const waitForLoadTool = defineBrowserTool({
|
|
|
87
84
|
promptSnippet: "Wait for the page to finish loading",
|
|
88
85
|
promptGuidelines: [
|
|
89
86
|
"Call after browser_navigate / browser_open_urls before extracting data.",
|
|
90
|
-
"Returns when readyState becomes 'complete' OR the timeout elapses.",
|
|
91
87
|
],
|
|
92
88
|
parameters: WaitForLoadArgs,
|
|
93
89
|
concurrency: "serialized",
|
|
@@ -122,8 +118,6 @@ export const waitForTool = defineBrowserTool({
|
|
|
122
118
|
promptSnippet: "Wait for an element or text to appear (or an element to disappear)",
|
|
123
119
|
promptGuidelines: [
|
|
124
120
|
"Use before browser_fill / browser_click on dynamic pages so you act after the element renders.",
|
|
125
|
-
"Provide selector (default: wait until present; gone:true: wait until absent) or text.",
|
|
126
|
-
"Returns a typed timeout error if the condition isn't met within the timeout.",
|
|
127
121
|
],
|
|
128
122
|
parameters: WaitForArgs,
|
|
129
123
|
concurrency: "parallel",
|
|
@@ -157,7 +157,6 @@ export const readPageTool = defineBrowserTool({
|
|
|
157
157
|
"Read a page as clean article text — main content with nav/ads/boilerplate stripped. Pass a url (opened in an isolated tab, read, then closed) or a targetId of an owned tab. Reader-mode counterpart to browser_web_search.",
|
|
158
158
|
promptSnippet: "Read a page's main content as clean text",
|
|
159
159
|
promptGuidelines: [
|
|
160
|
-
"Pass url to read an arbitrary page (opened + closed in its own tab, never disturbing your current tab), or targetId to read an already-open owned tab.",
|
|
161
160
|
"Returns readable main-article text — use this over browser_snapshot/browser_execute_js when you want an article's content for reading or research.",
|
|
162
161
|
"Boilerplate-heavy or structure-less pages fall back to bounded body text rather than erroring.",
|
|
163
162
|
],
|
|
@@ -20,10 +20,6 @@ export const screenshotTool = defineBrowserTool({
|
|
|
20
20
|
"Capture the page as a JPEG/PNG image. NOT a default exploration tool. Use ONLY when (a) you need to verify visual properties (layout, color, spacing, rendered text legibility), or (b) browser_snapshot and browser_execute_js cannot answer your question. For understanding what's on the page use browser_snapshot. For specific element values use browser_execute_js. For click coordinates use the @(x,y) hints in browser_snapshot's outline.",
|
|
21
21
|
promptSnippet: "Capture a screenshot — visual verification only",
|
|
22
22
|
promptGuidelines: [
|
|
23
|
-
"DO NOT use as a default exploration tool. browser_snapshot is the default for understanding page structure — far cheaper and more reliable.",
|
|
24
|
-
"DO NOT use to find click coordinates. browser_snapshot already includes @(x,y) for every interactive element. If a target isn't there, use browser_execute_js with element.getBoundingClientRect().",
|
|
25
|
-
"DO use to verify visual rendering after an action: did the modal animate in correctly, did the chart render, are colors right, did the layout reflow as expected.",
|
|
26
|
-
"DO use as a debugging fallback when browser_execute_js or browser_snapshot return surprising results and you suspect the rendered page differs from the DOM.",
|
|
27
23
|
"format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.",
|
|
28
24
|
],
|
|
29
25
|
parameters: ScreenshotArgs,
|
package/src/domains/scroll.ts
CHANGED
|
@@ -20,7 +20,6 @@ export const scrollTool = defineBrowserTool({
|
|
|
20
20
|
"Scroll the page at given coordinates. deltaY follows W3C wheel-event convention: positive = scroll down, negative = scroll up. Default scrolls down (deltaY = 300).",
|
|
21
21
|
promptSnippet: "Scroll the page (deltaY positive=down, negative=up)",
|
|
22
22
|
promptGuidelines: [
|
|
23
|
-
"Default behavior scrolls down 300px (deltaY=300). Pass a negative deltaY to scroll up.",
|
|
24
23
|
"Pass x/y to target a specific scrollable region (e.g., a div with overflow); otherwise scrolls the page at viewport center.",
|
|
25
24
|
],
|
|
26
25
|
parameters: ScrollArgs,
|
|
@@ -77,7 +77,6 @@ export const webSearchTool = defineBrowserTool({
|
|
|
77
77
|
"Search the web and return ranked results (title, url, snippet, rank). Scrapes a Google results page in the real Chrome — no API key. Returns links only; use browser_read_page to read a result's content.",
|
|
78
78
|
promptSnippet: "Search the web for a query",
|
|
79
79
|
promptGuidelines: [
|
|
80
|
-
"Returns ranked {title, url, snippet, rank} — links only, no page content. Follow up with browser_read_page.",
|
|
81
80
|
"On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.",
|
|
82
81
|
"On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
|
|
83
82
|
"Runs in its own isolated tab and never disturbs the user's current tab.",
|
package/src/domains/setup.ts
CHANGED
|
@@ -15,8 +15,6 @@ export const setupTool = defineBrowserTool({
|
|
|
15
15
|
"Call when browser tools fail with 'Browser harness not initialized'. Idempotent.",
|
|
16
16
|
promptSnippet: "Initialize browser connection",
|
|
17
17
|
promptGuidelines: [
|
|
18
|
-
"Call browser_setup when you get a 'Browser harness not initialized' error from any browser tool.",
|
|
19
|
-
"This tool is idempotent — calling it when already connected is harmless.",
|
|
20
18
|
"After browser_setup succeeds, retry the browser tool that failed.",
|
|
21
19
|
],
|
|
22
20
|
parameters: SetupArgs,
|
package/src/domains/snapshot.ts
CHANGED
|
@@ -98,10 +98,7 @@ export const snapshotTool = defineBrowserTool({
|
|
|
98
98
|
"DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy). Every interactive element gets a stable ref shown as [eN] plus click coordinates @(x,y). Pass the ref to browser_click/browser_fill/etc. — refs survive re-renders, coordinates don't. Use this BEFORE deciding whether you need a screenshot. Pair with browser_execute_js for surgical reads of specific element values.",
|
|
99
99
|
promptSnippet: "Get accessibility-tree snapshot with stable element refs (default for page inspection)",
|
|
100
100
|
promptGuidelines: [
|
|
101
|
-
"
|
|
102
|
-
"Refs come for free: every interactive element shows '[eN]' in the outline. Pass eN as `ref` to browser_click/browser_fill/browser_select_option/browser_focus/browser_upload_file — refs survive re-renders, so prefer them over the '@(x,y)' coordinates (a fallback).",
|
|
103
|
-
"DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, refs, and click targets.",
|
|
104
|
-
"Re-run after a navigation or a major re-render to get fresh refs; a 'ref is stale' error from an interaction tool means you need a new snapshot.",
|
|
101
|
+
"Re-run browser_snapshot after a navigation or a major re-render to get fresh refs.",
|
|
105
102
|
"Pass includeScreenshot:true ONLY if you also need to verify visual rendering (rare).",
|
|
106
103
|
"format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.",
|
|
107
104
|
],
|
package/src/domains/tabs.ts
CHANGED
|
@@ -19,10 +19,7 @@ export const listTabsTool = defineBrowserTool({
|
|
|
19
19
|
description: "List browser tabs. Defaults to tabs this harness session opened (scope:'owned'). Pass scope:'all' to see the user's other tabs too.",
|
|
20
20
|
promptSnippet: "List browser tabs",
|
|
21
21
|
promptGuidelines: [
|
|
22
|
-
"Defaults to scope:'owned' — only tabs this session created. Pass scope:'all' to inspect the user's other tabs.",
|
|
23
|
-
"browser_switch_tab and browser_close_tab refuse non-owned tabs; use browser_new_tab to open one this session controls.",
|
|
24
22
|
"Each tab shows its full 32-char hex targetId — use the exact value with browser_switch_tab.",
|
|
25
|
-
"Internal tabs (chrome://) included by default; pass includeInternal=false to exclude them.",
|
|
26
23
|
],
|
|
27
24
|
parameters: ListTabsArgs,
|
|
28
25
|
concurrency: "parallel",
|
|
@@ -83,7 +80,6 @@ export const switchTabTool = defineBrowserTool({
|
|
|
83
80
|
promptGuidelines: [
|
|
84
81
|
"Get a targetId via browser_list_tabs first.",
|
|
85
82
|
"Accepts exact targetId or a unique prefix of at least 8 hex characters.",
|
|
86
|
-
"Refuses tabs not owned by this session — use browser_new_tab to open a controllable tab instead.",
|
|
87
83
|
],
|
|
88
84
|
parameters: SwitchTabArgs,
|
|
89
85
|
concurrency: "serialized",
|
|
@@ -138,10 +134,7 @@ export const newTabTool = defineBrowserTool({
|
|
|
138
134
|
label: "Browser New Tab",
|
|
139
135
|
description: "Open a new tab in the harness's dedicated Chrome window and switch to it. Optionally navigate to a URL.",
|
|
140
136
|
promptSnippet: "Open a new tab",
|
|
141
|
-
promptGuidelines: [
|
|
142
|
-
"All tabs opened by this tool live in a single dedicated Chrome window separate from the user's main browsing.",
|
|
143
|
-
"Pass url to navigate immediately.",
|
|
144
|
-
],
|
|
137
|
+
promptGuidelines: [],
|
|
145
138
|
parameters: NewTabArgs,
|
|
146
139
|
concurrency: "serialized",
|
|
147
140
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
@@ -166,7 +159,6 @@ export const closeTabTool = defineBrowserTool({
|
|
|
166
159
|
description: "Close a tab the harness owns. Refuses tabs not opened by this session.",
|
|
167
160
|
promptSnippet: "Close an owned tab",
|
|
168
161
|
promptGuidelines: [
|
|
169
|
-
"Only closes tabs this session opened (visible in browser_list_tabs default scope).",
|
|
170
162
|
"Accepts exact targetId or a unique hex prefix (>=8 chars).",
|
|
171
163
|
],
|
|
172
164
|
parameters: CloseTabArgs,
|
package/src/domains/viewport.ts
CHANGED
|
@@ -17,7 +17,6 @@ export const viewportResizeTool = defineBrowserTool({
|
|
|
17
17
|
description: "Override the viewport size and device pixel ratio for responsive testing.",
|
|
18
18
|
promptSnippet: "Resize the viewport (responsive testing)",
|
|
19
19
|
promptGuidelines: [
|
|
20
|
-
"Width/height in CSS pixels (e.g., 375x667 for iPhone SE).",
|
|
21
20
|
"Pass deviceScaleFactor=2 to simulate retina displays.",
|
|
22
21
|
],
|
|
23
22
|
parameters: ViewportArgs,
|
package/src/index.ts
CHANGED
|
@@ -4,9 +4,10 @@ import { getBrowserSystemPrompt } from "./prompt";
|
|
|
4
4
|
import { registerSetupCommand } from "./setup";
|
|
5
5
|
import { registerProfileCommand } from "./profile/command";
|
|
6
6
|
import { readPin } from "./profile/store";
|
|
7
|
-
import { type BrowserState,
|
|
7
|
+
import { type BrowserState, createStatePersister, defaultState, restoreState } from "./state";
|
|
8
8
|
import { registerAllTools } from "./registry";
|
|
9
9
|
import { cleanupTempDirs } from "./util/truncate";
|
|
10
|
+
import { restoreDownloadBehavior } from "./domains/files";
|
|
10
11
|
import { createDaemonTransport } from "./daemon/transport";
|
|
11
12
|
import { setDebugClicks } from "./util/debug";
|
|
12
13
|
import { asBoolean, asString } from "./util/guards";
|
|
@@ -16,6 +17,7 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
16
17
|
const namespace = flagNs ?? `pi-${Math.random().toString(36).slice(2, 10)}`;
|
|
17
18
|
|
|
18
19
|
let state: BrowserState = defaultState(namespace);
|
|
20
|
+
const persister = createStatePersister(pi);
|
|
19
21
|
let client: BrowserClient | null = null;
|
|
20
22
|
let toolsRegistered = false;
|
|
21
23
|
|
|
@@ -83,6 +85,7 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
83
85
|
|
|
84
86
|
pi.on("session_start", async (_event, ctx) => {
|
|
85
87
|
state = restoreState(ctx, state.namespace);
|
|
88
|
+
persister.markPersisted(state);
|
|
86
89
|
setDebugClicks(asBoolean(pi.getFlag("browser-debug-clicks")) ?? false);
|
|
87
90
|
|
|
88
91
|
// Re-read the on-disk profile pin every session start: another pi session may have changed it since this client was created.
|
|
@@ -112,7 +115,7 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
112
115
|
? { harnessWindowId: snap.harnessWindowId }
|
|
113
116
|
: {}),
|
|
114
117
|
};
|
|
115
|
-
try {
|
|
118
|
+
try { persister.persist(state); } catch {}
|
|
116
119
|
},
|
|
117
120
|
});
|
|
118
121
|
} else {
|
|
@@ -130,6 +133,11 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
130
133
|
pi.on("session_shutdown", async () => {
|
|
131
134
|
if (client) {
|
|
132
135
|
try {
|
|
136
|
+
// Warn but keep going: detach and tab cleanup still have to run, and a throw here would skip both.
|
|
137
|
+
const restored = await restoreDownloadBehavior(client);
|
|
138
|
+
if (!restored.success) {
|
|
139
|
+
console.warn(`[pi-browser-harness] ${restored.error.message}`);
|
|
140
|
+
}
|
|
133
141
|
await client.detach();
|
|
134
142
|
await client.closeOwnedTabs();
|
|
135
143
|
} catch (e) {
|
|
@@ -137,23 +145,19 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
137
145
|
}
|
|
138
146
|
// Do NOT stop the client or null it out: the surviving daemon connection is what eliminates the per-session "Allow Remote Debugging" prompt.
|
|
139
147
|
}
|
|
140
|
-
|
|
148
|
+
persister.persist(state);
|
|
141
149
|
await cleanupTempDirs();
|
|
142
150
|
});
|
|
143
151
|
|
|
152
|
+
// The branch already holds the restored snapshot: re-appending it only grows the session file.
|
|
144
153
|
pi.on("session_tree", async (_event, ctx) => {
|
|
145
154
|
state = restoreState(ctx, client?.namespace);
|
|
146
|
-
|
|
155
|
+
persister.markPersisted(state);
|
|
147
156
|
});
|
|
148
157
|
|
|
149
158
|
pi.on("before_agent_start", async (event) => {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
event.systemPrompt +
|
|
154
|
-
`\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser is not connected. Run /browser-setup.`,
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
return { systemPrompt: event.systemPrompt + getBrowserSystemPrompt() };
|
|
159
|
+
return {
|
|
160
|
+
systemPrompt: event.systemPrompt + getBrowserSystemPrompt(client?.status().alive ?? false),
|
|
161
|
+
};
|
|
158
162
|
});
|
|
159
163
|
}
|