pi-browser-harness 0.8.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +28 -7
- package/package.json +1 -1
- package/skills/deep-research/SKILL.md +66 -0
- package/skills/pi-browser-harness/SKILL.md +8 -0
- package/src/cdp/discovery.ts +171 -9
- package/src/cdp/ownership.ts +19 -1
- package/src/cdp/session.ts +55 -0
- package/src/cdp/window-binding.ts +15 -0
- package/src/client.ts +52 -9
- package/src/daemon/protocol.ts +12 -2
- package/src/daemon/spawn.ts +46 -13
- package/src/deep-research.ts +41 -0
- package/src/domains/forms.ts +355 -0
- package/src/domains/isolated-tab.ts +121 -0
- package/src/domains/navigate.ts +25 -10
- package/src/domains/readpage/capture.ts +46 -0
- package/src/domains/readpage/read-page.ts +119 -0
- package/src/domains/readpage/readability.ts +89 -0
- package/src/domains/render.ts +64 -0
- package/src/domains/search/extract.ts +44 -0
- package/src/domains/search/google-serp.ts +114 -0
- package/src/domains/search/web-search.ts +116 -0
- package/src/index.ts +17 -7
- package/src/prompt.ts +92 -116
- package/src/registry.ts +6 -0
- package/src/setup.ts +26 -10
- package/src/state.ts +1 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../../util/result";
|
|
3
|
+
import { applyTruncation } from "../../util/truncate";
|
|
4
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
|
|
5
|
+
import {
|
|
6
|
+
closeIsolatedTab,
|
|
7
|
+
evalInIsolatedTab,
|
|
8
|
+
navigateIsolatedTab,
|
|
9
|
+
openIsolatedTab,
|
|
10
|
+
waitForIsolatedLoad,
|
|
11
|
+
} from "../isolated-tab";
|
|
12
|
+
import { renderExpandableText } from "../render";
|
|
13
|
+
import { buildSerpExtractionExpr } from "./extract";
|
|
14
|
+
import { classifySerp, parseGoogleSerp, type SearchResult, type SerpExtraction } from "./google-serp";
|
|
15
|
+
|
|
16
|
+
const LOAD_TIMEOUT_MS = 15_000;
|
|
17
|
+
|
|
18
|
+
const WebSearchArgs = Type.Object({
|
|
19
|
+
query: Type.String({ description: "The search query." }),
|
|
20
|
+
limit: Type.Optional(
|
|
21
|
+
Type.Integer({ default: 10, minimum: 1, maximum: 30, description: "Max results to return. Default: 10." }),
|
|
22
|
+
),
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const buildSerpUrl = (query: string, limit: number): string =>
|
|
26
|
+
`https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
|
|
27
|
+
|
|
28
|
+
const isSerpExtraction = (value: unknown): value is SerpExtraction => {
|
|
29
|
+
if (typeof value !== "object" || value === null) return false;
|
|
30
|
+
const v = value as Record<string, unknown>;
|
|
31
|
+
return Array.isArray(v["anchors"]) && typeof v["pageText"] === "string";
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
|
|
35
|
+
const lines = results.map((r) => `${r.rank}. ${r.title}\n ${r.url}\n ${r.snippet}`);
|
|
36
|
+
return `${results.length} result(s) for "${query}":\n\n${lines.join("\n\n")}`;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export const webSearchTool = defineBrowserTool({
|
|
40
|
+
name: "browser_web_search",
|
|
41
|
+
label: "Browser Web Search",
|
|
42
|
+
description:
|
|
43
|
+
"Search the web and return ranked results (title, url, snippet, rank). Scrapes a Google results page in the real Chrome — no API key. Returns links only; use browser_read_page to read a result's content.",
|
|
44
|
+
promptSnippet: "Search the web for a query",
|
|
45
|
+
promptGuidelines: [
|
|
46
|
+
"Returns ranked {title, url, snippet, rank} — links only, no page content. Follow up with browser_read_page.",
|
|
47
|
+
"On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.",
|
|
48
|
+
"On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
|
|
49
|
+
"Runs in its own isolated tab and never disturbs the user's current tab.",
|
|
50
|
+
],
|
|
51
|
+
parameters: WebSearchArgs,
|
|
52
|
+
async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
|
|
53
|
+
const limit = args.limit ?? 10;
|
|
54
|
+
const serpUrl = buildSerpUrl(args.query, limit);
|
|
55
|
+
|
|
56
|
+
const opened = await openIsolatedTab(client);
|
|
57
|
+
if (!opened.success) return err(opened.error);
|
|
58
|
+
const tab = opened.data;
|
|
59
|
+
|
|
60
|
+
try {
|
|
61
|
+
const navigated = await navigateIsolatedTab(client, tab, serpUrl);
|
|
62
|
+
if (!navigated.success) return err(navigated.error);
|
|
63
|
+
|
|
64
|
+
const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
|
|
65
|
+
if (!loaded.success) return err(loaded.error);
|
|
66
|
+
|
|
67
|
+
const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr());
|
|
68
|
+
if (!extracted.success) return err(extracted.error);
|
|
69
|
+
|
|
70
|
+
const parsedJson = typeof extracted.data === "string" ? safeParse(extracted.data) : extracted.data;
|
|
71
|
+
if (!isSerpExtraction(parsedJson)) {
|
|
72
|
+
return err({ kind: "internal", message: "SERP extraction returned an unexpected shape" });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const results = parseGoogleSerp(parsedJson.anchors, limit);
|
|
76
|
+
const verdict = classifySerp(parsedJson.pageText, results.length);
|
|
77
|
+
if (verdict !== "ok") {
|
|
78
|
+
return err({
|
|
79
|
+
kind: "invalid_state",
|
|
80
|
+
message: verdict === "captcha" ? "Google served a CAPTCHA / bot wall" : "no results found",
|
|
81
|
+
details: { reason: verdict, serpUrl },
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const body = formatResults(args.query, results);
|
|
86
|
+
const truncated = await applyTruncation(body, "search");
|
|
87
|
+
return ok({
|
|
88
|
+
text: truncated.text,
|
|
89
|
+
details: {
|
|
90
|
+
results,
|
|
91
|
+
engine: "google",
|
|
92
|
+
query: args.query,
|
|
93
|
+
render: {
|
|
94
|
+
summary: `${results.length} result(s) · google · "${args.query}"`,
|
|
95
|
+
body,
|
|
96
|
+
...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
|
|
97
|
+
},
|
|
98
|
+
...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
} finally {
|
|
102
|
+
await closeIsolatedTab(client, tab);
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
renderResult(result, expanded, theme) {
|
|
106
|
+
return renderExpandableText("web_search", result, expanded, theme);
|
|
107
|
+
},
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
const safeParse = (text: string): unknown => {
|
|
111
|
+
try {
|
|
112
|
+
return JSON.parse(text) as unknown;
|
|
113
|
+
} catch {
|
|
114
|
+
return undefined;
|
|
115
|
+
}
|
|
116
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* /browser-setup — guided setup wizard
|
|
15
15
|
* /browser-status — show client status and current page
|
|
16
16
|
* /browser-reload-daemon — restart the browser client
|
|
17
|
+
* /deep-research <q> — research a question on the web, write a cited report
|
|
17
18
|
*
|
|
18
19
|
* Flags:
|
|
19
20
|
* --browser-namespace <name> — override namespace (default: auto)
|
|
@@ -24,6 +25,7 @@ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
|
24
25
|
import { type BrowserClient, createBrowserClient } from "./client";
|
|
25
26
|
import { getBrowserSystemPrompt } from "./prompt";
|
|
26
27
|
import { registerSetupCommand } from "./setup";
|
|
28
|
+
import { registerDeepResearchCommand } from "./deep-research";
|
|
27
29
|
import { type BrowserState, defaultState, persistState, restoreState } from "./state";
|
|
28
30
|
import { registerAllTools } from "./registry";
|
|
29
31
|
import { cleanupTempDirs } from "./util/truncate";
|
|
@@ -107,9 +109,10 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
107
109
|
if (!client) {
|
|
108
110
|
const transport = createDaemonTransport(state.namespace);
|
|
109
111
|
|
|
110
|
-
const initialOwnership: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {};
|
|
112
|
+
const initialOwnership: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string; harnessWindowId?: number } = {};
|
|
111
113
|
if (state.ownedTargetIds !== undefined) initialOwnership.ownedTargetIds = state.ownedTargetIds;
|
|
112
114
|
if (state.harnessWindowTargetId !== undefined) initialOwnership.harnessWindowTargetId = state.harnessWindowTargetId;
|
|
115
|
+
if (state.harnessWindowId !== undefined) initialOwnership.harnessWindowId = state.harnessWindowId;
|
|
113
116
|
|
|
114
117
|
client = createBrowserClient({
|
|
115
118
|
namespace: state.namespace,
|
|
@@ -122,6 +125,9 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
122
125
|
...(snap.harnessWindowTargetId !== undefined
|
|
123
126
|
? { harnessWindowTargetId: snap.harnessWindowTargetId }
|
|
124
127
|
: {}),
|
|
128
|
+
...(snap.harnessWindowId !== undefined
|
|
129
|
+
? { harnessWindowId: snap.harnessWindowId }
|
|
130
|
+
: {}),
|
|
125
131
|
};
|
|
126
132
|
// Ownership changes can fire from CDP events at any time,
|
|
127
133
|
// including after session replacement when pi's ctx is stale.
|
|
@@ -141,27 +147,31 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
141
147
|
toolsRegistered = true;
|
|
142
148
|
}
|
|
143
149
|
registerSetupCommand(pi, client);
|
|
144
|
-
|
|
145
|
-
"browser",
|
|
146
|
-
client.status().alive ? "🟢 Browser connected" : "🔴 Browser — run /browser-setup",
|
|
147
|
-
);
|
|
150
|
+
registerDeepResearchCommand(pi);
|
|
148
151
|
});
|
|
149
152
|
|
|
150
153
|
pi.on("session_shutdown", async () => {
|
|
151
|
-
persistState(pi, state);
|
|
152
154
|
if (client) {
|
|
153
155
|
try {
|
|
154
156
|
// Detach from the page target (removes the "Chrome is being controlled"
|
|
155
157
|
// banner) but keep the transport alive for the next session.
|
|
156
158
|
await client.detach();
|
|
159
|
+
// Close every tab this session opened so no stale harness tabs linger
|
|
160
|
+
// in the user's browser between sessions. This also clears the persisted
|
|
161
|
+
// ownership + window binding via onOwnershipChange, so the next session
|
|
162
|
+
// starts with a fresh dedicated window rather than reattaching to leftovers.
|
|
163
|
+
await client.closeOwnedTabs();
|
|
157
164
|
} catch (e) {
|
|
158
|
-
console.warn("[pi-browser-harness]
|
|
165
|
+
console.warn("[pi-browser-harness] browser teardown failed during shutdown:", e);
|
|
159
166
|
}
|
|
160
167
|
// ponytail: keep the transport alive across sessions.
|
|
161
168
|
// Do NOT call client.stop() or null out client — the daemon connection
|
|
162
169
|
// persists and eliminates the per-session "Allow Remote Debugging" prompt.
|
|
163
170
|
// Only Chrome restart or daemon death triggers a new prompt.
|
|
164
171
|
}
|
|
172
|
+
// Persist last — closeOwnedTabs mutates `state` through onOwnershipChange,
|
|
173
|
+
// so this captures the cleared ownership set.
|
|
174
|
+
persistState(pi, state);
|
|
165
175
|
await cleanupTempDirs();
|
|
166
176
|
});
|
|
167
177
|
|
package/src/prompt.ts
CHANGED
|
@@ -2,150 +2,126 @@
|
|
|
2
2
|
* System prompt guidance for browser usage.
|
|
3
3
|
*
|
|
4
4
|
* Injects a compact browser control reference into the pi system prompt.
|
|
5
|
-
*
|
|
6
|
-
*
|
|
5
|
+
* This prompt carries only cross-cutting knowledge — the mental model, tool
|
|
6
|
+
* routing, workflows, and gotchas. Per-tool specifics (schemas, detailed usage)
|
|
7
|
+
* are injected separately via promptGuidelines on each tool registration, so this
|
|
8
|
+
* file intentionally does NOT restate full tool descriptions.
|
|
7
9
|
*/
|
|
8
10
|
|
|
9
11
|
export function getBrowserSystemPrompt(): string {
|
|
10
12
|
return `
|
|
11
13
|
## Browser Control
|
|
12
14
|
|
|
13
|
-
You
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
**These tools coexist with all other standard pi tools** — read, bash, edit,
|
|
17
|
-
write, and any other built-in tools are still available. Use browser_* tools
|
|
18
|
-
to interact with pages visually, and other tools alongside them.
|
|
19
|
-
|
|
20
|
-
### Browser-Specific Tools
|
|
21
|
-
|
|
22
|
-
| Tool | Purpose |
|
|
23
|
-
|------|---------|
|
|
24
|
-
| browser_snapshot | Accessibility-tree snapshot. Assigns every interactive element a stable **ref** (\`[eN]\`) plus \`@(x,y)\`. Refs are the primary handle for interaction tools — they survive re-renders, coordinates don't. |
|
|
25
|
-
| browser_screenshot | Capture a PNG or JPEG screenshot of the current page (rendered inline in the TUI; the LLM reads the image via the saved file path) |
|
|
26
|
-
| browser_click | Click an element by **ref** (preferred — re-resolves position, survives re-renders) or viewport x/y. Appends a compact diff of page changes. |
|
|
27
|
-
| browser_type | Type text into the focused element (real keystrokes; requires a focused field) |
|
|
28
|
-
| browser_fill | Fill an input/textarea by **ref** (preferred) or selector — updates React/Vue controlled components, verifies the value, appends a page-changes diff |
|
|
29
|
-
| browser_select_option | Select an option in a native <select> by **ref** (preferred) or selector, choosing by value/label/index (fires change) |
|
|
30
|
-
| browser_focus | Focus an element by **ref** (preferred) or selector (deterministic; use before browser_type) |
|
|
31
|
-
| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) |
|
|
32
|
-
| browser_scroll | Scroll the page at coordinates |
|
|
33
|
-
| browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") |
|
|
34
|
-
| browser_new_tab | Open a new tab, optionally navigate |
|
|
35
|
-
| browser_open_urls | Open multiple URLs in new tabs (parallel) |
|
|
36
|
-
| browser_go_back / browser_go_forward / browser_reload | History navigation |
|
|
37
|
-
| browser_page_info | Get page URL, title, viewport, scroll position, or dialog info |
|
|
38
|
-
| browser_list_tabs | List all open browser tabs |
|
|
39
|
-
| browser_current_tab | Get current tab info |
|
|
40
|
-
| browser_switch_tab | Switch to a different tab by targetId |
|
|
41
|
-
| browser_execute_js | Execute JavaScript and return the result |
|
|
42
|
-
| browser_http_get | Direct HTTP GET (outside browser, for APIs) |
|
|
43
|
-
| browser_console | Read JS errors / console output on the current tab (diagnostic — use when an action looks broken; pass sinceSeq from the previous nextCursor to see only new messages) |
|
|
44
|
-
| browser_wait | Wait N seconds |
|
|
45
|
-
| browser_wait_for | Wait until a selector/text appears (or a selector disappears) — use for SPA content before interacting |
|
|
46
|
-
| browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) |
|
|
47
|
-
| browser_handle_dialog | Accept or dismiss a JS dialog |
|
|
48
|
-
| browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) |
|
|
49
|
-
|
|
50
|
-
### Temporary Scripts
|
|
51
|
-
|
|
52
|
-
When the built-in tools aren't enough for a multi-step workflow, write a temporary
|
|
53
|
-
script to disk and execute it with browser_run_script. The script runs in the
|
|
54
|
-
harness process with direct access to the browser daemon and Node.js APIs.
|
|
15
|
+
You drive a **real Chrome** (the user's running instance, not headless) through
|
|
16
|
+
\`browser_*\` tools. These coexist with every standard pi tool — read, bash, edit,
|
|
17
|
+
write — so mix them freely: \`browser_*\` for the page, everything else alongside.
|
|
55
18
|
|
|
56
|
-
|
|
57
|
-
write("/tmp/scrape-pages.js", \`
|
|
58
|
-
const results = [];
|
|
59
|
-
for (const url of params.urls) {
|
|
60
|
-
await daemon.cdp("Page.navigate", { url });
|
|
61
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
62
|
-
const data = await daemon.evaluateJS("document.title");
|
|
63
|
-
results.push({ url, title: data });
|
|
64
|
-
}
|
|
65
|
-
return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] };
|
|
66
|
-
\`)
|
|
67
|
-
browser_run_script("/tmp/scrape-pages.js", { urls: [...] })
|
|
68
|
-
\`\`\`
|
|
19
|
+
### The core handle: refs
|
|
69
20
|
|
|
70
|
-
|
|
21
|
+
Interaction is ref-first. Call \`browser_snapshot\` to get an accessibility tree
|
|
22
|
+
where every interactive element has a stable ref (\`[eN]\`) and \`@(x,y)\`. **Target
|
|
23
|
+
elements by ref, not by coordinates or guessed selectors** — refs are keyed to
|
|
24
|
+
element identity, so they survive the re-renders that React/Vue forms trigger on
|
|
25
|
+
every edit and save. Coordinates go stale; guessed selectors miss.
|
|
71
26
|
|
|
72
|
-
|
|
27
|
+
- Prefer \`browser_fill\` for text — it fires input/change so controlled components
|
|
28
|
+
update, and returns the value for verification. Use \`browser_click\` + \`browser_type\`
|
|
29
|
+
only for keystroke-sensitive widgets (autocomplete, masked inputs).
|
|
30
|
+
- Mutating calls append a compact **"Page changes"** diff — read it to confirm the
|
|
31
|
+
change landed (form closed, new \`*[eN]\` appeared).
|
|
32
|
+
- \`"ref is stale"\` means the page changed: re-run \`browser_snapshot\` for fresh refs.
|
|
73
33
|
|
|
74
|
-
|
|
34
|
+
### Choosing a tool
|
|
75
35
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
36
|
+
| To… | Reach for |
|
|
37
|
+
|-----|-----------|
|
|
38
|
+
| See the page structure (for interaction) | \`browser_snapshot\` (refs) |
|
|
39
|
+
| See the page visually | \`browser_screenshot\` |
|
|
40
|
+
| Search the web | \`browser_web_search\` → ranked {title, url, snippet} |
|
|
41
|
+
| Read an article's clean text | \`browser_read_page\` (reader mode; url or owned tab) |
|
|
42
|
+
| Extract specific DOM values | \`browser_execute_js\` |
|
|
43
|
+
| Hit a JSON/API endpoint | \`browser_http_get\` (raw GET, outside the browser) |
|
|
44
|
+
| Click / fill / type / select / focus / press a key | \`browser_click\` · \`browser_fill\` · \`browser_type\` · \`browser_select_option\` · \`browser_focus\` · \`browser_press_key\` |
|
|
45
|
+
| Upload a file · drag · resize viewport | \`browser_upload_file\` · \`browser_drag_and_drop\` · \`browser_viewport_resize\` |
|
|
46
|
+
| Go to a URL / open many / manage tabs | \`browser_navigate\` · \`browser_new_tab\` · \`browser_open_urls\` · \`browser_list_tabs\` · \`browser_switch_tab\` · \`browser_go_back\`/\`go_forward\`/\`reload\` |
|
|
47
|
+
| Wait for content / load / a fixed delay | \`browser_wait_for\` (selector/text) · \`browser_wait_for_load\` (readyState) · \`browser_wait\` |
|
|
48
|
+
| Inspect page / handle a dialog | \`browser_page_info\` · \`browser_current_tab\` · \`browser_handle_dialog\` |
|
|
49
|
+
| Diagnose a broken action | \`browser_console\` (JS errors) · \`browser_get_network_log\` |
|
|
50
|
+
| Save output / config downloads | \`browser_print_to_pdf\` · \`browser_download\` |
|
|
51
|
+
| Anything the tools can't express | \`browser_run_script\` (see Extending) |
|
|
81
52
|
|
|
82
|
-
|
|
53
|
+
Each tool carries its own detailed guidelines — this table is only for routing.
|
|
83
54
|
|
|
84
|
-
###
|
|
55
|
+
### Workflows
|
|
85
56
|
|
|
86
|
-
**
|
|
57
|
+
**Research (search → read):**
|
|
87
58
|
\`\`\`
|
|
88
|
-
|
|
59
|
+
browser_web_search("query") // ranked links, no page content
|
|
60
|
+
→ browser_read_page(url) // clean article text for the top hits
|
|
61
|
+
// fall back to browser_open_urls + browser_execute_js only when reader mode misses
|
|
89
62
|
\`\`\`
|
|
90
63
|
|
|
91
|
-
**Form filling (ref-first —
|
|
64
|
+
**Form filling (ref-first — the reliable path on SPA forms):**
|
|
92
65
|
\`\`\`
|
|
93
|
-
browser_wait_for({ selector: "#email" })
|
|
94
|
-
browser_snapshot()
|
|
95
|
-
→ browser_fill({ ref: "e7", value: "a@b.com" })
|
|
96
|
-
→ browser_select_option({ ref: "e9", label: "India" })
|
|
97
|
-
→ browser_click({ ref: "e12" })
|
|
98
|
-
// each
|
|
99
|
-
// the change landed (e.g. a save closed the form, a new field appeared as *[eN]).
|
|
66
|
+
browser_wait_for({ selector: "#email" }) // ensure the field is rendered
|
|
67
|
+
browser_snapshot() // interactive elements get [eN] refs
|
|
68
|
+
→ browser_fill({ ref: "e7", value: "a@b.com" })
|
|
69
|
+
→ browser_select_option({ ref: "e9", label: "India" })
|
|
70
|
+
→ browser_click({ ref: "e12" }) // re-resolves position even after reflow
|
|
71
|
+
// read each "Page changes" diff to confirm the step landed
|
|
100
72
|
\`\`\`
|
|
101
|
-
**Why ref over selector/coordinates:** refs are keyed to the element's identity, so
|
|
102
|
-
they survive the re-renders that React/Vue forms trigger on every edit and save —
|
|
103
|
-
coordinates go stale and selectors you guess often don't match. If a call returns
|
|
104
|
-
"ref is stale", the page changed: re-run browser_snapshot for fresh refs.
|
|
105
|
-
|
|
106
|
-
Prefer browser_fill for text inputs/textareas — it fires input/change so controlled
|
|
107
|
-
components update, and returns the field's value for verification. Use browser_click +
|
|
108
|
-
browser_type only for keystroke-sensitive widgets (autocomplete, masked inputs);
|
|
109
|
-
browser_type requires an already-focused field and errors if none is focused.
|
|
110
73
|
|
|
111
|
-
**
|
|
74
|
+
**Navigate + verify:**
|
|
112
75
|
\`\`\`
|
|
113
|
-
|
|
114
|
-
// or for APIs:
|
|
115
|
-
browser_http_get("https://api.example.com/data")
|
|
76
|
+
browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot()
|
|
116
77
|
\`\`\`
|
|
117
78
|
|
|
118
|
-
**
|
|
119
|
-
|
|
120
|
-
browser_screenshot() → browser_scroll({ deltaY: 500 }) → browser_screenshot()
|
|
121
|
-
\`\`\`
|
|
122
|
-
Note: deltaY follows W3C wheel-event convention: positive=down, negative=up. Default deltaY=300 scrolls down.
|
|
79
|
+
**Scroll:** \`browser_scroll({ deltaY: 500 })\` — W3C wheel convention: positive = down,
|
|
80
|
+
negative = up (default 300, down).
|
|
123
81
|
|
|
124
|
-
|
|
82
|
+
### Parallelism
|
|
83
|
+
|
|
84
|
+
Observation tools (\`browser_screenshot\`, \`browser_page_info\`, \`browser_execute_js\`,
|
|
85
|
+
\`browser_list_tabs\`, \`browser_http_get\`, \`browser_web_search\`, \`browser_read_page\`, …)
|
|
86
|
+
run in parallel with each other and with mutations. The harness serializes mutations
|
|
87
|
+
(click, type, scroll, navigate, switch_tab) so they never race. Emit independent calls
|
|
88
|
+
in one turn:
|
|
125
89
|
\`\`\`
|
|
126
|
-
|
|
127
|
-
browser_open_urls(urls: ["url1", "url2", ...]) → open result pages in parallel tabs
|
|
128
|
-
browser_list_tabs() → see all open tabs with targetIds
|
|
129
|
-
browser_switch_tab(targetId: "...") → switch to a tab
|
|
130
|
-
browser_screenshot() → visually inspect the page
|
|
131
|
-
browser_execute_js("document.querySelector('.main').innerText") → extract content
|
|
90
|
+
browser_screenshot() + browser_page_info() + browser_execute_js("document.title")
|
|
132
91
|
\`\`\`
|
|
133
92
|
|
|
134
|
-
**
|
|
93
|
+
**Multi-agent:** tab switching by one agent changes the active tab for all agents,
|
|
94
|
+
but per-tab data (console, network, dialogs) stays isolated. Call \`browser_current_tab\`
|
|
95
|
+
before a mutation to confirm you're on the expected tab. Handle any dialog reported by
|
|
96
|
+
\`browser_page_info\` promptly — dialogs block interaction and aren't queued across agents.
|
|
97
|
+
|
|
98
|
+
**Session boundary & tab hygiene:** you operate only inside this session's own Chrome
|
|
99
|
+
window. \`browser_switch_tab\`/\`browser_close_tab\` refuse tabs this session didn't open,
|
|
100
|
+
and \`browser_list_tabs\` defaults to owned tabs — never reach into the user's other tabs
|
|
101
|
+
or windows. Close tabs when you're done with them: call \`browser_close_tab\` on any tab
|
|
102
|
+
you opened (via \`browser_new_tab\`/\`browser_open_urls\`) once you've extracted what you
|
|
103
|
+
need. Don't leave a pile of stale tabs behind — keep only what a later step still needs.
|
|
104
|
+
(Leftover tabs are closed automatically when the session ends, but close them yourself
|
|
105
|
+
as you go.)
|
|
106
|
+
|
|
107
|
+
### Extending: temporary scripts
|
|
108
|
+
|
|
109
|
+
When built-in tools can't express a multi-step workflow, write a script to a temp file
|
|
110
|
+
and run it with \`browser_run_script\`. It executes in the harness process with direct
|
|
111
|
+
daemon + Node access.
|
|
135
112
|
\`\`\`
|
|
136
|
-
write(
|
|
113
|
+
write(<tempfile>.js, \`
|
|
114
|
+
const results = [];
|
|
115
|
+
for (const url of params.urls) {
|
|
116
|
+
await daemon.cdp("Page.navigate", { url });
|
|
117
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
118
|
+
results.push({ url, title: await daemon.evaluateJS("document.title") });
|
|
119
|
+
}
|
|
120
|
+
return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] };
|
|
121
|
+
\`)
|
|
122
|
+
browser_run_script(<tempfile>.js, { urls: [...] })
|
|
137
123
|
\`\`\`
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
| Tool | Purpose |
|
|
142
|
-
|------|---------|
|
|
143
|
-
| browser_upload_file | Upload a file to a file input by **ref** (preferred) or selector (bypasses file picker) |
|
|
144
|
-
| browser_dispatch_key | Dispatch a DOM KeyboardEvent on an element by **ref** (preferred) or selector (for React/Vue inputs); returns details.matched |
|
|
145
|
-
| browser_download | Configure download directory and disable save-as prompts |
|
|
146
|
-
| browser_viewport_resize | Resize the viewport for responsive testing |
|
|
147
|
-
| browser_drag_and_drop | Perform drag-and-drop from one coordinate to another |
|
|
148
|
-
| browser_print_to_pdf | Print the current page to a PDF file |
|
|
149
|
-
| browser_get_network_log | Get buffered network request/response events |
|
|
124
|
+
Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON,
|
|
125
|
+
Buffer, setTimeout, clearTimeout. Use a real temp path from the OS — don't hardcode \`/tmp\`.
|
|
150
126
|
`;
|
|
151
127
|
}
|
package/src/registry.ts
CHANGED
|
@@ -19,6 +19,8 @@ import { consoleTool } from "./domains/console";
|
|
|
19
19
|
import { snapshotTool } from "./domains/snapshot";
|
|
20
20
|
import { executeJsTool, runScriptTool } from "./domains/js";
|
|
21
21
|
import { setupTool } from "./domains/setup";
|
|
22
|
+
import { webSearchTool } from "./domains/search/web-search";
|
|
23
|
+
import { readPageTool } from "./domains/readpage/read-page";
|
|
22
24
|
|
|
23
25
|
// Mutation tools that modify page / browser state are marked `serialized: true`
|
|
24
26
|
// so they run through a shared async mutex. Observation / read-only tools are
|
|
@@ -61,6 +63,10 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
|
|
|
61
63
|
snapshotTool,
|
|
62
64
|
executeJsTool,
|
|
63
65
|
runScriptTool,
|
|
66
|
+
// Opens and owns an isolated tab to scrape a SERP — same class as openUrlsTool.
|
|
67
|
+
{ ...webSearchTool, serialized: true },
|
|
68
|
+
// Pure isolated read (own sessionId, own tab) — safe concurrent, like executeJsTool.
|
|
69
|
+
readPageTool,
|
|
64
70
|
];
|
|
65
71
|
|
|
66
72
|
export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => {
|
package/src/setup.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
|
8
8
|
import { execSync } from "node:child_process";
|
|
9
9
|
import type { BrowserClient } from "./client";
|
|
10
10
|
import { ensureDaemon } from "./daemon/spawn";
|
|
11
|
+
import { DAEMON_SOCKET_PATH } from "./daemon/protocol";
|
|
11
12
|
|
|
12
13
|
// ── Public: register the /browser-setup slash command ──────────────────────
|
|
13
14
|
|
|
@@ -41,7 +42,7 @@ export async function performSetup(client: BrowserClient): Promise<SetupResult>
|
|
|
41
42
|
// Step 2: Start the browser daemon (spawns if not running, silently reuses if alive)
|
|
42
43
|
const daemonReady = await ensureDaemon();
|
|
43
44
|
if (!daemonReady) {
|
|
44
|
-
return { success: false, error:
|
|
45
|
+
return { success: false, error: `Could not start the browser daemon. Check ${DAEMON_SOCKET_PATH}.` };
|
|
45
46
|
}
|
|
46
47
|
|
|
47
48
|
// Step 3: Connect to Chrome DevTools
|
|
@@ -59,9 +60,12 @@ export async function performSetup(client: BrowserClient): Promise<SetupResult>
|
|
|
59
60
|
return {
|
|
60
61
|
success: false,
|
|
61
62
|
error:
|
|
62
|
-
"
|
|
63
|
-
|
|
63
|
+
"Browser remote debugging needs to be enabled.\n\n" +
|
|
64
|
+
"Open chrome://inspect/#remote-debugging (or brave://inspect,\n" +
|
|
65
|
+
"edge://inspect) in your browser, tick the\n" +
|
|
64
66
|
'"Discover network targets" / Allow checkbox, then retry.\n\n' +
|
|
67
|
+
"If that doesn't expose DevTools, relaunch the browser with\n" +
|
|
68
|
+
"--remote-debugging-port=9222.\n\n" +
|
|
65
69
|
"Or set BU_CDP_WS to a remote browser WebSocket URL.",
|
|
66
70
|
};
|
|
67
71
|
}
|
|
@@ -89,19 +93,31 @@ export async function performSetup(client: BrowserClient): Promise<SetupResult>
|
|
|
89
93
|
function checkChromeRunning(): boolean {
|
|
90
94
|
try {
|
|
91
95
|
if (process.platform === "darwin") {
|
|
92
|
-
// ps -o comm= returns the full executable path on modern macOS.
|
|
93
|
-
//
|
|
94
|
-
//
|
|
96
|
+
// ps -o comm= returns the full executable path on modern macOS, e.g.
|
|
97
|
+
// "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser". We check
|
|
98
|
+
// for a known Chromium-family name anywhere in the path (case-insensitive),
|
|
99
|
+
// excluding helper/renderer/gpu/crashpad/updater subprocesses that linger
|
|
100
|
+
// after the user quits the browser.
|
|
95
101
|
const out = execSync("ps -A -o comm=", { timeout: 5000 }).toString().toLowerCase();
|
|
96
102
|
const lines = out.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
103
|
+
const browserNames = ["google chrome", "chromium", "microsoft edge", "brave browser"];
|
|
97
104
|
return lines.some((l) => {
|
|
98
|
-
if (
|
|
99
|
-
|
|
105
|
+
if (
|
|
106
|
+
l.includes("helper") ||
|
|
107
|
+
l.includes("renderer") ||
|
|
108
|
+
l.includes("crashpad") ||
|
|
109
|
+
l.includes(" gpu") ||
|
|
110
|
+
l.includes("updater")
|
|
111
|
+
) return false;
|
|
112
|
+
return browserNames.some((name) => l.includes(name));
|
|
100
113
|
});
|
|
101
114
|
} else if (process.platform === "linux") {
|
|
102
115
|
const out = execSync("ps -A -o comm=,args=", { timeout: 5000 }).toString().toLowerCase();
|
|
103
116
|
const lines = out.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
104
|
-
const browserComms = [
|
|
117
|
+
const browserComms = [
|
|
118
|
+
"chrome", "chromium", "chromium-browser", "msedge", "microsoft-edge",
|
|
119
|
+
"google-chrome", "brave", "brave-browser",
|
|
120
|
+
];
|
|
105
121
|
return lines.some((line) => {
|
|
106
122
|
const parts = line.split(/\s+/);
|
|
107
123
|
const comm = parts[0] ?? "";
|
|
@@ -111,7 +127,7 @@ function checkChromeRunning(): boolean {
|
|
|
111
127
|
});
|
|
112
128
|
} else if (process.platform === "win32") {
|
|
113
129
|
const out = execSync("tasklist", { timeout: 5000 }).toString().toLowerCase();
|
|
114
|
-
return ["chrome.exe", "msedge.exe"].some((n) => out.includes(n));
|
|
130
|
+
return ["chrome.exe", "msedge.exe", "brave.exe"].some((n) => out.includes(n));
|
|
115
131
|
}
|
|
116
132
|
} catch {
|
|
117
133
|
// best-effort
|
package/src/state.ts
CHANGED
|
@@ -12,6 +12,7 @@ export type BrowserState = {
|
|
|
12
12
|
readonly remoteBrowserId?: string;
|
|
13
13
|
readonly ownedTargetIds?: ReadonlyArray<string>;
|
|
14
14
|
readonly harnessWindowTargetId?: string;
|
|
15
|
+
readonly harnessWindowId?: number;
|
|
15
16
|
};
|
|
16
17
|
|
|
17
18
|
export const defaultState = (namespace = "default"): BrowserState => ({
|