pi-browser-harness 0.2.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/package.json +4 -1
- package/skills/pi-browser-harness/SKILL.md +64 -6
- package/src/cdp/discovery.ts +95 -0
- package/src/cdp/errors.ts +19 -0
- package/src/cdp/session.ts +139 -0
- package/src/cdp/transport.ts +221 -0
- package/src/cdp/types.ts +72 -0
- package/src/client.ts +238 -0
- package/src/domains/click.ts +73 -0
- package/src/domains/dialog.ts +31 -0
- package/src/domains/drag.ts +66 -0
- package/src/domains/files.ts +194 -0
- package/src/domains/history.ts +76 -0
- package/src/domains/js.ts +197 -0
- package/src/domains/keyboard.ts +127 -0
- package/src/domains/navigate.ts +146 -0
- package/src/domains/network.ts +80 -0
- package/src/domains/page.ts +88 -0
- package/src/domains/screenshot.ts +150 -0
- package/src/domains/scroll.ts +60 -0
- package/src/domains/tabs.ts +121 -0
- package/src/domains/viewport.ts +40 -0
- package/src/index.ts +65 -142
- package/src/prompt.ts +27 -4
- package/src/registry.ts +55 -0
- package/src/schemas/common.ts +11 -0
- package/src/setup.ts +43 -26
- package/src/state.ts +22 -54
- package/src/util/js-template.ts +27 -0
- package/src/util/keycodes.ts +10 -0
- package/src/util/mutex.ts +25 -0
- package/src/util/paths.ts +21 -0
- package/src/util/result.ts +15 -0
- package/src/util/sharp-shim.ts +49 -0
- package/src/util/time.ts +29 -0
- package/src/util/tool.ts +193 -0
- package/src/util/truncate.ts +35 -0
- package/tsconfig.json +10 -1
- package/src/daemon.ts +0 -881
- package/src/protocol.ts +0 -77
- package/src/renderers.ts +0 -21
- package/src/tools.ts +0 -1363
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
|
|
5
|
+
const ViewportArgs = Type.Object({
|
|
6
|
+
width: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel width" }),
|
|
7
|
+
height: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel height" }),
|
|
8
|
+
deviceScaleFactor: Type.Optional(
|
|
9
|
+
Type.Number({ minimum: 0.5, maximum: 4, default: 1, description: "Device pixel ratio" }),
|
|
10
|
+
),
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
export const viewportResizeTool = defineBrowserTool({
|
|
14
|
+
name: "browser_viewport_resize",
|
|
15
|
+
label: "Browser Viewport Resize",
|
|
16
|
+
description: "Override the viewport size and device pixel ratio for responsive testing.",
|
|
17
|
+
promptSnippet: "Resize the viewport (responsive testing)",
|
|
18
|
+
promptGuidelines: [
|
|
19
|
+
"Width/height in CSS pixels (e.g., 375x667 for iPhone SE).",
|
|
20
|
+
"Pass deviceScaleFactor=2 to simulate retina displays.",
|
|
21
|
+
],
|
|
22
|
+
parameters: ViewportArgs,
|
|
23
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
24
|
+
const r = await client.session().call("Emulation.setDeviceMetricsOverride", {
|
|
25
|
+
width: args.width,
|
|
26
|
+
height: args.height,
|
|
27
|
+
deviceScaleFactor: args.deviceScaleFactor ?? 1,
|
|
28
|
+
mobile: false,
|
|
29
|
+
});
|
|
30
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
31
|
+
return ok({
|
|
32
|
+
text: `Viewport set to ${args.width}x${args.height} @${args.deviceScaleFactor ?? 1}x`,
|
|
33
|
+
details: {
|
|
34
|
+
width: args.width,
|
|
35
|
+
height: args.height,
|
|
36
|
+
deviceScaleFactor: args.deviceScaleFactor ?? 1,
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
},
|
|
40
|
+
});
|
package/src/index.ts
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* pi-browser-harness — browser control extension for pi.
|
|
3
3
|
*
|
|
4
4
|
* Gives pi agents full control of a real Chrome browser via CDP.
|
|
5
|
-
*
|
|
6
|
-
* and injects browser control guidance into the
|
|
5
|
+
* Connects to the user's running Chrome (chrome://inspect/#remote-debugging),
|
|
6
|
+
* registers browser_* tools, and injects browser control guidance into the
|
|
7
|
+
* system prompt.
|
|
7
8
|
*
|
|
8
9
|
* Install:
|
|
9
10
|
* pi install npm:pi-browser-harness
|
|
@@ -11,39 +12,32 @@
|
|
|
11
12
|
*
|
|
12
13
|
* Commands:
|
|
13
14
|
* /browser-setup — guided setup wizard
|
|
14
|
-
* /browser-status — show
|
|
15
|
-
* /browser-reload-daemon — restart the
|
|
15
|
+
* /browser-status — show client status and current page
|
|
16
|
+
* /browser-reload-daemon — restart the browser client
|
|
16
17
|
*
|
|
17
18
|
* Flags:
|
|
18
|
-
* --browser-namespace <name>
|
|
19
|
+
* --browser-namespace <name> — override namespace (default: auto)
|
|
19
20
|
* --browser-debug-clicks — enable debug click overlay
|
|
20
21
|
*/
|
|
21
22
|
|
|
22
|
-
import type { ExtensionAPI
|
|
23
|
-
import {
|
|
23
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
24
|
+
import { type BrowserClient, createBrowserClient } from "./client";
|
|
24
25
|
import { getBrowserSystemPrompt } from "./prompt";
|
|
25
|
-
import { registerRenderers } from "./renderers";
|
|
26
26
|
import { registerSetupCommand } from "./setup";
|
|
27
27
|
import { type BrowserState, defaultState, persistState, restoreState } from "./state";
|
|
28
|
-
import {
|
|
28
|
+
import { registerAllTools } from "./registry";
|
|
29
|
+
import { cleanupTempDirs } from "./util/truncate";
|
|
29
30
|
|
|
30
|
-
export default function browserHarnessExtension(pi: ExtensionAPI) {
|
|
31
|
-
// ── Resolve namespace ──────────────────────────────────────────────────────
|
|
31
|
+
export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
32
32
|
const flagNs = pi.getFlag("browser-namespace") as string | undefined;
|
|
33
|
-
const namespace = flagNs
|
|
33
|
+
const namespace = flagNs ?? `pi-${Math.random().toString(36).slice(2, 10)}`;
|
|
34
34
|
|
|
35
|
-
// ── State ──────────────────────────────────────────────────────────────────
|
|
36
35
|
let state: BrowserState = defaultState(namespace);
|
|
37
|
-
let
|
|
38
|
-
|
|
39
|
-
// ── Tool initialization flag ───────────────────────────────────────────────
|
|
36
|
+
let client: BrowserClient | null = null;
|
|
40
37
|
let toolsRegistered = false;
|
|
41
38
|
|
|
42
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
43
|
-
// Flags
|
|
44
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
45
39
|
pi.registerFlag("browser-namespace", {
|
|
46
|
-
description: "Browser daemon namespace
|
|
40
|
+
description: "Browser daemon namespace. Default: auto-generated",
|
|
47
41
|
type: "string",
|
|
48
42
|
});
|
|
49
43
|
pi.registerFlag("browser-debug-clicks", {
|
|
@@ -52,172 +46,101 @@ export default function browserHarnessExtension(pi: ExtensionAPI) {
|
|
|
52
46
|
default: false,
|
|
53
47
|
});
|
|
54
48
|
|
|
55
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
56
|
-
// Commands
|
|
57
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
58
49
|
pi.registerCommand("browser-status", {
|
|
59
50
|
description: "Show browser connection status and current page",
|
|
60
51
|
handler: async (_args, ctx) => {
|
|
61
|
-
if (!
|
|
62
|
-
ctx.ui.notify("Browser
|
|
52
|
+
if (!client) {
|
|
53
|
+
ctx.ui.notify("Browser client not started. Run /browser-setup first.", "warning");
|
|
63
54
|
return;
|
|
64
55
|
}
|
|
65
|
-
|
|
66
|
-
const status = daemon.getStatus();
|
|
56
|
+
const s = client.status();
|
|
67
57
|
const lines = [
|
|
68
|
-
`Browser: ${
|
|
69
|
-
`Session: ${
|
|
58
|
+
`Browser: ${s.alive ? "🟢 Connected" : "🔴 Disconnected"}`,
|
|
59
|
+
`Session: ${s.sessionId ?? "none"}`,
|
|
70
60
|
];
|
|
71
|
-
|
|
72
|
-
if (
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (status.alive) {
|
|
78
|
-
try {
|
|
79
|
-
const info = await daemon.getPageInfo();
|
|
80
|
-
if ("dialog" in info) {
|
|
81
|
-
lines.push(`\n⚠️ Dialog open: ${info.dialog.type} — "${info.dialog.message}"`);
|
|
61
|
+
if (s.remoteBrowserId) lines.push(`Browser ID: ${s.remoteBrowserId}`);
|
|
62
|
+
if (s.alive) {
|
|
63
|
+
const info = await client.pageInfo();
|
|
64
|
+
if (info.success) {
|
|
65
|
+
if ("dialog" in info.data) {
|
|
66
|
+
lines.push(`\n⚠️ Dialog open: ${info.data.dialog.type} — "${info.data.dialog.message}"`);
|
|
82
67
|
} else {
|
|
83
|
-
lines.push(
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
68
|
+
lines.push(
|
|
69
|
+
`\nCurrent Page:`,
|
|
70
|
+
` URL: ${info.data.url}`,
|
|
71
|
+
` Title: ${info.data.title}`,
|
|
72
|
+
` Viewport: ${info.data.width}x${info.data.height}`,
|
|
73
|
+
);
|
|
87
74
|
}
|
|
88
|
-
} catch {
|
|
89
|
-
lines.push("\n(Could not read page info)");
|
|
90
75
|
}
|
|
91
76
|
}
|
|
92
|
-
|
|
93
77
|
ctx.ui.notify(lines.join("\n"), "info");
|
|
94
78
|
},
|
|
95
79
|
});
|
|
96
80
|
|
|
97
81
|
pi.registerCommand("browser-reload-daemon", {
|
|
98
|
-
description: "Restart the browser
|
|
82
|
+
description: "Restart the browser client",
|
|
99
83
|
handler: async (_args, ctx) => {
|
|
100
|
-
if (!
|
|
101
|
-
ctx.ui.notify("Browser
|
|
84
|
+
if (!client) {
|
|
85
|
+
ctx.ui.notify("Browser client not started.", "warning");
|
|
102
86
|
return;
|
|
103
87
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
ctx.ui.notify(
|
|
112
|
-
`Restart failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
113
|
-
"error",
|
|
114
|
-
);
|
|
88
|
+
ctx.ui.notify("Restarting browser client...", "info");
|
|
89
|
+
await client.stop();
|
|
90
|
+
const r = await client.start();
|
|
91
|
+
if (r.success) {
|
|
92
|
+
ctx.ui.notify("Browser client restarted ✓", "info");
|
|
93
|
+
} else {
|
|
94
|
+
ctx.ui.notify(`Restart failed: ${r.error.message}`, "error");
|
|
115
95
|
}
|
|
116
96
|
},
|
|
117
97
|
});
|
|
118
98
|
|
|
119
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
120
|
-
// Session lifecycle
|
|
121
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
122
99
|
pi.on("session_start", async (_event, ctx) => {
|
|
123
|
-
// Restore persisted state (preserving the current namespace from flag)
|
|
124
100
|
state = restoreState(ctx, state.namespace);
|
|
125
|
-
|
|
126
|
-
//
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
try {
|
|
131
|
-
await daemon.start();
|
|
132
|
-
} catch {
|
|
133
|
-
// Don't block — user can run /browser-setup later
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// Register tools (once)
|
|
137
|
-
if (!toolsRegistered && daemon) {
|
|
138
|
-
registerTools(pi, daemon);
|
|
101
|
+
client = createBrowserClient({ namespace: state.namespace });
|
|
102
|
+
// failure is fine — surfaced via tool errors when the agent first tries to use the browser
|
|
103
|
+
await client.start();
|
|
104
|
+
if (!toolsRegistered) {
|
|
105
|
+
registerAllTools(pi, client);
|
|
139
106
|
toolsRegistered = true;
|
|
140
107
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
if (daemon) {
|
|
147
|
-
registerSetupCommand(pi, daemon);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// Update status
|
|
151
|
-
if (daemon.getStatus().alive) {
|
|
152
|
-
ctx.ui.setStatus("browser", "🟢 Browser connected");
|
|
153
|
-
} else {
|
|
154
|
-
ctx.ui.setStatus("browser", "🔴 Browser — run /browser-setup");
|
|
155
|
-
}
|
|
108
|
+
registerSetupCommand(pi, client);
|
|
109
|
+
ctx.ui.setStatus(
|
|
110
|
+
"browser",
|
|
111
|
+
client.status().alive ? "🟢 Browser connected" : "🔴 Browser — run /browser-setup",
|
|
112
|
+
);
|
|
156
113
|
});
|
|
157
114
|
|
|
158
|
-
pi.on("session_shutdown", async (
|
|
159
|
-
|
|
160
|
-
if (
|
|
161
|
-
persistState(pi, state);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
// Stop daemon
|
|
165
|
-
if (daemon) {
|
|
115
|
+
pi.on("session_shutdown", async () => {
|
|
116
|
+
persistState(pi, state);
|
|
117
|
+
if (client) {
|
|
166
118
|
try {
|
|
167
|
-
await
|
|
168
|
-
} catch {
|
|
169
|
-
// best-effort
|
|
119
|
+
await client.stop();
|
|
120
|
+
} catch (e) {
|
|
121
|
+
// Shutdown is best-effort, but a stuck stop() points at a transport
|
|
122
|
+
// bug worth surfacing for debugging.
|
|
123
|
+
console.warn("[pi-browser-harness] client.stop() failed during shutdown:", e);
|
|
170
124
|
}
|
|
171
|
-
|
|
125
|
+
client = null;
|
|
172
126
|
}
|
|
173
|
-
|
|
174
127
|
toolsRegistered = false;
|
|
175
|
-
|
|
176
|
-
// Clean up temp files from truncated tool outputs
|
|
177
128
|
await cleanupTempDirs();
|
|
178
129
|
});
|
|
179
130
|
|
|
180
|
-
// Restore state on tree navigation
|
|
181
131
|
pi.on("session_tree", async (_event, ctx) => {
|
|
182
|
-
state = restoreState(ctx,
|
|
183
|
-
|
|
184
|
-
persistState(pi, state);
|
|
185
|
-
}
|
|
132
|
+
state = restoreState(ctx, client?.namespace);
|
|
133
|
+
persistState(pi, state);
|
|
186
134
|
});
|
|
187
135
|
|
|
188
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
189
|
-
// System prompt injection
|
|
190
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
191
136
|
pi.on("before_agent_start", async (event) => {
|
|
192
|
-
if (!
|
|
193
|
-
// Browser not connected — inject minimal note
|
|
137
|
+
if (!client || !client.status().alive) {
|
|
194
138
|
return {
|
|
195
139
|
systemPrompt:
|
|
196
140
|
event.systemPrompt +
|
|
197
|
-
`\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser
|
|
198
|
-
`Run /browser-setup to connect to Chrome, or /browser-status to check.`,
|
|
141
|
+
`\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser is not connected. Run /browser-setup.`,
|
|
199
142
|
};
|
|
200
143
|
}
|
|
201
|
-
|
|
202
|
-
return {
|
|
203
|
-
systemPrompt: event.systemPrompt + getBrowserSystemPrompt(),
|
|
204
|
-
};
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
208
|
-
// Tool result hook: track tab history
|
|
209
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
210
|
-
pi.on("tool_result", async (event) => {
|
|
211
|
-
if (event.toolName === "browser_new_tab" || event.toolName === "browser_navigate") {
|
|
212
|
-
// Track new tabs in history
|
|
213
|
-
const details = event.details as { targetId?: string } | undefined;
|
|
214
|
-
if (details?.targetId) {
|
|
215
|
-
state = {
|
|
216
|
-
...state,
|
|
217
|
-
tabHistory: [details.targetId, ...state.tabHistory.filter((id) => id !== details.targetId)].slice(0, 20),
|
|
218
|
-
};
|
|
219
|
-
persistState(pi, state);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
144
|
+
return { systemPrompt: event.systemPrompt + getBrowserSystemPrompt() };
|
|
222
145
|
});
|
|
223
146
|
}
|
package/src/prompt.ts
CHANGED
|
@@ -21,12 +21,12 @@ to interact with pages visually, and other tools alongside them.
|
|
|
21
21
|
|
|
22
22
|
| Tool | Purpose |
|
|
23
23
|
|------|---------|
|
|
24
|
-
| browser_screenshot | Capture a PNG screenshot of the current page |
|
|
24
|
+
| browser_screenshot | Capture a PNG or JPEG screenshot of the current page (rendered inline in the TUI; the LLM reads the image via the saved file path) |
|
|
25
25
|
| browser_click | Click at viewport coordinates |
|
|
26
26
|
| browser_type | Type text into the focused element |
|
|
27
27
|
| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) |
|
|
28
28
|
| browser_scroll | Scroll the page at coordinates |
|
|
29
|
-
| browser_navigate | Navigate to a URL |
|
|
29
|
+
| browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") |
|
|
30
30
|
| browser_new_tab | Open a new tab, optionally navigate |
|
|
31
31
|
| browser_open_urls | Open multiple URLs in new tabs (parallel) |
|
|
32
32
|
| browser_go_back / browser_go_forward / browser_reload | History navigation |
|
|
@@ -37,7 +37,7 @@ to interact with pages visually, and other tools alongside them.
|
|
|
37
37
|
| browser_execute_js | Execute JavaScript and return the result |
|
|
38
38
|
| browser_http_get | Direct HTTP GET (outside browser, for APIs) |
|
|
39
39
|
| browser_wait | Wait N seconds |
|
|
40
|
-
| browser_wait_for_load | Wait for document.readyState === 'complete' |
|
|
40
|
+
| browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) |
|
|
41
41
|
| browser_handle_dialog | Accept or dismiss a JS dialog |
|
|
42
42
|
| browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) |
|
|
43
43
|
|
|
@@ -63,6 +63,16 @@ browser_run_script("/tmp/scrape-pages.js", { urls: [...] })
|
|
|
63
63
|
|
|
64
64
|
Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.
|
|
65
65
|
|
|
66
|
+
### Parallel Execution
|
|
67
|
+
|
|
68
|
+
Observation tools (browser_screenshot, browser_page_info, browser_execute_js, browser_list_tabs, browser_http_get, etc.) can run in parallel with each other and with mutation tools. The harness automatically serializes mutation tools (click, type, scroll, navigate, switch_tab, etc.) so they never race on shared state. When operations are independent, emit them in the same turn for better performance.
|
|
69
|
+
|
|
70
|
+
**Examples of safe parallel calls:**
|
|
71
|
+
\`\`\`
|
|
72
|
+
browser_screenshot() + browser_page_info() + browser_execute_js("document.title")
|
|
73
|
+
browser_http_get("https://api.example.com/data") + browser_click(x, y)
|
|
74
|
+
\`\`\`
|
|
75
|
+
|
|
66
76
|
### Common Patterns
|
|
67
77
|
|
|
68
78
|
**Navigation:**
|
|
@@ -85,8 +95,9 @@ browser_http_get("https://api.example.com/data")
|
|
|
85
95
|
|
|
86
96
|
**Scrolling:**
|
|
87
97
|
\`\`\`
|
|
88
|
-
browser_screenshot() → browser_scroll({ deltaY:
|
|
98
|
+
browser_screenshot() → browser_scroll({ deltaY: 500 }) → browser_screenshot()
|
|
89
99
|
\`\`\`
|
|
100
|
+
Note: deltaY follows W3C wheel-event convention: positive=down, negative=up. Default deltaY=300 scrolls down.
|
|
90
101
|
|
|
91
102
|
**Research Workflow (search + browser):**
|
|
92
103
|
\`\`\`
|
|
@@ -102,5 +113,17 @@ browser_execute_js("document.querySelector('.main').innerText") → extract cont
|
|
|
102
113
|
\`\`\`
|
|
103
114
|
write("/tmp/extract.js", "...script with daemon access...") → browser_run_script("/tmp/extract.js")
|
|
104
115
|
\`\`\`
|
|
116
|
+
|
|
117
|
+
### Additional Tools
|
|
118
|
+
|
|
119
|
+
| Tool | Purpose |
|
|
120
|
+
|------|---------|
|
|
121
|
+
| browser_upload_file | Upload a file to a file input (bypasses file picker) |
|
|
122
|
+
| browser_dispatch_key | Dispatch a DOM KeyboardEvent on a specific element (for React/Vue inputs); returns details.matched (number of elements) |
|
|
123
|
+
| browser_download | Configure download directory and disable save-as prompts |
|
|
124
|
+
| browser_viewport_resize | Resize the viewport for responsive testing |
|
|
125
|
+
| browser_drag_and_drop | Perform drag-and-drop from one coordinate to another |
|
|
126
|
+
| browser_print_to_pdf | Print the current page to a PDF file |
|
|
127
|
+
| browser_get_network_log | Get buffered network request/response events |
|
|
105
128
|
`;
|
|
106
129
|
}
|
package/src/registry.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
2
|
+
import type { BrowserClient } from "./client";
|
|
3
|
+
import { type AnyBrowserToolDefinition, registerBrowserTool } from "./util/tool";
|
|
4
|
+
import { clickTool } from "./domains/click";
|
|
5
|
+
import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard";
|
|
6
|
+
import { pageInfoTool, waitTool, waitForLoadTool } from "./domains/page";
|
|
7
|
+
import { scrollTool } from "./domains/scroll";
|
|
8
|
+
import { handleDialogTool } from "./domains/dialog";
|
|
9
|
+
import { screenshotTool } from "./domains/screenshot";
|
|
10
|
+
import { navigateTool, openUrlsTool } from "./domains/navigate";
|
|
11
|
+
import { goBackTool, goForwardTool, reloadTool } from "./domains/history";
|
|
12
|
+
import { listTabsTool, currentTabTool, switchTabTool, newTabTool } from "./domains/tabs";
|
|
13
|
+
import { uploadFileTool, downloadTool, printToPdfTool } from "./domains/files";
|
|
14
|
+
import { viewportResizeTool } from "./domains/viewport";
|
|
15
|
+
import { dragAndDropTool } from "./domains/drag";
|
|
16
|
+
import { httpGetTool, getNetworkLogTool } from "./domains/network";
|
|
17
|
+
import { executeJsTool, runScriptTool } from "./domains/js";
|
|
18
|
+
|
|
19
|
+
// Mutation tools that modify page / browser state are marked `serialized: true`
|
|
20
|
+
// so they run through a shared async mutex. Observation / read-only tools are
|
|
21
|
+
// left unmarked and can execute in parallel.
|
|
22
|
+
const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
|
|
23
|
+
{ ...clickTool, serialized: true },
|
|
24
|
+
{ ...typeTool, serialized: true },
|
|
25
|
+
{ ...pressKeyTool, serialized: true },
|
|
26
|
+
{ ...dispatchKeyTool, serialized: true },
|
|
27
|
+
{ ...scrollTool, serialized: true },
|
|
28
|
+
pageInfoTool,
|
|
29
|
+
waitTool,
|
|
30
|
+
{ ...waitForLoadTool, serialized: true },
|
|
31
|
+
{ ...handleDialogTool, serialized: true },
|
|
32
|
+
screenshotTool,
|
|
33
|
+
{ ...navigateTool, serialized: true },
|
|
34
|
+
{ ...openUrlsTool, serialized: true },
|
|
35
|
+
{ ...goBackTool, serialized: true },
|
|
36
|
+
{ ...goForwardTool, serialized: true },
|
|
37
|
+
{ ...reloadTool, serialized: true },
|
|
38
|
+
listTabsTool,
|
|
39
|
+
currentTabTool,
|
|
40
|
+
{ ...switchTabTool, serialized: true },
|
|
41
|
+
{ ...newTabTool, serialized: true },
|
|
42
|
+
{ ...uploadFileTool, serialized: true },
|
|
43
|
+
{ ...downloadTool, serialized: true },
|
|
44
|
+
{ ...printToPdfTool, serialized: true },
|
|
45
|
+
{ ...viewportResizeTool, serialized: true },
|
|
46
|
+
{ ...dragAndDropTool, serialized: true },
|
|
47
|
+
httpGetTool,
|
|
48
|
+
getNetworkLogTool,
|
|
49
|
+
executeJsTool,
|
|
50
|
+
runScriptTool,
|
|
51
|
+
];
|
|
52
|
+
|
|
53
|
+
export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => {
|
|
54
|
+
for (const t of TOOLS) registerBrowserTool(pi, client, t);
|
|
55
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
|
|
3
|
+
export const Coords = Type.Object({
|
|
4
|
+
x: Type.Number({ description: "X coordinate in CSS pixels from left edge of viewport" }),
|
|
5
|
+
y: Type.Number({ description: "Y coordinate in CSS pixels from top edge of viewport" }),
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
export const MouseButton = Type.Union(
|
|
9
|
+
[Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")],
|
|
10
|
+
{ description: 'Mouse button: "left", "right", or "middle"' },
|
|
11
|
+
);
|
package/src/setup.ts
CHANGED
|
@@ -5,25 +5,25 @@
|
|
|
5
5
|
|
|
6
6
|
import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
|
|
7
7
|
import { execSync } from "node:child_process";
|
|
8
|
-
import type {
|
|
8
|
+
import type { BrowserClient } from "./client";
|
|
9
9
|
|
|
10
|
-
export function registerSetupCommand(pi: ExtensionAPI,
|
|
10
|
+
export function registerSetupCommand(pi: ExtensionAPI, client: BrowserClient): void {
|
|
11
11
|
pi.registerCommand("browser-setup", {
|
|
12
12
|
description: "Connect pi to your Chrome browser",
|
|
13
13
|
handler: async (_args, ctx) => {
|
|
14
|
-
await runSetup(ctx,
|
|
14
|
+
await runSetup(ctx, client);
|
|
15
15
|
},
|
|
16
16
|
});
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
async function runSetup(ctx: ExtensionContext,
|
|
19
|
+
async function runSetup(ctx: ExtensionContext, client: BrowserClient): Promise<void> {
|
|
20
20
|
ctx.ui.notify("browser setup: checking Chrome...", "info");
|
|
21
21
|
|
|
22
22
|
// Step 1: Check Chrome is running
|
|
23
23
|
const chromeRunning = checkChromeRunning();
|
|
24
24
|
if (!chromeRunning) {
|
|
25
25
|
ctx.ui.notify(
|
|
26
|
-
"
|
|
26
|
+
"No browser instance running. Please open your browser and then run /browser-setup.",
|
|
27
27
|
"error",
|
|
28
28
|
);
|
|
29
29
|
return;
|
|
@@ -32,11 +32,9 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v
|
|
|
32
32
|
|
|
33
33
|
// Step 2: Try to connect to Chrome DevTools
|
|
34
34
|
ctx.ui.notify("Connecting to Chrome DevTools...", "info");
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
} catch (err) {
|
|
39
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
35
|
+
const startResult = await client.start();
|
|
36
|
+
if (!startResult.success) {
|
|
37
|
+
const msg = startResult.error.message;
|
|
40
38
|
const lower = msg.toLowerCase();
|
|
41
39
|
|
|
42
40
|
if (
|
|
@@ -58,34 +56,53 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v
|
|
|
58
56
|
ctx.ui.notify(`Connection failed: ${msg}`, "error");
|
|
59
57
|
return;
|
|
60
58
|
}
|
|
59
|
+
ctx.ui.notify("Connected to Chrome ✓", "info");
|
|
61
60
|
|
|
62
61
|
// Step 3: Verify with test navigation
|
|
63
62
|
ctx.ui.notify("Testing browser control...", "info");
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const info = await daemon.getPageInfo();
|
|
67
|
-
if ("dialog" in info) {
|
|
68
|
-
await daemon.cdp("Page.handleJavaScriptDialog", { accept: true });
|
|
69
|
-
}
|
|
70
|
-
ctx.ui.notify(
|
|
71
|
-
`Browser control verified ✓\nNavigated to: ${"url" in info ? info.url : "github.com"}`,
|
|
72
|
-
"info",
|
|
73
|
-
);
|
|
74
|
-
} catch (err) {
|
|
63
|
+
const tabResult = await client.newTab("https://github.com");
|
|
64
|
+
if (!tabResult.success) {
|
|
75
65
|
ctx.ui.notify(
|
|
76
|
-
`Browser connected but test navigation failed: ${
|
|
66
|
+
`Browser connected but test navigation failed: ${tabResult.error.message}`,
|
|
77
67
|
"warning",
|
|
78
68
|
);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const info = await client.pageInfo();
|
|
73
|
+
if (info.success && "dialog" in info.data) {
|
|
74
|
+
await client.session().call("Page.handleJavaScriptDialog", { accept: true });
|
|
79
75
|
}
|
|
76
|
+
|
|
77
|
+
const pageUrl = info.success && !("dialog" in info.data) ? info.data.url : "github.com";
|
|
78
|
+
ctx.ui.notify(`Browser control verified ✓\nNavigated to: ${pageUrl}`, "info");
|
|
80
79
|
}
|
|
81
80
|
|
|
82
81
|
function checkChromeRunning(): boolean {
|
|
83
82
|
try {
|
|
84
|
-
if (process.platform === "darwin"
|
|
83
|
+
if (process.platform === "darwin") {
|
|
84
|
+
// Exact line matching — excludes "Google Chrome Helper" etc.
|
|
85
|
+
// that linger after the user quits the browser.
|
|
85
86
|
const out = execSync("ps -A -o comm=", { timeout: 5000 }).toString().toLowerCase();
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
)
|
|
87
|
+
const lines = out.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
88
|
+
const browserProcesses = ["google chrome", "chromium", "microsoft edge"];
|
|
89
|
+
return lines.some((l) => {
|
|
90
|
+
if (l.includes("helper") || l.includes("renderer")) return false;
|
|
91
|
+
return browserProcesses.includes(l);
|
|
92
|
+
});
|
|
93
|
+
} else if (process.platform === "linux") {
|
|
94
|
+
// Use args to distinguish the main browser process from sub-processes
|
|
95
|
+
// (GPU, renderer, utility, etc.) which carry --type= flags.
|
|
96
|
+
const out = execSync("ps -A -o comm=,args=", { timeout: 5000 }).toString().toLowerCase();
|
|
97
|
+
const lines = out.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
98
|
+
const browserComms = ["chrome", "chromium", "chromium-browser", "msedge", "microsoft-edge", "google-chrome"];
|
|
99
|
+
return lines.some((line) => {
|
|
100
|
+
const parts = line.split(/\s+/);
|
|
101
|
+
const comm = parts[0] ?? "";
|
|
102
|
+
if (!comm || !browserComms.includes(comm)) return false;
|
|
103
|
+
const isSubprocess = line.includes("--type=") && !line.includes("--type=browser");
|
|
104
|
+
return !isSubprocess;
|
|
105
|
+
});
|
|
89
106
|
} else if (process.platform === "win32") {
|
|
90
107
|
const out = execSync("tasklist", { timeout: 5000 }).toString().toLowerCase();
|
|
91
108
|
return ["chrome.exe", "msedge.exe"].some((n) => out.includes(n));
|