pi-browser-harness 0.1.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +134 -112
  3. package/package.json +4 -1
  4. package/skills/pi-browser-harness/SKILL.md +17 -4
  5. package/src/cdp/discovery.ts +95 -0
  6. package/src/cdp/errors.ts +19 -0
  7. package/src/cdp/session.ts +139 -0
  8. package/src/cdp/transport.ts +221 -0
  9. package/src/cdp/types.ts +72 -0
  10. package/src/client.ts +232 -0
  11. package/src/domains/click.ts +73 -0
  12. package/src/domains/dialog.ts +31 -0
  13. package/src/domains/drag.ts +66 -0
  14. package/src/domains/files.ts +194 -0
  15. package/src/domains/history.ts +76 -0
  16. package/src/domains/js.ts +197 -0
  17. package/src/domains/keyboard.ts +127 -0
  18. package/src/domains/navigate.ts +146 -0
  19. package/src/domains/network.ts +80 -0
  20. package/src/domains/page.ts +88 -0
  21. package/src/domains/screenshot.ts +114 -0
  22. package/src/domains/scroll.ts +53 -0
  23. package/src/domains/tabs.ts +121 -0
  24. package/src/domains/viewport.ts +40 -0
  25. package/src/index.ts +65 -142
  26. package/src/prompt.ts +16 -3
  27. package/src/registry.ts +52 -0
  28. package/src/schemas/common.ts +11 -0
  29. package/src/setup.ts +20 -21
  30. package/src/state.ts +22 -54
  31. package/src/util/js-template.ts +27 -0
  32. package/src/util/keycodes.ts +10 -0
  33. package/src/util/paths.ts +21 -0
  34. package/src/util/result.ts +15 -0
  35. package/src/util/sharp-shim.ts +49 -0
  36. package/src/util/time.ts +29 -0
  37. package/src/util/tool.ts +169 -0
  38. package/src/util/truncate.ts +35 -0
  39. package/tsconfig.json +10 -1
  40. package/src/daemon.ts +0 -767
  41. package/src/protocol.ts +0 -77
  42. package/src/renderers.ts +0 -21
  43. package/src/tools.ts +0 -1358
@@ -0,0 +1,121 @@
1
+ import { Type } from "typebox";
2
+ import { type Result, err, ok } from "../util/result";
3
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
4
+
5
+ const ListTabsArgs = Type.Object({
6
+ includeInternal: Type.Optional(Type.Boolean({ default: true, description: "Include chrome:// pages" })),
7
+ });
8
+
9
+ export const listTabsTool = defineBrowserTool({
10
+ name: "browser_list_tabs",
11
+ label: "Browser List Tabs",
12
+ description: "List all open browser tabs (page targets).",
13
+ promptSnippet: "List browser tabs",
14
+ promptGuidelines: [
15
+ "Use to find a full targetId for browser_switch_tab.",
16
+ "Internal tabs (chrome://) included by default; pass includeInternal=false to exclude them.",
17
+ "Each tab shows its full 32-char hex targetId — use the exact value with browser_switch_tab.",
18
+ ],
19
+ parameters: ListTabsArgs,
20
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
21
+ const r = await client.listTabs(args.includeInternal ?? true);
22
+ if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
23
+ const lines = r.data.map((t, i) => ` [${i}] ${t.targetId} ${t.url}\n ${t.title}`);
24
+ return ok({
25
+ text: `Tabs (${r.data.length}):\n${lines.join("\n")}`,
26
+ details: { tabs: r.data },
27
+ });
28
+ },
29
+ });
30
+
31
+ export const currentTabTool = defineBrowserTool({
32
+ name: "browser_current_tab",
33
+ label: "Browser Current Tab",
34
+ description: "Get info about the currently attached tab.",
35
+ promptSnippet: "Get current tab info",
36
+ promptGuidelines: ["Returns targetId, url, title."],
37
+ parameters: Type.Object({}),
38
+ async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
39
+ const cur = client.current();
40
+ if (!cur) return err({ kind: "invalid_state", message: "No tab attached." });
41
+ const ti = await client.session().callBrowser("Target.getTargetInfo", { targetId: cur.targetId });
42
+ if (!ti.success) return err({ kind: "cdp_error", message: ti.error.message });
43
+ const info = (ti.data as { targetInfo: { targetId: string; url: string; title: string } }).targetInfo; // CDP boundary cast: Target.getTargetInfo returns { targetInfo: { targetId, url, title } }
44
+ return ok({
45
+ text: `Current tab:\n ${info.targetId}\n ${info.url}\n ${info.title}`,
46
+ details: { targetId: info.targetId, url: info.url, title: info.title },
47
+ });
48
+ },
49
+ });
50
+
51
+ const SwitchTabArgs = Type.Object({
52
+ targetId: Type.String({ description: "Target ID from browser_list_tabs" }),
53
+ });
54
+
55
+ export const switchTabTool = defineBrowserTool({
56
+ name: "browser_switch_tab",
57
+ label: "Browser Switch Tab",
58
+ description: "Switch to and attach to a different tab by targetId.",
59
+ promptSnippet: "Switch tabs",
60
+ promptGuidelines: [
61
+ "Get a targetId via browser_list_tabs first.",
62
+ "Accepts exact targetId or a unique prefix of at least 8 hex characters.",
63
+ ],
64
+ parameters: SwitchTabArgs,
65
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
66
+ // Try exact match first
67
+ let r = await client.switchTab(args.targetId);
68
+ if (r.success) {
69
+ return ok({ text: `Switched to ${args.targetId}`, details: { targetId: args.targetId } });
70
+ }
71
+ // If exact match failed and input looks like a short prefix (≥8 hex chars), try prefix resolution
72
+ const isHexPrefix = /^[0-9A-Fa-f]{8,}$/.test(args.targetId);
73
+ if (!isHexPrefix) {
74
+ return err({ kind: "cdp_error", message: r.error.message });
75
+ }
76
+ const tabs = await client.listTabs(true);
77
+ if (!tabs.success) return err({ kind: "cdp_error", message: r.error.message });
78
+ const matches = tabs.data.filter((t) => t.targetId.startsWith(args.targetId));
79
+ if (matches.length === 0) {
80
+ return err({ kind: "cdp_error", message: `No tab found with prefix "${args.targetId}"` });
81
+ }
82
+ if (matches.length > 1) {
83
+ const ids = matches.map((t) => t.targetId).join(", ");
84
+ return err({
85
+ kind: "invalid_state",
86
+ message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} tabs: ${ids}`,
87
+ details: { matches: matches.map((t) => ({ targetId: t.targetId, url: t.url })) },
88
+ });
89
+ }
90
+ const resolved = matches[0]!.targetId;
91
+ r = await client.switchTab(resolved);
92
+ if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
93
+ return ok({
94
+ text: `Switched to ${resolved} (resolved from prefix "${args.targetId}")`,
95
+ details: { targetId: resolved, resolvedPrefix: args.targetId },
96
+ });
97
+ },
98
+ });
99
+
100
+ const NewTabArgs = Type.Object({
101
+ url: Type.Optional(Type.String({ description: "Optional URL to navigate to" })),
102
+ });
103
+
104
+ export const newTabTool = defineBrowserTool({
105
+ name: "browser_new_tab",
106
+ label: "Browser New Tab",
107
+ description: "Open a new tab and switch to it. Optionally navigate to a URL.",
108
+ promptSnippet: "Open a new tab",
109
+ promptGuidelines: ["Pass url to navigate immediately."],
110
+ parameters: NewTabArgs,
111
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
112
+ const r = await client.newTab(args.url);
113
+ if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
114
+ return ok({
115
+ text: `New tab: ${r.data}${args.url ? ` (${args.url})` : ""}`,
116
+ details: args.url !== undefined
117
+ ? { targetId: r.data, url: args.url }
118
+ : { targetId: r.data },
119
+ });
120
+ },
121
+ });
@@ -0,0 +1,40 @@
1
+ import { Type } from "typebox";
2
+ import { type Result, err, ok } from "../util/result";
3
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
4
+
5
+ const ViewportArgs = Type.Object({
6
+ width: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel width" }),
7
+ height: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel height" }),
8
+ deviceScaleFactor: Type.Optional(
9
+ Type.Number({ minimum: 0.5, maximum: 4, default: 1, description: "Device pixel ratio" }),
10
+ ),
11
+ });
12
+
13
+ export const viewportResizeTool = defineBrowserTool({
14
+ name: "browser_viewport_resize",
15
+ label: "Browser Viewport Resize",
16
+ description: "Override the viewport size and device pixel ratio for responsive testing.",
17
+ promptSnippet: "Resize the viewport (responsive testing)",
18
+ promptGuidelines: [
19
+ "Width/height in CSS pixels (e.g., 375x667 for iPhone SE).",
20
+ "Pass deviceScaleFactor=2 to simulate retina displays.",
21
+ ],
22
+ parameters: ViewportArgs,
23
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
24
+ const r = await client.session().call("Emulation.setDeviceMetricsOverride", {
25
+ width: args.width,
26
+ height: args.height,
27
+ deviceScaleFactor: args.deviceScaleFactor ?? 1,
28
+ mobile: false,
29
+ });
30
+ if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
31
+ return ok({
32
+ text: `Viewport set to ${args.width}x${args.height} @${args.deviceScaleFactor ?? 1}x`,
33
+ details: {
34
+ width: args.width,
35
+ height: args.height,
36
+ deviceScaleFactor: args.deviceScaleFactor ?? 1,
37
+ },
38
+ });
39
+ },
40
+ });
package/src/index.ts CHANGED
@@ -2,8 +2,9 @@
2
2
  * pi-browser-harness — browser control extension for pi.
3
3
  *
4
4
  * Gives pi agents full control of a real Chrome browser via CDP.
5
- * Spawns the browser-harness daemon, registers browser_* tools,
6
- * and injects browser control guidance into the system prompt.
5
+ * Connects to the user's running Chrome (chrome://inspect/#remote-debugging),
6
+ * registers browser_* tools, and injects browser control guidance into the
7
+ * system prompt.
7
8
  *
8
9
  * Install:
9
10
  * pi install npm:pi-browser-harness
@@ -11,39 +12,32 @@
11
12
  *
12
13
  * Commands:
13
14
  * /browser-setup — guided setup wizard
14
- * /browser-status — show daemon status and current page
15
- * /browser-reload-daemon — restart the daemon
15
+ * /browser-status — show client status and current page
16
+ * /browser-reload-daemon — restart the browser client
16
17
  *
17
18
  * Flags:
18
- * --browser-namespace <name> — override BU_NAME (default: auto)
19
+ * --browser-namespace <name> — override namespace (default: auto)
19
20
  * --browser-debug-clicks — enable debug click overlay
20
21
  */
21
22
 
22
- import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
23
- import { BrowserDaemon } from "./daemon";
23
+ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
24
+ import { type BrowserClient, createBrowserClient } from "./client";
24
25
  import { getBrowserSystemPrompt } from "./prompt";
25
- import { registerRenderers } from "./renderers";
26
26
  import { registerSetupCommand } from "./setup";
27
27
  import { type BrowserState, defaultState, persistState, restoreState } from "./state";
28
- import { cleanupTempDirs, registerTools } from "./tools";
28
+ import { registerAllTools } from "./registry";
29
+ import { cleanupTempDirs } from "./util/truncate";
29
30
 
30
- export default function browserHarnessExtension(pi: ExtensionAPI) {
31
- // ── Resolve namespace ──────────────────────────────────────────────────────
31
+ export default function browserHarnessExtension(pi: ExtensionAPI): void {
32
32
  const flagNs = pi.getFlag("browser-namespace") as string | undefined;
33
- const namespace = flagNs || `pi-${Math.random().toString(36).slice(2, 10)}`;
33
+ const namespace = flagNs ?? `pi-${Math.random().toString(36).slice(2, 10)}`;
34
34
 
35
- // ── State ──────────────────────────────────────────────────────────────────
36
35
  let state: BrowserState = defaultState(namespace);
37
- let daemon: BrowserDaemon | null = null;
38
-
39
- // ── Tool initialization flag ───────────────────────────────────────────────
36
+ let client: BrowserClient | null = null;
40
37
  let toolsRegistered = false;
41
38
 
42
- // ═══════════════════════════════════════════════════════════════════════════
43
- // Flags
44
- // ═══════════════════════════════════════════════════════════════════════════
45
39
  pi.registerFlag("browser-namespace", {
46
- description: "Browser daemon namespace (BU_NAME). Default: auto-generated",
40
+ description: "Browser daemon namespace. Default: auto-generated",
47
41
  type: "string",
48
42
  });
49
43
  pi.registerFlag("browser-debug-clicks", {
@@ -52,172 +46,101 @@ export default function browserHarnessExtension(pi: ExtensionAPI) {
52
46
  default: false,
53
47
  });
54
48
 
55
- // ═══════════════════════════════════════════════════════════════════════════
56
- // Commands
57
- // ═══════════════════════════════════════════════════════════════════════════
58
49
  pi.registerCommand("browser-status", {
59
50
  description: "Show browser connection status and current page",
60
51
  handler: async (_args, ctx) => {
61
- if (!daemon) {
62
- ctx.ui.notify("Browser daemon not started. Run /browser-setup first.", "warning");
52
+ if (!client) {
53
+ ctx.ui.notify("Browser client not started. Run /browser-setup first.", "warning");
63
54
  return;
64
55
  }
65
-
66
- const status = daemon.getStatus();
56
+ const s = client.status();
67
57
  const lines = [
68
- `Browser: ${status.alive ? "🟢 Connected" : "🔴 Disconnected"}`,
69
- `Session: ${status.sessionId || "none"}`,
58
+ `Browser: ${s.alive ? "🟢 Connected" : "🔴 Disconnected"}`,
59
+ `Session: ${s.sessionId ?? "none"}`,
70
60
  ];
71
-
72
- if (status.remoteBrowserId) {
73
- lines.push(`Browser ID: ${status.remoteBrowserId}`);
74
- }
75
-
76
- // Try to get current page info
77
- if (status.alive) {
78
- try {
79
- const info = await daemon.getPageInfo();
80
- if ("dialog" in info) {
81
- lines.push(`\n⚠️ Dialog open: ${info.dialog.type} — "${info.dialog.message}"`);
61
+ if (s.remoteBrowserId) lines.push(`Browser ID: ${s.remoteBrowserId}`);
62
+ if (s.alive) {
63
+ const info = await client.pageInfo();
64
+ if (info.success) {
65
+ if ("dialog" in info.data) {
66
+ lines.push(`\n⚠️ Dialog open: ${info.data.dialog.type} — "${info.data.dialog.message}"`);
82
67
  } else {
83
- lines.push(`\nCurrent Page:`);
84
- lines.push(` URL: ${info.url}`);
85
- lines.push(` Title: ${info.title}`);
86
- lines.push(` Viewport: ${info.width}x${info.height}`);
68
+ lines.push(
69
+ `\nCurrent Page:`,
70
+ ` URL: ${info.data.url}`,
71
+ ` Title: ${info.data.title}`,
72
+ ` Viewport: ${info.data.width}x${info.data.height}`,
73
+ );
87
74
  }
88
- } catch {
89
- lines.push("\n(Could not read page info)");
90
75
  }
91
76
  }
92
-
93
77
  ctx.ui.notify(lines.join("\n"), "info");
94
78
  },
95
79
  });
96
80
 
97
81
  pi.registerCommand("browser-reload-daemon", {
98
- description: "Restart the browser daemon",
82
+ description: "Restart the browser client",
99
83
  handler: async (_args, ctx) => {
100
- if (!daemon) {
101
- ctx.ui.notify("Browser daemon not started.", "warning");
84
+ if (!client) {
85
+ ctx.ui.notify("Browser client not started.", "warning");
102
86
  return;
103
87
  }
104
-
105
- ctx.ui.notify("Restarting browser daemon...", "info");
106
- try {
107
- await daemon.stop();
108
- await daemon.start();
109
- ctx.ui.notify("Browser daemon restarted ✓", "info");
110
- } catch (err) {
111
- ctx.ui.notify(
112
- `Restart failed: ${err instanceof Error ? err.message : String(err)}`,
113
- "error",
114
- );
88
+ ctx.ui.notify("Restarting browser client...", "info");
89
+ await client.stop();
90
+ const r = await client.start();
91
+ if (r.success) {
92
+ ctx.ui.notify("Browser client restarted ✓", "info");
93
+ } else {
94
+ ctx.ui.notify(`Restart failed: ${r.error.message}`, "error");
115
95
  }
116
96
  },
117
97
  });
118
98
 
119
- // ═══════════════════════════════════════════════════════════════════════════
120
- // Session lifecycle
121
- // ═══════════════════════════════════════════════════════════════════════════
122
99
  pi.on("session_start", async (_event, ctx) => {
123
- // Restore persisted state (preserving the current namespace from flag)
124
100
  state = restoreState(ctx, state.namespace);
125
-
126
- // Create daemon instance
127
- daemon = new BrowserDaemon(state.namespace);
128
-
129
- // Attempt to start daemon (will fail gracefully if Chrome not connected)
130
- try {
131
- await daemon.start();
132
- } catch {
133
- // Don't block — user can run /browser-setup later
134
- }
135
-
136
- // Register tools (once)
137
- if (!toolsRegistered && daemon) {
138
- registerTools(pi, daemon);
101
+ client = createBrowserClient({ namespace: state.namespace });
102
+ // failure is fine — surfaced via tool errors when the agent first tries to use the browser
103
+ await client.start();
104
+ if (!toolsRegistered) {
105
+ registerAllTools(pi, client);
139
106
  toolsRegistered = true;
140
107
  }
141
-
142
- // Register renderers
143
- registerRenderers(pi);
144
-
145
- // Setup command (always available)
146
- if (daemon) {
147
- registerSetupCommand(pi, daemon);
148
- }
149
-
150
- // Update status
151
- if (daemon.getStatus().alive) {
152
- ctx.ui.setStatus("browser", "🟢 Browser connected");
153
- } else {
154
- ctx.ui.setStatus("browser", "🔴 Browser — run /browser-setup");
155
- }
108
+ registerSetupCommand(pi, client);
109
+ ctx.ui.setStatus(
110
+ "browser",
111
+ client.status().alive ? "🟢 Browser connected" : "🔴 Browser — run /browser-setup",
112
+ );
156
113
  });
157
114
 
158
- pi.on("session_shutdown", async (_event) => {
159
- // Persist current state
160
- if (state) {
161
- persistState(pi, state);
162
- }
163
-
164
- // Stop daemon
165
- if (daemon) {
115
+ pi.on("session_shutdown", async () => {
116
+ persistState(pi, state);
117
+ if (client) {
166
118
  try {
167
- await daemon.stop();
168
- } catch {
169
- // best-effort
119
+ await client.stop();
120
+ } catch (e) {
121
+ // Shutdown is best-effort, but a stuck stop() points at a transport
122
+ // bug worth surfacing for debugging.
123
+ console.warn("[pi-browser-harness] client.stop() failed during shutdown:", e);
170
124
  }
171
- daemon = null;
125
+ client = null;
172
126
  }
173
-
174
127
  toolsRegistered = false;
175
-
176
- // Clean up temp files from truncated tool outputs
177
128
  await cleanupTempDirs();
178
129
  });
179
130
 
180
- // Restore state on tree navigation
181
131
  pi.on("session_tree", async (_event, ctx) => {
182
- state = restoreState(ctx, daemon?.namespace);
183
- if (daemon) {
184
- persistState(pi, state);
185
- }
132
+ state = restoreState(ctx, client?.namespace);
133
+ persistState(pi, state);
186
134
  });
187
135
 
188
- // ═══════════════════════════════════════════════════════════════════════════
189
- // System prompt injection
190
- // ═══════════════════════════════════════════════════════════════════════════
191
136
  pi.on("before_agent_start", async (event) => {
192
- if (!daemon || !daemon.getStatus().alive) {
193
- // Browser not connected — inject minimal note
137
+ if (!client || !client.status().alive) {
194
138
  return {
195
139
  systemPrompt:
196
140
  event.systemPrompt +
197
- `\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser daemon is not connected. ` +
198
- `Run /browser-setup to connect to Chrome, or /browser-status to check.`,
141
+ `\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser is not connected. Run /browser-setup.`,
199
142
  };
200
143
  }
201
-
202
- return {
203
- systemPrompt: event.systemPrompt + getBrowserSystemPrompt(),
204
- };
205
- });
206
-
207
- // ═══════════════════════════════════════════════════════════════════════════
208
- // Tool result hook: track tab history
209
- // ═══════════════════════════════════════════════════════════════════════════
210
- pi.on("tool_result", async (event) => {
211
- if (event.toolName === "browser_new_tab" || event.toolName === "browser_navigate") {
212
- // Track new tabs in history
213
- const details = event.details as { targetId?: string } | undefined;
214
- if (details?.targetId) {
215
- state = {
216
- ...state,
217
- tabHistory: [details.targetId, ...state.tabHistory.filter((id) => id !== details.targetId)].slice(0, 20),
218
- };
219
- persistState(pi, state);
220
- }
221
- }
144
+ return { systemPrompt: event.systemPrompt + getBrowserSystemPrompt() };
222
145
  });
223
146
  }
package/src/prompt.ts CHANGED
@@ -21,12 +21,12 @@ to interact with pages visually, and other tools alongside them.
21
21
 
22
22
  | Tool | Purpose |
23
23
  |------|---------|
24
- | browser_screenshot | Capture a PNG screenshot of the current page |
24
+ | browser_screenshot | Capture a PNG screenshot of the current page (result details include path and attached: false — the LLM reads the image via the saved file path) |
25
25
  | browser_click | Click at viewport coordinates |
26
26
  | browser_type | Type text into the focused element |
27
27
  | browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) |
28
28
  | browser_scroll | Scroll the page at coordinates |
29
- | browser_navigate | Navigate to a URL |
29
+ | browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") |
30
30
  | browser_new_tab | Open a new tab, optionally navigate |
31
31
  | browser_open_urls | Open multiple URLs in new tabs (parallel) |
32
32
  | browser_go_back / browser_go_forward / browser_reload | History navigation |
@@ -37,7 +37,7 @@ to interact with pages visually, and other tools alongside them.
37
37
  | browser_execute_js | Execute JavaScript and return the result |
38
38
  | browser_http_get | Direct HTTP GET (outside browser, for APIs) |
39
39
  | browser_wait | Wait N seconds |
40
- | browser_wait_for_load | Wait for document.readyState === 'complete' |
40
+ | browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) |
41
41
  | browser_handle_dialog | Accept or dismiss a JS dialog |
42
42
  | browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) |
43
43
 
@@ -87,6 +87,7 @@ browser_http_get("https://api.example.com/data")
87
87
  \`\`\`
88
88
  browser_screenshot() → browser_scroll({ deltaY: -500 }) → browser_screenshot()
89
89
  \`\`\`
90
+ Note: deltaY follows W3C wheel-event convention: positive=up, negative=down. Default deltaY=-300 scrolls down.
90
91
 
91
92
  **Research Workflow (search + browser):**
92
93
  \`\`\`
@@ -102,5 +103,17 @@ browser_execute_js("document.querySelector('.main').innerText") → extract cont
102
103
  \`\`\`
103
104
  write("/tmp/extract.js", "...script with daemon access...") → browser_run_script("/tmp/extract.js")
104
105
  \`\`\`
106
+
107
+ ### Additional Tools
108
+
109
+ | Tool | Purpose |
110
+ |------|---------|
111
+ | browser_upload_file | Upload a file to a file input (bypasses file picker) |
112
+ | browser_dispatch_key | Dispatch a DOM KeyboardEvent on a specific element (for React/Vue inputs); returns details.matched (number of elements) |
113
+ | browser_download | Configure download directory and disable save-as prompts |
114
+ | browser_viewport_resize | Resize the viewport for responsive testing |
115
+ | browser_drag_and_drop | Perform drag-and-drop from one coordinate to another |
116
+ | browser_print_to_pdf | Print the current page to a PDF file |
117
+ | browser_get_network_log | Get buffered network request/response events |
105
118
  `;
106
119
  }
@@ -0,0 +1,52 @@
1
+ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
2
+ import type { BrowserClient } from "./client";
3
+ import { type AnyBrowserToolDefinition, registerBrowserTool } from "./util/tool";
4
+ import { clickTool } from "./domains/click";
5
+ import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard";
6
+ import { pageInfoTool, waitTool, waitForLoadTool } from "./domains/page";
7
+ import { scrollTool } from "./domains/scroll";
8
+ import { handleDialogTool } from "./domains/dialog";
9
+ import { screenshotTool } from "./domains/screenshot";
10
+ import { navigateTool, openUrlsTool } from "./domains/navigate";
11
+ import { goBackTool, goForwardTool, reloadTool } from "./domains/history";
12
+ import { listTabsTool, currentTabTool, switchTabTool, newTabTool } from "./domains/tabs";
13
+ import { uploadFileTool, downloadTool, printToPdfTool } from "./domains/files";
14
+ import { viewportResizeTool } from "./domains/viewport";
15
+ import { dragAndDropTool } from "./domains/drag";
16
+ import { httpGetTool, getNetworkLogTool } from "./domains/network";
17
+ import { executeJsTool, runScriptTool } from "./domains/js";
18
+
19
+ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
20
+ clickTool,
21
+ typeTool,
22
+ pressKeyTool,
23
+ dispatchKeyTool,
24
+ scrollTool,
25
+ pageInfoTool,
26
+ waitTool,
27
+ waitForLoadTool,
28
+ handleDialogTool,
29
+ screenshotTool,
30
+ navigateTool,
31
+ openUrlsTool,
32
+ goBackTool,
33
+ goForwardTool,
34
+ reloadTool,
35
+ listTabsTool,
36
+ currentTabTool,
37
+ switchTabTool,
38
+ newTabTool,
39
+ uploadFileTool,
40
+ downloadTool,
41
+ printToPdfTool,
42
+ viewportResizeTool,
43
+ dragAndDropTool,
44
+ httpGetTool,
45
+ getNetworkLogTool,
46
+ executeJsTool,
47
+ runScriptTool,
48
+ ];
49
+
50
+ export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => {
51
+ for (const t of TOOLS) registerBrowserTool(pi, client, t);
52
+ };
@@ -0,0 +1,11 @@
1
+ import { Type } from "typebox";
2
+
3
+ export const Coords = Type.Object({
4
+ x: Type.Number({ description: "X coordinate in CSS pixels from left edge of viewport" }),
5
+ y: Type.Number({ description: "Y coordinate in CSS pixels from top edge of viewport" }),
6
+ });
7
+
8
+ export const MouseButton = Type.Union(
9
+ [Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")],
10
+ { description: 'Mouse button: "left", "right", or "middle"' },
11
+ );
package/src/setup.ts CHANGED
@@ -5,18 +5,18 @@
5
5
 
6
6
  import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
7
7
  import { execSync } from "node:child_process";
8
- import type { BrowserDaemon } from "./daemon";
8
+ import type { BrowserClient } from "./client";
9
9
 
10
- export function registerSetupCommand(pi: ExtensionAPI, daemon: BrowserDaemon): void {
10
+ export function registerSetupCommand(pi: ExtensionAPI, client: BrowserClient): void {
11
11
  pi.registerCommand("browser-setup", {
12
12
  description: "Connect pi to your Chrome browser",
13
13
  handler: async (_args, ctx) => {
14
- await runSetup(ctx, daemon);
14
+ await runSetup(ctx, client);
15
15
  },
16
16
  });
17
17
  }
18
18
 
19
- async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<void> {
19
+ async function runSetup(ctx: ExtensionContext, client: BrowserClient): Promise<void> {
20
20
  ctx.ui.notify("browser setup: checking Chrome...", "info");
21
21
 
22
22
  // Step 1: Check Chrome is running
@@ -32,11 +32,9 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v
32
32
 
33
33
  // Step 2: Try to connect to Chrome DevTools
34
34
  ctx.ui.notify("Connecting to Chrome DevTools...", "info");
35
- try {
36
- await daemon.start();
37
- ctx.ui.notify("Connected to Chrome ✓", "info");
38
- } catch (err) {
39
- const msg = err instanceof Error ? err.message : String(err);
35
+ const startResult = await client.start();
36
+ if (!startResult.success) {
37
+ const msg = startResult.error.message;
40
38
  const lower = msg.toLowerCase();
41
39
 
42
40
  if (
@@ -58,25 +56,26 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v
58
56
  ctx.ui.notify(`Connection failed: ${msg}`, "error");
59
57
  return;
60
58
  }
59
+ ctx.ui.notify("Connected to Chrome ✓", "info");
61
60
 
62
61
  // Step 3: Verify with test navigation
63
62
  ctx.ui.notify("Testing browser control...", "info");
64
- try {
65
- await daemon.newTab("https://github.com");
66
- const info = await daemon.getPageInfo();
67
- if ("dialog" in info) {
68
- await daemon.cdp("Page.handleJavaScriptDialog", { accept: true });
69
- }
70
- ctx.ui.notify(
71
- `Browser control verified ✓\nNavigated to: ${"url" in info ? info.url : "github.com"}`,
72
- "info",
73
- );
74
- } catch (err) {
63
+ const tabResult = await client.newTab("https://github.com");
64
+ if (!tabResult.success) {
75
65
  ctx.ui.notify(
76
- `Browser connected but test navigation failed: ${err instanceof Error ? err.message : String(err)}`,
66
+ `Browser connected but test navigation failed: ${tabResult.error.message}`,
77
67
  "warning",
78
68
  );
69
+ return;
70
+ }
71
+
72
+ const info = await client.pageInfo();
73
+ if (info.success && "dialog" in info.data) {
74
+ await client.session().call("Page.handleJavaScriptDialog", { accept: true });
79
75
  }
76
+
77
+ const pageUrl = info.success && !("dialog" in info.data) ? info.data.url : "github.com";
78
+ ctx.ui.notify(`Browser control verified ✓\nNavigated to: ${pageUrl}`, "info");
80
79
  }
81
80
 
82
81
  function checkChromeRunning(): boolean {