pi-browser-harness 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +134 -112
- package/package.json +4 -1
- package/skills/pi-browser-harness/SKILL.md +17 -4
- package/src/cdp/discovery.ts +95 -0
- package/src/cdp/errors.ts +19 -0
- package/src/cdp/session.ts +139 -0
- package/src/cdp/transport.ts +221 -0
- package/src/cdp/types.ts +72 -0
- package/src/client.ts +232 -0
- package/src/domains/click.ts +73 -0
- package/src/domains/dialog.ts +31 -0
- package/src/domains/drag.ts +66 -0
- package/src/domains/files.ts +194 -0
- package/src/domains/history.ts +76 -0
- package/src/domains/js.ts +197 -0
- package/src/domains/keyboard.ts +127 -0
- package/src/domains/navigate.ts +146 -0
- package/src/domains/network.ts +80 -0
- package/src/domains/page.ts +88 -0
- package/src/domains/screenshot.ts +114 -0
- package/src/domains/scroll.ts +53 -0
- package/src/domains/tabs.ts +121 -0
- package/src/domains/viewport.ts +40 -0
- package/src/index.ts +65 -142
- package/src/prompt.ts +16 -3
- package/src/registry.ts +52 -0
- package/src/schemas/common.ts +11 -0
- package/src/setup.ts +20 -21
- package/src/state.ts +22 -54
- package/src/util/js-template.ts +27 -0
- package/src/util/keycodes.ts +10 -0
- package/src/util/paths.ts +21 -0
- package/src/util/result.ts +15 -0
- package/src/util/sharp-shim.ts +49 -0
- package/src/util/time.ts +29 -0
- package/src/util/tool.ts +169 -0
- package/src/util/truncate.ts +35 -0
- package/tsconfig.json +10 -1
- package/src/daemon.ts +0 -767
- package/src/protocol.ts +0 -77
- package/src/renderers.ts +0 -21
- package/src/tools.ts +0 -1358
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
|
|
5
|
+
const ListTabsArgs = Type.Object({
|
|
6
|
+
includeInternal: Type.Optional(Type.Boolean({ default: true, description: "Include chrome:// pages" })),
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
export const listTabsTool = defineBrowserTool({
|
|
10
|
+
name: "browser_list_tabs",
|
|
11
|
+
label: "Browser List Tabs",
|
|
12
|
+
description: "List all open browser tabs (page targets).",
|
|
13
|
+
promptSnippet: "List browser tabs",
|
|
14
|
+
promptGuidelines: [
|
|
15
|
+
"Use to find a full targetId for browser_switch_tab.",
|
|
16
|
+
"Internal tabs (chrome://) included by default; pass includeInternal=false to exclude them.",
|
|
17
|
+
"Each tab shows its full 32-char hex targetId — use the exact value with browser_switch_tab.",
|
|
18
|
+
],
|
|
19
|
+
parameters: ListTabsArgs,
|
|
20
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
21
|
+
const r = await client.listTabs(args.includeInternal ?? true);
|
|
22
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
23
|
+
const lines = r.data.map((t, i) => ` [${i}] ${t.targetId} ${t.url}\n ${t.title}`);
|
|
24
|
+
return ok({
|
|
25
|
+
text: `Tabs (${r.data.length}):\n${lines.join("\n")}`,
|
|
26
|
+
details: { tabs: r.data },
|
|
27
|
+
});
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
export const currentTabTool = defineBrowserTool({
|
|
32
|
+
name: "browser_current_tab",
|
|
33
|
+
label: "Browser Current Tab",
|
|
34
|
+
description: "Get info about the currently attached tab.",
|
|
35
|
+
promptSnippet: "Get current tab info",
|
|
36
|
+
promptGuidelines: ["Returns targetId, url, title."],
|
|
37
|
+
parameters: Type.Object({}),
|
|
38
|
+
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
39
|
+
const cur = client.current();
|
|
40
|
+
if (!cur) return err({ kind: "invalid_state", message: "No tab attached." });
|
|
41
|
+
const ti = await client.session().callBrowser("Target.getTargetInfo", { targetId: cur.targetId });
|
|
42
|
+
if (!ti.success) return err({ kind: "cdp_error", message: ti.error.message });
|
|
43
|
+
const info = (ti.data as { targetInfo: { targetId: string; url: string; title: string } }).targetInfo; // CDP boundary cast: Target.getTargetInfo returns { targetInfo: { targetId, url, title } }
|
|
44
|
+
return ok({
|
|
45
|
+
text: `Current tab:\n ${info.targetId}\n ${info.url}\n ${info.title}`,
|
|
46
|
+
details: { targetId: info.targetId, url: info.url, title: info.title },
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
const SwitchTabArgs = Type.Object({
|
|
52
|
+
targetId: Type.String({ description: "Target ID from browser_list_tabs" }),
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
export const switchTabTool = defineBrowserTool({
|
|
56
|
+
name: "browser_switch_tab",
|
|
57
|
+
label: "Browser Switch Tab",
|
|
58
|
+
description: "Switch to and attach to a different tab by targetId.",
|
|
59
|
+
promptSnippet: "Switch tabs",
|
|
60
|
+
promptGuidelines: [
|
|
61
|
+
"Get a targetId via browser_list_tabs first.",
|
|
62
|
+
"Accepts exact targetId or a unique prefix of at least 8 hex characters.",
|
|
63
|
+
],
|
|
64
|
+
parameters: SwitchTabArgs,
|
|
65
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
66
|
+
// Try exact match first
|
|
67
|
+
let r = await client.switchTab(args.targetId);
|
|
68
|
+
if (r.success) {
|
|
69
|
+
return ok({ text: `Switched to ${args.targetId}`, details: { targetId: args.targetId } });
|
|
70
|
+
}
|
|
71
|
+
// If exact match failed and input looks like a short prefix (≥8 hex chars), try prefix resolution
|
|
72
|
+
const isHexPrefix = /^[0-9A-Fa-f]{8,}$/.test(args.targetId);
|
|
73
|
+
if (!isHexPrefix) {
|
|
74
|
+
return err({ kind: "cdp_error", message: r.error.message });
|
|
75
|
+
}
|
|
76
|
+
const tabs = await client.listTabs(true);
|
|
77
|
+
if (!tabs.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
78
|
+
const matches = tabs.data.filter((t) => t.targetId.startsWith(args.targetId));
|
|
79
|
+
if (matches.length === 0) {
|
|
80
|
+
return err({ kind: "cdp_error", message: `No tab found with prefix "${args.targetId}"` });
|
|
81
|
+
}
|
|
82
|
+
if (matches.length > 1) {
|
|
83
|
+
const ids = matches.map((t) => t.targetId).join(", ");
|
|
84
|
+
return err({
|
|
85
|
+
kind: "invalid_state",
|
|
86
|
+
message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} tabs: ${ids}`,
|
|
87
|
+
details: { matches: matches.map((t) => ({ targetId: t.targetId, url: t.url })) },
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
const resolved = matches[0]!.targetId;
|
|
91
|
+
r = await client.switchTab(resolved);
|
|
92
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
93
|
+
return ok({
|
|
94
|
+
text: `Switched to ${resolved} (resolved from prefix "${args.targetId}")`,
|
|
95
|
+
details: { targetId: resolved, resolvedPrefix: args.targetId },
|
|
96
|
+
});
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
const NewTabArgs = Type.Object({
|
|
101
|
+
url: Type.Optional(Type.String({ description: "Optional URL to navigate to" })),
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
export const newTabTool = defineBrowserTool({
|
|
105
|
+
name: "browser_new_tab",
|
|
106
|
+
label: "Browser New Tab",
|
|
107
|
+
description: "Open a new tab and switch to it. Optionally navigate to a URL.",
|
|
108
|
+
promptSnippet: "Open a new tab",
|
|
109
|
+
promptGuidelines: ["Pass url to navigate immediately."],
|
|
110
|
+
parameters: NewTabArgs,
|
|
111
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
112
|
+
const r = await client.newTab(args.url);
|
|
113
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
114
|
+
return ok({
|
|
115
|
+
text: `New tab: ${r.data}${args.url ? ` (${args.url})` : ""}`,
|
|
116
|
+
details: args.url !== undefined
|
|
117
|
+
? { targetId: r.data, url: args.url }
|
|
118
|
+
: { targetId: r.data },
|
|
119
|
+
});
|
|
120
|
+
},
|
|
121
|
+
});
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
|
|
5
|
+
const ViewportArgs = Type.Object({
|
|
6
|
+
width: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel width" }),
|
|
7
|
+
height: Type.Integer({ minimum: 100, maximum: 8000, description: "Viewport CSS pixel height" }),
|
|
8
|
+
deviceScaleFactor: Type.Optional(
|
|
9
|
+
Type.Number({ minimum: 0.5, maximum: 4, default: 1, description: "Device pixel ratio" }),
|
|
10
|
+
),
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
export const viewportResizeTool = defineBrowserTool({
|
|
14
|
+
name: "browser_viewport_resize",
|
|
15
|
+
label: "Browser Viewport Resize",
|
|
16
|
+
description: "Override the viewport size and device pixel ratio for responsive testing.",
|
|
17
|
+
promptSnippet: "Resize the viewport (responsive testing)",
|
|
18
|
+
promptGuidelines: [
|
|
19
|
+
"Width/height in CSS pixels (e.g., 375x667 for iPhone SE).",
|
|
20
|
+
"Pass deviceScaleFactor=2 to simulate retina displays.",
|
|
21
|
+
],
|
|
22
|
+
parameters: ViewportArgs,
|
|
23
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
24
|
+
const r = await client.session().call("Emulation.setDeviceMetricsOverride", {
|
|
25
|
+
width: args.width,
|
|
26
|
+
height: args.height,
|
|
27
|
+
deviceScaleFactor: args.deviceScaleFactor ?? 1,
|
|
28
|
+
mobile: false,
|
|
29
|
+
});
|
|
30
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
31
|
+
return ok({
|
|
32
|
+
text: `Viewport set to ${args.width}x${args.height} @${args.deviceScaleFactor ?? 1}x`,
|
|
33
|
+
details: {
|
|
34
|
+
width: args.width,
|
|
35
|
+
height: args.height,
|
|
36
|
+
deviceScaleFactor: args.deviceScaleFactor ?? 1,
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
},
|
|
40
|
+
});
|
package/src/index.ts
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* pi-browser-harness — browser control extension for pi.
|
|
3
3
|
*
|
|
4
4
|
* Gives pi agents full control of a real Chrome browser via CDP.
|
|
5
|
-
*
|
|
6
|
-
* and injects browser control guidance into the
|
|
5
|
+
* Connects to the user's running Chrome (chrome://inspect/#remote-debugging),
|
|
6
|
+
* registers browser_* tools, and injects browser control guidance into the
|
|
7
|
+
* system prompt.
|
|
7
8
|
*
|
|
8
9
|
* Install:
|
|
9
10
|
* pi install npm:pi-browser-harness
|
|
@@ -11,39 +12,32 @@
|
|
|
11
12
|
*
|
|
12
13
|
* Commands:
|
|
13
14
|
* /browser-setup — guided setup wizard
|
|
14
|
-
* /browser-status — show
|
|
15
|
-
* /browser-reload-daemon — restart the
|
|
15
|
+
* /browser-status — show client status and current page
|
|
16
|
+
* /browser-reload-daemon — restart the browser client
|
|
16
17
|
*
|
|
17
18
|
* Flags:
|
|
18
|
-
* --browser-namespace <name>
|
|
19
|
+
* --browser-namespace <name> — override namespace (default: auto)
|
|
19
20
|
* --browser-debug-clicks — enable debug click overlay
|
|
20
21
|
*/
|
|
21
22
|
|
|
22
|
-
import type { ExtensionAPI
|
|
23
|
-
import {
|
|
23
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
24
|
+
import { type BrowserClient, createBrowserClient } from "./client";
|
|
24
25
|
import { getBrowserSystemPrompt } from "./prompt";
|
|
25
|
-
import { registerRenderers } from "./renderers";
|
|
26
26
|
import { registerSetupCommand } from "./setup";
|
|
27
27
|
import { type BrowserState, defaultState, persistState, restoreState } from "./state";
|
|
28
|
-
import {
|
|
28
|
+
import { registerAllTools } from "./registry";
|
|
29
|
+
import { cleanupTempDirs } from "./util/truncate";
|
|
29
30
|
|
|
30
|
-
export default function browserHarnessExtension(pi: ExtensionAPI) {
|
|
31
|
-
// ── Resolve namespace ──────────────────────────────────────────────────────
|
|
31
|
+
export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
32
32
|
const flagNs = pi.getFlag("browser-namespace") as string | undefined;
|
|
33
|
-
const namespace = flagNs
|
|
33
|
+
const namespace = flagNs ?? `pi-${Math.random().toString(36).slice(2, 10)}`;
|
|
34
34
|
|
|
35
|
-
// ── State ──────────────────────────────────────────────────────────────────
|
|
36
35
|
let state: BrowserState = defaultState(namespace);
|
|
37
|
-
let
|
|
38
|
-
|
|
39
|
-
// ── Tool initialization flag ───────────────────────────────────────────────
|
|
36
|
+
let client: BrowserClient | null = null;
|
|
40
37
|
let toolsRegistered = false;
|
|
41
38
|
|
|
42
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
43
|
-
// Flags
|
|
44
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
45
39
|
pi.registerFlag("browser-namespace", {
|
|
46
|
-
description: "Browser daemon namespace
|
|
40
|
+
description: "Browser daemon namespace. Default: auto-generated",
|
|
47
41
|
type: "string",
|
|
48
42
|
});
|
|
49
43
|
pi.registerFlag("browser-debug-clicks", {
|
|
@@ -52,172 +46,101 @@ export default function browserHarnessExtension(pi: ExtensionAPI) {
|
|
|
52
46
|
default: false,
|
|
53
47
|
});
|
|
54
48
|
|
|
55
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
56
|
-
// Commands
|
|
57
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
58
49
|
pi.registerCommand("browser-status", {
|
|
59
50
|
description: "Show browser connection status and current page",
|
|
60
51
|
handler: async (_args, ctx) => {
|
|
61
|
-
if (!
|
|
62
|
-
ctx.ui.notify("Browser
|
|
52
|
+
if (!client) {
|
|
53
|
+
ctx.ui.notify("Browser client not started. Run /browser-setup first.", "warning");
|
|
63
54
|
return;
|
|
64
55
|
}
|
|
65
|
-
|
|
66
|
-
const status = daemon.getStatus();
|
|
56
|
+
const s = client.status();
|
|
67
57
|
const lines = [
|
|
68
|
-
`Browser: ${
|
|
69
|
-
`Session: ${
|
|
58
|
+
`Browser: ${s.alive ? "🟢 Connected" : "🔴 Disconnected"}`,
|
|
59
|
+
`Session: ${s.sessionId ?? "none"}`,
|
|
70
60
|
];
|
|
71
|
-
|
|
72
|
-
if (
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (status.alive) {
|
|
78
|
-
try {
|
|
79
|
-
const info = await daemon.getPageInfo();
|
|
80
|
-
if ("dialog" in info) {
|
|
81
|
-
lines.push(`\n⚠️ Dialog open: ${info.dialog.type} — "${info.dialog.message}"`);
|
|
61
|
+
if (s.remoteBrowserId) lines.push(`Browser ID: ${s.remoteBrowserId}`);
|
|
62
|
+
if (s.alive) {
|
|
63
|
+
const info = await client.pageInfo();
|
|
64
|
+
if (info.success) {
|
|
65
|
+
if ("dialog" in info.data) {
|
|
66
|
+
lines.push(`\n⚠️ Dialog open: ${info.data.dialog.type} — "${info.data.dialog.message}"`);
|
|
82
67
|
} else {
|
|
83
|
-
lines.push(
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
68
|
+
lines.push(
|
|
69
|
+
`\nCurrent Page:`,
|
|
70
|
+
` URL: ${info.data.url}`,
|
|
71
|
+
` Title: ${info.data.title}`,
|
|
72
|
+
` Viewport: ${info.data.width}x${info.data.height}`,
|
|
73
|
+
);
|
|
87
74
|
}
|
|
88
|
-
} catch {
|
|
89
|
-
lines.push("\n(Could not read page info)");
|
|
90
75
|
}
|
|
91
76
|
}
|
|
92
|
-
|
|
93
77
|
ctx.ui.notify(lines.join("\n"), "info");
|
|
94
78
|
},
|
|
95
79
|
});
|
|
96
80
|
|
|
97
81
|
pi.registerCommand("browser-reload-daemon", {
|
|
98
|
-
description: "Restart the browser
|
|
82
|
+
description: "Restart the browser client",
|
|
99
83
|
handler: async (_args, ctx) => {
|
|
100
|
-
if (!
|
|
101
|
-
ctx.ui.notify("Browser
|
|
84
|
+
if (!client) {
|
|
85
|
+
ctx.ui.notify("Browser client not started.", "warning");
|
|
102
86
|
return;
|
|
103
87
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
ctx.ui.notify(
|
|
112
|
-
`Restart failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
113
|
-
"error",
|
|
114
|
-
);
|
|
88
|
+
ctx.ui.notify("Restarting browser client...", "info");
|
|
89
|
+
await client.stop();
|
|
90
|
+
const r = await client.start();
|
|
91
|
+
if (r.success) {
|
|
92
|
+
ctx.ui.notify("Browser client restarted ✓", "info");
|
|
93
|
+
} else {
|
|
94
|
+
ctx.ui.notify(`Restart failed: ${r.error.message}`, "error");
|
|
115
95
|
}
|
|
116
96
|
},
|
|
117
97
|
});
|
|
118
98
|
|
|
119
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
120
|
-
// Session lifecycle
|
|
121
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
122
99
|
pi.on("session_start", async (_event, ctx) => {
|
|
123
|
-
// Restore persisted state (preserving the current namespace from flag)
|
|
124
100
|
state = restoreState(ctx, state.namespace);
|
|
125
|
-
|
|
126
|
-
//
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
try {
|
|
131
|
-
await daemon.start();
|
|
132
|
-
} catch {
|
|
133
|
-
// Don't block — user can run /browser-setup later
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// Register tools (once)
|
|
137
|
-
if (!toolsRegistered && daemon) {
|
|
138
|
-
registerTools(pi, daemon);
|
|
101
|
+
client = createBrowserClient({ namespace: state.namespace });
|
|
102
|
+
// failure is fine — surfaced via tool errors when the agent first tries to use the browser
|
|
103
|
+
await client.start();
|
|
104
|
+
if (!toolsRegistered) {
|
|
105
|
+
registerAllTools(pi, client);
|
|
139
106
|
toolsRegistered = true;
|
|
140
107
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
if (daemon) {
|
|
147
|
-
registerSetupCommand(pi, daemon);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// Update status
|
|
151
|
-
if (daemon.getStatus().alive) {
|
|
152
|
-
ctx.ui.setStatus("browser", "🟢 Browser connected");
|
|
153
|
-
} else {
|
|
154
|
-
ctx.ui.setStatus("browser", "🔴 Browser — run /browser-setup");
|
|
155
|
-
}
|
|
108
|
+
registerSetupCommand(pi, client);
|
|
109
|
+
ctx.ui.setStatus(
|
|
110
|
+
"browser",
|
|
111
|
+
client.status().alive ? "🟢 Browser connected" : "🔴 Browser — run /browser-setup",
|
|
112
|
+
);
|
|
156
113
|
});
|
|
157
114
|
|
|
158
|
-
pi.on("session_shutdown", async (
|
|
159
|
-
|
|
160
|
-
if (
|
|
161
|
-
persistState(pi, state);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
// Stop daemon
|
|
165
|
-
if (daemon) {
|
|
115
|
+
pi.on("session_shutdown", async () => {
|
|
116
|
+
persistState(pi, state);
|
|
117
|
+
if (client) {
|
|
166
118
|
try {
|
|
167
|
-
await
|
|
168
|
-
} catch {
|
|
169
|
-
// best-effort
|
|
119
|
+
await client.stop();
|
|
120
|
+
} catch (e) {
|
|
121
|
+
// Shutdown is best-effort, but a stuck stop() points at a transport
|
|
122
|
+
// bug worth surfacing for debugging.
|
|
123
|
+
console.warn("[pi-browser-harness] client.stop() failed during shutdown:", e);
|
|
170
124
|
}
|
|
171
|
-
|
|
125
|
+
client = null;
|
|
172
126
|
}
|
|
173
|
-
|
|
174
127
|
toolsRegistered = false;
|
|
175
|
-
|
|
176
|
-
// Clean up temp files from truncated tool outputs
|
|
177
128
|
await cleanupTempDirs();
|
|
178
129
|
});
|
|
179
130
|
|
|
180
|
-
// Restore state on tree navigation
|
|
181
131
|
pi.on("session_tree", async (_event, ctx) => {
|
|
182
|
-
state = restoreState(ctx,
|
|
183
|
-
|
|
184
|
-
persistState(pi, state);
|
|
185
|
-
}
|
|
132
|
+
state = restoreState(ctx, client?.namespace);
|
|
133
|
+
persistState(pi, state);
|
|
186
134
|
});
|
|
187
135
|
|
|
188
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
189
|
-
// System prompt injection
|
|
190
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
191
136
|
pi.on("before_agent_start", async (event) => {
|
|
192
|
-
if (!
|
|
193
|
-
// Browser not connected — inject minimal note
|
|
137
|
+
if (!client || !client.status().alive) {
|
|
194
138
|
return {
|
|
195
139
|
systemPrompt:
|
|
196
140
|
event.systemPrompt +
|
|
197
|
-
`\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser
|
|
198
|
-
`Run /browser-setup to connect to Chrome, or /browser-status to check.`,
|
|
141
|
+
`\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser is not connected. Run /browser-setup.`,
|
|
199
142
|
};
|
|
200
143
|
}
|
|
201
|
-
|
|
202
|
-
return {
|
|
203
|
-
systemPrompt: event.systemPrompt + getBrowserSystemPrompt(),
|
|
204
|
-
};
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
208
|
-
// Tool result hook: track tab history
|
|
209
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
210
|
-
pi.on("tool_result", async (event) => {
|
|
211
|
-
if (event.toolName === "browser_new_tab" || event.toolName === "browser_navigate") {
|
|
212
|
-
// Track new tabs in history
|
|
213
|
-
const details = event.details as { targetId?: string } | undefined;
|
|
214
|
-
if (details?.targetId) {
|
|
215
|
-
state = {
|
|
216
|
-
...state,
|
|
217
|
-
tabHistory: [details.targetId, ...state.tabHistory.filter((id) => id !== details.targetId)].slice(0, 20),
|
|
218
|
-
};
|
|
219
|
-
persistState(pi, state);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
144
|
+
return { systemPrompt: event.systemPrompt + getBrowserSystemPrompt() };
|
|
222
145
|
});
|
|
223
146
|
}
|
package/src/prompt.ts
CHANGED
|
@@ -21,12 +21,12 @@ to interact with pages visually, and other tools alongside them.
|
|
|
21
21
|
|
|
22
22
|
| Tool | Purpose |
|
|
23
23
|
|------|---------|
|
|
24
|
-
| browser_screenshot | Capture a PNG screenshot of the current page |
|
|
24
|
+
| browser_screenshot | Capture a PNG screenshot of the current page (result details include path and attached: false — the LLM reads the image via the saved file path) |
|
|
25
25
|
| browser_click | Click at viewport coordinates |
|
|
26
26
|
| browser_type | Type text into the focused element |
|
|
27
27
|
| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) |
|
|
28
28
|
| browser_scroll | Scroll the page at coordinates |
|
|
29
|
-
| browser_navigate | Navigate to a URL |
|
|
29
|
+
| browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") |
|
|
30
30
|
| browser_new_tab | Open a new tab, optionally navigate |
|
|
31
31
|
| browser_open_urls | Open multiple URLs in new tabs (parallel) |
|
|
32
32
|
| browser_go_back / browser_go_forward / browser_reload | History navigation |
|
|
@@ -37,7 +37,7 @@ to interact with pages visually, and other tools alongside them.
|
|
|
37
37
|
| browser_execute_js | Execute JavaScript and return the result |
|
|
38
38
|
| browser_http_get | Direct HTTP GET (outside browser, for APIs) |
|
|
39
39
|
| browser_wait | Wait N seconds |
|
|
40
|
-
| browser_wait_for_load | Wait for document.readyState === 'complete' |
|
|
40
|
+
| browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) |
|
|
41
41
|
| browser_handle_dialog | Accept or dismiss a JS dialog |
|
|
42
42
|
| browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) |
|
|
43
43
|
|
|
@@ -87,6 +87,7 @@ browser_http_get("https://api.example.com/data")
|
|
|
87
87
|
\`\`\`
|
|
88
88
|
browser_screenshot() → browser_scroll({ deltaY: -500 }) → browser_screenshot()
|
|
89
89
|
\`\`\`
|
|
90
|
+
Note: deltaY follows W3C wheel-event convention: positive=up, negative=down. Default deltaY=-300 scrolls down.
|
|
90
91
|
|
|
91
92
|
**Research Workflow (search + browser):**
|
|
92
93
|
\`\`\`
|
|
@@ -102,5 +103,17 @@ browser_execute_js("document.querySelector('.main').innerText") → extract cont
|
|
|
102
103
|
\`\`\`
|
|
103
104
|
write("/tmp/extract.js", "...script with daemon access...") → browser_run_script("/tmp/extract.js")
|
|
104
105
|
\`\`\`
|
|
106
|
+
|
|
107
|
+
### Additional Tools
|
|
108
|
+
|
|
109
|
+
| Tool | Purpose |
|
|
110
|
+
|------|---------|
|
|
111
|
+
| browser_upload_file | Upload a file to a file input (bypasses file picker) |
|
|
112
|
+
| browser_dispatch_key | Dispatch a DOM KeyboardEvent on a specific element (for React/Vue inputs); returns details.matched (number of elements) |
|
|
113
|
+
| browser_download | Configure download directory and disable save-as prompts |
|
|
114
|
+
| browser_viewport_resize | Resize the viewport for responsive testing |
|
|
115
|
+
| browser_drag_and_drop | Perform drag-and-drop from one coordinate to another |
|
|
116
|
+
| browser_print_to_pdf | Print the current page to a PDF file |
|
|
117
|
+
| browser_get_network_log | Get buffered network request/response events |
|
|
105
118
|
`;
|
|
106
119
|
}
|
package/src/registry.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
2
|
+
import type { BrowserClient } from "./client";
|
|
3
|
+
import { type AnyBrowserToolDefinition, registerBrowserTool } from "./util/tool";
|
|
4
|
+
import { clickTool } from "./domains/click";
|
|
5
|
+
import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard";
|
|
6
|
+
import { pageInfoTool, waitTool, waitForLoadTool } from "./domains/page";
|
|
7
|
+
import { scrollTool } from "./domains/scroll";
|
|
8
|
+
import { handleDialogTool } from "./domains/dialog";
|
|
9
|
+
import { screenshotTool } from "./domains/screenshot";
|
|
10
|
+
import { navigateTool, openUrlsTool } from "./domains/navigate";
|
|
11
|
+
import { goBackTool, goForwardTool, reloadTool } from "./domains/history";
|
|
12
|
+
import { listTabsTool, currentTabTool, switchTabTool, newTabTool } from "./domains/tabs";
|
|
13
|
+
import { uploadFileTool, downloadTool, printToPdfTool } from "./domains/files";
|
|
14
|
+
import { viewportResizeTool } from "./domains/viewport";
|
|
15
|
+
import { dragAndDropTool } from "./domains/drag";
|
|
16
|
+
import { httpGetTool, getNetworkLogTool } from "./domains/network";
|
|
17
|
+
import { executeJsTool, runScriptTool } from "./domains/js";
|
|
18
|
+
|
|
19
|
+
const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
|
|
20
|
+
clickTool,
|
|
21
|
+
typeTool,
|
|
22
|
+
pressKeyTool,
|
|
23
|
+
dispatchKeyTool,
|
|
24
|
+
scrollTool,
|
|
25
|
+
pageInfoTool,
|
|
26
|
+
waitTool,
|
|
27
|
+
waitForLoadTool,
|
|
28
|
+
handleDialogTool,
|
|
29
|
+
screenshotTool,
|
|
30
|
+
navigateTool,
|
|
31
|
+
openUrlsTool,
|
|
32
|
+
goBackTool,
|
|
33
|
+
goForwardTool,
|
|
34
|
+
reloadTool,
|
|
35
|
+
listTabsTool,
|
|
36
|
+
currentTabTool,
|
|
37
|
+
switchTabTool,
|
|
38
|
+
newTabTool,
|
|
39
|
+
uploadFileTool,
|
|
40
|
+
downloadTool,
|
|
41
|
+
printToPdfTool,
|
|
42
|
+
viewportResizeTool,
|
|
43
|
+
dragAndDropTool,
|
|
44
|
+
httpGetTool,
|
|
45
|
+
getNetworkLogTool,
|
|
46
|
+
executeJsTool,
|
|
47
|
+
runScriptTool,
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => {
|
|
51
|
+
for (const t of TOOLS) registerBrowserTool(pi, client, t);
|
|
52
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
|
|
3
|
+
export const Coords = Type.Object({
|
|
4
|
+
x: Type.Number({ description: "X coordinate in CSS pixels from left edge of viewport" }),
|
|
5
|
+
y: Type.Number({ description: "Y coordinate in CSS pixels from top edge of viewport" }),
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
export const MouseButton = Type.Union(
|
|
9
|
+
[Type.Literal("left"), Type.Literal("right"), Type.Literal("middle")],
|
|
10
|
+
{ description: 'Mouse button: "left", "right", or "middle"' },
|
|
11
|
+
);
|
package/src/setup.ts
CHANGED
|
@@ -5,18 +5,18 @@
|
|
|
5
5
|
|
|
6
6
|
import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
|
|
7
7
|
import { execSync } from "node:child_process";
|
|
8
|
-
import type {
|
|
8
|
+
import type { BrowserClient } from "./client";
|
|
9
9
|
|
|
10
|
-
export function registerSetupCommand(pi: ExtensionAPI,
|
|
10
|
+
export function registerSetupCommand(pi: ExtensionAPI, client: BrowserClient): void {
|
|
11
11
|
pi.registerCommand("browser-setup", {
|
|
12
12
|
description: "Connect pi to your Chrome browser",
|
|
13
13
|
handler: async (_args, ctx) => {
|
|
14
|
-
await runSetup(ctx,
|
|
14
|
+
await runSetup(ctx, client);
|
|
15
15
|
},
|
|
16
16
|
});
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
async function runSetup(ctx: ExtensionContext,
|
|
19
|
+
async function runSetup(ctx: ExtensionContext, client: BrowserClient): Promise<void> {
|
|
20
20
|
ctx.ui.notify("browser setup: checking Chrome...", "info");
|
|
21
21
|
|
|
22
22
|
// Step 1: Check Chrome is running
|
|
@@ -32,11 +32,9 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v
|
|
|
32
32
|
|
|
33
33
|
// Step 2: Try to connect to Chrome DevTools
|
|
34
34
|
ctx.ui.notify("Connecting to Chrome DevTools...", "info");
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
} catch (err) {
|
|
39
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
35
|
+
const startResult = await client.start();
|
|
36
|
+
if (!startResult.success) {
|
|
37
|
+
const msg = startResult.error.message;
|
|
40
38
|
const lower = msg.toLowerCase();
|
|
41
39
|
|
|
42
40
|
if (
|
|
@@ -58,25 +56,26 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v
|
|
|
58
56
|
ctx.ui.notify(`Connection failed: ${msg}`, "error");
|
|
59
57
|
return;
|
|
60
58
|
}
|
|
59
|
+
ctx.ui.notify("Connected to Chrome ✓", "info");
|
|
61
60
|
|
|
62
61
|
// Step 3: Verify with test navigation
|
|
63
62
|
ctx.ui.notify("Testing browser control...", "info");
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const info = await daemon.getPageInfo();
|
|
67
|
-
if ("dialog" in info) {
|
|
68
|
-
await daemon.cdp("Page.handleJavaScriptDialog", { accept: true });
|
|
69
|
-
}
|
|
70
|
-
ctx.ui.notify(
|
|
71
|
-
`Browser control verified ✓\nNavigated to: ${"url" in info ? info.url : "github.com"}`,
|
|
72
|
-
"info",
|
|
73
|
-
);
|
|
74
|
-
} catch (err) {
|
|
63
|
+
const tabResult = await client.newTab("https://github.com");
|
|
64
|
+
if (!tabResult.success) {
|
|
75
65
|
ctx.ui.notify(
|
|
76
|
-
`Browser connected but test navigation failed: ${
|
|
66
|
+
`Browser connected but test navigation failed: ${tabResult.error.message}`,
|
|
77
67
|
"warning",
|
|
78
68
|
);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const info = await client.pageInfo();
|
|
73
|
+
if (info.success && "dialog" in info.data) {
|
|
74
|
+
await client.session().call("Page.handleJavaScriptDialog", { accept: true });
|
|
79
75
|
}
|
|
76
|
+
|
|
77
|
+
const pageUrl = info.success && !("dialog" in info.data) ? info.data.url : "github.com";
|
|
78
|
+
ctx.ui.notify(`Browser control verified ✓\nNavigated to: ${pageUrl}`, "info");
|
|
80
79
|
}
|
|
81
80
|
|
|
82
81
|
function checkChromeRunning(): boolean {
|