pi-browser-harness 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +150 -56
- package/package.json +1 -1
- package/skills/pi-browser-harness/SKILL.md +37 -248
- package/src/cdp/console-buffer.ts +263 -0
- package/src/cdp/network-buffer.ts +213 -0
- package/src/cdp/ownership.ts +72 -0
- package/src/cdp/session.ts +173 -26
- package/src/cdp/types.ts +1 -0
- package/src/client.ts +88 -12
- package/src/domains/click.ts +6 -5
- package/src/domains/console.ts +173 -0
- package/src/domains/js.ts +65 -6
- package/src/domains/navigate.ts +1 -1
- package/src/domains/network.ts +191 -20
- package/src/domains/screenshot.ts +8 -5
- package/src/domains/snapshot.ts +421 -0
- package/src/domains/tabs.ts +113 -39
- package/src/index.ts +17 -1
- package/src/prompt.ts +3 -0
- package/src/registry.ts +8 -3
- package/src/state.ts +6 -1
package/src/domains/tabs.ts
CHANGED
|
@@ -4,26 +4,41 @@ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
|
4
4
|
|
|
5
5
|
const ListTabsArgs = Type.Object({
|
|
6
6
|
includeInternal: Type.Optional(Type.Boolean({ default: true, description: "Include chrome:// pages" })),
|
|
7
|
+
scope: Type.Optional(
|
|
8
|
+
Type.Union([Type.Literal("owned"), Type.Literal("all")], {
|
|
9
|
+
default: "owned",
|
|
10
|
+
description: "'owned' (default) shows only tabs this harness session opened; 'all' shows every tab in the browser.",
|
|
11
|
+
}),
|
|
12
|
+
),
|
|
7
13
|
});
|
|
8
14
|
|
|
9
15
|
export const listTabsTool = defineBrowserTool({
|
|
10
16
|
name: "browser_list_tabs",
|
|
11
17
|
label: "Browser List Tabs",
|
|
12
|
-
description: "List
|
|
18
|
+
description: "List browser tabs. Defaults to tabs this harness session opened (scope:'owned'). Pass scope:'all' to see the user's other tabs too.",
|
|
13
19
|
promptSnippet: "List browser tabs",
|
|
14
20
|
promptGuidelines: [
|
|
15
|
-
"
|
|
16
|
-
"
|
|
21
|
+
"Defaults to scope:'owned' — only tabs this session created. Pass scope:'all' to inspect the user's other tabs.",
|
|
22
|
+
"browser_switch_tab and browser_close_tab refuse non-owned tabs; use browser_new_tab to open one this session controls.",
|
|
17
23
|
"Each tab shows its full 32-char hex targetId — use the exact value with browser_switch_tab.",
|
|
24
|
+
"Internal tabs (chrome://) included by default; pass includeInternal=false to exclude them.",
|
|
18
25
|
],
|
|
19
26
|
parameters: ListTabsArgs,
|
|
20
27
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
21
28
|
const r = await client.listTabs(args.includeInternal ?? true);
|
|
22
29
|
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
23
|
-
const
|
|
30
|
+
const scope = args.scope ?? "owned";
|
|
31
|
+
const filtered = scope === "owned" ? r.data.filter((t) => t.owned) : r.data;
|
|
32
|
+
const lines = filtered.map((t, i) => {
|
|
33
|
+
const flag = t.owned ? "★" : " ";
|
|
34
|
+
return ` ${flag} [${i}] ${t.targetId} ${t.url}\n ${t.title}`;
|
|
35
|
+
});
|
|
36
|
+
const header = scope === "owned"
|
|
37
|
+
? `Owned tabs (${filtered.length}):`
|
|
38
|
+
: `All tabs (${filtered.length}, ★ = owned by this session):`;
|
|
24
39
|
return ok({
|
|
25
|
-
text:
|
|
26
|
-
details: { tabs:
|
|
40
|
+
text: `${header}\n${lines.join("\n")}`,
|
|
41
|
+
details: { tabs: filtered, scope },
|
|
27
42
|
});
|
|
28
43
|
},
|
|
29
44
|
});
|
|
@@ -33,7 +48,7 @@ export const currentTabTool = defineBrowserTool({
|
|
|
33
48
|
label: "Browser Current Tab",
|
|
34
49
|
description: "Get info about the currently attached tab.",
|
|
35
50
|
promptSnippet: "Get current tab info",
|
|
36
|
-
promptGuidelines: ["Returns targetId, url, title."],
|
|
51
|
+
promptGuidelines: ["Returns targetId, url, title, and whether the tab is owned by this harness session."],
|
|
37
52
|
parameters: Type.Object({}),
|
|
38
53
|
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
39
54
|
const cur = client.current();
|
|
@@ -41,9 +56,10 @@ export const currentTabTool = defineBrowserTool({
|
|
|
41
56
|
const ti = await client.session().callBrowser("Target.getTargetInfo", { targetId: cur.targetId });
|
|
42
57
|
if (!ti.success) return err({ kind: "cdp_error", message: ti.error.message });
|
|
43
58
|
const info = (ti.data as { targetInfo: { targetId: string; url: string; title: string } }).targetInfo; // CDP boundary cast: Target.getTargetInfo returns { targetInfo: { targetId, url, title } }
|
|
59
|
+
const owned = client.owns(info.targetId);
|
|
44
60
|
return ok({
|
|
45
|
-
text: `Current tab:\n ${info.targetId}\n ${info.url}\n ${info.title}`,
|
|
46
|
-
details: { targetId: info.targetId, url: info.url, title: info.title },
|
|
61
|
+
text: `Current tab${owned ? " (owned)" : " (NOT owned by this session)"}:\n ${info.targetId}\n ${info.url}\n ${info.title}`,
|
|
62
|
+
details: { targetId: info.targetId, url: info.url, title: info.title, owned },
|
|
47
63
|
});
|
|
48
64
|
},
|
|
49
65
|
});
|
|
@@ -52,47 +68,59 @@ const SwitchTabArgs = Type.Object({
|
|
|
52
68
|
targetId: Type.String({ description: "Target ID from browser_list_tabs" }),
|
|
53
69
|
});
|
|
54
70
|
|
|
71
|
+
const ownershipDeniedMessage = (id: string, namespace: string): string =>
|
|
72
|
+
`Tab ${id} is not owned by this harness session (namespace: ${namespace}). ` +
|
|
73
|
+
`Use browser_list_tabs with scope:'all' to see it, or browser_new_tab to open one this session can control.`;
|
|
74
|
+
|
|
55
75
|
export const switchTabTool = defineBrowserTool({
|
|
56
76
|
name: "browser_switch_tab",
|
|
57
77
|
label: "Browser Switch Tab",
|
|
58
|
-
description: "Switch to and attach to a different tab by targetId.",
|
|
78
|
+
description: "Switch to and attach to a different tab by targetId. Only tabs this harness session opened are accepted.",
|
|
59
79
|
promptSnippet: "Switch tabs",
|
|
60
80
|
promptGuidelines: [
|
|
61
81
|
"Get a targetId via browser_list_tabs first.",
|
|
62
82
|
"Accepts exact targetId or a unique prefix of at least 8 hex characters.",
|
|
83
|
+
"Refuses tabs not owned by this session — use browser_new_tab to open a controllable tab instead.",
|
|
63
84
|
],
|
|
64
85
|
parameters: SwitchTabArgs,
|
|
65
86
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
66
|
-
//
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
87
|
+
// Resolve to a concrete targetId (exact, or unique hex prefix). We resolve
|
|
88
|
+
// first so the ownership check sees the canonical id, not the prefix.
|
|
89
|
+
let resolved = args.targetId;
|
|
90
|
+
if (!client.owns(resolved)) {
|
|
91
|
+
const isHexPrefix = /^[0-9A-Fa-f]{8,}$/.test(args.targetId);
|
|
92
|
+
if (isHexPrefix) {
|
|
93
|
+
const tabs = await client.listTabs(true);
|
|
94
|
+
if (!tabs.success) return err({ kind: "cdp_error", message: tabs.error.message });
|
|
95
|
+
const matches = tabs.data.filter((t) => t.targetId.startsWith(args.targetId));
|
|
96
|
+
if (matches.length === 0) {
|
|
97
|
+
return err({ kind: "cdp_error", message: `No tab found with prefix "${args.targetId}"` });
|
|
98
|
+
}
|
|
99
|
+
if (matches.length > 1) {
|
|
100
|
+
const ids = matches.map((t) => t.targetId).join(", ");
|
|
101
|
+
return err({
|
|
102
|
+
kind: "invalid_state",
|
|
103
|
+
message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} tabs: ${ids}`,
|
|
104
|
+
details: { matches: matches.map((t) => ({ targetId: t.targetId, url: t.url })) },
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
resolved = matches[0]!.targetId;
|
|
108
|
+
}
|
|
109
|
+
if (!client.owns(resolved)) {
|
|
110
|
+
return err({
|
|
111
|
+
kind: "invalid_state",
|
|
112
|
+
message: ownershipDeniedMessage(resolved, client.namespace),
|
|
113
|
+
details: { targetId: resolved, namespace: client.namespace },
|
|
114
|
+
});
|
|
115
|
+
}
|
|
81
116
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
return err({
|
|
85
|
-
kind: "invalid_state",
|
|
86
|
-
message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} tabs: ${ids}`,
|
|
87
|
-
details: { matches: matches.map((t) => ({ targetId: t.targetId, url: t.url })) },
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
const resolved = matches[0]!.targetId;
|
|
91
|
-
r = await client.switchTab(resolved);
|
|
117
|
+
|
|
118
|
+
const r = await client.switchTab(resolved);
|
|
92
119
|
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
120
|
+
const note = resolved === args.targetId ? "" : ` (resolved from prefix "${args.targetId}")`;
|
|
93
121
|
return ok({
|
|
94
|
-
text: `Switched to ${resolved}
|
|
95
|
-
details: { targetId: resolved
|
|
122
|
+
text: `Switched to ${resolved}${note}`,
|
|
123
|
+
details: { targetId: resolved },
|
|
96
124
|
});
|
|
97
125
|
},
|
|
98
126
|
});
|
|
@@ -104,9 +132,12 @@ const NewTabArgs = Type.Object({
|
|
|
104
132
|
export const newTabTool = defineBrowserTool({
|
|
105
133
|
name: "browser_new_tab",
|
|
106
134
|
label: "Browser New Tab",
|
|
107
|
-
description: "Open a new tab and switch to it. Optionally navigate to a URL.",
|
|
135
|
+
description: "Open a new tab in the harness's dedicated Chrome window and switch to it. Optionally navigate to a URL.",
|
|
108
136
|
promptSnippet: "Open a new tab",
|
|
109
|
-
promptGuidelines: [
|
|
137
|
+
promptGuidelines: [
|
|
138
|
+
"All tabs opened by this tool live in a single dedicated Chrome window separate from the user's main browsing.",
|
|
139
|
+
"Pass url to navigate immediately.",
|
|
140
|
+
],
|
|
110
141
|
parameters: NewTabArgs,
|
|
111
142
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
112
143
|
const r = await client.newTab(args.url);
|
|
@@ -119,3 +150,46 @@ export const newTabTool = defineBrowserTool({
|
|
|
119
150
|
});
|
|
120
151
|
},
|
|
121
152
|
});
|
|
153
|
+
|
|
154
|
+
const CloseTabArgs = Type.Object({
|
|
155
|
+
targetId: Type.String({ description: "Target ID from browser_list_tabs" }),
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
export const closeTabTool = defineBrowserTool({
|
|
159
|
+
name: "browser_close_tab",
|
|
160
|
+
label: "Browser Close Tab",
|
|
161
|
+
description: "Close a tab the harness owns. Refuses tabs not opened by this session.",
|
|
162
|
+
promptSnippet: "Close an owned tab",
|
|
163
|
+
promptGuidelines: [
|
|
164
|
+
"Only closes tabs this session opened (visible in browser_list_tabs default scope).",
|
|
165
|
+
"Accepts exact targetId or a unique hex prefix (>=8 chars).",
|
|
166
|
+
],
|
|
167
|
+
parameters: CloseTabArgs,
|
|
168
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
169
|
+
let resolved = args.targetId;
|
|
170
|
+
if (!client.owns(resolved)) {
|
|
171
|
+
const isHexPrefix = /^[0-9A-Fa-f]{8,}$/.test(args.targetId);
|
|
172
|
+
if (isHexPrefix) {
|
|
173
|
+
const owned = client.ownership().list();
|
|
174
|
+
const matches = owned.filter((id) => id.startsWith(args.targetId));
|
|
175
|
+
if (matches.length === 1) resolved = matches[0]!;
|
|
176
|
+
else if (matches.length > 1) {
|
|
177
|
+
return err({
|
|
178
|
+
kind: "invalid_state",
|
|
179
|
+
message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} owned tabs: ${matches.join(", ")}`,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
if (!client.owns(resolved)) {
|
|
184
|
+
return err({
|
|
185
|
+
kind: "invalid_state",
|
|
186
|
+
message: ownershipDeniedMessage(resolved, client.namespace),
|
|
187
|
+
details: { targetId: resolved, namespace: client.namespace },
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const r = await client.closeTab(resolved);
|
|
192
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
193
|
+
return ok({ text: `Closed tab ${resolved}`, details: { targetId: resolved } });
|
|
194
|
+
},
|
|
195
|
+
});
|
package/src/index.ts
CHANGED
|
@@ -98,7 +98,23 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
|
|
|
98
98
|
|
|
99
99
|
pi.on("session_start", async (_event, ctx) => {
|
|
100
100
|
state = restoreState(ctx, state.namespace);
|
|
101
|
-
|
|
101
|
+
const initialOwnership: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {};
|
|
102
|
+
if (state.ownedTargetIds !== undefined) initialOwnership.ownedTargetIds = state.ownedTargetIds;
|
|
103
|
+
if (state.harnessWindowTargetId !== undefined) initialOwnership.harnessWindowTargetId = state.harnessWindowTargetId;
|
|
104
|
+
client = createBrowserClient({
|
|
105
|
+
namespace: state.namespace,
|
|
106
|
+
...(Object.keys(initialOwnership).length > 0 ? { initialOwnership } : {}),
|
|
107
|
+
onOwnershipChange: (snap) => {
|
|
108
|
+
state = {
|
|
109
|
+
...state,
|
|
110
|
+
ownedTargetIds: snap.ownedTargetIds,
|
|
111
|
+
...(snap.harnessWindowTargetId !== undefined
|
|
112
|
+
? { harnessWindowTargetId: snap.harnessWindowTargetId }
|
|
113
|
+
: {}),
|
|
114
|
+
};
|
|
115
|
+
persistState(pi, state);
|
|
116
|
+
},
|
|
117
|
+
});
|
|
102
118
|
// failure is fine — surfaced via tool errors when the agent first tries to use the browser
|
|
103
119
|
await client.start();
|
|
104
120
|
if (!toolsRegistered) {
|
package/src/prompt.ts
CHANGED
|
@@ -36,6 +36,7 @@ to interact with pages visually, and other tools alongside them.
|
|
|
36
36
|
| browser_switch_tab | Switch to a different tab by targetId |
|
|
37
37
|
| browser_execute_js | Execute JavaScript and return the result |
|
|
38
38
|
| browser_http_get | Direct HTTP GET (outside browser, for APIs) |
|
|
39
|
+
| browser_console | Read JS errors / console output on the current tab (diagnostic — use when an action looks broken; pass sinceSeq from the previous nextCursor to see only new messages) |
|
|
39
40
|
| browser_wait | Wait N seconds |
|
|
40
41
|
| browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) |
|
|
41
42
|
| browser_handle_dialog | Accept or dismiss a JS dialog |
|
|
@@ -73,6 +74,8 @@ browser_screenshot() + browser_page_info() + browser_execute_js("document.title"
|
|
|
73
74
|
browser_http_get("https://api.example.com/data") + browser_click(x, y)
|
|
74
75
|
\`\`\`
|
|
75
76
|
|
|
77
|
+
**Multi-agent note**: When multiple subagents use the browser, tab switching by one agent changes the active tab for all agents. Per-tab data (console buffers, network traces, dialogs, page info cache) is isolated — switching tabs does not destroy another agent's collected data. Call \`browser_current_tab\` before mutation tools to confirm you're on the expected tab. If \`browser_page_info\` returns a dialog, handle it with \`browser_handle_dialog\` promptly — dialogs block page interaction and are not queued across agents.
|
|
78
|
+
|
|
76
79
|
### Common Patterns
|
|
77
80
|
|
|
78
81
|
**Navigation:**
|
package/src/registry.ts
CHANGED
|
@@ -9,11 +9,13 @@ import { handleDialogTool } from "./domains/dialog";
|
|
|
9
9
|
import { screenshotTool } from "./domains/screenshot";
|
|
10
10
|
import { navigateTool, openUrlsTool } from "./domains/navigate";
|
|
11
11
|
import { goBackTool, goForwardTool, reloadTool } from "./domains/history";
|
|
12
|
-
import { listTabsTool, currentTabTool, switchTabTool, newTabTool } from "./domains/tabs";
|
|
12
|
+
import { listTabsTool, currentTabTool, switchTabTool, newTabTool, closeTabTool } from "./domains/tabs";
|
|
13
13
|
import { uploadFileTool, downloadTool, printToPdfTool } from "./domains/files";
|
|
14
14
|
import { viewportResizeTool } from "./domains/viewport";
|
|
15
15
|
import { dragAndDropTool } from "./domains/drag";
|
|
16
|
-
import { httpGetTool,
|
|
16
|
+
import { httpGetTool, networkRequestsTool } from "./domains/network";
|
|
17
|
+
import { consoleTool } from "./domains/console";
|
|
18
|
+
import { snapshotTool } from "./domains/snapshot";
|
|
17
19
|
import { executeJsTool, runScriptTool } from "./domains/js";
|
|
18
20
|
|
|
19
21
|
// Mutation tools that modify page / browser state are marked `serialized: true`
|
|
@@ -39,13 +41,16 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
|
|
|
39
41
|
currentTabTool,
|
|
40
42
|
{ ...switchTabTool, serialized: true },
|
|
41
43
|
{ ...newTabTool, serialized: true },
|
|
44
|
+
{ ...closeTabTool, serialized: true },
|
|
42
45
|
{ ...uploadFileTool, serialized: true },
|
|
43
46
|
{ ...downloadTool, serialized: true },
|
|
44
47
|
{ ...printToPdfTool, serialized: true },
|
|
45
48
|
{ ...viewportResizeTool, serialized: true },
|
|
46
49
|
{ ...dragAndDropTool, serialized: true },
|
|
47
50
|
httpGetTool,
|
|
48
|
-
|
|
51
|
+
networkRequestsTool,
|
|
52
|
+
consoleTool,
|
|
53
|
+
snapshotTool,
|
|
49
54
|
executeJsTool,
|
|
50
55
|
runScriptTool,
|
|
51
56
|
];
|
package/src/state.ts
CHANGED
|
@@ -10,9 +10,14 @@ import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-age
|
|
|
10
10
|
export type BrowserState = {
|
|
11
11
|
readonly namespace: string;
|
|
12
12
|
readonly remoteBrowserId?: string;
|
|
13
|
+
readonly ownedTargetIds?: ReadonlyArray<string>;
|
|
14
|
+
readonly harnessWindowTargetId?: string;
|
|
13
15
|
};
|
|
14
16
|
|
|
15
|
-
export const defaultState = (namespace = "default"): BrowserState => ({
|
|
17
|
+
export const defaultState = (namespace = "default"): BrowserState => ({
|
|
18
|
+
namespace,
|
|
19
|
+
ownedTargetIds: [],
|
|
20
|
+
});
|
|
16
21
|
|
|
17
22
|
export const persistState = (pi: ExtensionAPI, state: BrowserState): void => {
|
|
18
23
|
pi.appendEntry<BrowserState>("browser-harness-state", state);
|