pi-browser-harness 0.2.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/package.json +4 -1
- package/skills/pi-browser-harness/SKILL.md +64 -6
- package/src/cdp/discovery.ts +95 -0
- package/src/cdp/errors.ts +19 -0
- package/src/cdp/session.ts +139 -0
- package/src/cdp/transport.ts +221 -0
- package/src/cdp/types.ts +72 -0
- package/src/client.ts +238 -0
- package/src/domains/click.ts +73 -0
- package/src/domains/dialog.ts +31 -0
- package/src/domains/drag.ts +66 -0
- package/src/domains/files.ts +194 -0
- package/src/domains/history.ts +76 -0
- package/src/domains/js.ts +197 -0
- package/src/domains/keyboard.ts +127 -0
- package/src/domains/navigate.ts +146 -0
- package/src/domains/network.ts +80 -0
- package/src/domains/page.ts +88 -0
- package/src/domains/screenshot.ts +150 -0
- package/src/domains/scroll.ts +60 -0
- package/src/domains/tabs.ts +121 -0
- package/src/domains/viewport.ts +40 -0
- package/src/index.ts +65 -142
- package/src/prompt.ts +27 -4
- package/src/registry.ts +55 -0
- package/src/schemas/common.ts +11 -0
- package/src/setup.ts +43 -26
- package/src/state.ts +22 -54
- package/src/util/js-template.ts +27 -0
- package/src/util/keycodes.ts +10 -0
- package/src/util/mutex.ts +25 -0
- package/src/util/paths.ts +21 -0
- package/src/util/result.ts +15 -0
- package/src/util/sharp-shim.ts +49 -0
- package/src/util/time.ts +29 -0
- package/src/util/tool.ts +193 -0
- package/src/util/truncate.ts +35 -0
- package/tsconfig.json +10 -1
- package/src/daemon.ts +0 -881
- package/src/protocol.ts +0 -77
- package/src/renderers.ts +0 -21
- package/src/tools.ts +0 -1363
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
|
|
5
|
+
const DragArgs = Type.Object({
|
|
6
|
+
startX: Type.Number(),
|
|
7
|
+
startY: Type.Number(),
|
|
8
|
+
endX: Type.Number(),
|
|
9
|
+
endY: Type.Number(),
|
|
10
|
+
dataTransfer: Type.Optional(
|
|
11
|
+
Type.Record(Type.String(), Type.String(), {
|
|
12
|
+
description: "Optional MIME → data map for DataTransfer (e.g., { 'text/plain': 'hello' })",
|
|
13
|
+
}),
|
|
14
|
+
),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
export const dragAndDropTool = defineBrowserTool({
|
|
18
|
+
name: "browser_drag_and_drop",
|
|
19
|
+
label: "Browser Drag & Drop",
|
|
20
|
+
description: "Drag from (startX, startY) to (endX, endY) using CDP Input.dispatchDragEvent.",
|
|
21
|
+
promptSnippet: "Drag and drop between two coordinates",
|
|
22
|
+
promptGuidelines: [
|
|
23
|
+
"Coordinates in CSS pixels.",
|
|
24
|
+
"dataTransfer is an optional MIME→data map (e.g., { 'text/plain': 'hello' }).",
|
|
25
|
+
],
|
|
26
|
+
parameters: DragArgs,
|
|
27
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
28
|
+
const data = args.dataTransfer
|
|
29
|
+
? {
|
|
30
|
+
items: Object.entries(args.dataTransfer).map(([mimeType, payload]) => ({
|
|
31
|
+
mimeType,
|
|
32
|
+
data: Buffer.from(payload).toString("base64"),
|
|
33
|
+
})),
|
|
34
|
+
dragOperationsMask: 1,
|
|
35
|
+
}
|
|
36
|
+
: { items: [], dragOperationsMask: 1 };
|
|
37
|
+
const cdp = client.session();
|
|
38
|
+
const calls: ReadonlyArray<readonly [string, Record<string, unknown>]> = [
|
|
39
|
+
["Input.dispatchMouseEvent", { type: "mousePressed", x: args.startX, y: args.startY, button: "left", clickCount: 1 }],
|
|
40
|
+
["Input.dispatchDragEvent", { type: "dragEnter", x: args.startX, y: args.startY, data, modifiers: 0 }],
|
|
41
|
+
// 5 interpolated dragOver steps from start → end so the page sees a smooth drag.
|
|
42
|
+
...Array.from({ length: 5 }, (_, i): readonly [string, Record<string, unknown>] => {
|
|
43
|
+
const t = (i + 1) / 5;
|
|
44
|
+
return [
|
|
45
|
+
"Input.dispatchDragEvent",
|
|
46
|
+
{
|
|
47
|
+
type: "dragOver",
|
|
48
|
+
x: Math.round(args.startX + (args.endX - args.startX) * t),
|
|
49
|
+
y: Math.round(args.startY + (args.endY - args.startY) * t),
|
|
50
|
+
data,
|
|
51
|
+
modifiers: 0,
|
|
52
|
+
},
|
|
53
|
+
];
|
|
54
|
+
}),
|
|
55
|
+
["Input.dispatchDragEvent", { type: "drop", x: args.endX, y: args.endY, data, modifiers: 0 }],
|
|
56
|
+
["Input.dispatchMouseEvent", { type: "mouseReleased", x: args.endX, y: args.endY, button: "left", clickCount: 1 }],
|
|
57
|
+
];
|
|
58
|
+
for (const [method, params] of calls) {
|
|
59
|
+
const r = await cdp.call(method, params);
|
|
60
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
61
|
+
}
|
|
62
|
+
return ok({
|
|
63
|
+
text: `Dragged from (${args.startX},${args.startY}) to (${args.endX},${args.endY})`,
|
|
64
|
+
});
|
|
65
|
+
},
|
|
66
|
+
});
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { mkdir, access, constants, stat, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
import { Type } from "typebox";
|
|
4
|
+
import type { BrowserClient } from "../client";
|
|
5
|
+
import { type Result, err, ok } from "../util/result";
|
|
6
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
7
|
+
import { safeJs } from "../util/js-template";
|
|
8
|
+
import { pdfPath } from "../util/paths";
|
|
9
|
+
|
|
10
|
+
const UploadArgs = Type.Object({
|
|
11
|
+
selector: Type.String({ description: "CSS selector of the file <input>" }),
|
|
12
|
+
filePath: Type.String({ description: "Absolute path to the file to upload" }),
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => {
|
|
16
|
+
try {
|
|
17
|
+
await access(filePath, constants.R_OK);
|
|
18
|
+
return ok(undefined);
|
|
19
|
+
} catch (e) {
|
|
20
|
+
return err({
|
|
21
|
+
kind: "io_error",
|
|
22
|
+
message: `Cannot read file: ${filePath} (${e instanceof Error ? e.message : String(e)})`,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const tryCdpUpload = async (
|
|
28
|
+
client: BrowserClient,
|
|
29
|
+
selector: string,
|
|
30
|
+
filePath: string,
|
|
31
|
+
): Promise<Result<void, ToolErr>> => {
|
|
32
|
+
const doc = await client.session().call("DOM.getDocument", { depth: -1 });
|
|
33
|
+
if (!doc.success) return err({ kind: "cdp_error", message: doc.error.message });
|
|
34
|
+
// CDP boundary cast: DOM.getDocument returns { root: { nodeId: number } }
|
|
35
|
+
const root = (doc.data as { root: { nodeId: number } }).root;
|
|
36
|
+
const q = await client.session().call("DOM.querySelector", { nodeId: root.nodeId, selector });
|
|
37
|
+
if (!q.success) return err({ kind: "cdp_error", message: q.error.message });
|
|
38
|
+
// CDP boundary cast: DOM.querySelector returns { nodeId: number }
|
|
39
|
+
const nodeId = (q.data as { nodeId: number }).nodeId;
|
|
40
|
+
if (!nodeId) return err({ kind: "invalid_state", message: `Selector matched 0 file inputs: ${selector}` });
|
|
41
|
+
const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], nodeId });
|
|
42
|
+
if (!set.success) return err({ kind: "cdp_error", message: set.error.message });
|
|
43
|
+
const verify = await client.evaluateJs(safeJs`document.querySelector(${selector})?.files?.length || 0`);
|
|
44
|
+
if (!verify.success) return err({ kind: "cdp_error", message: verify.error.message });
|
|
45
|
+
if (Number(verify.data ?? 0) === 0) {
|
|
46
|
+
return err({ kind: "invalid_state", message: "CDP upload reported success but file count is 0" });
|
|
47
|
+
}
|
|
48
|
+
return ok(undefined);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const inferMime = (name: string): string => {
|
|
52
|
+
if (name.endsWith(".png")) return "image/png";
|
|
53
|
+
if (name.endsWith(".jpg") || name.endsWith(".jpeg")) return "image/jpeg";
|
|
54
|
+
if (name.endsWith(".pdf")) return "application/pdf";
|
|
55
|
+
if (name.endsWith(".json")) return "application/json";
|
|
56
|
+
return "text/plain";
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const jsFallbackUpload = async (
|
|
60
|
+
client: BrowserClient,
|
|
61
|
+
selector: string,
|
|
62
|
+
filePath: string,
|
|
63
|
+
): Promise<Result<void, ToolErr>> => {
|
|
64
|
+
const buf = await readFile(filePath);
|
|
65
|
+
const st = await stat(filePath);
|
|
66
|
+
const name = basename(filePath);
|
|
67
|
+
const mime = inferMime(name);
|
|
68
|
+
const expr = safeJs`
|
|
69
|
+
(() => {
|
|
70
|
+
const input = document.querySelector(${selector});
|
|
71
|
+
if (!input || input.type !== 'file') throw new Error('File input not found');
|
|
72
|
+
const bin = atob(${buf.toString("base64")});
|
|
73
|
+
const bytes = new Uint8Array(bin.length);
|
|
74
|
+
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
|
75
|
+
const file = new File([bytes], ${name}, { type: ${mime}, lastModified: ${st.mtimeMs} });
|
|
76
|
+
const dt = new DataTransfer();
|
|
77
|
+
dt.items.add(file);
|
|
78
|
+
input.files = dt.files;
|
|
79
|
+
input.dispatchEvent(new Event('change', { bubbles: true }));
|
|
80
|
+
return input.files.length;
|
|
81
|
+
})()
|
|
82
|
+
`;
|
|
83
|
+
const r = await client.evaluateJs(expr);
|
|
84
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
85
|
+
if (!Number(r.data ?? 0)) return err({ kind: "invalid_state", message: "JS fallback set 0 files" });
|
|
86
|
+
return ok(undefined);
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
export const uploadFileTool = defineBrowserTool({
|
|
90
|
+
name: "browser_upload_file",
|
|
91
|
+
label: "Browser Upload File",
|
|
92
|
+
description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages.",
|
|
93
|
+
promptSnippet: "Upload a file to a file input",
|
|
94
|
+
promptGuidelines: [
|
|
95
|
+
"File path must be absolute and readable.",
|
|
96
|
+
"Selector must match a file input.",
|
|
97
|
+
],
|
|
98
|
+
parameters: UploadArgs,
|
|
99
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
100
|
+
const readable = await verifyReadable(args.filePath);
|
|
101
|
+
if (!readable.success) return readable;
|
|
102
|
+
const cdp = await tryCdpUpload(client, args.selector, args.filePath);
|
|
103
|
+
if (cdp.success) {
|
|
104
|
+
return ok({
|
|
105
|
+
text: `Uploaded ${args.filePath} via CDP`,
|
|
106
|
+
details: { mode: "cdp", filePath: args.filePath },
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
const js = await jsFallbackUpload(client, args.selector, args.filePath);
|
|
110
|
+
if (js.success) {
|
|
111
|
+
return ok({
|
|
112
|
+
text: `Uploaded ${args.filePath} via JS DataTransfer`,
|
|
113
|
+
details: { mode: "js", filePath: args.filePath },
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return err({
|
|
117
|
+
kind: "cdp_error",
|
|
118
|
+
message: `Both CDP and JS fallback failed. CDP: ${cdp.error.message}; JS: ${js.error.message}`,
|
|
119
|
+
});
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
const DownloadArgs = Type.Object({
|
|
124
|
+
downloadPath: Type.String({ description: "Absolute path to a writable directory where downloads should be saved" }),
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
export const downloadTool = defineBrowserTool({
|
|
128
|
+
name: "browser_download",
|
|
129
|
+
label: "Browser Download",
|
|
130
|
+
description: "Configure Chrome's download behavior: set the save directory and disable the save-as prompt.",
|
|
131
|
+
promptSnippet: "Configure download directory",
|
|
132
|
+
promptGuidelines: [
|
|
133
|
+
"Pass an absolute path to an existing writable directory.",
|
|
134
|
+
"Affects all subsequent downloads on this browser.",
|
|
135
|
+
],
|
|
136
|
+
parameters: DownloadArgs,
|
|
137
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
138
|
+
try {
|
|
139
|
+
await mkdir(args.downloadPath, { recursive: true });
|
|
140
|
+
} catch (e) {
|
|
141
|
+
return err({
|
|
142
|
+
kind: "io_error",
|
|
143
|
+
message: `Cannot create download directory: ${args.downloadPath} (${e instanceof Error ? e.message : String(e)})`,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
const s = await stat(args.downloadPath);
|
|
148
|
+
if (!s.isDirectory()) {
|
|
149
|
+
return err({ kind: "io_error", message: `Not a directory: ${args.downloadPath}` });
|
|
150
|
+
}
|
|
151
|
+
await access(args.downloadPath, constants.W_OK);
|
|
152
|
+
} catch (e) {
|
|
153
|
+
return err({
|
|
154
|
+
kind: "io_error",
|
|
155
|
+
message: `Download path unusable: ${e instanceof Error ? e.message : String(e)}`,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
const r = await client.session().callBrowser("Browser.setDownloadBehavior", {
|
|
159
|
+
behavior: "allow",
|
|
160
|
+
downloadPath: args.downloadPath,
|
|
161
|
+
eventsEnabled: true,
|
|
162
|
+
});
|
|
163
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
164
|
+
return ok({
|
|
165
|
+
text: `Downloads will save to: ${args.downloadPath}`,
|
|
166
|
+
details: { downloadPath: args.downloadPath },
|
|
167
|
+
});
|
|
168
|
+
},
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
const PrintPdfArgs = Type.Object({
|
|
172
|
+
outputPath: Type.Optional(Type.String({ description: "Where to save the PDF. Default: tmpdir + uuid." })),
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
export const printToPdfTool = defineBrowserTool({
|
|
176
|
+
name: "browser_print_to_pdf",
|
|
177
|
+
label: "Browser Print to PDF",
|
|
178
|
+
description: "Print the current page to a PDF file using Chrome's Page.printToPDF.",
|
|
179
|
+
promptSnippet: "Print the current page to PDF",
|
|
180
|
+
promptGuidelines: ["Default output path is in tmpdir; pass outputPath to control."],
|
|
181
|
+
parameters: PrintPdfArgs,
|
|
182
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
183
|
+
const r = await client.session().call("Page.printToPDF", {
|
|
184
|
+
printBackground: true,
|
|
185
|
+
preferCSSPageSize: true,
|
|
186
|
+
});
|
|
187
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
188
|
+
// CDP boundary cast: Page.printToPDF returns { data: string } (base64-encoded PDF)
|
|
189
|
+
const data = (r.data as { data: string }).data;
|
|
190
|
+
const path = args.outputPath ?? pdfPath(client.namespace);
|
|
191
|
+
await writeFile(path, Buffer.from(data, "base64"));
|
|
192
|
+
return ok({ text: `PDF saved: ${path}`, details: { path } });
|
|
193
|
+
},
|
|
194
|
+
});
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import type { BrowserClient } from "../client";
|
|
3
|
+
import { type Result, err, ok } from "../util/result";
|
|
4
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
5
|
+
|
|
6
|
+
type HistoryEntry = { readonly id: number; readonly url: string; readonly title: string };
|
|
7
|
+
type History = { readonly entries: ReadonlyArray<HistoryEntry>; readonly currentIndex: number };
|
|
8
|
+
|
|
9
|
+
const fetchHistory = async (client: BrowserClient): Promise<Result<History, ToolErr>> => {
|
|
10
|
+
const r = await client.session().call("Page.getNavigationHistory");
|
|
11
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
12
|
+
return ok(r.data as History);
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export const goBackTool = defineBrowserTool({
|
|
16
|
+
name: "browser_go_back",
|
|
17
|
+
label: "Browser Go Back",
|
|
18
|
+
description: "Navigate back one page in history.",
|
|
19
|
+
promptSnippet: "Go back one page",
|
|
20
|
+
promptGuidelines: ["Use to return to the previous page after navigating."],
|
|
21
|
+
parameters: Type.Object({}),
|
|
22
|
+
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
23
|
+
const h = await fetchHistory(client);
|
|
24
|
+
if (!h.success) return h;
|
|
25
|
+
if (h.data.currentIndex <= 0) return ok({ text: "Already at the beginning of history." });
|
|
26
|
+
const target = h.data.entries[h.data.currentIndex - 1];
|
|
27
|
+
if (!target) return err({ kind: "internal", message: "history entry missing at index" });
|
|
28
|
+
const nav = await client.session().call("Page.navigateToHistoryEntry", { entryId: target.id });
|
|
29
|
+
if (!nav.success) return err({ kind: "cdp_error", message: nav.error.message });
|
|
30
|
+
return ok({ text: `Navigated back to: ${target.url}`, details: { url: target.url } });
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
export const goForwardTool = defineBrowserTool({
|
|
35
|
+
name: "browser_go_forward",
|
|
36
|
+
label: "Browser Go Forward",
|
|
37
|
+
description: "Navigate forward one page in history.",
|
|
38
|
+
promptSnippet: "Go forward one page",
|
|
39
|
+
promptGuidelines: ["Use to undo a browser_go_back."],
|
|
40
|
+
parameters: Type.Object({}),
|
|
41
|
+
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
42
|
+
const h = await fetchHistory(client);
|
|
43
|
+
if (!h.success) return h;
|
|
44
|
+
if (h.data.currentIndex >= h.data.entries.length - 1) return ok({ text: "Already at the end of history." });
|
|
45
|
+
const target = h.data.entries[h.data.currentIndex + 1];
|
|
46
|
+
if (!target) return err({ kind: "internal", message: "history entry missing at index" });
|
|
47
|
+
const nav = await client.session().call("Page.navigateToHistoryEntry", { entryId: target.id });
|
|
48
|
+
if (!nav.success) return err({ kind: "cdp_error", message: nav.error.message });
|
|
49
|
+
return ok({ text: `Navigated forward to: ${target.url}`, details: { url: target.url } });
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
export const reloadTool = defineBrowserTool({
|
|
54
|
+
name: "browser_reload",
|
|
55
|
+
label: "Browser Reload",
|
|
56
|
+
description: "Reload the current page.",
|
|
57
|
+
promptSnippet: "Reload the page",
|
|
58
|
+
promptGuidelines: ["Use to refresh the page, e.g. after server-side changes."],
|
|
59
|
+
parameters: Type.Object({}),
|
|
60
|
+
async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
61
|
+
const r = await client.session().call("Page.reload");
|
|
62
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
63
|
+
const info = await client.pageInfo();
|
|
64
|
+
if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
|
|
65
|
+
if ("dialog" in info.data) {
|
|
66
|
+
return ok({
|
|
67
|
+
text: `Reloaded. ⚠️ Dialog open: ${info.data.dialog.type}`,
|
|
68
|
+
details: { dialog: info.data.dialog },
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
return ok({
|
|
72
|
+
text: `Reloaded: ${info.data.title} (${info.data.url})`,
|
|
73
|
+
details: { page: info.data },
|
|
74
|
+
});
|
|
75
|
+
},
|
|
76
|
+
});
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { resolve, isAbsolute } from "node:path";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { createRequire } from "node:module";
|
|
5
|
+
import { Type } from "typebox";
|
|
6
|
+
import { type Result, err, ok } from "../util/result";
|
|
7
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
8
|
+
import { applyTruncation } from "../util/truncate";
|
|
9
|
+
|
|
10
|
+
// AsyncFunction constructor — the documented Node.js mechanism for compiling
|
|
11
|
+
// arbitrary user-supplied source. Equivalent to `new Function` but produces an
|
|
12
|
+
// async function we can `await`. The full RCE risk is contained at the source
|
|
13
|
+
// boundary: see the path allowlist + timeout below. A plain `unknown` cast
|
|
14
|
+
// would lose the constructor signature; we keep this single typed cast.
|
|
15
|
+
const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor as
|
|
16
|
+
new (...args: ReadonlyArray<string>) => (...args: ReadonlyArray<unknown>) => Promise<unknown>;
|
|
17
|
+
|
|
18
|
+
// ESNext module — bare `require` is not in scope. createRequire provides the
|
|
19
|
+
// CJS resolver anchored to this file's URL so scripts can require() builtins
|
|
20
|
+
// and installed CJS packages.
|
|
21
|
+
const requireFromHere = createRequire(import.meta.url);
|
|
22
|
+
|
|
23
|
+
const ExecuteJsArgs = Type.Object({
|
|
24
|
+
expression: Type.String({ description: "JavaScript expression. `return X` is auto-wrapped in an IIFE." }),
|
|
25
|
+
targetId: Type.Optional(Type.String({ description: "Optional iframe targetId; default = current page." })),
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
export const executeJsTool = defineBrowserTool({
|
|
29
|
+
name: "browser_execute_js",
|
|
30
|
+
label: "Browser Execute JS",
|
|
31
|
+
description:
|
|
32
|
+
"Run JavaScript in the page (or a specific iframe target). `return X` gets auto-wrapped in an IIFE for convenience. Always use safeJs / JSON.stringify when interpolating untrusted strings into source.",
|
|
33
|
+
promptSnippet: "Execute JS in the page (or iframe)",
|
|
34
|
+
promptGuidelines: [
|
|
35
|
+
"`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.",
|
|
36
|
+
"For iframes, get a targetId via browser_execute_js with `Object.values(document.querySelectorAll('iframe'))` then call browser_execute_js with targetId.",
|
|
37
|
+
"Result must be JSON-serializable (Runtime.evaluate returnByValue=true).",
|
|
38
|
+
],
|
|
39
|
+
parameters: ExecuteJsArgs,
|
|
40
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
41
|
+
const r = await client.evaluateJs(args.expression, args.targetId);
|
|
42
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
43
|
+
const valueStr = r.data === undefined ? "undefined" : JSON.stringify(r.data);
|
|
44
|
+
const truncated = await applyTruncation(valueStr, "js");
|
|
45
|
+
return ok({
|
|
46
|
+
text: truncated.text,
|
|
47
|
+
details: truncated.fullOutputPath !== undefined
|
|
48
|
+
? { valueLength: valueStr.length, fullOutputPath: truncated.fullOutputPath }
|
|
49
|
+
: { valueLength: valueStr.length },
|
|
50
|
+
});
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
const RunScriptArgs = Type.Object({
|
|
55
|
+
path: Type.String({ description: "Absolute path to the script file (.js or .mjs)" }),
|
|
56
|
+
params: Type.Optional(
|
|
57
|
+
Type.Object({}, { additionalProperties: true, description: "Args passed to the script as `params`" }),
|
|
58
|
+
),
|
|
59
|
+
timeoutMs: Type.Optional(
|
|
60
|
+
Type.Integer({
|
|
61
|
+
default: 60_000,
|
|
62
|
+
minimum: 100,
|
|
63
|
+
maximum: 600_000,
|
|
64
|
+
description: "Hard timeout in ms. Default 60s, max 600s.",
|
|
65
|
+
}),
|
|
66
|
+
),
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const allowedRoots = (): ReadonlyArray<string> => {
|
|
70
|
+
const env = process.env["BH_SCRIPT_DIR"];
|
|
71
|
+
return [tmpdir(), process.cwd(), ...(env !== undefined ? [env] : [])].map((d) => resolve(d));
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
const isPathAllowed = (p: string): boolean => {
|
|
75
|
+
const abs = resolve(p);
|
|
76
|
+
return allowedRoots().some((root) => abs === root || abs.startsWith(`${root}/`));
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const MAX_SOURCE_BYTES = 1_000_000;
|
|
80
|
+
|
|
81
|
+
type ContentItem = { readonly type: "text"; readonly text: string };
|
|
82
|
+
|
|
83
|
+
const isContentItem = (v: unknown): v is ContentItem =>
|
|
84
|
+
typeof v === "object" && v !== null
|
|
85
|
+
&& (v as Readonly<Record<string, unknown>>)["type"] === "text"
|
|
86
|
+
&& typeof (v as Readonly<Record<string, unknown>>)["text"] === "string";
|
|
87
|
+
|
|
88
|
+
export const runScriptTool = defineBrowserTool({
|
|
89
|
+
name: "browser_run_script",
|
|
90
|
+
label: "Browser Run Script",
|
|
91
|
+
description:
|
|
92
|
+
"Execute a temporary JavaScript script with full Node.js + browser-daemon access. Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR. Mandatory timeout. The script is full RCE on the harness's process — only invoke scripts you wrote and reviewed.",
|
|
93
|
+
promptSnippet: "Run a temporary script with daemon + Node access (security-bounded)",
|
|
94
|
+
promptGuidelines: [
|
|
95
|
+
"Write the script with the write tool first; pass its absolute path.",
|
|
96
|
+
"Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR — otherwise rejected.",
|
|
97
|
+
"Default timeout is 60s; pass timeoutMs to extend (max 600s).",
|
|
98
|
+
"Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.",
|
|
99
|
+
"Script MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }. Throw on errors.",
|
|
100
|
+
],
|
|
101
|
+
parameters: RunScriptArgs,
|
|
102
|
+
async handler(args, { client, signal, onUpdate, extensionCtx }): Promise<Result<ToolOk, ToolErr>> {
|
|
103
|
+
if (!isAbsolute(args.path)) {
|
|
104
|
+
return err({ kind: "invalid_state", message: "Script path must be absolute" });
|
|
105
|
+
}
|
|
106
|
+
if (!isPathAllowed(args.path)) {
|
|
107
|
+
return err({
|
|
108
|
+
kind: "invalid_state",
|
|
109
|
+
message: `Script path outside allowed directories (allowed: ${allowedRoots().join(", ")})`,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
let source: string;
|
|
113
|
+
try {
|
|
114
|
+
source = await readFile(args.path, "utf8");
|
|
115
|
+
} catch (e) {
|
|
116
|
+
return err({
|
|
117
|
+
kind: "io_error",
|
|
118
|
+
message: `Failed to read script: ${e instanceof Error ? e.message : String(e)}`,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
if (source.length === 0) return err({ kind: "invalid_state", message: "Script is empty" });
|
|
122
|
+
if (source.length > MAX_SOURCE_BYTES) {
|
|
123
|
+
return err({ kind: "invalid_state", message: `Script exceeds ${MAX_SOURCE_BYTES}B size cap` });
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let executeFn: (...a: ReadonlyArray<unknown>) => Promise<unknown>;
|
|
127
|
+
try {
|
|
128
|
+
executeFn = new AsyncFunction(
|
|
129
|
+
"params", "daemon", "require", "signal", "onUpdate", "ctx",
|
|
130
|
+
"console", "fetch", "JSON", "Buffer", "setTimeout", "clearTimeout",
|
|
131
|
+
`"use strict";\n${source}`,
|
|
132
|
+
);
|
|
133
|
+
} catch (e) {
|
|
134
|
+
return err({
|
|
135
|
+
kind: "invalid_state",
|
|
136
|
+
message: `Syntax error: ${e instanceof Error ? e.message : String(e)}`,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const timeoutMs = args.timeoutMs ?? 60_000;
|
|
141
|
+
const ac = new AbortController();
|
|
142
|
+
const onAbort = (): void => ac.abort();
|
|
143
|
+
if (signal !== undefined) signal.addEventListener("abort", onAbort, { once: true });
|
|
144
|
+
const timeoutTimer = setTimeout(() => ac.abort(), timeoutMs);
|
|
145
|
+
|
|
146
|
+
try {
|
|
147
|
+
const scriptPromise = executeFn(
|
|
148
|
+
args.params ?? {},
|
|
149
|
+
client, // bound as `daemon` — name preserved for back-compat
|
|
150
|
+
requireFromHere,
|
|
151
|
+
ac.signal,
|
|
152
|
+
(u: unknown) => {
|
|
153
|
+
if (typeof u !== "object" || u === null) return;
|
|
154
|
+
const content = (u as Readonly<Record<string, unknown>>)["content"];
|
|
155
|
+
if (!Array.isArray(content) || content.length === 0) return;
|
|
156
|
+
const first = content[0];
|
|
157
|
+
if (typeof first !== "object" || first === null) return;
|
|
158
|
+
const txt = (first as Readonly<Record<string, unknown>>)["text"];
|
|
159
|
+
if (typeof txt !== "string") return;
|
|
160
|
+
try { onUpdate({ text: txt }); } catch { /* swallow */ }
|
|
161
|
+
},
|
|
162
|
+
extensionCtx ?? { cwd: process.cwd() },
|
|
163
|
+
console, fetch, JSON, Buffer, setTimeout, clearTimeout,
|
|
164
|
+
);
|
|
165
|
+
const abortPromise = new Promise<never>((_, reject) => {
|
|
166
|
+
ac.signal.addEventListener("abort", () => reject(new Error("script aborted (timeout or cancellation)")), { once: true });
|
|
167
|
+
});
|
|
168
|
+
const result = await Promise.race([scriptPromise, abortPromise]);
|
|
169
|
+
clearTimeout(timeoutTimer);
|
|
170
|
+
if (signal !== undefined) signal.removeEventListener("abort", onAbort);
|
|
171
|
+
|
|
172
|
+
if (typeof result !== "object" || result === null) {
|
|
173
|
+
return err({ kind: "invalid_state", message: `Script must return an object; got ${JSON.stringify(result)}` });
|
|
174
|
+
}
|
|
175
|
+
const content = (result as Readonly<Record<string, unknown>>)["content"];
|
|
176
|
+
if (!Array.isArray(content)) {
|
|
177
|
+
return err({ kind: "invalid_state", message: "Script return value must have a content array" });
|
|
178
|
+
}
|
|
179
|
+
if (!content.every(isContentItem)) {
|
|
180
|
+
return err({ kind: "invalid_state", message: "Script content array must contain { type: 'text', text: string } items" });
|
|
181
|
+
}
|
|
182
|
+
const textOut = content.map((c) => c.text).join("\n");
|
|
183
|
+
const details = (result as Readonly<Record<string, unknown>>)["details"];
|
|
184
|
+
const isPlainObject = typeof details === "object" && details !== null && !Array.isArray(details);
|
|
185
|
+
return isPlainObject
|
|
186
|
+
? ok({ text: textOut, details: details as Readonly<Record<string, unknown>> })
|
|
187
|
+
: ok({ text: textOut });
|
|
188
|
+
} catch (e) {
|
|
189
|
+
clearTimeout(timeoutTimer);
|
|
190
|
+
if (signal !== undefined) signal.removeEventListener("abort", onAbort);
|
|
191
|
+
return err({
|
|
192
|
+
kind: "internal",
|
|
193
|
+
message: `Script execution failed: ${e instanceof Error ? e.message : String(e)}`,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
},
|
|
197
|
+
});
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { safeJs } from "../util/js-template";
|
|
3
|
+
import { virtualKeyCode } from "../util/keycodes";
|
|
4
|
+
import { type Result, err, ok } from "../util/result";
|
|
5
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
6
|
+
|
|
7
|
+
const TypeArgs = Type.Object({
|
|
8
|
+
text: Type.String({ description: "Text to type" }),
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
export const typeTool = defineBrowserTool({
|
|
12
|
+
name: "browser_type",
|
|
13
|
+
label: "Browser Type",
|
|
14
|
+
description: "Type text into the currently focused element. Use browser_click first to focus an input field.",
|
|
15
|
+
promptSnippet: "Type text into the focused element",
|
|
16
|
+
promptGuidelines: [
|
|
17
|
+
"Use browser_type to enter text. Click on an input field with browser_click first to focus it.",
|
|
18
|
+
"For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.",
|
|
19
|
+
],
|
|
20
|
+
parameters: TypeArgs,
|
|
21
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
22
|
+
const r = await client.session().call("Input.insertText", { text: args.text });
|
|
23
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
24
|
+
return ok({ text: `Typed: "${args.text}"` });
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
const PressKeyArgs = Type.Object({
|
|
29
|
+
key: Type.String({
|
|
30
|
+
description:
|
|
31
|
+
'Key to press. Special keys: Enter, Tab, Backspace, Escape, Delete, ArrowLeft/Up/Right/Down, Home, End, PageUp, PageDown. Space as " "',
|
|
32
|
+
}),
|
|
33
|
+
modifiers: Type.Optional(
|
|
34
|
+
Type.Integer({
|
|
35
|
+
minimum: 0,
|
|
36
|
+
maximum: 15,
|
|
37
|
+
description: "Modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with OR.",
|
|
38
|
+
}),
|
|
39
|
+
),
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
export const pressKeyTool = defineBrowserTool({
|
|
43
|
+
name: "browser_press_key",
|
|
44
|
+
label: "Browser Press Key",
|
|
45
|
+
description:
|
|
46
|
+
"Press a keyboard key. Supports special keys (Enter, Tab, Backspace, Escape, Delete, arrows, Home, End, PageUp, PageDown, Space as ' ') and regular characters. Optional modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta/Cmd, 8=Shift.",
|
|
47
|
+
promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)",
|
|
48
|
+
promptGuidelines: [
|
|
49
|
+
"Use browser_press_key for keyboard shortcuts and navigation keys.",
|
|
50
|
+
"Special key names: Enter, Tab, Backspace, Escape, Delete, ArrowLeft, ArrowUp, ArrowRight, ArrowDown, Home, End, PageUp, PageDown.",
|
|
51
|
+
"Use Space as ' ' (a single space character).",
|
|
52
|
+
"Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
|
|
53
|
+
],
|
|
54
|
+
parameters: PressKeyArgs,
|
|
55
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
56
|
+
const k = args.key;
|
|
57
|
+
const code = virtualKeyCode(k);
|
|
58
|
+
const modifiers = args.modifiers ?? 0;
|
|
59
|
+
const isChar = k.length === 1;
|
|
60
|
+
const downParams: Record<string, unknown> = {
|
|
61
|
+
type: "keyDown",
|
|
62
|
+
key: k,
|
|
63
|
+
code: k,
|
|
64
|
+
modifiers,
|
|
65
|
+
...(code ? { windowsVirtualKeyCode: code, nativeVirtualKeyCode: code } : {}),
|
|
66
|
+
...(isChar ? { text: k, unmodifiedText: k } : {}),
|
|
67
|
+
};
|
|
68
|
+
const down = await client.session().call("Input.dispatchKeyEvent", downParams);
|
|
69
|
+
if (!down.success) return err({ kind: "cdp_error", message: down.error.message });
|
|
70
|
+
const up = await client.session().call("Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers });
|
|
71
|
+
if (!up.success) return err({ kind: "cdp_error", message: up.error.message });
|
|
72
|
+
return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` });
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const DispatchKeyArgs = Type.Object({
|
|
77
|
+
selector: Type.String({ description: "CSS selector of the target element" }),
|
|
78
|
+
key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }),
|
|
79
|
+
eventType: Type.Optional(
|
|
80
|
+
Type.Union(
|
|
81
|
+
[Type.Literal("keydown"), Type.Literal("keyup"), Type.Literal("keypress")],
|
|
82
|
+
{ default: "keydown", description: "Event type to dispatch. Default: keydown" },
|
|
83
|
+
),
|
|
84
|
+
),
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
export const dispatchKeyTool = defineBrowserTool({
|
|
88
|
+
name: "browser_dispatch_key",
|
|
89
|
+
label: "Browser Dispatch Key",
|
|
90
|
+
description:
|
|
91
|
+
"Dispatch a DOM KeyboardEvent on a specific element via JS injection. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
|
|
92
|
+
promptSnippet: "Dispatch a DOM KeyboardEvent on a specific element",
|
|
93
|
+
promptGuidelines: [
|
|
94
|
+
"Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text into inputs (use browser_type or browser_press_key for actual typing).",
|
|
95
|
+
"Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
|
|
96
|
+
"The selector must match exactly one or more elements; zero matches is reported as an error.",
|
|
97
|
+
"eventType defaults to 'keydown'.",
|
|
98
|
+
],
|
|
99
|
+
parameters: DispatchKeyArgs,
|
|
100
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
101
|
+
const eventType = args.eventType ?? "keydown";
|
|
102
|
+
const expr = safeJs`
|
|
103
|
+
(() => {
|
|
104
|
+
const els = document.querySelectorAll(${args.selector});
|
|
105
|
+
if (els.length === 0) return 0;
|
|
106
|
+
for (const el of els) {
|
|
107
|
+
el.dispatchEvent(new KeyboardEvent(${eventType}, { key: ${args.key}, bubbles: true, cancelable: true }));
|
|
108
|
+
}
|
|
109
|
+
return els.length;
|
|
110
|
+
})()
|
|
111
|
+
`;
|
|
112
|
+
const r = await client.evaluateJs(expr);
|
|
113
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
114
|
+
const matched = Number(r.data ?? 0);
|
|
115
|
+
if (matched === 0) {
|
|
116
|
+
return err({
|
|
117
|
+
kind: "invalid_state",
|
|
118
|
+
message: `Selector matched 0 elements: ${args.selector}`,
|
|
119
|
+
details: { matched: 0, selector: args.selector },
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return ok({
|
|
123
|
+
text: `Dispatched ${eventType}('${args.key}') on ${matched} element(s)`,
|
|
124
|
+
details: { matched, selector: args.selector },
|
|
125
|
+
});
|
|
126
|
+
},
|
|
127
|
+
});
|