pi-browser-harness 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/package.json +4 -1
- package/skills/pi-browser-harness/SKILL.md +17 -4
- package/src/cdp/discovery.ts +95 -0
- package/src/cdp/errors.ts +19 -0
- package/src/cdp/session.ts +139 -0
- package/src/cdp/transport.ts +221 -0
- package/src/cdp/types.ts +72 -0
- package/src/client.ts +232 -0
- package/src/domains/click.ts +73 -0
- package/src/domains/dialog.ts +31 -0
- package/src/domains/drag.ts +66 -0
- package/src/domains/files.ts +194 -0
- package/src/domains/history.ts +76 -0
- package/src/domains/js.ts +197 -0
- package/src/domains/keyboard.ts +127 -0
- package/src/domains/navigate.ts +146 -0
- package/src/domains/network.ts +80 -0
- package/src/domains/page.ts +88 -0
- package/src/domains/screenshot.ts +114 -0
- package/src/domains/scroll.ts +53 -0
- package/src/domains/tabs.ts +121 -0
- package/src/domains/viewport.ts +40 -0
- package/src/index.ts +65 -142
- package/src/prompt.ts +16 -3
- package/src/registry.ts +52 -0
- package/src/schemas/common.ts +11 -0
- package/src/setup.ts +20 -21
- package/src/state.ts +22 -54
- package/src/util/js-template.ts +27 -0
- package/src/util/keycodes.ts +10 -0
- package/src/util/paths.ts +21 -0
- package/src/util/result.ts +15 -0
- package/src/util/sharp-shim.ts +49 -0
- package/src/util/time.ts +29 -0
- package/src/util/tool.ts +169 -0
- package/src/util/truncate.ts +35 -0
- package/tsconfig.json +10 -1
- package/src/daemon.ts +0 -881
- package/src/protocol.ts +0 -77
- package/src/renderers.ts +0 -21
- package/src/tools.ts +0 -1363
package/src/cdp/types.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export type CdpEvent = {
|
|
2
|
+
readonly method: string;
|
|
3
|
+
readonly params: unknown;
|
|
4
|
+
readonly sessionId?: string;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
export type CdpRawMessage = {
|
|
8
|
+
readonly id?: number;
|
|
9
|
+
readonly method?: string;
|
|
10
|
+
readonly params?: Record<string, unknown>;
|
|
11
|
+
readonly result?: unknown;
|
|
12
|
+
readonly error?: { readonly message: string; readonly code?: number };
|
|
13
|
+
readonly sessionId?: string;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export type DialogInfo = {
|
|
17
|
+
readonly type: "alert" | "confirm" | "prompt" | "beforeunload";
|
|
18
|
+
readonly message: string;
|
|
19
|
+
readonly defaultPrompt?: string;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export type TabInfo = {
|
|
23
|
+
readonly targetId: string;
|
|
24
|
+
readonly title: string;
|
|
25
|
+
readonly url: string;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type PageInfo = {
|
|
29
|
+
readonly url: string;
|
|
30
|
+
readonly title: string;
|
|
31
|
+
readonly width: number;
|
|
32
|
+
readonly height: number;
|
|
33
|
+
readonly scrollX: number;
|
|
34
|
+
readonly scrollY: number;
|
|
35
|
+
readonly pageWidth: number;
|
|
36
|
+
readonly pageHeight: number;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export type DaemonStatus = {
|
|
40
|
+
readonly alive: boolean;
|
|
41
|
+
readonly sessionId: string | null;
|
|
42
|
+
readonly namespace: string;
|
|
43
|
+
readonly remoteBrowserId?: string;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const INTERNAL_PREFIXES: ReadonlyArray<string> = [
|
|
47
|
+
"chrome://", "chrome-untrusted://", "devtools://", "chrome-extension://", "about:",
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
export const isInternalUrl = (url: string): boolean =>
|
|
51
|
+
INTERNAL_PREFIXES.some((p) => url.startsWith(p));
|
|
52
|
+
|
|
53
|
+
const isObject = (v: unknown): v is Readonly<Record<string, unknown>> =>
|
|
54
|
+
typeof v === "object" && v !== null && !Array.isArray(v);
|
|
55
|
+
|
|
56
|
+
export const isCdpRawMessage = (v: unknown): v is CdpRawMessage => {
|
|
57
|
+
if (!isObject(v)) return false;
|
|
58
|
+
const id = v["id"];
|
|
59
|
+
if (id !== undefined && typeof id !== "number") return false;
|
|
60
|
+
const method = v["method"];
|
|
61
|
+
if (method !== undefined && typeof method !== "string") return false;
|
|
62
|
+
const sessionId = v["sessionId"];
|
|
63
|
+
if (sessionId !== undefined && typeof sessionId !== "string") return false;
|
|
64
|
+
const errVal = v["error"];
|
|
65
|
+
if (errVal !== undefined) {
|
|
66
|
+
if (!isObject(errVal)) return false;
|
|
67
|
+
if (typeof errVal["message"] !== "string") return false;
|
|
68
|
+
const code = errVal["code"];
|
|
69
|
+
if (code !== undefined && typeof code !== "number") return false;
|
|
70
|
+
}
|
|
71
|
+
return true;
|
|
72
|
+
};
|
package/src/client.ts
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { type Result, err, ok } from "./util/result";
|
|
2
|
+
import { safeJs } from "./util/js-template";
|
|
3
|
+
import { discoverWsUrl } from "./cdp/discovery";
|
|
4
|
+
import { type CdpError, cdpError } from "./cdp/errors";
|
|
5
|
+
import type { CdpTransport } from "./cdp/transport";
|
|
6
|
+
import { createCdpTransport } from "./cdp/transport";
|
|
7
|
+
import { type CdpSession, createCdpSession } from "./cdp/session";
|
|
8
|
+
import type { DaemonStatus, DialogInfo, PageInfo, TabInfo } from "./cdp/types";
|
|
9
|
+
|
|
10
|
+
export type BrowserClientOptions = {
|
|
11
|
+
readonly namespace: string;
|
|
12
|
+
readonly remote?: { readonly cdpUrl: string; readonly browserId: string };
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export type BrowserClient = {
|
|
16
|
+
readonly namespace: string;
|
|
17
|
+
ensureAlive(): Promise<Result<void, CdpError>>;
|
|
18
|
+
status(): DaemonStatus;
|
|
19
|
+
start(): Promise<Result<void, CdpError>>;
|
|
20
|
+
stop(): Promise<void>;
|
|
21
|
+
evaluateJs(expression: string, sessionId?: string): Promise<Result<unknown, CdpError>>;
|
|
22
|
+
pageInfo(): Promise<Result<PageInfo | { readonly dialog: DialogInfo }, CdpError>>;
|
|
23
|
+
takeDialog(): DialogInfo | null;
|
|
24
|
+
listTabs(includeInternal?: boolean): Promise<Result<ReadonlyArray<TabInfo>, CdpError>>;
|
|
25
|
+
switchTab(targetId: string): Promise<Result<void, CdpError>>;
|
|
26
|
+
newTab(url?: string): Promise<Result<string, CdpError>>;
|
|
27
|
+
current(): { readonly sessionId: string; readonly targetId: string } | null;
|
|
28
|
+
session(): CdpSession;
|
|
29
|
+
transport(): CdpTransport;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const HEALTH_TTL_MS = 30_000;
|
|
33
|
+
const PAGE_INFO_TTL_MS = 1_000;
|
|
34
|
+
|
|
35
|
+
const parsePageInfoPayload = (v: unknown): Result<PageInfo, CdpError> => {
|
|
36
|
+
if (typeof v !== "object" || v === null) {
|
|
37
|
+
return err(cdpError("invalid_response", "page info payload is not an object"));
|
|
38
|
+
}
|
|
39
|
+
const o = v as Readonly<Record<string, unknown>>;
|
|
40
|
+
const fields: ReadonlyArray<readonly [string, "string" | "number"]> = [
|
|
41
|
+
["url", "string"], ["title", "string"],
|
|
42
|
+
["w", "number"], ["h", "number"],
|
|
43
|
+
["sx", "number"], ["sy", "number"],
|
|
44
|
+
["pw", "number"], ["ph", "number"],
|
|
45
|
+
];
|
|
46
|
+
for (const [k, t] of fields) {
|
|
47
|
+
if (typeof o[k] !== t) {
|
|
48
|
+
return err(cdpError("invalid_response", `page info field ${k} has wrong type (expected ${t})`));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return ok({
|
|
52
|
+
url: o["url"] as string,
|
|
53
|
+
title: o["title"] as string,
|
|
54
|
+
width: o["w"] as number,
|
|
55
|
+
height: o["h"] as number,
|
|
56
|
+
scrollX: o["sx"] as number,
|
|
57
|
+
scrollY: o["sy"] as number,
|
|
58
|
+
pageWidth: o["pw"] as number,
|
|
59
|
+
pageHeight: o["ph"] as number,
|
|
60
|
+
});
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient => {
|
|
64
|
+
const transport = createCdpTransport();
|
|
65
|
+
const session = createCdpSession(transport);
|
|
66
|
+
let lastHealth = 0;
|
|
67
|
+
let pageCache: { readonly info: PageInfo; readonly at: number } | null = null;
|
|
68
|
+
let remote: BrowserClientOptions["remote"] | null = opts.remote ?? null;
|
|
69
|
+
|
|
70
|
+
const start = async (): Promise<Result<void, CdpError>> => {
|
|
71
|
+
if (transport.state() === "open" && session.current()) return ok(undefined);
|
|
72
|
+
const envUrl = process.env["BU_CDP_WS"];
|
|
73
|
+
let wsUrl: string;
|
|
74
|
+
if (remote?.cdpUrl) {
|
|
75
|
+
wsUrl = remote.cdpUrl;
|
|
76
|
+
} else if (envUrl) {
|
|
77
|
+
wsUrl = envUrl;
|
|
78
|
+
} else {
|
|
79
|
+
const discovered = await discoverWsUrl();
|
|
80
|
+
if (!discovered.success) return discovered;
|
|
81
|
+
wsUrl = discovered.data;
|
|
82
|
+
}
|
|
83
|
+
const connected = await transport.connect(wsUrl, { timeoutMs: 10_000 });
|
|
84
|
+
if (!connected.success) return connected;
|
|
85
|
+
if (!remote) {
|
|
86
|
+
remote = { cdpUrl: wsUrl, browserId: wsUrl.split("/").pop() ?? "unknown" };
|
|
87
|
+
}
|
|
88
|
+
const attached = await session.attachFirstPage();
|
|
89
|
+
if (!attached.success) {
|
|
90
|
+
await transport.close();
|
|
91
|
+
return attached;
|
|
92
|
+
}
|
|
93
|
+
lastHealth = Date.now();
|
|
94
|
+
pageCache = null;
|
|
95
|
+
return ok(undefined);
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const stop = async (): Promise<void> => {
|
|
99
|
+
await transport.close();
|
|
100
|
+
pageCache = null;
|
|
101
|
+
lastHealth = 0;
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
const ensureAlive = async (): Promise<Result<void, CdpError>> => {
|
|
105
|
+
if (transport.state() !== "open" || !session.current()) {
|
|
106
|
+
await stop();
|
|
107
|
+
return start();
|
|
108
|
+
}
|
|
109
|
+
if (Date.now() - lastHealth < HEALTH_TTL_MS) return ok(undefined);
|
|
110
|
+
const probe = await transport.request("Target.getTargets", {}, { sessionId: null, timeoutMs: 2_000 });
|
|
111
|
+
if (!probe.success) {
|
|
112
|
+
await stop();
|
|
113
|
+
return start();
|
|
114
|
+
}
|
|
115
|
+
lastHealth = Date.now();
|
|
116
|
+
// Verify the page session is still responsive (handles the case where the
|
|
117
|
+
// browser transport is alive but the page target crashed, e.g. localhost died).
|
|
118
|
+
const jsProbe = await session.call("Runtime.evaluate", {
|
|
119
|
+
expression: "1", returnByValue: true,
|
|
120
|
+
}, { timeoutMs: 2_000 });
|
|
121
|
+
if (!jsProbe.success && jsProbe.error.kind === "session_not_found") {
|
|
122
|
+
const reattached = await session.attachFirstPage();
|
|
123
|
+
if (reattached.success) {
|
|
124
|
+
pageCache = null;
|
|
125
|
+
return ok(undefined);
|
|
126
|
+
}
|
|
127
|
+
await stop();
|
|
128
|
+
return start();
|
|
129
|
+
}
|
|
130
|
+
return ok(undefined);
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
const evaluateJs = async (expression: string, sessionId?: string): Promise<Result<unknown, CdpError>> => {
|
|
134
|
+
// Heuristic IIFE wrap: legacy convenience that lets agents write
|
|
135
|
+
// `return foo` instead of `(() => foo)()`. False-positive on substring
|
|
136
|
+
// `"return "` inside string literals or comments — agents that hit this
|
|
137
|
+
// can pass an explicit IIFE instead.
|
|
138
|
+
const wrapped = expression.includes("return ") && !expression.trim().startsWith("(")
|
|
139
|
+
? `(function(){${expression}})()`
|
|
140
|
+
: expression;
|
|
141
|
+
const r = sessionId
|
|
142
|
+
? await session.callOnTarget("Runtime.evaluate", { expression: wrapped, returnByValue: true, awaitPromise: true }, sessionId)
|
|
143
|
+
: await session.call("Runtime.evaluate", { expression: wrapped, returnByValue: true, awaitPromise: true });
|
|
144
|
+
if (!r.success) return r;
|
|
145
|
+
const data = r.data as { result?: { value?: unknown }; exceptionDetails?: unknown };
|
|
146
|
+
if (data.exceptionDetails) {
|
|
147
|
+
return err(cdpError("remote_error", `JS evaluation failed: ${JSON.stringify(data.exceptionDetails)}`, "Runtime.evaluate"));
|
|
148
|
+
}
|
|
149
|
+
return ok(data.result?.value);
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const readPageInfo = async (): Promise<Result<PageInfo, CdpError>> => {
|
|
153
|
+
const dirty = session.drainPageInfoInvalidations();
|
|
154
|
+
if (pageCache && !dirty && Date.now() - pageCache.at < PAGE_INFO_TTL_MS) return ok(pageCache.info);
|
|
155
|
+
const expr = safeJs`JSON.stringify({url:location.href,title:document.title,w:innerWidth,h:innerHeight,sx:scrollX,sy:scrollY,pw:document.documentElement.scrollWidth,ph:document.documentElement.scrollHeight})`;
|
|
156
|
+
const raw = await evaluateJs(expr);
|
|
157
|
+
if (!raw.success) return raw;
|
|
158
|
+
if (typeof raw.data !== "string") return err(cdpError("invalid_response", "page info evaluation did not return a string"));
|
|
159
|
+
let parsedRaw: unknown;
|
|
160
|
+
try {
|
|
161
|
+
parsedRaw = JSON.parse(raw.data);
|
|
162
|
+
} catch (e) {
|
|
163
|
+
return err(cdpError("invalid_response", `page info JSON.parse failed: ${e instanceof Error ? e.message : String(e)}`));
|
|
164
|
+
}
|
|
165
|
+
const info = parsePageInfoPayload(parsedRaw);
|
|
166
|
+
if (!info.success) return info;
|
|
167
|
+
pageCache = { info: info.data, at: Date.now() };
|
|
168
|
+
return ok(info.data);
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
const pageInfo = async (): Promise<Result<PageInfo | { readonly dialog: DialogInfo }, CdpError>> => {
|
|
172
|
+
const d = session.takeDialog();
|
|
173
|
+
if (d) return ok({ dialog: d });
|
|
174
|
+
return readPageInfo();
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
const listTabs = async (includeInternal = true): Promise<Result<ReadonlyArray<TabInfo>, CdpError>> => {
|
|
178
|
+
const r = await session.callBrowser("Target.getTargets");
|
|
179
|
+
if (!r.success) return r;
|
|
180
|
+
const data = r.data as { targetInfos: ReadonlyArray<{ targetId: string; type: string; title: string; url: string }> };
|
|
181
|
+
const tabs = data.targetInfos
|
|
182
|
+
.filter((t) => t.type === "page")
|
|
183
|
+
.filter((t) => includeInternal || !t.url.startsWith("chrome://"))
|
|
184
|
+
.map((t): TabInfo => ({ targetId: t.targetId, title: t.title, url: t.url }));
|
|
185
|
+
return ok(tabs);
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
const switchTab = async (targetId: string): Promise<Result<void, CdpError>> => {
|
|
189
|
+
const r = await session.switchTo(targetId);
|
|
190
|
+
if (!r.success) return r;
|
|
191
|
+
pageCache = null;
|
|
192
|
+
// Best-effort: mark the tab title with a green circle so the user can
|
|
193
|
+
// see which tab the agent attached to. CSP or detached frames may
|
|
194
|
+
// block the eval; we don't surface that as a switchTab failure.
|
|
195
|
+
await session.call("Runtime.evaluate", {
|
|
196
|
+
expression: safeJs`if(!document.title.startsWith('🟢'))document.title='🟢 '+document.title`,
|
|
197
|
+
});
|
|
198
|
+
return ok(undefined);
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
const newTab = async (url?: string): Promise<Result<string, CdpError>> => {
|
|
202
|
+
const created = await session.callBrowser("Target.createTarget", { url: "about:blank" });
|
|
203
|
+
if (!created.success) return created;
|
|
204
|
+
const c = created.data as { targetId: string };
|
|
205
|
+
const switched = await switchTab(c.targetId);
|
|
206
|
+
if (!switched.success) return switched;
|
|
207
|
+
if (url && url !== "about:blank") {
|
|
208
|
+
const nav = await session.call("Page.navigate", { url });
|
|
209
|
+
if (!nav.success) return nav;
|
|
210
|
+
pageCache = null;
|
|
211
|
+
}
|
|
212
|
+
return ok(c.targetId);
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
const status = (): DaemonStatus => ({
|
|
216
|
+
alive: transport.state() === "open" && session.current() !== null,
|
|
217
|
+
sessionId: session.current()?.sessionId ?? null,
|
|
218
|
+
namespace: opts.namespace,
|
|
219
|
+
...(remote?.browserId !== undefined ? { remoteBrowserId: remote.browserId } : {}),
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
return {
|
|
223
|
+
namespace: opts.namespace,
|
|
224
|
+
ensureAlive, status, start, stop,
|
|
225
|
+
evaluateJs, pageInfo,
|
|
226
|
+
takeDialog: () => session.takeDialog(),
|
|
227
|
+
listTabs, switchTab, newTab,
|
|
228
|
+
current: () => session.current(),
|
|
229
|
+
session: () => session,
|
|
230
|
+
transport: () => transport,
|
|
231
|
+
};
|
|
232
|
+
};
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { Type, type Static } from "typebox";
|
|
2
|
+
import { Text } from "@mariozechner/pi-tui";
|
|
3
|
+
import type { BrowserClient } from "../client";
|
|
4
|
+
import { Coords, MouseButton } from "../schemas/common";
|
|
5
|
+
import { type Result, err, ok } from "../util/result";
|
|
6
|
+
import { defineBrowserTool, type ToolErr } from "../util/tool";
|
|
7
|
+
import { captureWithCrosshair } from "./screenshot";
|
|
8
|
+
|
|
9
|
+
const ClickArgs = Type.Object({
|
|
10
|
+
...Coords.properties,
|
|
11
|
+
button: Type.Optional(MouseButton),
|
|
12
|
+
count: Type.Optional(
|
|
13
|
+
Type.Integer({
|
|
14
|
+
minimum: 1,
|
|
15
|
+
maximum: 3,
|
|
16
|
+
default: 1,
|
|
17
|
+
description: "Number of clicks (1 = single, 2 = double). Default: 1",
|
|
18
|
+
}),
|
|
19
|
+
),
|
|
20
|
+
});
|
|
21
|
+
type ClickArgsT = Static<typeof ClickArgs>;
|
|
22
|
+
|
|
23
|
+
const dispatchClick = async (
|
|
24
|
+
client: BrowserClient,
|
|
25
|
+
args: ClickArgsT,
|
|
26
|
+
): Promise<Result<void, ToolErr>> => {
|
|
27
|
+
const button = args.button ?? "left";
|
|
28
|
+
const count = args.count ?? 1;
|
|
29
|
+
const pressed = await client.session().call("Input.dispatchMouseEvent", {
|
|
30
|
+
type: "mousePressed", x: args.x, y: args.y, button, clickCount: count,
|
|
31
|
+
});
|
|
32
|
+
if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
|
|
33
|
+
const released = await client.session().call("Input.dispatchMouseEvent", {
|
|
34
|
+
type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count,
|
|
35
|
+
});
|
|
36
|
+
if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
|
|
37
|
+
return ok(undefined);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const clickTool = defineBrowserTool({
|
|
41
|
+
name: "browser_click",
|
|
42
|
+
label: "Browser Click",
|
|
43
|
+
description:
|
|
44
|
+
"Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.",
|
|
45
|
+
promptSnippet: "Click at pixel coordinates (x, y) on the page",
|
|
46
|
+
promptGuidelines: [
|
|
47
|
+
"Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).",
|
|
48
|
+
"Capture a browser_screenshot BEFORE clicking to find the right coordinates.",
|
|
49
|
+
"Capture another browser_screenshot AFTER clicking to verify the action worked.",
|
|
50
|
+
"Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
|
|
51
|
+
"If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.",
|
|
52
|
+
"For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
|
|
53
|
+
],
|
|
54
|
+
parameters: ClickArgs,
|
|
55
|
+
renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0),
|
|
56
|
+
async handler(args, { client }) {
|
|
57
|
+
const clicked = await dispatchClick(client, args);
|
|
58
|
+
if (!clicked.success) return clicked;
|
|
59
|
+
if (process.env["BH_DEBUG_CLICKS"]) {
|
|
60
|
+
const debug = await captureWithCrosshair(client, { x: args.x, y: args.y });
|
|
61
|
+
if (debug.success) {
|
|
62
|
+
return ok({
|
|
63
|
+
text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`,
|
|
64
|
+
details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y },
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return ok({
|
|
69
|
+
text: `Clicked at (${args.x}, ${args.y})`,
|
|
70
|
+
details: { x: args.x, y: args.y },
|
|
71
|
+
});
|
|
72
|
+
},
|
|
73
|
+
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
|
|
5
|
+
const HandleDialogArgs = Type.Object({
|
|
6
|
+
accept: Type.Boolean({ description: "true = accept, false = dismiss" }),
|
|
7
|
+
promptText: Type.Optional(Type.String({ description: "Text to type if dialog is a prompt()" })),
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
export const handleDialogTool = defineBrowserTool({
|
|
11
|
+
name: "browser_handle_dialog",
|
|
12
|
+
label: "Browser Handle Dialog",
|
|
13
|
+
description:
|
|
14
|
+
"Accept or dismiss the currently open JS dialog (alert/confirm/prompt/beforeunload).",
|
|
15
|
+
promptSnippet: "Accept or dismiss a JS dialog",
|
|
16
|
+
promptGuidelines: [
|
|
17
|
+
"Use after browser_page_info reports a dialog is open. Until handled, no other browser action will work.",
|
|
18
|
+
"For prompt() dialogs, supply promptText with the value to submit.",
|
|
19
|
+
],
|
|
20
|
+
parameters: HandleDialogArgs,
|
|
21
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
22
|
+
const params: Record<string, unknown> = { accept: args.accept };
|
|
23
|
+
if (args.promptText !== undefined) params["promptText"] = args.promptText;
|
|
24
|
+
const r = await client.session().call("Page.handleJavaScriptDialog", params);
|
|
25
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
26
|
+
return ok({
|
|
27
|
+
text: `Dialog ${args.accept ? "accepted" : "dismissed"}`,
|
|
28
|
+
details: { accept: args.accept },
|
|
29
|
+
});
|
|
30
|
+
},
|
|
31
|
+
});
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
|
|
5
|
+
const DragArgs = Type.Object({
|
|
6
|
+
startX: Type.Number(),
|
|
7
|
+
startY: Type.Number(),
|
|
8
|
+
endX: Type.Number(),
|
|
9
|
+
endY: Type.Number(),
|
|
10
|
+
dataTransfer: Type.Optional(
|
|
11
|
+
Type.Record(Type.String(), Type.String(), {
|
|
12
|
+
description: "Optional MIME → data map for DataTransfer (e.g., { 'text/plain': 'hello' })",
|
|
13
|
+
}),
|
|
14
|
+
),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
export const dragAndDropTool = defineBrowserTool({
|
|
18
|
+
name: "browser_drag_and_drop",
|
|
19
|
+
label: "Browser Drag & Drop",
|
|
20
|
+
description: "Drag from (startX, startY) to (endX, endY) using CDP Input.dispatchDragEvent.",
|
|
21
|
+
promptSnippet: "Drag and drop between two coordinates",
|
|
22
|
+
promptGuidelines: [
|
|
23
|
+
"Coordinates in CSS pixels.",
|
|
24
|
+
"dataTransfer is an optional MIME→data map (e.g., { 'text/plain': 'hello' }).",
|
|
25
|
+
],
|
|
26
|
+
parameters: DragArgs,
|
|
27
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
28
|
+
const data = args.dataTransfer
|
|
29
|
+
? {
|
|
30
|
+
items: Object.entries(args.dataTransfer).map(([mimeType, payload]) => ({
|
|
31
|
+
mimeType,
|
|
32
|
+
data: Buffer.from(payload).toString("base64"),
|
|
33
|
+
})),
|
|
34
|
+
dragOperationsMask: 1,
|
|
35
|
+
}
|
|
36
|
+
: { items: [], dragOperationsMask: 1 };
|
|
37
|
+
const cdp = client.session();
|
|
38
|
+
const calls: ReadonlyArray<readonly [string, Record<string, unknown>]> = [
|
|
39
|
+
["Input.dispatchMouseEvent", { type: "mousePressed", x: args.startX, y: args.startY, button: "left", clickCount: 1 }],
|
|
40
|
+
["Input.dispatchDragEvent", { type: "dragEnter", x: args.startX, y: args.startY, data, modifiers: 0 }],
|
|
41
|
+
// 5 interpolated dragOver steps from start → end so the page sees a smooth drag.
|
|
42
|
+
...Array.from({ length: 5 }, (_, i): readonly [string, Record<string, unknown>] => {
|
|
43
|
+
const t = (i + 1) / 5;
|
|
44
|
+
return [
|
|
45
|
+
"Input.dispatchDragEvent",
|
|
46
|
+
{
|
|
47
|
+
type: "dragOver",
|
|
48
|
+
x: Math.round(args.startX + (args.endX - args.startX) * t),
|
|
49
|
+
y: Math.round(args.startY + (args.endY - args.startY) * t),
|
|
50
|
+
data,
|
|
51
|
+
modifiers: 0,
|
|
52
|
+
},
|
|
53
|
+
];
|
|
54
|
+
}),
|
|
55
|
+
["Input.dispatchDragEvent", { type: "drop", x: args.endX, y: args.endY, data, modifiers: 0 }],
|
|
56
|
+
["Input.dispatchMouseEvent", { type: "mouseReleased", x: args.endX, y: args.endY, button: "left", clickCount: 1 }],
|
|
57
|
+
];
|
|
58
|
+
for (const [method, params] of calls) {
|
|
59
|
+
const r = await cdp.call(method, params);
|
|
60
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
61
|
+
}
|
|
62
|
+
return ok({
|
|
63
|
+
text: `Dragged from (${args.startX},${args.startY}) to (${args.endX},${args.endY})`,
|
|
64
|
+
});
|
|
65
|
+
},
|
|
66
|
+
});
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { mkdir, access, constants, stat, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
import { Type } from "typebox";
|
|
4
|
+
import type { BrowserClient } from "../client";
|
|
5
|
+
import { type Result, err, ok } from "../util/result";
|
|
6
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
7
|
+
import { safeJs } from "../util/js-template";
|
|
8
|
+
import { pdfPath } from "../util/paths";
|
|
9
|
+
|
|
10
|
+
const UploadArgs = Type.Object({
|
|
11
|
+
selector: Type.String({ description: "CSS selector of the file <input>" }),
|
|
12
|
+
filePath: Type.String({ description: "Absolute path to the file to upload" }),
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => {
|
|
16
|
+
try {
|
|
17
|
+
await access(filePath, constants.R_OK);
|
|
18
|
+
return ok(undefined);
|
|
19
|
+
} catch (e) {
|
|
20
|
+
return err({
|
|
21
|
+
kind: "io_error",
|
|
22
|
+
message: `Cannot read file: ${filePath} (${e instanceof Error ? e.message : String(e)})`,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const tryCdpUpload = async (
|
|
28
|
+
client: BrowserClient,
|
|
29
|
+
selector: string,
|
|
30
|
+
filePath: string,
|
|
31
|
+
): Promise<Result<void, ToolErr>> => {
|
|
32
|
+
const doc = await client.session().call("DOM.getDocument", { depth: -1 });
|
|
33
|
+
if (!doc.success) return err({ kind: "cdp_error", message: doc.error.message });
|
|
34
|
+
// CDP boundary cast: DOM.getDocument returns { root: { nodeId: number } }
|
|
35
|
+
const root = (doc.data as { root: { nodeId: number } }).root;
|
|
36
|
+
const q = await client.session().call("DOM.querySelector", { nodeId: root.nodeId, selector });
|
|
37
|
+
if (!q.success) return err({ kind: "cdp_error", message: q.error.message });
|
|
38
|
+
// CDP boundary cast: DOM.querySelector returns { nodeId: number }
|
|
39
|
+
const nodeId = (q.data as { nodeId: number }).nodeId;
|
|
40
|
+
if (!nodeId) return err({ kind: "invalid_state", message: `Selector matched 0 file inputs: ${selector}` });
|
|
41
|
+
const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], nodeId });
|
|
42
|
+
if (!set.success) return err({ kind: "cdp_error", message: set.error.message });
|
|
43
|
+
const verify = await client.evaluateJs(safeJs`document.querySelector(${selector})?.files?.length || 0`);
|
|
44
|
+
if (!verify.success) return err({ kind: "cdp_error", message: verify.error.message });
|
|
45
|
+
if (Number(verify.data ?? 0) === 0) {
|
|
46
|
+
return err({ kind: "invalid_state", message: "CDP upload reported success but file count is 0" });
|
|
47
|
+
}
|
|
48
|
+
return ok(undefined);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const inferMime = (name: string): string => {
|
|
52
|
+
if (name.endsWith(".png")) return "image/png";
|
|
53
|
+
if (name.endsWith(".jpg") || name.endsWith(".jpeg")) return "image/jpeg";
|
|
54
|
+
if (name.endsWith(".pdf")) return "application/pdf";
|
|
55
|
+
if (name.endsWith(".json")) return "application/json";
|
|
56
|
+
return "text/plain";
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const jsFallbackUpload = async (
|
|
60
|
+
client: BrowserClient,
|
|
61
|
+
selector: string,
|
|
62
|
+
filePath: string,
|
|
63
|
+
): Promise<Result<void, ToolErr>> => {
|
|
64
|
+
const buf = await readFile(filePath);
|
|
65
|
+
const st = await stat(filePath);
|
|
66
|
+
const name = basename(filePath);
|
|
67
|
+
const mime = inferMime(name);
|
|
68
|
+
const expr = safeJs`
|
|
69
|
+
(() => {
|
|
70
|
+
const input = document.querySelector(${selector});
|
|
71
|
+
if (!input || input.type !== 'file') throw new Error('File input not found');
|
|
72
|
+
const bin = atob(${buf.toString("base64")});
|
|
73
|
+
const bytes = new Uint8Array(bin.length);
|
|
74
|
+
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
|
75
|
+
const file = new File([bytes], ${name}, { type: ${mime}, lastModified: ${st.mtimeMs} });
|
|
76
|
+
const dt = new DataTransfer();
|
|
77
|
+
dt.items.add(file);
|
|
78
|
+
input.files = dt.files;
|
|
79
|
+
input.dispatchEvent(new Event('change', { bubbles: true }));
|
|
80
|
+
return input.files.length;
|
|
81
|
+
})()
|
|
82
|
+
`;
|
|
83
|
+
const r = await client.evaluateJs(expr);
|
|
84
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
85
|
+
if (!Number(r.data ?? 0)) return err({ kind: "invalid_state", message: "JS fallback set 0 files" });
|
|
86
|
+
return ok(undefined);
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
export const uploadFileTool = defineBrowserTool({
|
|
90
|
+
name: "browser_upload_file",
|
|
91
|
+
label: "Browser Upload File",
|
|
92
|
+
description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages.",
|
|
93
|
+
promptSnippet: "Upload a file to a file input",
|
|
94
|
+
promptGuidelines: [
|
|
95
|
+
"File path must be absolute and readable.",
|
|
96
|
+
"Selector must match a file input.",
|
|
97
|
+
],
|
|
98
|
+
parameters: UploadArgs,
|
|
99
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
100
|
+
const readable = await verifyReadable(args.filePath);
|
|
101
|
+
if (!readable.success) return readable;
|
|
102
|
+
const cdp = await tryCdpUpload(client, args.selector, args.filePath);
|
|
103
|
+
if (cdp.success) {
|
|
104
|
+
return ok({
|
|
105
|
+
text: `Uploaded ${args.filePath} via CDP`,
|
|
106
|
+
details: { mode: "cdp", filePath: args.filePath },
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
const js = await jsFallbackUpload(client, args.selector, args.filePath);
|
|
110
|
+
if (js.success) {
|
|
111
|
+
return ok({
|
|
112
|
+
text: `Uploaded ${args.filePath} via JS DataTransfer`,
|
|
113
|
+
details: { mode: "js", filePath: args.filePath },
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return err({
|
|
117
|
+
kind: "cdp_error",
|
|
118
|
+
message: `Both CDP and JS fallback failed. CDP: ${cdp.error.message}; JS: ${js.error.message}`,
|
|
119
|
+
});
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
const DownloadArgs = Type.Object({
|
|
124
|
+
downloadPath: Type.String({ description: "Absolute path to a writable directory where downloads should be saved" }),
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
export const downloadTool = defineBrowserTool({
|
|
128
|
+
name: "browser_download",
|
|
129
|
+
label: "Browser Download",
|
|
130
|
+
description: "Configure Chrome's download behavior: set the save directory and disable the save-as prompt.",
|
|
131
|
+
promptSnippet: "Configure download directory",
|
|
132
|
+
promptGuidelines: [
|
|
133
|
+
"Pass an absolute path to an existing writable directory.",
|
|
134
|
+
"Affects all subsequent downloads on this browser.",
|
|
135
|
+
],
|
|
136
|
+
parameters: DownloadArgs,
|
|
137
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
138
|
+
try {
|
|
139
|
+
await mkdir(args.downloadPath, { recursive: true });
|
|
140
|
+
} catch (e) {
|
|
141
|
+
return err({
|
|
142
|
+
kind: "io_error",
|
|
143
|
+
message: `Cannot create download directory: ${args.downloadPath} (${e instanceof Error ? e.message : String(e)})`,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
const s = await stat(args.downloadPath);
|
|
148
|
+
if (!s.isDirectory()) {
|
|
149
|
+
return err({ kind: "io_error", message: `Not a directory: ${args.downloadPath}` });
|
|
150
|
+
}
|
|
151
|
+
await access(args.downloadPath, constants.W_OK);
|
|
152
|
+
} catch (e) {
|
|
153
|
+
return err({
|
|
154
|
+
kind: "io_error",
|
|
155
|
+
message: `Download path unusable: ${e instanceof Error ? e.message : String(e)}`,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
const r = await client.session().callBrowser("Browser.setDownloadBehavior", {
|
|
159
|
+
behavior: "allow",
|
|
160
|
+
downloadPath: args.downloadPath,
|
|
161
|
+
eventsEnabled: true,
|
|
162
|
+
});
|
|
163
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
164
|
+
return ok({
|
|
165
|
+
text: `Downloads will save to: ${args.downloadPath}`,
|
|
166
|
+
details: { downloadPath: args.downloadPath },
|
|
167
|
+
});
|
|
168
|
+
},
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
const PrintPdfArgs = Type.Object({
|
|
172
|
+
outputPath: Type.Optional(Type.String({ description: "Where to save the PDF. Default: tmpdir + uuid." })),
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
export const printToPdfTool = defineBrowserTool({
|
|
176
|
+
name: "browser_print_to_pdf",
|
|
177
|
+
label: "Browser Print to PDF",
|
|
178
|
+
description: "Print the current page to a PDF file using Chrome's Page.printToPDF.",
|
|
179
|
+
promptSnippet: "Print the current page to PDF",
|
|
180
|
+
promptGuidelines: ["Default output path is in tmpdir; pass outputPath to control."],
|
|
181
|
+
parameters: PrintPdfArgs,
|
|
182
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
183
|
+
const r = await client.session().call("Page.printToPDF", {
|
|
184
|
+
printBackground: true,
|
|
185
|
+
preferCSSPageSize: true,
|
|
186
|
+
});
|
|
187
|
+
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
188
|
+
// CDP boundary cast: Page.printToPDF returns { data: string } (base64-encoded PDF)
|
|
189
|
+
const data = (r.data as { data: string }).data;
|
|
190
|
+
const path = args.outputPath ?? pdfPath(client.namespace);
|
|
191
|
+
await writeFile(path, Buffer.from(data, "base64"));
|
|
192
|
+
return ok({ text: `PDF saved: ${path}`, details: { path } });
|
|
193
|
+
},
|
|
194
|
+
});
|