pi-browser-harness 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +5 -9
- package/package.json +5 -2
- package/skills/pi-browser-harness/SKILL.md +1 -1
- package/src/cdp/commands.ts +256 -0
- package/src/cdp/console-buffer.ts +10 -85
- package/src/cdp/discovery.ts +28 -71
- package/src/cdp/errors.ts +4 -0
- package/src/cdp/events.ts +170 -0
- package/src/cdp/network-buffer.ts +42 -109
- package/src/cdp/ownership.ts +0 -17
- package/src/cdp/record-store.ts +61 -0
- package/src/cdp/session.ts +205 -193
- package/src/cdp/target-factory.ts +10 -80
- package/src/cdp/types.ts +21 -12
- package/src/client.ts +39 -95
- package/src/daemon/bridge.ts +150 -203
- package/src/daemon/index.ts +0 -28
- package/src/daemon/protocol.ts +6 -68
- package/src/daemon/queue.ts +85 -0
- package/src/daemon/server.ts +24 -99
- package/src/daemon/spawn.ts +18 -68
- package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +18 -51
- package/src/domains/ax-tree.ts +179 -0
- package/src/domains/box.ts +51 -0
- package/src/domains/cdp-call.ts +55 -0
- package/src/domains/click.ts +19 -32
- package/src/domains/console.ts +48 -2
- package/src/domains/dialog.ts +8 -5
- package/src/domains/drag.ts +8 -7
- package/src/domains/element-call.ts +117 -0
- package/src/domains/files.ts +22 -26
- package/src/domains/fill-engine.ts +80 -0
- package/src/domains/form-batch.ts +65 -0
- package/src/domains/form.ts +116 -105
- package/src/domains/history.ts +13 -9
- package/src/domains/isolated-tab.ts +26 -55
- package/src/domains/js.ts +29 -42
- package/src/domains/keyboard.ts +36 -27
- package/src/domains/navigate.ts +16 -23
- package/src/domains/network.ts +60 -10
- package/src/domains/page.ts +24 -3
- package/src/domains/readpage/read-page.ts +74 -19
- package/src/domains/readpage/readability.ts +0 -24
- package/src/domains/ref-resolve.ts +24 -58
- package/src/domains/render.ts +6 -27
- package/src/domains/screenshot-capture.ts +80 -0
- package/src/domains/screenshot.ts +9 -86
- package/src/domains/scroll.ts +10 -11
- package/src/domains/search/google-serp.ts +0 -29
- package/src/domains/search/web-search.ts +48 -20
- package/src/domains/setup.ts +1 -11
- package/src/domains/snapshot.ts +35 -276
- package/src/domains/tabs.ts +14 -7
- package/src/domains/target.ts +35 -0
- package/src/domains/viewport.ts +5 -3
- package/src/index.ts +8 -58
- package/src/profile/bind.ts +8 -57
- package/src/profile/browser-process.ts +75 -126
- package/src/profile/command.ts +50 -29
- package/src/profile/launch.ts +22 -40
- package/src/profile/list.ts +14 -54
- package/src/profile/paths.ts +2 -65
- package/src/profile/store.ts +35 -56
- package/src/registry.ts +30 -33
- package/src/schemas/parse.ts +23 -0
- package/src/setup.ts +0 -25
- package/src/state.ts +16 -16
- package/src/util/debug.ts +8 -0
- package/src/util/guards.ts +30 -0
- package/src/util/js-template.ts +1 -16
- package/src/util/mutex.ts +0 -5
- package/src/util/result.ts +0 -6
- package/src/util/sharp-shim.ts +12 -19
- package/src/util/tool.ts +17 -42
- package/src/util/truncate.ts +6 -0
- package/tsconfig.json +1 -16
- package/skills/deep-research/SKILL.md +0 -66
- package/src/cdp/event-queue.ts +0 -128
- package/src/cdp/transport.ts +0 -156
- package/src/cdp/window-binding.ts +0 -15
- package/src/deep-research.ts +0 -41
- package/src/domains/forms.ts +0 -355
- package/src/domains/readpage/capture.ts +0 -46
- package/src/domains/search/extract.ts +0 -44
- package/src/profile/picker.ts +0 -73
- package/src/util/keycodes.ts +0 -10
- package/src/util/time.ts +0 -29
|
@@ -1,32 +1,17 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* DaemonTransport — CdpTransport backed by a Unix socket to the browser daemon.
|
|
3
|
-
*
|
|
4
|
-
* Satisfies the exact CdpTransport interface so BrowserClient, CdpSession, and
|
|
5
|
-
* all domain tools work without changes. Internally uses newline-delimited JSON
|
|
6
|
-
* over a Unix domain socket instead of a WebSocket to Chrome.
|
|
7
|
-
*
|
|
8
|
-
* References:
|
|
9
|
-
* - CdpTransport interface: src/cdp/transport.ts
|
|
10
|
-
* - net.createConnection IPC: https://nodejs.org/docs/latest-v22.x/api/net.html#netcreateconnection
|
|
11
|
-
* - readline.createInterface: https://nodejs.org/docs/latest-v22.x/api/readline.html#readlinecreateinterfaceoptions
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
1
|
import { createConnection, type Socket } from "node:net";
|
|
15
2
|
import { createInterface } from "node:readline";
|
|
16
3
|
import { type Result, err, ok } from "../util/result";
|
|
17
|
-
import { type CdpError, cdpError } from "../cdp/errors";
|
|
18
|
-
import type
|
|
4
|
+
import { type CdpError, cdpError, classifyRemoteError } from "../cdp/errors";
|
|
5
|
+
import { DEFAULT_TIMEOUT_MS, type CdpTransport } from "../cdp/types";
|
|
19
6
|
import type { CdpEvent } from "../cdp/types";
|
|
20
|
-
import { type Pending, makeEventQueue, makeOnClose, rejectAllPending, sendWithTimeout } from "./
|
|
7
|
+
import { type Pending, makeEventQueue, makeOnClose, rejectAllPending, sendWithTimeout } from "./queue";
|
|
21
8
|
import {
|
|
22
9
|
DAEMON_SOCKET_PATH,
|
|
23
10
|
type WireRequest,
|
|
24
11
|
type WireControl,
|
|
25
12
|
deserialize,
|
|
26
13
|
serialize,
|
|
27
|
-
} from "
|
|
28
|
-
|
|
29
|
-
// ── Implementation ─────────────────────────────────────────────────────────────
|
|
14
|
+
} from "./protocol";
|
|
30
15
|
|
|
31
16
|
export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
32
17
|
let socket: Socket | null = null;
|
|
@@ -49,10 +34,7 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
|
49
34
|
for (const cb of closeListeners) cb();
|
|
50
35
|
};
|
|
51
36
|
|
|
52
|
-
// ── Connect ────────────────────────────────────────────────────────────
|
|
53
|
-
|
|
54
37
|
const connect = (_url: string, opts?: { timeoutMs?: number }): Promise<Result<void, CdpError>> => {
|
|
55
|
-
// url is ignored — we always connect to the daemon socket
|
|
56
38
|
const timeoutMs = opts?.timeoutMs ?? 10_000;
|
|
57
39
|
|
|
58
40
|
if (socket && !socket.destroyed && registered) {
|
|
@@ -67,21 +49,23 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
|
67
49
|
resolve(r);
|
|
68
50
|
};
|
|
69
51
|
|
|
52
|
+
let sock: Socket;
|
|
70
53
|
try {
|
|
71
|
-
|
|
54
|
+
sock = createConnection(DAEMON_SOCKET_PATH);
|
|
72
55
|
} catch (e) {
|
|
73
56
|
settle(err(cdpError("transport_closed", e instanceof Error ? e.message : String(e))));
|
|
74
57
|
return;
|
|
75
58
|
}
|
|
59
|
+
socket = sock;
|
|
76
60
|
|
|
77
61
|
const connectTimer = setTimeout(() => {
|
|
78
62
|
cleanup("Connection timeout");
|
|
79
63
|
settle(err(cdpError("timeout", `Daemon connection timed out after ${timeoutMs}ms`)));
|
|
80
64
|
}, timeoutMs);
|
|
81
65
|
|
|
82
|
-
|
|
66
|
+
sock.on("connect", () => {
|
|
83
67
|
clearTimeout(connectTimer);
|
|
84
|
-
rl = createInterface({ input:
|
|
68
|
+
rl = createInterface({ input: sock, crlfDelay: Infinity });
|
|
85
69
|
|
|
86
70
|
rl.on("line", (line: string) => {
|
|
87
71
|
const msg = deserialize(line);
|
|
@@ -107,13 +91,7 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
|
107
91
|
clearTimeout(p.timer);
|
|
108
92
|
|
|
109
93
|
if (msg.error) {
|
|
110
|
-
p.resolve(err(cdpError(
|
|
111
|
-
msg.error.message.includes("Session with given id not found")
|
|
112
|
-
? "session_not_found"
|
|
113
|
-
: "remote_error",
|
|
114
|
-
msg.error.message,
|
|
115
|
-
p.method,
|
|
116
|
-
)));
|
|
94
|
+
p.resolve(err(cdpError(classifyRemoteError(msg.error.message), msg.error.message, p.method)));
|
|
117
95
|
} else {
|
|
118
96
|
p.resolve(ok(msg.result));
|
|
119
97
|
}
|
|
@@ -130,23 +108,19 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
|
130
108
|
}
|
|
131
109
|
});
|
|
132
110
|
|
|
133
|
-
|
|
134
|
-
// Errors are surfaced via 'close'
|
|
135
|
-
});
|
|
111
|
+
sock.on("error", () => {});
|
|
136
112
|
|
|
137
|
-
|
|
113
|
+
sock.on("close", () => {
|
|
138
114
|
clearTimeout(connectTimer);
|
|
139
115
|
cleanup("Daemon socket closed");
|
|
140
|
-
// If connect hasn't settled yet, settle with error
|
|
141
116
|
settle(err(cdpError("transport_closed", "Daemon socket closed before registration")));
|
|
142
117
|
});
|
|
143
118
|
|
|
144
|
-
// Send registration
|
|
145
119
|
const regMsg: WireControl = { type: "control", action: "register", clientId };
|
|
146
|
-
|
|
120
|
+
sock.write(serialize(regMsg) + "\n");
|
|
147
121
|
});
|
|
148
122
|
|
|
149
|
-
|
|
123
|
+
sock.on("error", (e: NodeJS.ErrnoException) => {
|
|
150
124
|
clearTimeout(connectTimer);
|
|
151
125
|
const msg = e.code === "ENOENT"
|
|
152
126
|
? "Daemon not running — socket not found"
|
|
@@ -156,8 +130,6 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
|
156
130
|
});
|
|
157
131
|
};
|
|
158
132
|
|
|
159
|
-
// ── Close ──────────────────────────────────────────────────────────────
|
|
160
|
-
|
|
161
133
|
const close = async (): Promise<void> => {
|
|
162
134
|
if (registered && socket && !socket.destroyed) {
|
|
163
135
|
const dereg: WireControl = { type: "control", action: "deregister", clientId };
|
|
@@ -166,17 +138,16 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
|
166
138
|
cleanup("close() called");
|
|
167
139
|
};
|
|
168
140
|
|
|
169
|
-
// ── Request ────────────────────────────────────────────────────────────
|
|
170
|
-
|
|
171
141
|
const request = (
|
|
172
142
|
method: string,
|
|
173
143
|
params: Record<string, unknown>,
|
|
174
144
|
opts?: { sessionId?: string | null; timeoutMs?: number },
|
|
175
145
|
): Promise<Result<unknown, CdpError>> => {
|
|
176
|
-
const timeoutMs = opts?.timeoutMs ??
|
|
146
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
177
147
|
const sessionId = opts?.sessionId ?? null;
|
|
178
148
|
|
|
179
|
-
|
|
149
|
+
const sock = socket;
|
|
150
|
+
if (sock === null || sock.destroyed || !registered) {
|
|
180
151
|
return Promise.resolve(err(cdpError("transport_closed", "Daemon not connected", method)));
|
|
181
152
|
}
|
|
182
153
|
|
|
@@ -189,15 +160,11 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
|
|
|
189
160
|
...(sessionId !== null && sessionId !== undefined ? { sessionId } : {}),
|
|
190
161
|
};
|
|
191
162
|
|
|
192
|
-
return sendWithTimeout(pending, id, method, timeoutMs, "Daemon", () =>
|
|
163
|
+
return sendWithTimeout(pending, id, method, timeoutMs, "Daemon", () => sock.write(serialize(req) + "\n"));
|
|
193
164
|
};
|
|
194
165
|
|
|
195
|
-
// ── Events ─────────────────────────────────────────────────────────────
|
|
196
|
-
|
|
197
166
|
const events = (): AsyncIterable<CdpEvent> => queue.iter;
|
|
198
167
|
|
|
199
|
-
// ── State ──────────────────────────────────────────────────────────────
|
|
200
|
-
|
|
201
168
|
const state = (): "open" | "closed" | "connecting" => {
|
|
202
169
|
if (!socket) return "closed";
|
|
203
170
|
if (!registered) return "connecting";
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import type { AxNodeResult } from "../cdp/commands";
|
|
2
|
+
import type { BrowserClient } from "../client";
|
|
3
|
+
import type { Box } from "./box";
|
|
4
|
+
import { cdpCall } from "./cdp-call";
|
|
5
|
+
|
|
6
|
+
export type RawAxNode = AxNodeResult;
|
|
7
|
+
type AxValue = NonNullable<RawAxNode["role"]>;
|
|
8
|
+
|
|
9
|
+
export type SlimNode = {
|
|
10
|
+
role: string;
|
|
11
|
+
name?: string;
|
|
12
|
+
value?: string;
|
|
13
|
+
description?: string;
|
|
14
|
+
state?: string;
|
|
15
|
+
children: SlimNode[];
|
|
16
|
+
_backendId?: number;
|
|
17
|
+
ref?: string;
|
|
18
|
+
box?: Box;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export const INTERACTIVE_ROLES = new Set([
|
|
22
|
+
"button",
|
|
23
|
+
"link",
|
|
24
|
+
"textbox",
|
|
25
|
+
"checkbox",
|
|
26
|
+
"radio",
|
|
27
|
+
"combobox",
|
|
28
|
+
"menuitem",
|
|
29
|
+
"tab",
|
|
30
|
+
"switch",
|
|
31
|
+
"slider",
|
|
32
|
+
"searchbox",
|
|
33
|
+
"spinbutton",
|
|
34
|
+
]);
|
|
35
|
+
|
|
36
|
+
// The AX `value` is stale for freshly-typed or controlled inputs, so for these roles the live DOM property is read instead.
|
|
37
|
+
export const VALUE_ROLES = new Set([
|
|
38
|
+
"textbox",
|
|
39
|
+
"searchbox",
|
|
40
|
+
"spinbutton",
|
|
41
|
+
"combobox",
|
|
42
|
+
"checkbox",
|
|
43
|
+
"radio",
|
|
44
|
+
"switch",
|
|
45
|
+
"slider",
|
|
46
|
+
]);
|
|
47
|
+
|
|
48
|
+
const stringOf = (v: AxValue | undefined): string | undefined => {
|
|
49
|
+
const x = v?.value;
|
|
50
|
+
if (typeof x === "string" && x !== "") return x;
|
|
51
|
+
if (typeof x === "number") return String(x);
|
|
52
|
+
return undefined;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const collectState = (props: RawAxNode["properties"]): string | undefined => {
|
|
56
|
+
if (!props || props.length === 0) return undefined;
|
|
57
|
+
const flags: string[] = [];
|
|
58
|
+
for (const p of props) {
|
|
59
|
+
const name = p.name;
|
|
60
|
+
if (!name) continue;
|
|
61
|
+
const raw = p.value?.value;
|
|
62
|
+
if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
|
|
63
|
+
flags.push(name);
|
|
64
|
+
}
|
|
65
|
+
if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
|
|
66
|
+
}
|
|
67
|
+
return flags.length > 0 ? flags.join(", ") : undefined;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
export const buildTree = (
|
|
71
|
+
rawNodes: ReadonlyArray<RawAxNode>,
|
|
72
|
+
opts: { interestingOnly: boolean; maxNodes: number },
|
|
73
|
+
): SlimNode[] => {
|
|
74
|
+
const byId = new Map<string, RawAxNode>();
|
|
75
|
+
for (const n of rawNodes) byId.set(n.nodeId, n);
|
|
76
|
+
|
|
77
|
+
const rootIds: string[] = [];
|
|
78
|
+
for (const n of rawNodes) {
|
|
79
|
+
if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
let budget = opts.maxNodes;
|
|
83
|
+
|
|
84
|
+
const slim = (node: RawAxNode): SlimNode | undefined => {
|
|
85
|
+
if (budget <= 0) return undefined;
|
|
86
|
+
if (opts.interestingOnly && node.ignored) {
|
|
87
|
+
// Chrome often ignores wrapper divs while their descendants remain meaningful, so recurse through an ignored node's children.
|
|
88
|
+
const out: SlimNode[] = [];
|
|
89
|
+
for (const cid of node.childIds ?? []) {
|
|
90
|
+
const child = byId.get(cid);
|
|
91
|
+
if (!child) continue;
|
|
92
|
+
const slimChild = slim(child);
|
|
93
|
+
if (slimChild) out.push(slimChild);
|
|
94
|
+
}
|
|
95
|
+
return out.length > 0 ? { role: "_hoist", children: out } : undefined;
|
|
96
|
+
}
|
|
97
|
+
budget--;
|
|
98
|
+
const role = stringOf(node.role) ?? "unknown";
|
|
99
|
+
const name = stringOf(node.name);
|
|
100
|
+
const value = stringOf(node.value);
|
|
101
|
+
const description = stringOf(node.description);
|
|
102
|
+
const state = collectState(node.properties);
|
|
103
|
+
const children: SlimNode[] = [];
|
|
104
|
+
for (const cid of node.childIds ?? []) {
|
|
105
|
+
const child = byId.get(cid);
|
|
106
|
+
if (!child) continue;
|
|
107
|
+
const slimChild = slim(child);
|
|
108
|
+
if (!slimChild) continue;
|
|
109
|
+
if (slimChild.role === "_hoist") children.push(...slimChild.children);
|
|
110
|
+
else children.push(slimChild);
|
|
111
|
+
}
|
|
112
|
+
const out: SlimNode = { role, children };
|
|
113
|
+
if (name !== undefined) out.name = name;
|
|
114
|
+
if (value !== undefined) out.value = value;
|
|
115
|
+
if (description !== undefined) out.description = description;
|
|
116
|
+
if (state !== undefined) out.state = state;
|
|
117
|
+
if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
|
|
118
|
+
return out;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const result: SlimNode[] = [];
|
|
122
|
+
for (const rid of rootIds) {
|
|
123
|
+
const root = byId.get(rid);
|
|
124
|
+
if (!root) continue;
|
|
125
|
+
const s = slim(root);
|
|
126
|
+
if (!s) continue;
|
|
127
|
+
if (s.role === "_hoist") result.push(...s.children);
|
|
128
|
+
else result.push(s);
|
|
129
|
+
}
|
|
130
|
+
return result;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
export const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
|
|
134
|
+
let n = 0;
|
|
135
|
+
for (const node of nodes) n += 1 + countNodes(node.children);
|
|
136
|
+
return n;
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
export const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
|
|
140
|
+
nodes.map((n) => {
|
|
141
|
+
const { _backendId: _bid, ...rest } = n;
|
|
142
|
+
return { ...rest, children: stripInternals(n.children) };
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
export const collectInteractiveTargets = (
|
|
146
|
+
nodes: ReadonlyArray<SlimNode>,
|
|
147
|
+
): Array<{ node: SlimNode; backendId: number }> => {
|
|
148
|
+
const out: Array<{ node: SlimNode; backendId: number }> = [];
|
|
149
|
+
const walk = (ns: ReadonlyArray<SlimNode>): void => {
|
|
150
|
+
for (const n of ns) {
|
|
151
|
+
if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
|
|
152
|
+
out.push({ node: n, backendId: n._backendId });
|
|
153
|
+
}
|
|
154
|
+
walk(n.children);
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
walk(nodes);
|
|
158
|
+
return out;
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
export const liveValue = async (client: BrowserClient, backendId: number): Promise<string | undefined> => {
|
|
162
|
+
const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId });
|
|
163
|
+
if (!resolved.success) return undefined;
|
|
164
|
+
const objectId = resolved.data.object.objectId;
|
|
165
|
+
if (!objectId) return undefined;
|
|
166
|
+
const fn = `function () {
|
|
167
|
+
if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
|
|
168
|
+
if (typeof this.value === "string") return this.value;
|
|
169
|
+
return null;
|
|
170
|
+
}`;
|
|
171
|
+
const r = await cdpCall(client, "Runtime.callFunctionOn", {
|
|
172
|
+
objectId,
|
|
173
|
+
functionDeclaration: fn,
|
|
174
|
+
returnByValue: true,
|
|
175
|
+
});
|
|
176
|
+
if (!r.success) return undefined;
|
|
177
|
+
const value = r.data.result.value;
|
|
178
|
+
return typeof value === "string" ? value : undefined;
|
|
179
|
+
};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { BrowserClient } from "../client";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import type { ToolErr } from "../util/tool";
|
|
4
|
+
import { cdpCall } from "./cdp-call";
|
|
5
|
+
|
|
6
|
+
export type Box = {
|
|
7
|
+
readonly x: number;
|
|
8
|
+
readonly y: number;
|
|
9
|
+
readonly width: number;
|
|
10
|
+
readonly height: number;
|
|
11
|
+
readonly cx: number;
|
|
12
|
+
readonly cy: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
type BoxModel = {
|
|
16
|
+
readonly content: ReadonlyArray<number>;
|
|
17
|
+
readonly width: number;
|
|
18
|
+
readonly height: number;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export const centreOf = (model: BoxModel): Box => {
|
|
22
|
+
const x = model.content[0] ?? 0;
|
|
23
|
+
const y = model.content[1] ?? 0;
|
|
24
|
+
return {
|
|
25
|
+
x: Math.round(x),
|
|
26
|
+
y: Math.round(y),
|
|
27
|
+
width: Math.round(model.width),
|
|
28
|
+
height: Math.round(model.height),
|
|
29
|
+
cx: Math.round(x + model.width / 2),
|
|
30
|
+
cy: Math.round(y + model.height / 2),
|
|
31
|
+
};
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export const boxOf = async (
|
|
35
|
+
client: BrowserClient,
|
|
36
|
+
backendNodeId: number,
|
|
37
|
+
): Promise<Result<Box, ToolErr>> => {
|
|
38
|
+
const r = await cdpCall(client, "DOM.getBoxModel", { backendNodeId });
|
|
39
|
+
if (!r.success) return r;
|
|
40
|
+
const model = r.data.model;
|
|
41
|
+
if (model.content.length < 8) {
|
|
42
|
+
return err({ kind: "invalid_state", message: "DOM.getBoxModel returned an incomplete content quad" });
|
|
43
|
+
}
|
|
44
|
+
if (model.width <= 0 || model.height <= 0) {
|
|
45
|
+
return err({
|
|
46
|
+
kind: "invalid_state",
|
|
47
|
+
message: "element has a zero-size box and cannot be clicked",
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return ok(centreOf(model));
|
|
51
|
+
};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { BrowserClient } from "../client";
|
|
2
|
+
import type { CdpMethod, ParamsOf, ResultOf } from "../cdp/commands";
|
|
3
|
+
import type { CdpError } from "../cdp/errors";
|
|
4
|
+
import { type Result, err } from "../util/result";
|
|
5
|
+
import type { ToolErr } from "../util/tool";
|
|
6
|
+
|
|
7
|
+
export const cdpErrToToolErr = (e: CdpError, method: string): ToolErr => ({
|
|
8
|
+
kind: e.kind === "timeout" ? "timeout" : "cdp_error",
|
|
9
|
+
message: e.message,
|
|
10
|
+
details: { method },
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
export const cdpCall = async <M extends CdpMethod>(
|
|
14
|
+
client: BrowserClient,
|
|
15
|
+
method: M,
|
|
16
|
+
params: ParamsOf<M>,
|
|
17
|
+
opts?: { timeoutMs?: number },
|
|
18
|
+
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
19
|
+
const r = await client.session().call(method, params, opts);
|
|
20
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
21
|
+
return r;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export const cdpCallOnTarget = async <M extends CdpMethod>(
|
|
25
|
+
client: BrowserClient,
|
|
26
|
+
method: M,
|
|
27
|
+
params: ParamsOf<M>,
|
|
28
|
+
sessionId: string,
|
|
29
|
+
opts?: { timeoutMs?: number },
|
|
30
|
+
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
31
|
+
const r = await client.session().callOnTarget(method, params, sessionId, opts);
|
|
32
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
33
|
+
return r;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export const cdpCallBrowser = async <M extends CdpMethod>(
|
|
37
|
+
client: BrowserClient,
|
|
38
|
+
method: M,
|
|
39
|
+
params?: ParamsOf<M>,
|
|
40
|
+
opts?: { timeoutMs?: number },
|
|
41
|
+
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
42
|
+
const r = await client.session().callBrowser(method, params, opts);
|
|
43
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
44
|
+
return r;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export const evalJs = async (
|
|
48
|
+
client: BrowserClient,
|
|
49
|
+
expression: string,
|
|
50
|
+
sessionId?: string,
|
|
51
|
+
): Promise<Result<unknown, ToolErr>> => {
|
|
52
|
+
const r = await client.evaluateJs(expression, sessionId);
|
|
53
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, "Runtime.evaluate"));
|
|
54
|
+
return r;
|
|
55
|
+
};
|
package/src/domains/click.ts
CHANGED
|
@@ -2,10 +2,13 @@ import { Type } from "typebox";
|
|
|
2
2
|
import { Text } from "@mariozechner/pi-tui";
|
|
3
3
|
import type { BrowserClient } from "../client";
|
|
4
4
|
import { Coords, MouseButton } from "../schemas/common";
|
|
5
|
-
import { type Result,
|
|
5
|
+
import { type Result, ok } from "../util/result";
|
|
6
|
+
import { debugClicksEnabled } from "../util/debug";
|
|
6
7
|
import { defineBrowserTool, type ToolErr } from "../util/tool";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
8
|
+
import { cdpCall } from "./cdp-call";
|
|
9
|
+
import { captureWithCrosshair } from "./screenshot-capture";
|
|
10
|
+
import { interactiveDiff } from "./ref-resolve";
|
|
11
|
+
import { resolveTarget } from "./target";
|
|
9
12
|
|
|
10
13
|
const ClickArgs = Type.Object({
|
|
11
14
|
ref: Type.Optional(
|
|
@@ -34,21 +37,17 @@ const dispatchClick = async (
|
|
|
34
37
|
button: "left" | "right" | "middle",
|
|
35
38
|
count: number,
|
|
36
39
|
): Promise<Result<void, ToolErr>> => {
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
type: "mouseMoved", x, y,
|
|
42
|
-
});
|
|
43
|
-
if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message });
|
|
44
|
-
const pressed = await client.session().call("Input.dispatchMouseEvent", {
|
|
40
|
+
// Some focus/hover handlers and React synthetic events only fire when a mousemove precedes the press, so without it a click lands without focusing.
|
|
41
|
+
const moved = await cdpCall(client, "Input.dispatchMouseEvent", { type: "mouseMoved", x, y });
|
|
42
|
+
if (!moved.success) return moved;
|
|
43
|
+
const pressed = await cdpCall(client, "Input.dispatchMouseEvent", {
|
|
45
44
|
type: "mousePressed", x, y, button, clickCount: count,
|
|
46
45
|
});
|
|
47
|
-
if (!pressed.success) return
|
|
48
|
-
const released = await client
|
|
46
|
+
if (!pressed.success) return pressed;
|
|
47
|
+
const released = await cdpCall(client, "Input.dispatchMouseEvent", {
|
|
49
48
|
type: "mouseReleased", x, y, button, clickCount: count,
|
|
50
49
|
});
|
|
51
|
-
if (!released.success) return
|
|
50
|
+
if (!released.success) return released;
|
|
52
51
|
return ok(undefined);
|
|
53
52
|
};
|
|
54
53
|
|
|
@@ -64,36 +63,24 @@ export const clickTool = defineBrowserTool({
|
|
|
64
63
|
"A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
|
|
65
64
|
"After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
|
|
66
65
|
"Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
|
|
67
|
-
"If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots
|
|
66
|
+
"If a click doesn't register, run pi with --browser-debug-clicks (or set BH_DEBUG_CLICKS=1) to get annotated screenshots. For React/Vue components ignoring clicks, try browser_dispatch_key.",
|
|
68
67
|
],
|
|
69
68
|
parameters: ClickArgs,
|
|
69
|
+
concurrency: "serialized",
|
|
70
70
|
renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0),
|
|
71
71
|
async handler(args, { client }) {
|
|
72
72
|
const button = args.button ?? "left";
|
|
73
73
|
const count = args.count ?? 1;
|
|
74
74
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
let y: number;
|
|
79
|
-
if (args.ref !== undefined) {
|
|
80
|
-
const box = await resolveRefToBox(client, args.ref);
|
|
81
|
-
if (!box.success) return box;
|
|
82
|
-
x = box.data.cx;
|
|
83
|
-
y = box.data.cy;
|
|
84
|
-
} else if (args.x !== undefined && args.y !== undefined) {
|
|
85
|
-
x = args.x;
|
|
86
|
-
y = args.y;
|
|
87
|
-
} else {
|
|
88
|
-
return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." });
|
|
89
|
-
}
|
|
75
|
+
const resolved = await resolveTarget(client, args);
|
|
76
|
+
if (!resolved.success) return resolved;
|
|
77
|
+
const { x, y, label: target } = resolved.data;
|
|
90
78
|
|
|
91
79
|
const clicked = await dispatchClick(client, x, y, button, count);
|
|
92
80
|
if (!clicked.success) return clicked;
|
|
93
81
|
|
|
94
|
-
const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`;
|
|
95
82
|
const diff = await interactiveDiff(client);
|
|
96
|
-
if (
|
|
83
|
+
if (debugClicksEnabled()) {
|
|
97
84
|
const debug = await captureWithCrosshair(client, { x, y });
|
|
98
85
|
if (debug.success) {
|
|
99
86
|
return ok({
|
package/src/domains/console.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { type Result, ok } from "../util/result";
|
|
|
5
5
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
6
6
|
import { applyTruncation } from "../util/truncate";
|
|
7
7
|
import type { ConsoleRecord, ConsoleLevel } from "../cdp/console-buffer";
|
|
8
|
+
import { asArrayOf, asBoolean, asNumber, asRecord, asString } from "../util/guards";
|
|
8
9
|
|
|
9
10
|
const ConsoleArgs = Type.Object({
|
|
10
11
|
levels: Type.Optional(
|
|
@@ -106,6 +107,50 @@ type ConsoleDetails = {
|
|
|
106
107
|
nextCursor?: number;
|
|
107
108
|
};
|
|
108
109
|
|
|
110
|
+
const CONSOLE_LEVELS: ReadonlyArray<ConsoleLevel> = ["log", "info", "warn", "error", "debug"];
|
|
111
|
+
|
|
112
|
+
const asConsoleLevel = (v: unknown): ConsoleLevel | undefined =>
|
|
113
|
+
CONSOLE_LEVELS.find((l) => l === v);
|
|
114
|
+
|
|
115
|
+
const asConsoleRecord = (v: unknown): ConsoleRecord | undefined => {
|
|
116
|
+
const o = asRecord(v);
|
|
117
|
+
if (o === undefined) return undefined;
|
|
118
|
+
const level = asConsoleLevel(o["level"]);
|
|
119
|
+
const text = asString(o["text"]);
|
|
120
|
+
if (level === undefined || text === undefined) return undefined;
|
|
121
|
+
const url = asString(o["url"]);
|
|
122
|
+
const lineNumber = asNumber(o["lineNumber"]);
|
|
123
|
+
const stackTrace = asString(o["stackTrace"]);
|
|
124
|
+
return {
|
|
125
|
+
seq: asNumber(o["seq"]) ?? 0,
|
|
126
|
+
timestampMs: asNumber(o["timestampMs"]) ?? 0,
|
|
127
|
+
level,
|
|
128
|
+
text,
|
|
129
|
+
source: o["source"] === "log-entry" ? "log-entry" : "console-api",
|
|
130
|
+
...(url !== undefined ? { url } : {}),
|
|
131
|
+
...(lineNumber !== undefined ? { lineNumber } : {}),
|
|
132
|
+
...(stackTrace !== undefined ? { stackTrace } : {}),
|
|
133
|
+
};
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
const asConsoleDetails = (v: unknown): ConsoleDetails | undefined => {
|
|
137
|
+
const o = asRecord(v);
|
|
138
|
+
if (o === undefined) return undefined;
|
|
139
|
+
const records = asArrayOf(o["records"], asConsoleRecord);
|
|
140
|
+
if (records === undefined) return undefined;
|
|
141
|
+
const fullOutputPath = asString(o["fullOutputPath"]);
|
|
142
|
+
const nextCursor = asNumber(o["nextCursor"]);
|
|
143
|
+
return {
|
|
144
|
+
total: asNumber(o["total"]) ?? records.length,
|
|
145
|
+
returned: asNumber(o["returned"]) ?? records.length,
|
|
146
|
+
bufferOverflowed: asBoolean(o["bufferOverflowed"]) ?? false,
|
|
147
|
+
truncated: asBoolean(o["truncated"]) ?? false,
|
|
148
|
+
records,
|
|
149
|
+
...(fullOutputPath !== undefined ? { fullOutputPath } : {}),
|
|
150
|
+
...(nextCursor !== undefined ? { nextCursor } : {}),
|
|
151
|
+
};
|
|
152
|
+
};
|
|
153
|
+
|
|
109
154
|
export const consoleTool = defineBrowserTool({
|
|
110
155
|
name: "browser_console",
|
|
111
156
|
label: "Browser Console",
|
|
@@ -119,6 +164,7 @@ export const consoleTool = defineBrowserTool({
|
|
|
119
164
|
"Don't use as a default observation tool — prefer browser_snapshot for page structure.",
|
|
120
165
|
],
|
|
121
166
|
parameters: ConsoleArgs,
|
|
167
|
+
concurrency: "parallel",
|
|
122
168
|
|
|
123
169
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
124
170
|
const session = client.session();
|
|
@@ -136,7 +182,7 @@ export const consoleTool = defineBrowserTool({
|
|
|
136
182
|
const text = renderConsoleMarkdown(drained.records, {
|
|
137
183
|
total: drained.total,
|
|
138
184
|
bufferOverflowed: drained.bufferOverflowed,
|
|
139
|
-
expanded: true,
|
|
185
|
+
expanded: true,
|
|
140
186
|
nextCursor,
|
|
141
187
|
});
|
|
142
188
|
const trunc = await applyTruncation(text, "console");
|
|
@@ -155,7 +201,7 @@ export const consoleTool = defineBrowserTool({
|
|
|
155
201
|
},
|
|
156
202
|
|
|
157
203
|
renderResult(result, expanded, theme) {
|
|
158
|
-
const details = result.details
|
|
204
|
+
const details = asConsoleDetails(result.details);
|
|
159
205
|
if (!details) return new Text(theme.fg("error", "console: no details"), 0, 0);
|
|
160
206
|
|
|
161
207
|
const md = renderConsoleMarkdown(details.records, {
|
package/src/domains/dialog.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
-
import { type Result,
|
|
2
|
+
import { type Result, ok } from "../util/result";
|
|
3
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
import { cdpCall } from "./cdp-call";
|
|
4
5
|
|
|
5
6
|
const HandleDialogArgs = Type.Object({
|
|
6
7
|
accept: Type.Boolean({ description: "true = accept, false = dismiss" }),
|
|
@@ -18,11 +19,13 @@ export const handleDialogTool = defineBrowserTool({
|
|
|
18
19
|
"For prompt() dialogs, supply promptText with the value to submit.",
|
|
19
20
|
],
|
|
20
21
|
parameters: HandleDialogArgs,
|
|
22
|
+
concurrency: "serialized",
|
|
21
23
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
const r = await cdpCall(client, "Page.handleJavaScriptDialog", {
|
|
25
|
+
accept: args.accept,
|
|
26
|
+
...(args.promptText !== undefined ? { promptText: args.promptText } : {}),
|
|
27
|
+
});
|
|
28
|
+
if (!r.success) return r;
|
|
26
29
|
return ok({
|
|
27
30
|
text: `Dialog ${args.accept ? "accepted" : "dismissed"}`,
|
|
28
31
|
details: { accept: args.accept },
|