pi-browser-harness 0.10.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +5 -9
  3. package/package.json +5 -2
  4. package/skills/pi-browser-harness/SKILL.md +1 -1
  5. package/src/cdp/commands.ts +256 -0
  6. package/src/cdp/console-buffer.ts +10 -85
  7. package/src/cdp/discovery.ts +28 -71
  8. package/src/cdp/errors.ts +4 -0
  9. package/src/cdp/events.ts +170 -0
  10. package/src/cdp/network-buffer.ts +42 -109
  11. package/src/cdp/ownership.ts +0 -17
  12. package/src/cdp/record-store.ts +61 -0
  13. package/src/cdp/session.ts +205 -193
  14. package/src/cdp/target-factory.ts +10 -80
  15. package/src/cdp/types.ts +21 -12
  16. package/src/client.ts +39 -95
  17. package/src/daemon/bridge.ts +150 -203
  18. package/src/daemon/index.ts +0 -28
  19. package/src/daemon/protocol.ts +6 -68
  20. package/src/daemon/queue.ts +85 -0
  21. package/src/daemon/server.ts +24 -99
  22. package/src/daemon/spawn.ts +18 -68
  23. package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +18 -51
  24. package/src/domains/ax-tree.ts +179 -0
  25. package/src/domains/box.ts +51 -0
  26. package/src/domains/cdp-call.ts +55 -0
  27. package/src/domains/click.ts +19 -32
  28. package/src/domains/console.ts +48 -2
  29. package/src/domains/dialog.ts +8 -5
  30. package/src/domains/drag.ts +8 -7
  31. package/src/domains/element-call.ts +117 -0
  32. package/src/domains/files.ts +22 -26
  33. package/src/domains/fill-engine.ts +80 -0
  34. package/src/domains/form-batch.ts +65 -0
  35. package/src/domains/form.ts +116 -105
  36. package/src/domains/history.ts +13 -9
  37. package/src/domains/isolated-tab.ts +26 -55
  38. package/src/domains/js.ts +29 -42
  39. package/src/domains/keyboard.ts +36 -27
  40. package/src/domains/navigate.ts +16 -23
  41. package/src/domains/network.ts +60 -10
  42. package/src/domains/page.ts +24 -3
  43. package/src/domains/readpage/read-page.ts +74 -19
  44. package/src/domains/readpage/readability.ts +0 -24
  45. package/src/domains/ref-resolve.ts +24 -58
  46. package/src/domains/render.ts +6 -27
  47. package/src/domains/screenshot-capture.ts +80 -0
  48. package/src/domains/screenshot.ts +9 -86
  49. package/src/domains/scroll.ts +10 -11
  50. package/src/domains/search/google-serp.ts +0 -29
  51. package/src/domains/search/web-search.ts +48 -20
  52. package/src/domains/setup.ts +1 -11
  53. package/src/domains/snapshot.ts +35 -276
  54. package/src/domains/tabs.ts +14 -7
  55. package/src/domains/target.ts +35 -0
  56. package/src/domains/viewport.ts +5 -3
  57. package/src/index.ts +8 -58
  58. package/src/profile/bind.ts +8 -57
  59. package/src/profile/browser-process.ts +75 -126
  60. package/src/profile/command.ts +50 -29
  61. package/src/profile/launch.ts +22 -40
  62. package/src/profile/list.ts +14 -54
  63. package/src/profile/paths.ts +2 -65
  64. package/src/profile/store.ts +35 -56
  65. package/src/registry.ts +30 -33
  66. package/src/schemas/parse.ts +23 -0
  67. package/src/setup.ts +0 -25
  68. package/src/state.ts +16 -16
  69. package/src/util/debug.ts +8 -0
  70. package/src/util/guards.ts +30 -0
  71. package/src/util/js-template.ts +1 -16
  72. package/src/util/mutex.ts +0 -5
  73. package/src/util/result.ts +0 -6
  74. package/src/util/sharp-shim.ts +12 -19
  75. package/src/util/tool.ts +17 -42
  76. package/src/util/truncate.ts +6 -0
  77. package/tsconfig.json +1 -16
  78. package/skills/deep-research/SKILL.md +0 -66
  79. package/src/cdp/event-queue.ts +0 -128
  80. package/src/cdp/transport.ts +0 -156
  81. package/src/cdp/window-binding.ts +0 -15
  82. package/src/deep-research.ts +0 -41
  83. package/src/domains/forms.ts +0 -355
  84. package/src/domains/readpage/capture.ts +0 -46
  85. package/src/domains/search/extract.ts +0 -44
  86. package/src/profile/picker.ts +0 -73
  87. package/src/util/keycodes.ts +0 -10
  88. package/src/util/time.ts +0 -29
@@ -1,32 +1,17 @@
1
- /**
2
- * DaemonTransport — CdpTransport backed by a Unix socket to the browser daemon.
3
- *
4
- * Satisfies the exact CdpTransport interface so BrowserClient, CdpSession, and
5
- * all domain tools work without changes. Internally uses newline-delimited JSON
6
- * over a Unix domain socket instead of a WebSocket to Chrome.
7
- *
8
- * References:
9
- * - CdpTransport interface: src/cdp/transport.ts
10
- * - net.createConnection IPC: https://nodejs.org/docs/latest-v22.x/api/net.html#netcreateconnection
11
- * - readline.createInterface: https://nodejs.org/docs/latest-v22.x/api/readline.html#readlinecreateinterfaceoptions
12
- */
13
-
14
1
  import { createConnection, type Socket } from "node:net";
15
2
  import { createInterface } from "node:readline";
16
3
  import { type Result, err, ok } from "../util/result";
17
- import { type CdpError, cdpError } from "../cdp/errors";
18
- import type { CdpTransport } from "../cdp/transport";
4
+ import { type CdpError, cdpError, classifyRemoteError } from "../cdp/errors";
5
+ import { DEFAULT_TIMEOUT_MS, type CdpTransport } from "../cdp/types";
19
6
  import type { CdpEvent } from "../cdp/types";
20
- import { type Pending, makeEventQueue, makeOnClose, rejectAllPending, sendWithTimeout } from "./event-queue";
7
+ import { type Pending, makeEventQueue, makeOnClose, rejectAllPending, sendWithTimeout } from "./queue";
21
8
  import {
22
9
  DAEMON_SOCKET_PATH,
23
10
  type WireRequest,
24
11
  type WireControl,
25
12
  deserialize,
26
13
  serialize,
27
- } from "../daemon/protocol";
28
-
29
- // ── Implementation ─────────────────────────────────────────────────────────────
14
+ } from "./protocol";
30
15
 
31
16
  export const createDaemonTransport = (clientId: string): CdpTransport => {
32
17
  let socket: Socket | null = null;
@@ -49,10 +34,7 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
49
34
  for (const cb of closeListeners) cb();
50
35
  };
51
36
 
52
- // ── Connect ────────────────────────────────────────────────────────────
53
-
54
37
  const connect = (_url: string, opts?: { timeoutMs?: number }): Promise<Result<void, CdpError>> => {
55
- // url is ignored — we always connect to the daemon socket
56
38
  const timeoutMs = opts?.timeoutMs ?? 10_000;
57
39
 
58
40
  if (socket && !socket.destroyed && registered) {
@@ -67,21 +49,23 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
67
49
  resolve(r);
68
50
  };
69
51
 
52
+ let sock: Socket;
70
53
  try {
71
- socket = createConnection(DAEMON_SOCKET_PATH);
54
+ sock = createConnection(DAEMON_SOCKET_PATH);
72
55
  } catch (e) {
73
56
  settle(err(cdpError("transport_closed", e instanceof Error ? e.message : String(e))));
74
57
  return;
75
58
  }
59
+ socket = sock;
76
60
 
77
61
  const connectTimer = setTimeout(() => {
78
62
  cleanup("Connection timeout");
79
63
  settle(err(cdpError("timeout", `Daemon connection timed out after ${timeoutMs}ms`)));
80
64
  }, timeoutMs);
81
65
 
82
- socket.on("connect", () => {
66
+ sock.on("connect", () => {
83
67
  clearTimeout(connectTimer);
84
- rl = createInterface({ input: socket!, crlfDelay: Infinity });
68
+ rl = createInterface({ input: sock, crlfDelay: Infinity });
85
69
 
86
70
  rl.on("line", (line: string) => {
87
71
  const msg = deserialize(line);
@@ -107,13 +91,7 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
107
91
  clearTimeout(p.timer);
108
92
 
109
93
  if (msg.error) {
110
- p.resolve(err(cdpError(
111
- msg.error.message.includes("Session with given id not found")
112
- ? "session_not_found"
113
- : "remote_error",
114
- msg.error.message,
115
- p.method,
116
- )));
94
+ p.resolve(err(cdpError(classifyRemoteError(msg.error.message), msg.error.message, p.method)));
117
95
  } else {
118
96
  p.resolve(ok(msg.result));
119
97
  }
@@ -130,23 +108,19 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
130
108
  }
131
109
  });
132
110
 
133
- socket!.on("error", () => {
134
- // Errors are surfaced via 'close'
135
- });
111
+ sock.on("error", () => {});
136
112
 
137
- socket!.on("close", () => {
113
+ sock.on("close", () => {
138
114
  clearTimeout(connectTimer);
139
115
  cleanup("Daemon socket closed");
140
- // If connect hasn't settled yet, settle with error
141
116
  settle(err(cdpError("transport_closed", "Daemon socket closed before registration")));
142
117
  });
143
118
 
144
- // Send registration
145
119
  const regMsg: WireControl = { type: "control", action: "register", clientId };
146
- socket!.write(serialize(regMsg) + "\n");
120
+ sock.write(serialize(regMsg) + "\n");
147
121
  });
148
122
 
149
- socket.on("error", (e: NodeJS.ErrnoException) => {
123
+ sock.on("error", (e: NodeJS.ErrnoException) => {
150
124
  clearTimeout(connectTimer);
151
125
  const msg = e.code === "ENOENT"
152
126
  ? "Daemon not running — socket not found"
@@ -156,8 +130,6 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
156
130
  });
157
131
  };
158
132
 
159
- // ── Close ──────────────────────────────────────────────────────────────
160
-
161
133
  const close = async (): Promise<void> => {
162
134
  if (registered && socket && !socket.destroyed) {
163
135
  const dereg: WireControl = { type: "control", action: "deregister", clientId };
@@ -166,17 +138,16 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
166
138
  cleanup("close() called");
167
139
  };
168
140
 
169
- // ── Request ────────────────────────────────────────────────────────────
170
-
171
141
  const request = (
172
142
  method: string,
173
143
  params: Record<string, unknown>,
174
144
  opts?: { sessionId?: string | null; timeoutMs?: number },
175
145
  ): Promise<Result<unknown, CdpError>> => {
176
- const timeoutMs = opts?.timeoutMs ?? 15_000;
146
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
177
147
  const sessionId = opts?.sessionId ?? null;
178
148
 
179
- if (!socket || socket.destroyed || !registered) {
149
+ const sock = socket;
150
+ if (sock === null || sock.destroyed || !registered) {
180
151
  return Promise.resolve(err(cdpError("transport_closed", "Daemon not connected", method)));
181
152
  }
182
153
 
@@ -189,15 +160,11 @@ export const createDaemonTransport = (clientId: string): CdpTransport => {
189
160
  ...(sessionId !== null && sessionId !== undefined ? { sessionId } : {}),
190
161
  };
191
162
 
192
- return sendWithTimeout(pending, id, method, timeoutMs, "Daemon", () => socket!.write(serialize(req) + "\n"));
163
+ return sendWithTimeout(pending, id, method, timeoutMs, "Daemon", () => sock.write(serialize(req) + "\n"));
193
164
  };
194
165
 
195
- // ── Events ─────────────────────────────────────────────────────────────
196
-
197
166
  const events = (): AsyncIterable<CdpEvent> => queue.iter;
198
167
 
199
- // ── State ──────────────────────────────────────────────────────────────
200
-
201
168
  const state = (): "open" | "closed" | "connecting" => {
202
169
  if (!socket) return "closed";
203
170
  if (!registered) return "connecting";
@@ -0,0 +1,179 @@
1
+ import type { AxNodeResult } from "../cdp/commands";
2
+ import type { BrowserClient } from "../client";
3
+ import type { Box } from "./box";
4
+ import { cdpCall } from "./cdp-call";
5
+
6
+ export type RawAxNode = AxNodeResult;
7
+ type AxValue = NonNullable<RawAxNode["role"]>;
8
+
9
+ export type SlimNode = {
10
+ role: string;
11
+ name?: string;
12
+ value?: string;
13
+ description?: string;
14
+ state?: string;
15
+ children: SlimNode[];
16
+ _backendId?: number;
17
+ ref?: string;
18
+ box?: Box;
19
+ };
20
+
21
+ export const INTERACTIVE_ROLES = new Set([
22
+ "button",
23
+ "link",
24
+ "textbox",
25
+ "checkbox",
26
+ "radio",
27
+ "combobox",
28
+ "menuitem",
29
+ "tab",
30
+ "switch",
31
+ "slider",
32
+ "searchbox",
33
+ "spinbutton",
34
+ ]);
35
+
36
+ // The AX `value` is stale for freshly-typed or controlled inputs, so for these roles the live DOM property is read instead.
37
+ export const VALUE_ROLES = new Set([
38
+ "textbox",
39
+ "searchbox",
40
+ "spinbutton",
41
+ "combobox",
42
+ "checkbox",
43
+ "radio",
44
+ "switch",
45
+ "slider",
46
+ ]);
47
+
48
+ const stringOf = (v: AxValue | undefined): string | undefined => {
49
+ const x = v?.value;
50
+ if (typeof x === "string" && x !== "") return x;
51
+ if (typeof x === "number") return String(x);
52
+ return undefined;
53
+ };
54
+
55
+ const collectState = (props: RawAxNode["properties"]): string | undefined => {
56
+ if (!props || props.length === 0) return undefined;
57
+ const flags: string[] = [];
58
+ for (const p of props) {
59
+ const name = p.name;
60
+ if (!name) continue;
61
+ const raw = p.value?.value;
62
+ if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
63
+ flags.push(name);
64
+ }
65
+ if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
66
+ }
67
+ return flags.length > 0 ? flags.join(", ") : undefined;
68
+ };
69
+
70
+ export const buildTree = (
71
+ rawNodes: ReadonlyArray<RawAxNode>,
72
+ opts: { interestingOnly: boolean; maxNodes: number },
73
+ ): SlimNode[] => {
74
+ const byId = new Map<string, RawAxNode>();
75
+ for (const n of rawNodes) byId.set(n.nodeId, n);
76
+
77
+ const rootIds: string[] = [];
78
+ for (const n of rawNodes) {
79
+ if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
80
+ }
81
+
82
+ let budget = opts.maxNodes;
83
+
84
+ const slim = (node: RawAxNode): SlimNode | undefined => {
85
+ if (budget <= 0) return undefined;
86
+ if (opts.interestingOnly && node.ignored) {
87
+ // Chrome often ignores wrapper divs while their descendants remain meaningful, so recurse through an ignored node's children.
88
+ const out: SlimNode[] = [];
89
+ for (const cid of node.childIds ?? []) {
90
+ const child = byId.get(cid);
91
+ if (!child) continue;
92
+ const slimChild = slim(child);
93
+ if (slimChild) out.push(slimChild);
94
+ }
95
+ return out.length > 0 ? { role: "_hoist", children: out } : undefined;
96
+ }
97
+ budget--;
98
+ const role = stringOf(node.role) ?? "unknown";
99
+ const name = stringOf(node.name);
100
+ const value = stringOf(node.value);
101
+ const description = stringOf(node.description);
102
+ const state = collectState(node.properties);
103
+ const children: SlimNode[] = [];
104
+ for (const cid of node.childIds ?? []) {
105
+ const child = byId.get(cid);
106
+ if (!child) continue;
107
+ const slimChild = slim(child);
108
+ if (!slimChild) continue;
109
+ if (slimChild.role === "_hoist") children.push(...slimChild.children);
110
+ else children.push(slimChild);
111
+ }
112
+ const out: SlimNode = { role, children };
113
+ if (name !== undefined) out.name = name;
114
+ if (value !== undefined) out.value = value;
115
+ if (description !== undefined) out.description = description;
116
+ if (state !== undefined) out.state = state;
117
+ if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
118
+ return out;
119
+ };
120
+
121
+ const result: SlimNode[] = [];
122
+ for (const rid of rootIds) {
123
+ const root = byId.get(rid);
124
+ if (!root) continue;
125
+ const s = slim(root);
126
+ if (!s) continue;
127
+ if (s.role === "_hoist") result.push(...s.children);
128
+ else result.push(s);
129
+ }
130
+ return result;
131
+ };
132
+
133
+ export const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
134
+ let n = 0;
135
+ for (const node of nodes) n += 1 + countNodes(node.children);
136
+ return n;
137
+ };
138
+
139
+ export const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
140
+ nodes.map((n) => {
141
+ const { _backendId: _bid, ...rest } = n;
142
+ return { ...rest, children: stripInternals(n.children) };
143
+ });
144
+
145
+ export const collectInteractiveTargets = (
146
+ nodes: ReadonlyArray<SlimNode>,
147
+ ): Array<{ node: SlimNode; backendId: number }> => {
148
+ const out: Array<{ node: SlimNode; backendId: number }> = [];
149
+ const walk = (ns: ReadonlyArray<SlimNode>): void => {
150
+ for (const n of ns) {
151
+ if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
152
+ out.push({ node: n, backendId: n._backendId });
153
+ }
154
+ walk(n.children);
155
+ }
156
+ };
157
+ walk(nodes);
158
+ return out;
159
+ };
160
+
161
+ export const liveValue = async (client: BrowserClient, backendId: number): Promise<string | undefined> => {
162
+ const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId });
163
+ if (!resolved.success) return undefined;
164
+ const objectId = resolved.data.object.objectId;
165
+ if (!objectId) return undefined;
166
+ const fn = `function () {
167
+ if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
168
+ if (typeof this.value === "string") return this.value;
169
+ return null;
170
+ }`;
171
+ const r = await cdpCall(client, "Runtime.callFunctionOn", {
172
+ objectId,
173
+ functionDeclaration: fn,
174
+ returnByValue: true,
175
+ });
176
+ if (!r.success) return undefined;
177
+ const value = r.data.result.value;
178
+ return typeof value === "string" ? value : undefined;
179
+ };
@@ -0,0 +1,51 @@
1
+ import type { BrowserClient } from "../client";
2
+ import { type Result, err, ok } from "../util/result";
3
+ import type { ToolErr } from "../util/tool";
4
+ import { cdpCall } from "./cdp-call";
5
+
6
+ export type Box = {
7
+ readonly x: number;
8
+ readonly y: number;
9
+ readonly width: number;
10
+ readonly height: number;
11
+ readonly cx: number;
12
+ readonly cy: number;
13
+ };
14
+
15
+ type BoxModel = {
16
+ readonly content: ReadonlyArray<number>;
17
+ readonly width: number;
18
+ readonly height: number;
19
+ };
20
+
21
+ export const centreOf = (model: BoxModel): Box => {
22
+ const x = model.content[0] ?? 0;
23
+ const y = model.content[1] ?? 0;
24
+ return {
25
+ x: Math.round(x),
26
+ y: Math.round(y),
27
+ width: Math.round(model.width),
28
+ height: Math.round(model.height),
29
+ cx: Math.round(x + model.width / 2),
30
+ cy: Math.round(y + model.height / 2),
31
+ };
32
+ };
33
+
34
+ export const boxOf = async (
35
+ client: BrowserClient,
36
+ backendNodeId: number,
37
+ ): Promise<Result<Box, ToolErr>> => {
38
+ const r = await cdpCall(client, "DOM.getBoxModel", { backendNodeId });
39
+ if (!r.success) return r;
40
+ const model = r.data.model;
41
+ if (model.content.length < 8) {
42
+ return err({ kind: "invalid_state", message: "DOM.getBoxModel returned an incomplete content quad" });
43
+ }
44
+ if (model.width <= 0 || model.height <= 0) {
45
+ return err({
46
+ kind: "invalid_state",
47
+ message: "element has a zero-size box and cannot be clicked",
48
+ });
49
+ }
50
+ return ok(centreOf(model));
51
+ };
@@ -0,0 +1,55 @@
1
+ import type { BrowserClient } from "../client";
2
+ import type { CdpMethod, ParamsOf, ResultOf } from "../cdp/commands";
3
+ import type { CdpError } from "../cdp/errors";
4
+ import { type Result, err } from "../util/result";
5
+ import type { ToolErr } from "../util/tool";
6
+
7
+ export const cdpErrToToolErr = (e: CdpError, method: string): ToolErr => ({
8
+ kind: e.kind === "timeout" ? "timeout" : "cdp_error",
9
+ message: e.message,
10
+ details: { method },
11
+ });
12
+
13
+ export const cdpCall = async <M extends CdpMethod>(
14
+ client: BrowserClient,
15
+ method: M,
16
+ params: ParamsOf<M>,
17
+ opts?: { timeoutMs?: number },
18
+ ): Promise<Result<ResultOf<M>, ToolErr>> => {
19
+ const r = await client.session().call(method, params, opts);
20
+ if (!r.success) return err(cdpErrToToolErr(r.error, method));
21
+ return r;
22
+ };
23
+
24
+ export const cdpCallOnTarget = async <M extends CdpMethod>(
25
+ client: BrowserClient,
26
+ method: M,
27
+ params: ParamsOf<M>,
28
+ sessionId: string,
29
+ opts?: { timeoutMs?: number },
30
+ ): Promise<Result<ResultOf<M>, ToolErr>> => {
31
+ const r = await client.session().callOnTarget(method, params, sessionId, opts);
32
+ if (!r.success) return err(cdpErrToToolErr(r.error, method));
33
+ return r;
34
+ };
35
+
36
+ export const cdpCallBrowser = async <M extends CdpMethod>(
37
+ client: BrowserClient,
38
+ method: M,
39
+ params?: ParamsOf<M>,
40
+ opts?: { timeoutMs?: number },
41
+ ): Promise<Result<ResultOf<M>, ToolErr>> => {
42
+ const r = await client.session().callBrowser(method, params, opts);
43
+ if (!r.success) return err(cdpErrToToolErr(r.error, method));
44
+ return r;
45
+ };
46
+
47
+ export const evalJs = async (
48
+ client: BrowserClient,
49
+ expression: string,
50
+ sessionId?: string,
51
+ ): Promise<Result<unknown, ToolErr>> => {
52
+ const r = await client.evaluateJs(expression, sessionId);
53
+ if (!r.success) return err(cdpErrToToolErr(r.error, "Runtime.evaluate"));
54
+ return r;
55
+ };
@@ -2,10 +2,13 @@ import { Type } from "typebox";
2
2
  import { Text } from "@mariozechner/pi-tui";
3
3
  import type { BrowserClient } from "../client";
4
4
  import { Coords, MouseButton } from "../schemas/common";
5
- import { type Result, err, ok } from "../util/result";
5
+ import { type Result, ok } from "../util/result";
6
+ import { debugClicksEnabled } from "../util/debug";
6
7
  import { defineBrowserTool, type ToolErr } from "../util/tool";
7
- import { captureWithCrosshair } from "./screenshot";
8
- import { interactiveDiff, resolveRefToBox } from "./ref-resolve";
8
+ import { cdpCall } from "./cdp-call";
9
+ import { captureWithCrosshair } from "./screenshot-capture";
10
+ import { interactiveDiff } from "./ref-resolve";
11
+ import { resolveTarget } from "./target";
9
12
 
10
13
  const ClickArgs = Type.Object({
11
14
  ref: Type.Optional(
@@ -34,21 +37,17 @@ const dispatchClick = async (
34
37
  button: "left" | "right" | "middle",
35
38
  count: number,
36
39
  ): Promise<Result<void, ToolErr>> => {
37
- // Move the pointer to the target first. Some focus/hover handlers (and React
38
- // synthetic events) only fire when a mousemove precedes the press, so without
39
- // it a click can land without focusing the element.
40
- const moved = await client.session().call("Input.dispatchMouseEvent", {
41
- type: "mouseMoved", x, y,
42
- });
43
- if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message });
44
- const pressed = await client.session().call("Input.dispatchMouseEvent", {
40
+ // Some focus/hover handlers and React synthetic events only fire when a mousemove precedes the press, so without it a click lands without focusing.
41
+ const moved = await cdpCall(client, "Input.dispatchMouseEvent", { type: "mouseMoved", x, y });
42
+ if (!moved.success) return moved;
43
+ const pressed = await cdpCall(client, "Input.dispatchMouseEvent", {
45
44
  type: "mousePressed", x, y, button, clickCount: count,
46
45
  });
47
- if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
48
- const released = await client.session().call("Input.dispatchMouseEvent", {
46
+ if (!pressed.success) return pressed;
47
+ const released = await cdpCall(client, "Input.dispatchMouseEvent", {
49
48
  type: "mouseReleased", x, y, button, clickCount: count,
50
49
  });
51
- if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
50
+ if (!released.success) return released;
52
51
  return ok(undefined);
53
52
  };
54
53
 
@@ -64,36 +63,24 @@ export const clickTool = defineBrowserTool({
64
63
  "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
65
64
  "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
66
65
  "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
67
- "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.",
66
+ "If a click doesn't register, run pi with --browser-debug-clicks (or set BH_DEBUG_CLICKS=1) to get annotated screenshots. For React/Vue components ignoring clicks, try browser_dispatch_key.",
68
67
  ],
69
68
  parameters: ClickArgs,
69
+ concurrency: "serialized",
70
70
  renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0),
71
71
  async handler(args, { client }) {
72
72
  const button = args.button ?? "left";
73
73
  const count = args.count ?? 1;
74
74
 
75
- // Resolve the click point: ref re-resolves to the element's CURRENT box
76
- // (so re-renders don't matter); otherwise use the literal x/y.
77
- let x: number;
78
- let y: number;
79
- if (args.ref !== undefined) {
80
- const box = await resolveRefToBox(client, args.ref);
81
- if (!box.success) return box;
82
- x = box.data.cx;
83
- y = box.data.cy;
84
- } else if (args.x !== undefined && args.y !== undefined) {
85
- x = args.x;
86
- y = args.y;
87
- } else {
88
- return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." });
89
- }
75
+ const resolved = await resolveTarget(client, args);
76
+ if (!resolved.success) return resolved;
77
+ const { x, y, label: target } = resolved.data;
90
78
 
91
79
  const clicked = await dispatchClick(client, x, y, button, count);
92
80
  if (!clicked.success) return clicked;
93
81
 
94
- const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`;
95
82
  const diff = await interactiveDiff(client);
96
- if (process.env["BH_DEBUG_CLICKS"]) {
83
+ if (debugClicksEnabled()) {
97
84
  const debug = await captureWithCrosshair(client, { x, y });
98
85
  if (debug.success) {
99
86
  return ok({
@@ -5,6 +5,7 @@ import { type Result, ok } from "../util/result";
5
5
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
6
6
  import { applyTruncation } from "../util/truncate";
7
7
  import type { ConsoleRecord, ConsoleLevel } from "../cdp/console-buffer";
8
+ import { asArrayOf, asBoolean, asNumber, asRecord, asString } from "../util/guards";
8
9
 
9
10
  const ConsoleArgs = Type.Object({
10
11
  levels: Type.Optional(
@@ -106,6 +107,50 @@ type ConsoleDetails = {
106
107
  nextCursor?: number;
107
108
  };
108
109
 
110
+ const CONSOLE_LEVELS: ReadonlyArray<ConsoleLevel> = ["log", "info", "warn", "error", "debug"];
111
+
112
+ const asConsoleLevel = (v: unknown): ConsoleLevel | undefined =>
113
+ CONSOLE_LEVELS.find((l) => l === v);
114
+
115
+ const asConsoleRecord = (v: unknown): ConsoleRecord | undefined => {
116
+ const o = asRecord(v);
117
+ if (o === undefined) return undefined;
118
+ const level = asConsoleLevel(o["level"]);
119
+ const text = asString(o["text"]);
120
+ if (level === undefined || text === undefined) return undefined;
121
+ const url = asString(o["url"]);
122
+ const lineNumber = asNumber(o["lineNumber"]);
123
+ const stackTrace = asString(o["stackTrace"]);
124
+ return {
125
+ seq: asNumber(o["seq"]) ?? 0,
126
+ timestampMs: asNumber(o["timestampMs"]) ?? 0,
127
+ level,
128
+ text,
129
+ source: o["source"] === "log-entry" ? "log-entry" : "console-api",
130
+ ...(url !== undefined ? { url } : {}),
131
+ ...(lineNumber !== undefined ? { lineNumber } : {}),
132
+ ...(stackTrace !== undefined ? { stackTrace } : {}),
133
+ };
134
+ };
135
+
136
+ const asConsoleDetails = (v: unknown): ConsoleDetails | undefined => {
137
+ const o = asRecord(v);
138
+ if (o === undefined) return undefined;
139
+ const records = asArrayOf(o["records"], asConsoleRecord);
140
+ if (records === undefined) return undefined;
141
+ const fullOutputPath = asString(o["fullOutputPath"]);
142
+ const nextCursor = asNumber(o["nextCursor"]);
143
+ return {
144
+ total: asNumber(o["total"]) ?? records.length,
145
+ returned: asNumber(o["returned"]) ?? records.length,
146
+ bufferOverflowed: asBoolean(o["bufferOverflowed"]) ?? false,
147
+ truncated: asBoolean(o["truncated"]) ?? false,
148
+ records,
149
+ ...(fullOutputPath !== undefined ? { fullOutputPath } : {}),
150
+ ...(nextCursor !== undefined ? { nextCursor } : {}),
151
+ };
152
+ };
153
+
109
154
  export const consoleTool = defineBrowserTool({
110
155
  name: "browser_console",
111
156
  label: "Browser Console",
@@ -119,6 +164,7 @@ export const consoleTool = defineBrowserTool({
119
164
  "Don't use as a default observation tool — prefer browser_snapshot for page structure.",
120
165
  ],
121
166
  parameters: ConsoleArgs,
167
+ concurrency: "parallel",
122
168
 
123
169
  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
124
170
  const session = client.session();
@@ -136,7 +182,7 @@ export const consoleTool = defineBrowserTool({
136
182
  const text = renderConsoleMarkdown(drained.records, {
137
183
  total: drained.total,
138
184
  bufferOverflowed: drained.bufferOverflowed,
139
- expanded: true, // full text always returned to the LLM; renderResult handles compact view for humans
185
+ expanded: true,
140
186
  nextCursor,
141
187
  });
142
188
  const trunc = await applyTruncation(text, "console");
@@ -155,7 +201,7 @@ export const consoleTool = defineBrowserTool({
155
201
  },
156
202
 
157
203
  renderResult(result, expanded, theme) {
158
- const details = result.details as ConsoleDetails | undefined;
204
+ const details = asConsoleDetails(result.details);
159
205
  if (!details) return new Text(theme.fg("error", "console: no details"), 0, 0);
160
206
 
161
207
  const md = renderConsoleMarkdown(details.records, {
@@ -1,6 +1,7 @@
1
1
  import { Type } from "typebox";
2
- import { type Result, err, ok } from "../util/result";
2
+ import { type Result, ok } from "../util/result";
3
3
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
4
+ import { cdpCall } from "./cdp-call";
4
5
 
5
6
  const HandleDialogArgs = Type.Object({
6
7
  accept: Type.Boolean({ description: "true = accept, false = dismiss" }),
@@ -18,11 +19,13 @@ export const handleDialogTool = defineBrowserTool({
18
19
  "For prompt() dialogs, supply promptText with the value to submit.",
19
20
  ],
20
21
  parameters: HandleDialogArgs,
22
+ concurrency: "serialized",
21
23
  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
22
- const params: Record<string, unknown> = { accept: args.accept };
23
- if (args.promptText !== undefined) params["promptText"] = args.promptText;
24
- const r = await client.session().call("Page.handleJavaScriptDialog", params);
25
- if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
24
+ const r = await cdpCall(client, "Page.handleJavaScriptDialog", {
25
+ accept: args.accept,
26
+ ...(args.promptText !== undefined ? { promptText: args.promptText } : {}),
27
+ });
28
+ if (!r.success) return r;
26
29
  return ok({
27
30
  text: `Dialog ${args.accept ? "accepted" : "dismissed"}`,
28
31
  details: { accept: args.accept },