pi-browser-harness 0.2.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +84 -0
  2. package/package.json +4 -1
  3. package/skills/pi-browser-harness/SKILL.md +64 -6
  4. package/src/cdp/discovery.ts +95 -0
  5. package/src/cdp/errors.ts +19 -0
  6. package/src/cdp/session.ts +139 -0
  7. package/src/cdp/transport.ts +221 -0
  8. package/src/cdp/types.ts +72 -0
  9. package/src/client.ts +238 -0
  10. package/src/domains/click.ts +73 -0
  11. package/src/domains/dialog.ts +31 -0
  12. package/src/domains/drag.ts +66 -0
  13. package/src/domains/files.ts +194 -0
  14. package/src/domains/history.ts +76 -0
  15. package/src/domains/js.ts +197 -0
  16. package/src/domains/keyboard.ts +127 -0
  17. package/src/domains/navigate.ts +146 -0
  18. package/src/domains/network.ts +80 -0
  19. package/src/domains/page.ts +88 -0
  20. package/src/domains/screenshot.ts +150 -0
  21. package/src/domains/scroll.ts +60 -0
  22. package/src/domains/tabs.ts +121 -0
  23. package/src/domains/viewport.ts +40 -0
  24. package/src/index.ts +65 -142
  25. package/src/prompt.ts +27 -4
  26. package/src/registry.ts +55 -0
  27. package/src/schemas/common.ts +11 -0
  28. package/src/setup.ts +43 -26
  29. package/src/state.ts +22 -54
  30. package/src/util/js-template.ts +27 -0
  31. package/src/util/keycodes.ts +10 -0
  32. package/src/util/mutex.ts +25 -0
  33. package/src/util/paths.ts +21 -0
  34. package/src/util/result.ts +15 -0
  35. package/src/util/sharp-shim.ts +49 -0
  36. package/src/util/time.ts +29 -0
  37. package/src/util/tool.ts +193 -0
  38. package/src/util/truncate.ts +35 -0
  39. package/tsconfig.json +10 -1
  40. package/src/daemon.ts +0 -881
  41. package/src/protocol.ts +0 -77
  42. package/src/renderers.ts +0 -21
  43. package/src/tools.ts +0 -1363
@@ -0,0 +1,221 @@
1
+ import WebSocket from "ws";
2
+ import { type Result, err, ok } from "../util/result";
3
+ import { type CdpError, cdpError } from "./errors";
4
+ import { isCdpRawMessage } from "./types";
5
+ import type { CdpEvent, CdpRawMessage } from "./types";
6
+
7
+ const DEFAULT_TIMEOUT_MS = 15_000;
8
+
9
+ type Pending = {
10
+ readonly resolve: (v: Result<unknown, CdpError>) => void;
11
+ readonly timer: ReturnType<typeof setTimeout>;
12
+ readonly method: string;
13
+ };
14
+
15
+ export type CdpTransport = {
16
+ connect(url: string, opts?: { timeoutMs?: number }): Promise<Result<void, CdpError>>;
17
+ close(): Promise<void>;
18
+ request(
19
+ method: string,
20
+ params: Record<string, unknown>,
21
+ opts?: { sessionId?: string | null; timeoutMs?: number },
22
+ ): Promise<Result<unknown, CdpError>>;
23
+ // Returns the current event queue's iterator. After close()+reconnect, callers
24
+ // must re-call events() — the old iterator will have received done:true.
25
+ events(): AsyncIterable<CdpEvent>;
26
+ state(): "open" | "closed" | "connecting";
27
+ onClose(cb: () => void): () => void;
28
+ };
29
+
30
+ export const createCdpTransport = (): CdpTransport => {
31
+ let ws: WebSocket | null = null;
32
+ let nextId = 1;
33
+ const pending = new Map<number, Pending>();
34
+ const closeListeners = new Set<() => void>();
35
+
36
+ type EventQueue = {
37
+ readonly push: (e: CdpEvent) => void;
38
+ readonly end: () => void;
39
+ readonly iter: AsyncIterable<CdpEvent>;
40
+ };
41
+
42
+ const makeEventQueue = (): EventQueue => {
43
+ const buf: CdpEvent[] = [];
44
+ const waiters: Array<(v: IteratorResult<CdpEvent>) => void> = [];
45
+ let ended = false;
46
+ return {
47
+ push(e) {
48
+ if (ended) return;
49
+ const w = waiters.shift();
50
+ if (w) w({ value: e, done: false });
51
+ else buf.push(e);
52
+ },
53
+ end() {
54
+ ended = true;
55
+ // value: undefined as unknown as CdpEvent is the correct iterator-protocol
56
+ // pattern: when done=true the value field is conventionally undefined but
57
+ // the TS type still requires CdpEvent.
58
+ for (const w of waiters.splice(0)) w({ value: undefined as unknown as CdpEvent, done: true });
59
+ },
60
+ iter: {
61
+ [Symbol.asyncIterator]() {
62
+ return {
63
+ next: (): Promise<IteratorResult<CdpEvent>> =>
64
+ new Promise((resolve) => {
65
+ const next = buf.shift();
66
+ if (next) resolve({ value: next, done: false });
67
+ else if (ended) resolve({ value: undefined as unknown as CdpEvent, done: true });
68
+ else waiters.push(resolve);
69
+ }),
70
+ };
71
+ },
72
+ },
73
+ };
74
+ };
75
+
76
+ let queue = makeEventQueue();
77
+
78
+ const handleMessage = (raw: string): void => {
79
+ let parsed: unknown;
80
+ try {
81
+ parsed = JSON.parse(raw);
82
+ } catch {
83
+ return;
84
+ }
85
+ if (!isCdpRawMessage(parsed)) return;
86
+ const msg: CdpRawMessage = parsed;
87
+ if (msg.id === undefined) {
88
+ if (msg.method) {
89
+ queue.push({
90
+ method: msg.method,
91
+ params: msg.params,
92
+ ...(msg.sessionId !== undefined ? { sessionId: msg.sessionId } : {}),
93
+ });
94
+ }
95
+ return;
96
+ }
97
+ const p = pending.get(msg.id);
98
+ if (!p) return;
99
+ pending.delete(msg.id);
100
+ clearTimeout(p.timer);
101
+ if (msg.error) {
102
+ const kind = msg.error.message.includes("Session with given id not found")
103
+ ? "session_not_found"
104
+ : "remote_error";
105
+ p.resolve(err(cdpError(kind, msg.error.message, p.method)));
106
+ return;
107
+ }
108
+ p.resolve(ok(msg.result));
109
+ };
110
+
111
+ const cleanup = (reason: string): void => {
112
+ for (const [, p] of pending) {
113
+ clearTimeout(p.timer);
114
+ p.resolve(err(cdpError("transport_closed", reason, p.method)));
115
+ }
116
+ pending.clear();
117
+ queue.end();
118
+ queue = makeEventQueue();
119
+ for (const cb of closeListeners) cb();
120
+ };
121
+
122
+ return {
123
+ connect(url, opts = {}): Promise<Result<void, CdpError>> {
124
+ const timeoutMs = opts.timeoutMs ?? 10_000;
125
+ return new Promise((resolve) => {
126
+ let settled = false;
127
+ const settle = (r: Result<void, CdpError>): void => {
128
+ if (settled) return;
129
+ settled = true;
130
+ resolve(r);
131
+ };
132
+ try {
133
+ ws = new WebSocket(url);
134
+ } catch (e) {
135
+ settle(err(cdpError("transport_closed", e instanceof Error ? e.message : String(e))));
136
+ return;
137
+ }
138
+ const timer = setTimeout(() => {
139
+ ws?.close();
140
+ ws = null;
141
+ settle(err(cdpError("timeout", `CDP WebSocket connection timed out after ${timeoutMs}ms`)));
142
+ }, timeoutMs);
143
+ ws.onopen = () => {
144
+ clearTimeout(timer);
145
+ settle(ok(undefined));
146
+ };
147
+ ws.onmessage = (ev: WebSocket.MessageEvent) => {
148
+ const data = ev.data;
149
+ handleMessage(typeof data === "string" ? data : data.toString());
150
+ };
151
+ ws.onerror = () => {
152
+ clearTimeout(timer);
153
+ settle(err(cdpError("transport_closed", "CDP WebSocket error during connection")));
154
+ };
155
+ ws.onclose = () => {
156
+ clearTimeout(timer);
157
+ ws = null;
158
+ cleanup("WebSocket closed");
159
+ // If onopen already settled the connect promise, this settle is a no-op
160
+ // (guarded by `settled`). cleanup() above still runs unconditionally so
161
+ // pending requests are rejected and the event queue is rotated.
162
+ settle(err(cdpError("transport_closed", "CDP WebSocket closed")));
163
+ };
164
+ });
165
+ },
166
+ close(): Promise<void> {
167
+ if (ws) {
168
+ try { ws.close(1000, "Shutdown"); } catch { /* best effort */ }
169
+ ws = null;
170
+ }
171
+ cleanup("close() called");
172
+ return Promise.resolve();
173
+ },
174
+ request(method, params, opts = {}): Promise<Result<unknown, CdpError>> {
175
+ const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
176
+ const sessionId = opts.sessionId ?? null;
177
+ if (!ws || ws.readyState !== WebSocket.OPEN) {
178
+ return Promise.resolve(err(cdpError("transport_closed", "Browser not connected. Is Chrome running?", method)));
179
+ }
180
+ // Bind to a local const so the non-null is guaranteed even across microtasks
181
+ // between the readyState check and the send() call.
182
+ const sock = ws;
183
+ const id = nextId++;
184
+ const payload: Record<string, unknown> = { id, method, params };
185
+ if (sessionId) payload["sessionId"] = sessionId;
186
+ const json = JSON.stringify(payload);
187
+ return new Promise((resolve) => {
188
+ const timer = setTimeout(() => {
189
+ pending.delete(id);
190
+ resolve(err(cdpError("timeout", `CDP timeout after ${timeoutMs}ms: ${method}`, method)));
191
+ }, timeoutMs);
192
+ pending.set(id, { resolve, timer, method });
193
+ try {
194
+ sock.send(json);
195
+ } catch (e) {
196
+ clearTimeout(timer);
197
+ pending.delete(id);
198
+ resolve(err(cdpError("transport_closed", e instanceof Error ? e.message : String(e), method)));
199
+ }
200
+ });
201
+ },
202
+ events(): AsyncIterable<CdpEvent> {
203
+ // Single-consumer: each connection's event stream may be iterated by
204
+ // exactly one for-await loop. Calling events() multiple times returns
205
+ // the same iterable; the second consumer will silently steal events
206
+ // from the first. After a reconnect, the previous iterator is ended
207
+ // and callers must re-call events() to receive new events.
208
+ return queue.iter;
209
+ },
210
+ state(): "open" | "closed" | "connecting" {
211
+ if (!ws) return "closed";
212
+ if (ws.readyState === WebSocket.CONNECTING) return "connecting";
213
+ if (ws.readyState === WebSocket.OPEN) return "open";
214
+ return "closed";
215
+ },
216
+ onClose(cb): () => void {
217
+ closeListeners.add(cb);
218
+ return () => closeListeners.delete(cb);
219
+ },
220
+ };
221
+ };
@@ -0,0 +1,72 @@
1
+ export type CdpEvent = {
2
+ readonly method: string;
3
+ readonly params: unknown;
4
+ readonly sessionId?: string;
5
+ };
6
+
7
+ export type CdpRawMessage = {
8
+ readonly id?: number;
9
+ readonly method?: string;
10
+ readonly params?: Record<string, unknown>;
11
+ readonly result?: unknown;
12
+ readonly error?: { readonly message: string; readonly code?: number };
13
+ readonly sessionId?: string;
14
+ };
15
+
16
+ export type DialogInfo = {
17
+ readonly type: "alert" | "confirm" | "prompt" | "beforeunload";
18
+ readonly message: string;
19
+ readonly defaultPrompt?: string;
20
+ };
21
+
22
+ export type TabInfo = {
23
+ readonly targetId: string;
24
+ readonly title: string;
25
+ readonly url: string;
26
+ };
27
+
28
+ export type PageInfo = {
29
+ readonly url: string;
30
+ readonly title: string;
31
+ readonly width: number;
32
+ readonly height: number;
33
+ readonly scrollX: number;
34
+ readonly scrollY: number;
35
+ readonly pageWidth: number;
36
+ readonly pageHeight: number;
37
+ };
38
+
39
+ export type DaemonStatus = {
40
+ readonly alive: boolean;
41
+ readonly sessionId: string | null;
42
+ readonly namespace: string;
43
+ readonly remoteBrowserId?: string;
44
+ };
45
+
46
+ const INTERNAL_PREFIXES: ReadonlyArray<string> = [
47
+ "chrome://", "chrome-untrusted://", "devtools://", "chrome-extension://", "about:",
48
+ ];
49
+
50
+ export const isInternalUrl = (url: string): boolean =>
51
+ INTERNAL_PREFIXES.some((p) => url.startsWith(p));
52
+
53
+ const isObject = (v: unknown): v is Readonly<Record<string, unknown>> =>
54
+ typeof v === "object" && v !== null && !Array.isArray(v);
55
+
56
+ export const isCdpRawMessage = (v: unknown): v is CdpRawMessage => {
57
+ if (!isObject(v)) return false;
58
+ const id = v["id"];
59
+ if (id !== undefined && typeof id !== "number") return false;
60
+ const method = v["method"];
61
+ if (method !== undefined && typeof method !== "string") return false;
62
+ const sessionId = v["sessionId"];
63
+ if (sessionId !== undefined && typeof sessionId !== "string") return false;
64
+ const errVal = v["error"];
65
+ if (errVal !== undefined) {
66
+ if (!isObject(errVal)) return false;
67
+ if (typeof errVal["message"] !== "string") return false;
68
+ const code = errVal["code"];
69
+ if (code !== undefined && typeof code !== "number") return false;
70
+ }
71
+ return true;
72
+ };
package/src/client.ts ADDED
@@ -0,0 +1,238 @@
1
+ import { type Result, err, ok } from "./util/result";
2
+ import { safeJs } from "./util/js-template";
3
+ import { type Mutex, createMutex } from "./util/mutex";
4
+ import { discoverWsUrl } from "./cdp/discovery";
5
+ import { type CdpError, cdpError } from "./cdp/errors";
6
+ import type { CdpTransport } from "./cdp/transport";
7
+ import { createCdpTransport } from "./cdp/transport";
8
+ import { type CdpSession, createCdpSession } from "./cdp/session";
9
+ import type { DaemonStatus, DialogInfo, PageInfo, TabInfo } from "./cdp/types";
10
+
11
+ export type BrowserClientOptions = {
12
+ readonly namespace: string;
13
+ readonly remote?: { readonly cdpUrl: string; readonly browserId: string };
14
+ };
15
+
16
+ export type BrowserClient = {
17
+ readonly namespace: string;
18
+ ensureAlive(): Promise<Result<void, CdpError>>;
19
+ status(): DaemonStatus;
20
+ start(): Promise<Result<void, CdpError>>;
21
+ stop(): Promise<void>;
22
+ evaluateJs(expression: string, sessionId?: string): Promise<Result<unknown, CdpError>>;
23
+ pageInfo(): Promise<Result<PageInfo | { readonly dialog: DialogInfo }, CdpError>>;
24
+ takeDialog(): DialogInfo | null;
25
+ listTabs(includeInternal?: boolean): Promise<Result<ReadonlyArray<TabInfo>, CdpError>>;
26
+ switchTab(targetId: string): Promise<Result<void, CdpError>>;
27
+ newTab(url?: string): Promise<Result<string, CdpError>>;
28
+ current(): { readonly sessionId: string; readonly targetId: string } | null;
29
+ session(): CdpSession;
30
+ transport(): CdpTransport;
31
+ /** Returns the shared async mutex that serialized browser tools must acquire
32
+ * before performing mutations. Observation tools should not use this. */
33
+ mutationMutex(): Mutex;
34
+ };
35
+
36
+ const HEALTH_TTL_MS = 30_000;
37
+ const PAGE_INFO_TTL_MS = 1_000;
38
+
39
+ const parsePageInfoPayload = (v: unknown): Result<PageInfo, CdpError> => {
40
+ if (typeof v !== "object" || v === null) {
41
+ return err(cdpError("invalid_response", "page info payload is not an object"));
42
+ }
43
+ const o = v as Readonly<Record<string, unknown>>;
44
+ const fields: ReadonlyArray<readonly [string, "string" | "number"]> = [
45
+ ["url", "string"], ["title", "string"],
46
+ ["w", "number"], ["h", "number"],
47
+ ["sx", "number"], ["sy", "number"],
48
+ ["pw", "number"], ["ph", "number"],
49
+ ];
50
+ for (const [k, t] of fields) {
51
+ if (typeof o[k] !== t) {
52
+ return err(cdpError("invalid_response", `page info field ${k} has wrong type (expected ${t})`));
53
+ }
54
+ }
55
+ return ok({
56
+ url: o["url"] as string,
57
+ title: o["title"] as string,
58
+ width: o["w"] as number,
59
+ height: o["h"] as number,
60
+ scrollX: o["sx"] as number,
61
+ scrollY: o["sy"] as number,
62
+ pageWidth: o["pw"] as number,
63
+ pageHeight: o["ph"] as number,
64
+ });
65
+ };
66
+
67
+ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient => {
68
+ const transport = createCdpTransport();
69
+ const session = createCdpSession(transport);
70
+ const mutationMutex = createMutex();
71
+ let lastHealth = 0;
72
+ let pageCache: { readonly info: PageInfo; readonly at: number } | null = null;
73
+ let remote: BrowserClientOptions["remote"] | null = opts.remote ?? null;
74
+
75
+ const start = async (): Promise<Result<void, CdpError>> => {
76
+ if (transport.state() === "open" && session.current()) return ok(undefined);
77
+ const envUrl = process.env["BU_CDP_WS"];
78
+ let wsUrl: string;
79
+ if (remote?.cdpUrl) {
80
+ wsUrl = remote.cdpUrl;
81
+ } else if (envUrl) {
82
+ wsUrl = envUrl;
83
+ } else {
84
+ const discovered = await discoverWsUrl();
85
+ if (!discovered.success) return discovered;
86
+ wsUrl = discovered.data;
87
+ }
88
+ const connected = await transport.connect(wsUrl, { timeoutMs: 10_000 });
89
+ if (!connected.success) return connected;
90
+ if (!remote) {
91
+ remote = { cdpUrl: wsUrl, browserId: wsUrl.split("/").pop() ?? "unknown" };
92
+ }
93
+ const attached = await session.attachFirstPage();
94
+ if (!attached.success) {
95
+ await transport.close();
96
+ return attached;
97
+ }
98
+ lastHealth = Date.now();
99
+ pageCache = null;
100
+ return ok(undefined);
101
+ };
102
+
103
+ const stop = async (): Promise<void> => {
104
+ await transport.close();
105
+ pageCache = null;
106
+ lastHealth = 0;
107
+ };
108
+
109
+ const ensureAlive = async (): Promise<Result<void, CdpError>> => {
110
+ if (transport.state() !== "open" || !session.current()) {
111
+ await stop();
112
+ return start();
113
+ }
114
+ if (Date.now() - lastHealth < HEALTH_TTL_MS) return ok(undefined);
115
+ const probe = await transport.request("Target.getTargets", {}, { sessionId: null, timeoutMs: 2_000 });
116
+ if (!probe.success) {
117
+ await stop();
118
+ return start();
119
+ }
120
+ lastHealth = Date.now();
121
+ // Verify the page session is still responsive (handles the case where the
122
+ // browser transport is alive but the page target crashed, e.g. localhost died).
123
+ const jsProbe = await session.call("Runtime.evaluate", {
124
+ expression: "1", returnByValue: true,
125
+ }, { timeoutMs: 2_000 });
126
+ if (!jsProbe.success && jsProbe.error.kind === "session_not_found") {
127
+ const reattached = await session.attachFirstPage();
128
+ if (reattached.success) {
129
+ pageCache = null;
130
+ return ok(undefined);
131
+ }
132
+ await stop();
133
+ return start();
134
+ }
135
+ return ok(undefined);
136
+ };
137
+
138
+ const evaluateJs = async (expression: string, sessionId?: string): Promise<Result<unknown, CdpError>> => {
139
+ // Heuristic IIFE wrap: legacy convenience that lets agents write
140
+ // `return foo` instead of `(() => foo)()`. False-positive on substring
141
+ // `"return "` inside string literals or comments — agents that hit this
142
+ // can pass an explicit IIFE instead.
143
+ const wrapped = expression.includes("return ") && !expression.trim().startsWith("(")
144
+ ? `(function(){${expression}})()`
145
+ : expression;
146
+ const r = sessionId
147
+ ? await session.callOnTarget("Runtime.evaluate", { expression: wrapped, returnByValue: true, awaitPromise: true }, sessionId)
148
+ : await session.call("Runtime.evaluate", { expression: wrapped, returnByValue: true, awaitPromise: true });
149
+ if (!r.success) return r;
150
+ const data = r.data as { result?: { value?: unknown }; exceptionDetails?: unknown };
151
+ if (data.exceptionDetails) {
152
+ return err(cdpError("remote_error", `JS evaluation failed: ${JSON.stringify(data.exceptionDetails)}`, "Runtime.evaluate"));
153
+ }
154
+ return ok(data.result?.value);
155
+ };
156
+
157
+ const readPageInfo = async (): Promise<Result<PageInfo, CdpError>> => {
158
+ const dirty = session.drainPageInfoInvalidations();
159
+ if (pageCache && !dirty && Date.now() - pageCache.at < PAGE_INFO_TTL_MS) return ok(pageCache.info);
160
+ const expr = safeJs`JSON.stringify({url:location.href,title:document.title,w:innerWidth,h:innerHeight,sx:scrollX,sy:scrollY,pw:document.documentElement.scrollWidth,ph:document.documentElement.scrollHeight})`;
161
+ const raw = await evaluateJs(expr);
162
+ if (!raw.success) return raw;
163
+ if (typeof raw.data !== "string") return err(cdpError("invalid_response", "page info evaluation did not return a string"));
164
+ let parsedRaw: unknown;
165
+ try {
166
+ parsedRaw = JSON.parse(raw.data);
167
+ } catch (e) {
168
+ return err(cdpError("invalid_response", `page info JSON.parse failed: ${e instanceof Error ? e.message : String(e)}`));
169
+ }
170
+ const info = parsePageInfoPayload(parsedRaw);
171
+ if (!info.success) return info;
172
+ pageCache = { info: info.data, at: Date.now() };
173
+ return ok(info.data);
174
+ };
175
+
176
+ const pageInfo = async (): Promise<Result<PageInfo | { readonly dialog: DialogInfo }, CdpError>> => {
177
+ const d = session.takeDialog();
178
+ if (d) return ok({ dialog: d });
179
+ return readPageInfo();
180
+ };
181
+
182
+ const listTabs = async (includeInternal = true): Promise<Result<ReadonlyArray<TabInfo>, CdpError>> => {
183
+ const r = await session.callBrowser("Target.getTargets");
184
+ if (!r.success) return r;
185
+ const data = r.data as { targetInfos: ReadonlyArray<{ targetId: string; type: string; title: string; url: string }> };
186
+ const tabs = data.targetInfos
187
+ .filter((t) => t.type === "page")
188
+ .filter((t) => includeInternal || !t.url.startsWith("chrome://"))
189
+ .map((t): TabInfo => ({ targetId: t.targetId, title: t.title, url: t.url }));
190
+ return ok(tabs);
191
+ };
192
+
193
+ const switchTab = async (targetId: string): Promise<Result<void, CdpError>> => {
194
+ const r = await session.switchTo(targetId);
195
+ if (!r.success) return r;
196
+ pageCache = null;
197
+ // Best-effort: mark the tab title with a green circle so the user can
198
+ // see which tab the agent attached to. CSP or detached frames may
199
+ // block the eval; we don't surface that as a switchTab failure.
200
+ await session.call("Runtime.evaluate", {
201
+ expression: safeJs`if(!document.title.startsWith('🟢'))document.title='🟢 '+document.title`,
202
+ });
203
+ return ok(undefined);
204
+ };
205
+
206
+ const newTab = async (url?: string): Promise<Result<string, CdpError>> => {
207
+ const created = await session.callBrowser("Target.createTarget", { url: "about:blank" });
208
+ if (!created.success) return created;
209
+ const c = created.data as { targetId: string };
210
+ const switched = await switchTab(c.targetId);
211
+ if (!switched.success) return switched;
212
+ if (url && url !== "about:blank") {
213
+ const nav = await session.call("Page.navigate", { url });
214
+ if (!nav.success) return nav;
215
+ pageCache = null;
216
+ }
217
+ return ok(c.targetId);
218
+ };
219
+
220
+ const status = (): DaemonStatus => ({
221
+ alive: transport.state() === "open" && session.current() !== null,
222
+ sessionId: session.current()?.sessionId ?? null,
223
+ namespace: opts.namespace,
224
+ ...(remote?.browserId !== undefined ? { remoteBrowserId: remote.browserId } : {}),
225
+ });
226
+
227
+ return {
228
+ namespace: opts.namespace,
229
+ ensureAlive, status, start, stop,
230
+ evaluateJs, pageInfo,
231
+ takeDialog: () => session.takeDialog(),
232
+ listTabs, switchTab, newTab,
233
+ current: () => session.current(),
234
+ session: () => session,
235
+ transport: () => transport,
236
+ mutationMutex: () => mutationMutex,
237
+ };
238
+ };
@@ -0,0 +1,73 @@
1
+ import { Type, type Static } from "typebox";
2
+ import { Text } from "@mariozechner/pi-tui";
3
+ import type { BrowserClient } from "../client";
4
+ import { Coords, MouseButton } from "../schemas/common";
5
+ import { type Result, err, ok } from "../util/result";
6
+ import { defineBrowserTool, type ToolErr } from "../util/tool";
7
+ import { captureWithCrosshair } from "./screenshot";
8
+
9
+ const ClickArgs = Type.Object({
10
+ ...Coords.properties,
11
+ button: Type.Optional(MouseButton),
12
+ count: Type.Optional(
13
+ Type.Integer({
14
+ minimum: 1,
15
+ maximum: 3,
16
+ default: 1,
17
+ description: "Number of clicks (1 = single, 2 = double). Default: 1",
18
+ }),
19
+ ),
20
+ });
21
+ type ClickArgsT = Static<typeof ClickArgs>;
22
+
23
+ const dispatchClick = async (
24
+ client: BrowserClient,
25
+ args: ClickArgsT,
26
+ ): Promise<Result<void, ToolErr>> => {
27
+ const button = args.button ?? "left";
28
+ const count = args.count ?? 1;
29
+ const pressed = await client.session().call("Input.dispatchMouseEvent", {
30
+ type: "mousePressed", x: args.x, y: args.y, button, clickCount: count,
31
+ });
32
+ if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
33
+ const released = await client.session().call("Input.dispatchMouseEvent", {
34
+ type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count,
35
+ });
36
+ if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
37
+ return ok(undefined);
38
+ };
39
+
40
+ export const clickTool = defineBrowserTool({
41
+ name: "browser_click",
42
+ label: "Browser Click",
43
+ description:
44
+ "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.",
45
+ promptSnippet: "Click at pixel coordinates (x, y) on the page",
46
+ promptGuidelines: [
47
+ "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).",
48
+ "Capture a browser_screenshot BEFORE clicking to find the right coordinates.",
49
+ "Capture another browser_screenshot AFTER clicking to verify the action worked.",
50
+ "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
51
+ "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.",
52
+ "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
53
+ ],
54
+ parameters: ClickArgs,
55
+ renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0),
56
+ async handler(args, { client }) {
57
+ const clicked = await dispatchClick(client, args);
58
+ if (!clicked.success) return clicked;
59
+ if (process.env["BH_DEBUG_CLICKS"]) {
60
+ const debug = await captureWithCrosshair(client, { x: args.x, y: args.y });
61
+ if (debug.success) {
62
+ return ok({
63
+ text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`,
64
+ details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y },
65
+ });
66
+ }
67
+ }
68
+ return ok({
69
+ text: `Clicked at (${args.x}, ${args.y})`,
70
+ details: { x: args.x, y: args.y },
71
+ });
72
+ },
73
+ });
@@ -0,0 +1,31 @@
1
+ import { Type } from "typebox";
2
+ import { type Result, err, ok } from "../util/result";
3
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
4
+
5
+ const HandleDialogArgs = Type.Object({
6
+ accept: Type.Boolean({ description: "true = accept, false = dismiss" }),
7
+ promptText: Type.Optional(Type.String({ description: "Text to type if dialog is a prompt()" })),
8
+ });
9
+
10
+ export const handleDialogTool = defineBrowserTool({
11
+ name: "browser_handle_dialog",
12
+ label: "Browser Handle Dialog",
13
+ description:
14
+ "Accept or dismiss the currently open JS dialog (alert/confirm/prompt/beforeunload).",
15
+ promptSnippet: "Accept or dismiss a JS dialog",
16
+ promptGuidelines: [
17
+ "Use after browser_page_info reports a dialog is open. Until handled, no other browser action will work.",
18
+ "For prompt() dialogs, supply promptText with the value to submit.",
19
+ ],
20
+ parameters: HandleDialogArgs,
21
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
22
+ const params: Record<string, unknown> = { accept: args.accept };
23
+ if (args.promptText !== undefined) params["promptText"] = args.promptText;
24
+ const r = await client.session().call("Page.handleJavaScriptDialog", params);
25
+ if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
26
+ return ok({
27
+ text: `Dialog ${args.accept ? "accepted" : "dismissed"}`,
28
+ details: { accept: args.accept },
29
+ });
30
+ },
31
+ });