pi-browser-harness 0.8.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +28 -7
- package/package.json +1 -1
- package/skills/deep-research/SKILL.md +66 -0
- package/skills/pi-browser-harness/SKILL.md +8 -0
- package/src/cdp/discovery.ts +171 -9
- package/src/cdp/ownership.ts +19 -1
- package/src/cdp/session.ts +55 -0
- package/src/cdp/window-binding.ts +15 -0
- package/src/client.ts +52 -9
- package/src/daemon/protocol.ts +12 -2
- package/src/daemon/spawn.ts +46 -13
- package/src/deep-research.ts +41 -0
- package/src/domains/forms.ts +355 -0
- package/src/domains/isolated-tab.ts +121 -0
- package/src/domains/navigate.ts +25 -10
- package/src/domains/readpage/capture.ts +46 -0
- package/src/domains/readpage/read-page.ts +119 -0
- package/src/domains/readpage/readability.ts +89 -0
- package/src/domains/render.ts +64 -0
- package/src/domains/search/extract.ts +44 -0
- package/src/domains/search/google-serp.ts +114 -0
- package/src/domains/search/web-search.ts +116 -0
- package/src/index.ts +17 -7
- package/src/prompt.ts +92 -116
- package/src/registry.ts +6 -0
- package/src/setup.ts +26 -10
- package/src/state.ts +1 -0
package/src/client.ts
CHANGED
|
@@ -16,10 +16,12 @@ export type BrowserClientOptions = {
|
|
|
16
16
|
readonly initialOwnership?: {
|
|
17
17
|
readonly ownedTargetIds?: ReadonlyArray<string>;
|
|
18
18
|
readonly harnessWindowTargetId?: string;
|
|
19
|
+
readonly harnessWindowId?: number;
|
|
19
20
|
};
|
|
20
21
|
readonly onOwnershipChange?: (snapshot: {
|
|
21
22
|
readonly ownedTargetIds: ReadonlyArray<string>;
|
|
22
23
|
readonly harnessWindowTargetId: string | undefined;
|
|
24
|
+
readonly harnessWindowId: number | undefined;
|
|
23
25
|
}) => void;
|
|
24
26
|
};
|
|
25
27
|
|
|
@@ -32,6 +34,9 @@ export type BrowserClient = {
|
|
|
32
34
|
/** Detach from the current page target (removes the "Chrome is being controlled" banner)
|
|
33
35
|
* while keeping the transport connection alive. Call on session shutdown. */
|
|
34
36
|
detach(): Promise<void>;
|
|
37
|
+
/** Close every tab this session owns and clear the window binding. Best-effort —
|
|
38
|
+
* used on session shutdown so no stale harness tabs survive. */
|
|
39
|
+
closeOwnedTabs(): Promise<void>;
|
|
35
40
|
evaluateJs(expression: string, sessionId?: string): Promise<Result<unknown, CdpError>>;
|
|
36
41
|
pageInfo(): Promise<Result<PageInfo | { readonly dialog: DialogInfo }, CdpError>>;
|
|
37
42
|
takeDialog(): DialogInfo | null;
|
|
@@ -82,13 +87,16 @@ const parsePageInfoPayload = (v: unknown): Result<PageInfo, CdpError> => {
|
|
|
82
87
|
|
|
83
88
|
export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient => {
|
|
84
89
|
const transport = opts.transport ?? createCdpTransport();
|
|
85
|
-
const ownershipInit: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {};
|
|
90
|
+
const ownershipInit: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string; harnessWindowId?: number } = {};
|
|
86
91
|
if (opts.initialOwnership?.ownedTargetIds !== undefined) {
|
|
87
92
|
ownershipInit.ownedTargetIds = opts.initialOwnership.ownedTargetIds;
|
|
88
93
|
}
|
|
89
94
|
if (opts.initialOwnership?.harnessWindowTargetId !== undefined) {
|
|
90
95
|
ownershipInit.harnessWindowTargetId = opts.initialOwnership.harnessWindowTargetId;
|
|
91
96
|
}
|
|
97
|
+
if (opts.initialOwnership?.harnessWindowId !== undefined) {
|
|
98
|
+
ownershipInit.harnessWindowId = opts.initialOwnership.harnessWindowId;
|
|
99
|
+
}
|
|
92
100
|
const ownership = createOwnershipRegistry(ownershipInit);
|
|
93
101
|
if (opts.onOwnershipChange) {
|
|
94
102
|
const cb = opts.onOwnershipChange;
|
|
@@ -96,6 +104,7 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
|
|
|
96
104
|
cb({
|
|
97
105
|
ownedTargetIds: ownership.list(),
|
|
98
106
|
harnessWindowTargetId: ownership.harnessWindow(),
|
|
107
|
+
harnessWindowId: ownership.harnessWindowId(),
|
|
99
108
|
});
|
|
100
109
|
});
|
|
101
110
|
}
|
|
@@ -256,6 +265,15 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
|
|
|
256
265
|
return ok(undefined);
|
|
257
266
|
};
|
|
258
267
|
|
|
268
|
+
// Ask Chrome for the real windowId hosting a target. Undefined on failure —
|
|
269
|
+
// callers treat the window binding as best-effort.
|
|
270
|
+
const getWindowId = async (targetId: string): Promise<number | undefined> => {
|
|
271
|
+
const r = await session.callBrowser("Browser.getWindowForTarget", { targetId });
|
|
272
|
+
if (!r.success) return undefined;
|
|
273
|
+
const wid = (r.data as { windowId?: number }).windowId;
|
|
274
|
+
return typeof wid === "number" ? wid : undefined;
|
|
275
|
+
};
|
|
276
|
+
|
|
259
277
|
const newTab = async (url?: string): Promise<Result<string, CdpError>> => {
|
|
260
278
|
// Verify the recorded harness window still exists. Querying the live
|
|
261
279
|
// target list also gives listTabs's reconciliation a chance to run.
|
|
@@ -263,22 +281,35 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
|
|
|
263
281
|
if (!tabsResult.success) return tabsResult;
|
|
264
282
|
const live = new Set(tabsResult.data.map((t) => t.targetId));
|
|
265
283
|
const hw = ownership.harnessWindow();
|
|
284
|
+
// Pick an opener that keeps us inside the harness window. Prefer the seed
|
|
285
|
+
// tab; if the user closed it but other owned tabs survive, reuse one of
|
|
286
|
+
// those instead of spawning a second window (avoids window scatter).
|
|
287
|
+
const opener = hw && live.has(hw)
|
|
288
|
+
? hw
|
|
289
|
+
: ownership.list().find((id) => live.has(id));
|
|
266
290
|
|
|
267
291
|
const params: Record<string, unknown> = { url: "about:blank" };
|
|
268
|
-
if (
|
|
269
|
-
// Open as a child of
|
|
270
|
-
//
|
|
271
|
-
params["openerId"] =
|
|
292
|
+
if (opener) {
|
|
293
|
+
// Open as a child of an owned tab — Chrome places it in the same window,
|
|
294
|
+
// giving us visual grouping for free and keeping the session single-window.
|
|
295
|
+
params["openerId"] = opener;
|
|
272
296
|
} else {
|
|
273
|
-
//
|
|
274
|
-
// Spawn a new dedicated window.
|
|
297
|
+
// Fresh session, or the whole harness window was closed by the user.
|
|
298
|
+
// Spawn a new dedicated window and rebind to it.
|
|
275
299
|
params["newWindow"] = true;
|
|
276
300
|
}
|
|
277
301
|
const created = await session.callBrowser("Target.createTarget", params);
|
|
278
302
|
if (!created.success) return created;
|
|
279
303
|
const c = created.data as { targetId: string };
|
|
280
304
|
|
|
281
|
-
if (params["newWindow"])
|
|
305
|
+
if (params["newWindow"]) {
|
|
306
|
+
ownership.setHarnessWindow(c.targetId);
|
|
307
|
+
const wid = await getWindowId(c.targetId);
|
|
308
|
+
if (wid !== undefined) ownership.setHarnessWindowId(wid);
|
|
309
|
+
} else if (opener && opener !== hw) {
|
|
310
|
+
// Re-seeded onto a surviving owned tab; keep the seed pointer live.
|
|
311
|
+
ownership.setHarnessWindow(opener);
|
|
312
|
+
}
|
|
282
313
|
ownership.add(c.targetId);
|
|
283
314
|
|
|
284
315
|
const switched = await switchTab(c.targetId);
|
|
@@ -297,6 +328,18 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
|
|
|
297
328
|
return ok(undefined);
|
|
298
329
|
};
|
|
299
330
|
|
|
331
|
+
const closeOwnedTabs = async (): Promise<void> => {
|
|
332
|
+
// Snapshot first — remove() mutates the list, and targetDestroyed events
|
|
333
|
+
// may also prune concurrently. Failures are swallowed: a tab the user
|
|
334
|
+
// already closed, or a dead transport, must not block shutdown.
|
|
335
|
+
for (const id of ownership.list()) {
|
|
336
|
+
await session.callBrowser("Target.closeTarget", { targetId: id }).catch(() => {});
|
|
337
|
+
ownership.remove(id);
|
|
338
|
+
}
|
|
339
|
+
ownership.setHarnessWindow(undefined);
|
|
340
|
+
ownership.setHarnessWindowId(undefined);
|
|
341
|
+
};
|
|
342
|
+
|
|
300
343
|
const status = (): DaemonStatus => ({
|
|
301
344
|
alive: transport.state() === "open" && session.current() !== null,
|
|
302
345
|
sessionId: session.current()?.sessionId ?? null,
|
|
@@ -316,7 +359,7 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
|
|
|
316
359
|
|
|
317
360
|
return {
|
|
318
361
|
namespace: opts.namespace,
|
|
319
|
-
ensureAlive, status, start, stop, detach,
|
|
362
|
+
ensureAlive, status, start, stop, detach, closeOwnedTabs,
|
|
320
363
|
evaluateJs, pageInfo,
|
|
321
364
|
takeDialog: () => session.takeDialog(),
|
|
322
365
|
listTabs, switchTab, newTab, closeTab,
|
package/src/daemon/protocol.ts
CHANGED
|
@@ -133,8 +133,18 @@ export const deserialize = (line: string): WireMessage | null => {
|
|
|
133
133
|
|
|
134
134
|
// ── Constants ──────────────────────────────────────────────────────────────────
|
|
135
135
|
|
|
136
|
-
/**
|
|
137
|
-
|
|
136
|
+
/**
|
|
137
|
+
* IPC endpoint for the daemon.
|
|
138
|
+
*
|
|
139
|
+
* On POSIX platforms this is a Unix domain socket path. On Windows, Node's IPC
|
|
140
|
+
* requires a named pipe of the form `\\.\pipe\<name>` — a Unix path like
|
|
141
|
+
* `/tmp/…` is not a valid `net` listen/connect target there and causes setup
|
|
142
|
+
* to fail. See https://nodejs.org/api/net.html#ipc-support.
|
|
143
|
+
*/
|
|
144
|
+
export const DAEMON_SOCKET_PATH =
|
|
145
|
+
process.platform === "win32"
|
|
146
|
+
? "\\\\.\\pipe\\pi-browser-daemon"
|
|
147
|
+
: "/tmp/pi-browser-daemon.sock";
|
|
138
148
|
|
|
139
149
|
/**
|
|
140
150
|
* How long the daemon stays alive with zero connected clients before exiting
|
package/src/daemon/spawn.ts
CHANGED
|
@@ -19,6 +19,19 @@ import { DAEMON_SOCKET_PATH, serialize, type WireControl } from "./protocol";
|
|
|
19
19
|
|
|
20
20
|
const moduleDir = dirname(fileURLToPath(import.meta.url));
|
|
21
21
|
|
|
22
|
+
const isWindows = process.platform === "win32";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Remove a stale POSIX socket file. On Windows the daemon endpoint is a named
|
|
26
|
+
* pipe, which is not a filesystem entry and is torn down automatically when the
|
|
27
|
+
* owning process exits — so there is nothing to unlink, and calling unlink on a
|
|
28
|
+
* pipe path throws. This is a no-op on Windows.
|
|
29
|
+
*/
|
|
30
|
+
const cleanupStaleSocket = (): void => {
|
|
31
|
+
if (isWindows) return;
|
|
32
|
+
unlink(DAEMON_SOCKET_PATH).catch(() => {});
|
|
33
|
+
};
|
|
34
|
+
|
|
22
35
|
/**
|
|
23
36
|
* Check whether the daemon is running by probing its Unix socket.
|
|
24
37
|
* A file-existence check is unreliable — a crashed daemon leaves a stale socket.
|
|
@@ -26,11 +39,15 @@ const moduleDir = dirname(fileURLToPath(import.meta.url));
|
|
|
26
39
|
* On failure, clean up the stale socket so ensureDaemon can spawn a fresh one.
|
|
27
40
|
*/
|
|
28
41
|
export const isDaemonRunning = async (timeoutMs = 2_000): Promise<boolean> => {
|
|
29
|
-
//
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
42
|
+
// On POSIX, a missing socket file means the daemon definitely isn't running,
|
|
43
|
+
// so we can short-circuit. Windows named pipes are not filesystem entries —
|
|
44
|
+
// access() always fails there — so skip the pre-check and probe directly.
|
|
45
|
+
if (!isWindows) {
|
|
46
|
+
try {
|
|
47
|
+
await access(DAEMON_SOCKET_PATH);
|
|
48
|
+
} catch {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
34
51
|
}
|
|
35
52
|
|
|
36
53
|
return new Promise((resolve) => {
|
|
@@ -45,7 +62,7 @@ export const isDaemonRunning = async (timeoutMs = 2_000): Promise<boolean> => {
|
|
|
45
62
|
const timer = setTimeout(() => {
|
|
46
63
|
sock.destroy();
|
|
47
64
|
// ponytail: stale socket cleanup; ENOENT = already gone, fine
|
|
48
|
-
|
|
65
|
+
cleanupStaleSocket();
|
|
49
66
|
done(false);
|
|
50
67
|
}, timeoutMs);
|
|
51
68
|
|
|
@@ -70,7 +87,7 @@ export const isDaemonRunning = async (timeoutMs = 2_000): Promise<boolean> => {
|
|
|
70
87
|
clearTimeout(timer);
|
|
71
88
|
sock.destroy();
|
|
72
89
|
// ponytail: connection refused = daemon dead, clean up stale socket
|
|
73
|
-
|
|
90
|
+
cleanupStaleSocket();
|
|
74
91
|
done(false);
|
|
75
92
|
});
|
|
76
93
|
|
|
@@ -96,18 +113,34 @@ export const isDaemonRunning = async (timeoutMs = 2_000): Promise<boolean> => {
|
|
|
96
113
|
export const spawnDaemon = (): ChildProcess | null => {
|
|
97
114
|
// Resolve the daemon entry point and tsx binary.
|
|
98
115
|
const daemonScript = join(moduleDir, "index.ts");
|
|
99
|
-
const tsxBin = join(moduleDir, "..", "..", "node_modules", ".bin", "tsx");
|
|
100
116
|
|
|
101
|
-
//
|
|
102
|
-
//
|
|
117
|
+
// On Windows, npm installs binaries as `.cmd` shims, not bare executables,
|
|
118
|
+
// and the npx fallback is `npx.cmd`. A `.cmd` file can only be launched
|
|
119
|
+
// through a shell (spawning it directly yields EINVAL), so set shell: true
|
|
120
|
+
// whenever we invoke one.
|
|
121
|
+
const tsxBinBase = join(moduleDir, "..", "..", "node_modules", ".bin", "tsx");
|
|
122
|
+
const tsxBin = isWindows ? `${tsxBinBase}.cmd` : tsxBinBase;
|
|
123
|
+
const hasLocalTsx = existsSync(tsxBin);
|
|
124
|
+
|
|
125
|
+
// Use the local tsx binary to run the daemon TypeScript source directly.
|
|
126
|
+
// The spawned process inherits the parent's env so module resolution works.
|
|
103
127
|
// Fall back to `npx tsx` if the local binary isn't available.
|
|
104
|
-
const cmd =
|
|
105
|
-
const args =
|
|
128
|
+
const cmd = hasLocalTsx ? tsxBin : isWindows ? "npx.cmd" : "npx";
|
|
129
|
+
const args = hasLocalTsx ? [daemonScript] : ["tsx", daemonScript];
|
|
130
|
+
|
|
131
|
+
// With shell: true, Node passes the command line to the shell verbatim
|
|
132
|
+
// without quoting, so any path containing spaces (e.g. C:\Users\Some Name)
|
|
133
|
+
// would be split into multiple tokens. Wrap each token in double quotes on
|
|
134
|
+
// Windows to keep space-containing paths intact.
|
|
135
|
+
const quote = (s: string): string => (isWindows ? `"${s}"` : s);
|
|
106
136
|
|
|
107
|
-
const child = spawn(cmd, args, {
|
|
137
|
+
const child = spawn(quote(cmd), args.map(quote), {
|
|
108
138
|
detached: true,
|
|
109
139
|
stdio: "ignore",
|
|
110
140
|
env: { ...process.env },
|
|
141
|
+
// `.cmd` shims (local tsx or npx.cmd on Windows) must run via a shell.
|
|
142
|
+
shell: isWindows,
|
|
143
|
+
windowsVerbatimArguments: isWindows,
|
|
111
144
|
});
|
|
112
145
|
|
|
113
146
|
child.unref();
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deep research — the /deep-research slash command. An explicit entry point
|
|
3
|
+
* (alongside the auto-triggering deep-research skill) that kicks off a research
|
|
4
|
+
* run for a given question.
|
|
5
|
+
*
|
|
6
|
+
* The command is thin: it validates a non-empty question and injects a user
|
|
7
|
+
* message that starts the deep-research flow. The actual orchestration
|
|
8
|
+
* (decompose → fan out web-search-researcher subagents → synthesize a cited
|
|
9
|
+
* report) lives in skills/deep-research/SKILL.md, which the injected prompt
|
|
10
|
+
* triggers. pi.sendUserMessage always starts a turn.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
14
|
+
|
|
15
|
+
// ── Shared core: build the prompt that starts a deep-research run ───────────
|
|
16
|
+
|
|
17
|
+
export function buildDeepResearchPrompt(question: string): string {
|
|
18
|
+
return (
|
|
19
|
+
`Do deep research on the following question and produce a cited Markdown report. ` +
|
|
20
|
+
`Use the deep-research skill: decompose it into sub-questions, fan out web-search-researcher ` +
|
|
21
|
+
`subagents to research each against the live web, then synthesize a source-cited report file.\n\n` +
|
|
22
|
+
`Question: ${question}`
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// ── Public: register the /deep-research slash command ──────────────────────
|
|
27
|
+
|
|
28
|
+
export function registerDeepResearchCommand(pi: ExtensionAPI): void {
|
|
29
|
+
pi.registerCommand("deep-research", {
|
|
30
|
+
description: "Research a question on the web and write a cited report",
|
|
31
|
+
handler: async (args, ctx) => {
|
|
32
|
+
const question = args.trim();
|
|
33
|
+
if (question.length === 0) {
|
|
34
|
+
ctx.ui.notify('Usage: /deep-research <question>. Example: /deep-research "compare CDP vs WebDriver"', "warning");
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
ctx.ui.notify(`Starting deep research: ${question}`, "info");
|
|
38
|
+
pi.sendUserMessage(buildDeepResearchPrompt(question));
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
}
|
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import type { BrowserClient } from "../client";
|
|
3
|
+
import { type Result, err, ok } from "../util/result";
|
|
4
|
+
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
5
|
+
|
|
6
|
+
// ── Page-side functions ──────────────────────────────────────────────────────
|
|
7
|
+
// These run inside the page via Runtime.callFunctionOn with `this` bound to the
|
|
8
|
+
// resolved element. They are NOT string-interpolated with user data — the value
|
|
9
|
+
// is passed as a structured `arguments` entry, so safeJs is unnecessary here.
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Universal field setter. Auto-detects the element type and applies the right
|
|
13
|
+
* strategy, then reads the resulting value back so the caller can confirm.
|
|
14
|
+
*
|
|
15
|
+
* The <input>/<textarea> branch uses the "native setter" trick: React (and
|
|
16
|
+
* other frameworks) overload the element's `value` setter to track state, so a
|
|
17
|
+
* plain `el.value = x` updates the DOM but not the framework. Calling the
|
|
18
|
+
* prototype's original setter and then dispatching a bubbling `input` event is
|
|
19
|
+
* what makes React's onChange fire.
|
|
20
|
+
*/
|
|
21
|
+
const FILL_FN = `
|
|
22
|
+
function(value) {
|
|
23
|
+
const el = this;
|
|
24
|
+
if (!el || el.nodeType !== 1) return { ok: false, reason: "ref does not point to an element" };
|
|
25
|
+
if (el.disabled) return { ok: false, reason: "element is disabled" };
|
|
26
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
27
|
+
const type = (el.type || "").toLowerCase();
|
|
28
|
+
try { el.focus(); } catch (e) {}
|
|
29
|
+
const fire = function(t) { el.dispatchEvent(new Event(t, { bubbles: true })); };
|
|
30
|
+
|
|
31
|
+
if (tag === "select") {
|
|
32
|
+
const want = String(value);
|
|
33
|
+
let matched = null;
|
|
34
|
+
for (let i = 0; i < el.options.length; i++) {
|
|
35
|
+
const o = el.options[i];
|
|
36
|
+
if (o.value === want || o.label === want || o.text === want) { matched = o; break; }
|
|
37
|
+
}
|
|
38
|
+
if (!matched) {
|
|
39
|
+
const opts = [];
|
|
40
|
+
for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
|
|
41
|
+
return { ok: false, reason: "no matching option", kind: "select", options: opts };
|
|
42
|
+
}
|
|
43
|
+
el.value = matched.value;
|
|
44
|
+
fire("input"); fire("change");
|
|
45
|
+
return { ok: true, kind: "select", value: el.value, text: matched.text };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (tag === "input" && (type === "checkbox" || type === "radio")) {
|
|
49
|
+
const want = value === true || value === "true" || value === "on" || value === 1;
|
|
50
|
+
if (el.checked !== want) {
|
|
51
|
+
try { el.click(); } catch (e) {}
|
|
52
|
+
if (el.checked !== want) { el.checked = want; fire("input"); fire("change"); }
|
|
53
|
+
}
|
|
54
|
+
return { ok: true, kind: type, checked: el.checked };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (el.isContentEditable) {
|
|
58
|
+
el.textContent = String(value);
|
|
59
|
+
fire("input"); fire("change");
|
|
60
|
+
return { ok: true, kind: "contenteditable", value: el.textContent };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (tag === "input" || tag === "textarea") {
|
|
64
|
+
const proto = tag === "textarea" ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype;
|
|
65
|
+
const desc = Object.getOwnPropertyDescriptor(proto, "value");
|
|
66
|
+
if (desc && desc.set) desc.set.call(el, String(value));
|
|
67
|
+
else el.value = String(value);
|
|
68
|
+
fire("input"); fire("change");
|
|
69
|
+
return { ok: true, kind: tag, value: el.value };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return { ok: false, reason: "element is not a fillable field (tag=" + tag + ")" };
|
|
73
|
+
}`;
|
|
74
|
+
|
|
75
|
+
/** Specialized <select> setter — matches by value first, then by label/text. */
|
|
76
|
+
const SELECT_FN = `
|
|
77
|
+
function(value, label) {
|
|
78
|
+
const el = this;
|
|
79
|
+
if (!el || (el.tagName || "").toLowerCase() !== "select") return { ok: false, reason: "ref is not a <select> element" };
|
|
80
|
+
if (el.disabled) return { ok: false, reason: "select is disabled" };
|
|
81
|
+
let matched = null;
|
|
82
|
+
for (let i = 0; i < el.options.length; i++) {
|
|
83
|
+
const o = el.options[i];
|
|
84
|
+
if (value != null && o.value === String(value)) { matched = o; break; }
|
|
85
|
+
if (label != null && (o.text === String(label) || o.label === String(label))) { matched = o; break; }
|
|
86
|
+
}
|
|
87
|
+
if (!matched) {
|
|
88
|
+
const opts = [];
|
|
89
|
+
for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
|
|
90
|
+
return { ok: false, reason: "no matching option", options: opts };
|
|
91
|
+
}
|
|
92
|
+
try { el.focus(); } catch (e) {}
|
|
93
|
+
el.value = matched.value;
|
|
94
|
+
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
95
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
96
|
+
return { ok: true, value: el.value, text: matched.text };
|
|
97
|
+
}`;
|
|
98
|
+
|
|
99
|
+
/** Specialized checkbox/radio setter — only acts when the state differs. */
|
|
100
|
+
const CHECK_FN = `
|
|
101
|
+
function(checked) {
|
|
102
|
+
const el = this;
|
|
103
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
104
|
+
const type = (el.type || "").toLowerCase();
|
|
105
|
+
if (tag !== "input" || (type !== "checkbox" && type !== "radio")) return { ok: false, reason: "ref is not a checkbox or radio" };
|
|
106
|
+
if (el.disabled) return { ok: false, reason: "element is disabled" };
|
|
107
|
+
const want = checked === true;
|
|
108
|
+
let changed = false;
|
|
109
|
+
if (el.checked !== want) {
|
|
110
|
+
try { el.click(); } catch (e) {}
|
|
111
|
+
if (el.checked !== want) { el.checked = want; el.dispatchEvent(new Event("change", { bubbles: true })); }
|
|
112
|
+
changed = true;
|
|
113
|
+
}
|
|
114
|
+
return { ok: true, checked: el.checked, changed: changed };
|
|
115
|
+
}`;
|
|
116
|
+
|
|
117
|
+
// ── Resolver ─────────────────────────────────────────────────────────────────
|
|
118
|
+
|
|
119
|
+
/** Shape returned by the page-side functions above. */
|
|
120
|
+
type PageResult = {
|
|
121
|
+
readonly ok: boolean;
|
|
122
|
+
readonly reason?: string;
|
|
123
|
+
readonly kind?: string;
|
|
124
|
+
readonly value?: unknown;
|
|
125
|
+
readonly text?: string;
|
|
126
|
+
readonly checked?: boolean;
|
|
127
|
+
readonly changed?: boolean;
|
|
128
|
+
readonly options?: ReadonlyArray<{ value: string; text: string }>;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Resolve a ref (CDP backendDOMNodeId) to a live JS handle and invoke a
|
|
133
|
+
* page-side function on it. DOM + Runtime are enabled on every session
|
|
134
|
+
* (see cdp/session.ts enableDomains), so no extra setup is required.
|
|
135
|
+
*/
|
|
136
|
+
const resolveAndCall = async (
|
|
137
|
+
client: BrowserClient,
|
|
138
|
+
ref: number,
|
|
139
|
+
functionDeclaration: string,
|
|
140
|
+
args: ReadonlyArray<unknown>,
|
|
141
|
+
): Promise<Result<PageResult, ToolErr>> => {
|
|
142
|
+
const session = client.session();
|
|
143
|
+
|
|
144
|
+
const resolved = await session.call("DOM.resolveNode", { backendNodeId: ref });
|
|
145
|
+
if (!resolved.success) {
|
|
146
|
+
return err({
|
|
147
|
+
kind: "invalid_state",
|
|
148
|
+
message: `ref ${ref} could not be resolved — the page may have changed. Re-run browser_snapshot to get fresh refs.`,
|
|
149
|
+
details: { ref },
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId;
|
|
153
|
+
if (objectId === undefined) {
|
|
154
|
+
return err({
|
|
155
|
+
kind: "invalid_state",
|
|
156
|
+
message: `ref ${ref} resolved to no JS object — re-run browser_snapshot to get fresh refs.`,
|
|
157
|
+
details: { ref },
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
try {
|
|
162
|
+
const called = await session.call("Runtime.callFunctionOn", {
|
|
163
|
+
objectId,
|
|
164
|
+
functionDeclaration,
|
|
165
|
+
arguments: args.map((v) => ({ value: v })),
|
|
166
|
+
returnByValue: true,
|
|
167
|
+
awaitPromise: true,
|
|
168
|
+
});
|
|
169
|
+
if (!called.success) return err({ kind: "cdp_error", message: called.error.message, details: { ref } });
|
|
170
|
+
const data = called.data as { result?: { value?: unknown }; exceptionDetails?: unknown };
|
|
171
|
+
if (data.exceptionDetails !== undefined) {
|
|
172
|
+
return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(data.exceptionDetails)}`, details: { ref } });
|
|
173
|
+
}
|
|
174
|
+
const value = data.result?.value;
|
|
175
|
+
if (typeof value !== "object" || value === null) {
|
|
176
|
+
return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
|
|
177
|
+
}
|
|
178
|
+
return ok(value as PageResult);
|
|
179
|
+
} finally {
|
|
180
|
+
// Best-effort: release the remote handle so it can be GC'd. Failure is
|
|
181
|
+
// harmless (the page may have navigated), so we ignore the result.
|
|
182
|
+
await session.call("Runtime.releaseObject", { objectId });
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
/** Drop the `ok` flag before stashing a page result in tool `details` — the
|
|
187
|
+
* tool-result envelope reserves `ok` for its own success discriminant. */
|
|
188
|
+
const detailsOf = (ref: number, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
|
|
189
|
+
const { ok: _ok, ...rest } = res;
|
|
190
|
+
return { ref, ...rest, ...(extra ?? {}) };
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
// ── Tools ────────────────────────────────────────────────────────────────────
|
|
194
|
+
|
|
195
|
+
const FieldValue = Type.Union([Type.String(), Type.Boolean()], {
|
|
196
|
+
description: "Value to set. String for text inputs / textareas / selects / contenteditable; boolean for checkboxes & radios.",
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
const FillArgs = Type.Object({
|
|
200
|
+
ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }),
|
|
201
|
+
value: FieldValue,
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
export const fillTool = defineBrowserTool({
|
|
205
|
+
name: "browser_fill",
|
|
206
|
+
label: "Browser Fill",
|
|
207
|
+
description:
|
|
208
|
+
"Fill a single form field by ref. Auto-detects input/textarea/select/checkbox/radio/contenteditable, clears then sets the value, fires input+change events (drives React/Vue controlled inputs correctly), and reads the value back to confirm. Get the ref from browser_snapshot's [ref=N].",
|
|
209
|
+
promptSnippet: "Fill a form field by ref (text, select, checkbox) — React-safe, self-confirming",
|
|
210
|
+
promptGuidelines: [
|
|
211
|
+
"Get the ref from browser_snapshot's [ref=N] for the target field. No click-to-focus needed.",
|
|
212
|
+
"value is a string for text fields/selects/contenteditable, or a boolean for checkboxes/radios.",
|
|
213
|
+
"The result reports the value the field actually holds after filling — check it matches what you intended.",
|
|
214
|
+
"For autocomplete/typeahead that needs real keystrokes, click the field's @(x,y) then use browser_type instead.",
|
|
215
|
+
"If a ref no longer resolves, re-run browser_snapshot to get fresh refs.",
|
|
216
|
+
],
|
|
217
|
+
parameters: FillArgs,
|
|
218
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
219
|
+
const r = await resolveAndCall(client, args.ref, FILL_FN, [args.value]);
|
|
220
|
+
if (!r.success) return r;
|
|
221
|
+
const res = r.data;
|
|
222
|
+
if (!res.ok) {
|
|
223
|
+
return err({ kind: "invalid_state", message: `Could not fill ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
|
|
224
|
+
}
|
|
225
|
+
const wanted = typeof args.value === "boolean" ? undefined : String(args.value);
|
|
226
|
+
const got = res.value !== undefined ? String(res.value) : undefined;
|
|
227
|
+
const mismatch = wanted !== undefined && got !== undefined && got !== wanted;
|
|
228
|
+
let text: string;
|
|
229
|
+
if (mismatch) {
|
|
230
|
+
text = `Filled ref ${args.ref}, but it now reads ${JSON.stringify(got)} (wanted ${JSON.stringify(wanted)})`;
|
|
231
|
+
} else if (got !== undefined) {
|
|
232
|
+
text = `Filled ref ${args.ref} = ${JSON.stringify(got)}`;
|
|
233
|
+
} else if (res.checked !== undefined) {
|
|
234
|
+
text = `Filled ref ${args.ref} (checked=${res.checked})`;
|
|
235
|
+
} else {
|
|
236
|
+
text = `Filled ref ${args.ref}`;
|
|
237
|
+
}
|
|
238
|
+
return ok({ text, details: detailsOf(args.ref, res, { mismatch }) });
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
const FillFormArgs = Type.Object({
|
|
243
|
+
fields: Type.Array(
|
|
244
|
+
Type.Object({
|
|
245
|
+
ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }),
|
|
246
|
+
value: FieldValue,
|
|
247
|
+
}),
|
|
248
|
+
{ minItems: 1, description: "Fields to fill in one batch, each by ref." },
|
|
249
|
+
),
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
export const fillFormTool = defineBrowserTool({
|
|
253
|
+
name: "browser_fill_form",
|
|
254
|
+
label: "Browser Fill Form",
|
|
255
|
+
description:
|
|
256
|
+
"Fill multiple form fields in one call — the efficient way to complete a form. Each field is { ref, value }; refs come from browser_snapshot's [ref=N]. Handles text inputs, textareas, selects, checkboxes, radios, and contenteditable, drives React/Vue controlled inputs correctly, and reports the resulting value of every field.",
|
|
257
|
+
promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming",
|
|
258
|
+
promptGuidelines: [
|
|
259
|
+
"Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.",
|
|
260
|
+
"Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.",
|
|
261
|
+
"The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.",
|
|
262
|
+
"Re-run browser_snapshot afterwards to verify the form state if needed.",
|
|
263
|
+
],
|
|
264
|
+
parameters: FillFormArgs,
|
|
265
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
266
|
+
const results: Array<Readonly<Record<string, unknown>>> = [];
|
|
267
|
+
let okCount = 0;
|
|
268
|
+
const issues: string[] = [];
|
|
269
|
+
for (const f of args.fields) {
|
|
270
|
+
const r = await resolveAndCall(client, f.ref, FILL_FN, [f.value]);
|
|
271
|
+
if (!r.success) {
|
|
272
|
+
results.push({ ref: f.ref, ok: false, error: r.error.message });
|
|
273
|
+
issues.push(`ref ${f.ref} (${r.error.message})`);
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
const res = r.data;
|
|
277
|
+
const wanted = typeof f.value === "boolean" ? undefined : String(f.value);
|
|
278
|
+
const got = res.value !== undefined ? String(res.value) : undefined;
|
|
279
|
+
const mismatch = wanted !== undefined && got !== undefined && got !== wanted;
|
|
280
|
+
const fieldOk = res.ok === true && !mismatch;
|
|
281
|
+
if (fieldOk) okCount++;
|
|
282
|
+
else issues.push(`ref ${f.ref} (${res.ok === false ? (res.reason ?? "failed") : "value mismatch"})`);
|
|
283
|
+
results.push({ ref: f.ref, ok: fieldOk, mismatch, ...detailsOf(f.ref, res) });
|
|
284
|
+
}
|
|
285
|
+
const text =
|
|
286
|
+
`Filled ${okCount}/${args.fields.length} fields` +
|
|
287
|
+
(issues.length > 0 ? `; issues: ${issues.join(", ")}` : "");
|
|
288
|
+
return ok({ text, details: { results } });
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
const SelectOptionArgs = Type.Object({
|
|
293
|
+
ref: Type.Integer({ description: "Element ref of a <select> from browser_snapshot ([ref=N])." }),
|
|
294
|
+
value: Type.Optional(Type.String({ description: "Option value attribute to select." })),
|
|
295
|
+
label: Type.Optional(Type.String({ description: "Visible option text/label to select (used if value is omitted)." })),
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
export const selectOptionTool = defineBrowserTool({
|
|
299
|
+
name: "browser_select_option",
|
|
300
|
+
label: "Browser Select Option",
|
|
301
|
+
description:
|
|
302
|
+
"Select an option in a native <select> dropdown by ref. Matches by option value, then by visible label/text, and fires change. Native selects cannot be driven by clicking coordinates — use this instead. Get the ref from browser_snapshot's [ref=N].",
|
|
303
|
+
promptSnippet: "Choose an option in a <select> dropdown by ref",
|
|
304
|
+
promptGuidelines: [
|
|
305
|
+
"Provide value (the option's value attribute) or label (the visible text). value wins if both given.",
|
|
306
|
+
"On no match, the result lists the available options so you can retry with a valid one.",
|
|
307
|
+
"Use this for native <select> elements — clicking coordinates does not work for OS dropdowns.",
|
|
308
|
+
],
|
|
309
|
+
parameters: SelectOptionArgs,
|
|
310
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
311
|
+
if (args.value === undefined && args.label === undefined) {
|
|
312
|
+
return err({ kind: "invalid_state", message: "Provide value or label to select an option." });
|
|
313
|
+
}
|
|
314
|
+
const r = await resolveAndCall(client, args.ref, SELECT_FN, [args.value ?? null, args.label ?? null]);
|
|
315
|
+
if (!r.success) return r;
|
|
316
|
+
const res = r.data;
|
|
317
|
+
if (!res.ok) {
|
|
318
|
+
return err({ kind: "invalid_state", message: `Could not select option on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
|
|
319
|
+
}
|
|
320
|
+
return ok({
|
|
321
|
+
text: `Selected ${JSON.stringify(res.text)} (value=${JSON.stringify(res.value)}) on ref ${args.ref}`,
|
|
322
|
+
details: detailsOf(args.ref, res),
|
|
323
|
+
});
|
|
324
|
+
},
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
const SetCheckedArgs = Type.Object({
|
|
328
|
+
ref: Type.Integer({ description: "Element ref of a checkbox or radio from browser_snapshot ([ref=N])." }),
|
|
329
|
+
checked: Type.Boolean({ description: "Desired checked state." }),
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
export const setCheckedTool = defineBrowserTool({
|
|
333
|
+
name: "browser_set_checked",
|
|
334
|
+
label: "Browser Set Checked",
|
|
335
|
+
description:
|
|
336
|
+
"Set a checkbox or radio to a desired checked state by ref. Only acts if the current state differs, fires change, and reports the final state. Get the ref from browser_snapshot's [ref=N].",
|
|
337
|
+
promptSnippet: "Set a checkbox/radio checked state by ref",
|
|
338
|
+
promptGuidelines: [
|
|
339
|
+
"checked:true ticks the box, checked:false unticks it. Idempotent — no-op if already in the desired state.",
|
|
340
|
+
"For radios, set checked:true on the option you want; the group deselects the others.",
|
|
341
|
+
],
|
|
342
|
+
parameters: SetCheckedArgs,
|
|
343
|
+
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
344
|
+
const r = await resolveAndCall(client, args.ref, CHECK_FN, [args.checked]);
|
|
345
|
+
if (!r.success) return r;
|
|
346
|
+
const res = r.data;
|
|
347
|
+
if (!res.ok) {
|
|
348
|
+
return err({ kind: "invalid_state", message: `Could not set checked on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
|
|
349
|
+
}
|
|
350
|
+
return ok({
|
|
351
|
+
text: `Set ref ${args.ref} checked=${res.checked}` + (res.changed === false ? " (already)" : ""),
|
|
352
|
+
details: detailsOf(args.ref, res),
|
|
353
|
+
});
|
|
354
|
+
},
|
|
355
|
+
});
|