pi-browser-harness 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +62 -0
  2. package/README.md +102 -1
  3. package/package.json +13 -7
  4. package/skills/pi-browser-harness/SKILL.md +84 -0
  5. package/src/cdp/commands.ts +204 -14
  6. package/src/cdp/console-buffer.ts +37 -23
  7. package/src/cdp/discovery.ts +72 -37
  8. package/src/cdp/events.ts +8 -3
  9. package/src/cdp/network-buffer.ts +38 -32
  10. package/src/cdp/ownership.ts +22 -4
  11. package/src/cdp/record-store.ts +2 -2
  12. package/src/cdp/session.ts +320 -217
  13. package/src/cdp/target-factory.ts +34 -15
  14. package/src/cdp/types.ts +17 -1
  15. package/src/client.ts +76 -43
  16. package/src/daemon/bridge.ts +115 -191
  17. package/src/daemon/connection.ts +323 -0
  18. package/src/daemon/index.ts +77 -48
  19. package/src/daemon/lines.ts +40 -0
  20. package/src/daemon/main.ts +12 -0
  21. package/src/daemon/protocol.ts +55 -25
  22. package/src/daemon/server.ts +100 -63
  23. package/src/daemon/socket.ts +10 -0
  24. package/src/daemon/spawn.ts +84 -27
  25. package/src/daemon/transport.ts +88 -52
  26. package/src/domains/ax-tree.ts +8 -8
  27. package/src/domains/cdp-call.ts +14 -7
  28. package/src/domains/click.ts +2 -7
  29. package/src/domains/concurrency.ts +50 -0
  30. package/src/domains/console.ts +20 -66
  31. package/src/domains/drag.ts +26 -28
  32. package/src/domains/element-call.ts +102 -68
  33. package/src/domains/files.ts +64 -11
  34. package/src/domains/form-batch.ts +13 -11
  35. package/src/domains/form.ts +79 -116
  36. package/src/domains/headers.ts +89 -0
  37. package/src/domains/history.ts +7 -7
  38. package/src/domains/io.ts +35 -0
  39. package/src/domains/isolated-tab.ts +6 -12
  40. package/src/domains/js.ts +68 -53
  41. package/src/domains/keyboard.ts +16 -18
  42. package/src/domains/navigate.ts +102 -69
  43. package/src/domains/network.ts +56 -98
  44. package/src/domains/page/fill.page.ts +139 -0
  45. package/src/domains/page/form.page.ts +61 -0
  46. package/src/domains/page/inject.ts +37 -0
  47. package/src/domains/page/keyboard.page.ts +16 -0
  48. package/src/domains/page/live-value.page.ts +10 -0
  49. package/src/domains/page/read-page.page.ts +51 -0
  50. package/src/domains/page/serp.page.ts +39 -0
  51. package/src/domains/page.ts +19 -58
  52. package/src/domains/poll.ts +42 -0
  53. package/src/domains/readpage/read-page.ts +48 -54
  54. package/src/domains/ref-resolve.ts +43 -29
  55. package/src/domains/ref-signature.ts +11 -0
  56. package/src/domains/render.ts +12 -23
  57. package/src/domains/screenshot-capture.ts +4 -2
  58. package/src/domains/screenshot.ts +13 -14
  59. package/src/domains/scroll.ts +4 -5
  60. package/src/domains/search/google-serp.ts +1 -1
  61. package/src/domains/search/web-search.ts +38 -34
  62. package/src/domains/setup.ts +0 -2
  63. package/src/domains/snapshot.ts +72 -77
  64. package/src/domains/tab-resolve.ts +36 -0
  65. package/src/domains/tabs.ts +19 -78
  66. package/src/domains/target.ts +1 -1
  67. package/src/domains/viewport.ts +0 -1
  68. package/src/index.ts +100 -39
  69. package/src/profile/bind.ts +45 -14
  70. package/src/profile/browser-process.ts +50 -12
  71. package/src/profile/command.ts +20 -12
  72. package/src/profile/launch.ts +13 -9
  73. package/src/profile/list.ts +9 -13
  74. package/src/profile/store.ts +2 -1
  75. package/src/prompt.ts +42 -117
  76. package/src/registry.ts +68 -4
  77. package/src/schemas/parse.ts +2 -2
  78. package/src/setup.ts +28 -19
  79. package/src/state.ts +33 -2
  80. package/src/util/debug.ts +7 -1
  81. package/src/util/guards.ts +3 -0
  82. package/src/util/mutex.ts +1 -1
  83. package/src/util/paths.ts +12 -1
  84. package/src/util/result.ts +8 -0
  85. package/src/util/sharp-shim.ts +18 -11
  86. package/src/util/sleep.ts +18 -0
  87. package/src/util/tool.ts +163 -114
  88. package/src/util/truncate.ts +1 -1
  89. package/tsconfig.json +2 -1
  90. package/src/domains/fill-engine.ts +0 -80
@@ -2,7 +2,7 @@ import { Type } from "typebox";
2
2
  import type { BrowserClient } from "../client";
3
3
  import { type Result, err, ok } from "../util/result";
4
4
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
5
- import { cdpCall } from "./cdp-call";
5
+ import { cdpCall, fromClient } from "./cdp-call";
6
6
 
7
7
  type HistoryEntry = { readonly id: number; readonly url: string; readonly title: string };
8
8
  type History = { readonly entries: ReadonlyArray<HistoryEntry>; readonly currentIndex: number };
@@ -18,7 +18,7 @@ export const goBackTool = defineBrowserTool({
18
18
  label: "Browser Go Back",
19
19
  description: "Navigate back one page in history.",
20
20
  promptSnippet: "Go back one page",
21
- promptGuidelines: ["Use to return to the previous page after navigating."],
21
+ promptGuidelines: [],
22
22
  parameters: Type.Object({}),
23
23
  concurrency: "serialized",
24
24
  async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
@@ -38,7 +38,7 @@ export const goForwardTool = defineBrowserTool({
38
38
  label: "Browser Go Forward",
39
39
  description: "Navigate forward one page in history.",
40
40
  promptSnippet: "Go forward one page",
41
- promptGuidelines: ["Use to undo a browser_go_back."],
41
+ promptGuidelines: [],
42
42
  parameters: Type.Object({}),
43
43
  concurrency: "serialized",
44
44
  async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
@@ -58,15 +58,15 @@ export const reloadTool = defineBrowserTool({
58
58
  label: "Browser Reload",
59
59
  description: "Reload the current page.",
60
60
  promptSnippet: "Reload the page",
61
- promptGuidelines: ["Use to refresh the page, e.g. after server-side changes."],
61
+ promptGuidelines: [],
62
62
  parameters: Type.Object({}),
63
63
  concurrency: "serialized",
64
64
  async handler(_a, { client }): Promise<Result<ToolOk, ToolErr>> {
65
65
  const r = await cdpCall(client, "Page.reload", {});
66
66
  if (!r.success) return r;
67
- const info = await client.pageInfo();
68
- if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
69
- if ("dialog" in info.data) {
67
+ const info = fromClient(await client.pageInfo(), "pageInfo");
68
+ if (!info.success) return info;
69
+ if (info.data.kind === "dialog") {
70
70
  return ok({
71
71
  text: `Reloaded. ⚠️ Dialog open: ${info.data.dialog.type}`,
72
72
  details: { dialog: info.data.dialog },
@@ -0,0 +1,35 @@
1
+ import { readFile, stat, writeFile } from "node:fs/promises";
2
+ import type { Stats } from "node:fs";
3
+ import { type Result, err, ok } from "../util/result";
4
+ import type { ToolErr } from "../util/tool";
5
+
6
+ const ioErr = (what: string, path: string, e: unknown): ToolErr => ({
7
+ kind: "io_error",
8
+ message: `${what}: ${path} (${e instanceof Error ? e.message : String(e)})`,
9
+ details: { path },
10
+ });
11
+
12
+ export const writeFileResult = async (path: string, data: Uint8Array | string): Promise<Result<void, ToolErr>> => {
13
+ try {
14
+ await writeFile(path, data);
15
+ return ok(undefined);
16
+ } catch (e) {
17
+ return err(ioErr("Cannot write file", path, e));
18
+ }
19
+ };
20
+
21
+ export const readFileResult = async (path: string): Promise<Result<Buffer, ToolErr>> => {
22
+ try {
23
+ return ok(await readFile(path));
24
+ } catch (e) {
25
+ return err(ioErr("Cannot read file", path, e));
26
+ }
27
+ };
28
+
29
+ export const statResult = async (path: string): Promise<Result<Stats, ToolErr>> => {
30
+ try {
31
+ return ok(await stat(path));
32
+ } catch (e) {
33
+ return err(ioErr("Cannot stat file", path, e));
34
+ }
35
+ };
@@ -1,10 +1,10 @@
1
1
  import type { BrowserClient } from "../client";
2
2
  import { type Result, err, ok } from "../util/result";
3
- import { safeJs } from "../util/js-template";
4
3
  import { ensureHarnessWindow, openHarnessTab } from "../cdp/target-factory";
5
4
  import { evaluateJson } from "../cdp/session";
6
5
  import type { ToolErr } from "../util/tool";
7
- import { cdpCallBrowser, cdpCallOnTarget, cdpErrToToolErr, evalJs } from "./cdp-call";
6
+ import { cdpCallBrowser, cdpCallOnTarget, cdpErrToToolErr } from "./cdp-call";
7
+ import { pollUntil, READY_STATE_COMPLETE } from "./poll";
8
8
 
9
9
  export type IsolatedTab = {
10
10
  readonly targetId: string;
@@ -12,7 +12,6 @@ export type IsolatedTab = {
12
12
  };
13
13
 
14
14
  const EVAL_TIMEOUT_MS = 15_000;
15
- const READY_POLL_INTERVAL_MS = 50;
16
15
 
17
16
  export const openIsolatedTab = async (client: BrowserClient): Promise<Result<IsolatedTab, ToolErr>> => {
18
17
  // Routed through the target factory so an isolated tab lands in the pinned profile, not another profile's cookies.
@@ -60,14 +59,9 @@ export const waitForIsolatedLoad = async (
60
59
  timeoutMs: number,
61
60
  signal?: AbortSignal,
62
61
  ): Promise<Result<void, ToolErr>> => {
63
- const start = Date.now();
64
- while (Date.now() - start < timeoutMs) {
65
- if (signal?.aborted) return err({ kind: "internal", message: "aborted" });
66
- const r = await evalJs(client, safeJs`document.readyState`, tab.sessionId);
67
- if (r.success && r.data === "complete") return ok(undefined);
68
- await new Promise((res) => setTimeout(res, READY_POLL_INTERVAL_MS));
69
- }
70
- return err({ kind: "timeout", message: `page did not finish loading in ${Math.round(timeoutMs / 1000)}s` });
62
+ const r = await pollUntil(client, READY_STATE_COMPLETE, { timeoutMs, signal, sessionId: tab.sessionId, what: "page load" });
63
+ if (!r.success) return r;
64
+ return ok(undefined);
71
65
  };
72
66
 
73
67
  export const isJsonText = (v: unknown): v is string => typeof v === "string";
@@ -87,5 +81,5 @@ export const evalInIsolatedTab = async <T>(
87
81
  };
88
82
 
89
83
  export const closeIsolatedTab = async (client: BrowserClient, tab: IsolatedTab): Promise<void> => {
90
- await client.closeTab(tab.targetId).catch(() => {});
84
+ await client.closeTab(tab.targetId);
91
85
  };
package/src/domains/js.ts CHANGED
@@ -1,30 +1,56 @@
1
1
  import { readFile } from "node:fs/promises";
2
2
  import { resolve, isAbsolute } from "node:path";
3
+ import { isPathWithin } from "../util/paths";
3
4
  import { tmpdir } from "node:os";
4
5
  import { createRequire } from "node:module";
5
6
  import { Type } from "typebox";
6
- import { Markdown, Text } from "@mariozechner/pi-tui";
7
+ import { Markdown } from "@mariozechner/pi-tui";
7
8
  import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
8
9
  import { type Result, err, ok } from "../util/result";
9
10
  import { formatBytes } from "../util/truncate";
10
- import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
11
+ import { type DetailsOf, defineBrowserTool, isJsonObject, type ToolErr, type ToolOk } from "../util/tool";
11
12
  import { evalJs } from "./cdp-call";
12
13
  import { applyTruncation } from "../util/truncate";
13
- import { asNumber, asRecord, asString, isRecord } from "../util/guards";
14
+ import { isRecord } from "../util/guards";
14
15
 
15
16
  const COMPACT_PREVIEW_BYTES = 120;
16
17
 
17
- // The AsyncFunction constructor is the documented way to compile user source; a plain `unknown` cast would lose the constructor signature, so this typed cast stays.
18
- const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor as
19
- new (...args: ReadonlyArray<string>) => (...args: ReadonlyArray<unknown>) => Promise<unknown>;
18
+ type AsyncFunctionConstructor = new (...args: ReadonlyArray<string>) => (...args: ReadonlyArray<unknown>) => Promise<unknown>;
19
+
20
+ // The AsyncFunction constructor is the documented way to compile user source. It is only reachable through
21
+ // Object.getPrototypeOf, which returns `any`; this is the one sanctioned cast (budgeted in boundary-allowlist.ts)
22
+ // because a plain `unknown` would lose the constructor signature and nothing can narrow back to it.
23
+ /* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-type-assertion -- the constructor is only reachable through an `any`-typed prototype lookup */
24
+ const AsyncFunction = Object.getPrototypeOf(async () => { /* only its prototype is wanted */ }).constructor as
25
+ AsyncFunctionConstructor;
26
+ /* eslint-enable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-type-assertion */
20
27
 
21
28
  const requireFromHere = createRequire(import.meta.url);
22
29
 
30
+ // Pretty-prints for the expanded renderer; a truncated (unparseable) payload is shown as-is.
31
+ const prettyJson = (text: string): string => {
32
+ try {
33
+ const parsed: unknown = JSON.parse(text);
34
+ return JSON.stringify(parsed, null, 2);
35
+ } catch {
36
+ return text;
37
+ }
38
+ };
39
+
23
40
  const ExecuteJsArgs = Type.Object({
24
41
  expression: Type.String({ description: "JavaScript expression. `return X` is auto-wrapped in an IIFE." }),
25
42
  targetId: Type.Optional(Type.String({ description: "Optional iframe targetId; default = current page." })),
26
43
  });
27
44
 
45
+ const ExecuteJsDetailsSchema = Type.Object({
46
+ valueLength: Type.Number(),
47
+ preview: Type.String(),
48
+ isJson: Type.Boolean(),
49
+ fullOutputPath: Type.Optional(Type.String()),
50
+ });
51
+
52
+ export type ExecuteJsDetails = DetailsOf<typeof ExecuteJsDetailsSchema>;
53
+
28
54
  export const executeJsTool = defineBrowserTool({
29
55
  name: "browser_execute_js",
30
56
  label: "Browser Execute JS",
@@ -35,52 +61,39 @@ export const executeJsTool = defineBrowserTool({
35
61
  "DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.",
36
62
  "For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.",
37
63
  "For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.",
38
- "DO NOT call browser_screenshot to read a value or check element state. Use this tool.",
39
- "`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.",
40
64
  "For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.",
41
65
  "Result must be JSON-serializable (Runtime.evaluate returnByValue=true).",
42
66
  ],
43
67
  parameters: ExecuteJsArgs,
68
+ details: ExecuteJsDetailsSchema,
44
69
  concurrency: "parallel",
45
- async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
70
+ async handler(args, { client }): Promise<Result<ToolOk<ExecuteJsDetails>, ToolErr>> {
46
71
  const r = await evalJs(client, args.expression, args.targetId);
47
72
  if (!r.success) return r;
48
73
  const valueStr = r.data === undefined ? "undefined" : JSON.stringify(r.data);
74
+ // The full value lives in the text (and in fullOutputPath when truncated); details carry only what the
75
+ // renderer needs so a large result is not persisted twice.
76
+ const isJson = r.data !== undefined;
49
77
  const truncated = await applyTruncation(valueStr, "js");
50
- let pretty: string | undefined;
51
- try {
52
- const parsed: unknown = JSON.parse(valueStr);
53
- pretty = JSON.stringify(parsed, null, 2);
54
- } catch {
55
- pretty = undefined;
56
- }
78
+ const preview = valueStr.length > COMPACT_PREVIEW_BYTES ? valueStr.slice(0, COMPACT_PREVIEW_BYTES) + "…" : valueStr;
57
79
  return ok({
58
80
  text: truncated.text,
59
81
  details: {
60
82
  valueLength: valueStr.length,
61
- full: valueStr.length > 200_000 ? valueStr.slice(0, 200_000) : valueStr,
62
- ...(pretty !== undefined ? { pretty: pretty.length > 200_000 ? pretty.slice(0, 200_000) : pretty } : {}),
83
+ preview,
84
+ isJson,
63
85
  ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
64
86
  },
65
87
  });
66
88
  },
67
89
 
68
- renderResult(result, expanded, theme) {
69
- const raw = asRecord(result.details);
70
- if (raw === undefined) return new Text(theme.fg("error", "execute_js: no details"), 0, 0);
71
- const details = {
72
- valueLength: asNumber(raw["valueLength"]),
73
- full: asString(raw["full"]),
74
- pretty: asString(raw["pretty"]),
75
- fullOutputPath: asString(raw["fullOutputPath"]),
76
- };
77
-
78
- const len = details.valueLength ?? 0;
79
- const full = details.full ?? "";
80
- const isJson = details.pretty !== undefined;
90
+ renderResult(result, expanded) {
91
+ const { valueLength: len, preview, isJson, fullOutputPath } = result.details;
92
+ const content = result.content[0];
93
+ const fullText = content?.type === "text" ? content.text : preview;
94
+ const body = isJson ? prettyJson(fullText) : fullText;
81
95
 
82
96
  if (!expanded) {
83
- const preview = full.length > COMPACT_PREVIEW_BYTES ? full.slice(0, COMPACT_PREVIEW_BYTES) + "…" : full;
84
97
  const md = [
85
98
  `**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`,
86
99
  "```",
@@ -91,10 +104,9 @@ export const executeJsTool = defineBrowserTool({
91
104
  return new Markdown(md, 0, 0, getMarkdownTheme());
92
105
  }
93
106
 
94
- const body = details.pretty ?? full;
95
107
  const fence = isJson ? "```json" : "```";
96
- const tail = details.fullOutputPath
97
- ? `\n\nFull value at \`${details.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
108
+ const tail = fullOutputPath !== undefined
109
+ ? `\n\nFull value at \`${fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
98
110
  : `\n\n${keyHint("app.tools.expand", "to collapse")}`;
99
111
  const md = [`**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`, fence, body, "```", tail].join("\n");
100
112
  return new Markdown(md, 0, 0, getMarkdownTheme());
@@ -121,10 +133,7 @@ const allowedRoots = (): ReadonlyArray<string> => {
121
133
  return [tmpdir(), process.cwd(), ...(env !== undefined ? [env] : [])].map((d) => resolve(d));
122
134
  };
123
135
 
124
- const isPathAllowed = (p: string): boolean => {
125
- const abs = resolve(p);
126
- return allowedRoots().some((root) => abs === root || abs.startsWith(`${root}/`));
127
- };
136
+ const isPathAllowed = (p: string): boolean => allowedRoots().some((root) => isPathWithin(root, p));
128
137
 
129
138
  const MAX_SOURCE_BYTES = 1_000_000;
130
139
 
@@ -140,9 +149,7 @@ export const runScriptTool = defineBrowserTool({
140
149
  "Execute a temporary JavaScript script with full Node.js + browser-daemon access. Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR. Mandatory timeout. The script is full RCE on the harness's process — only invoke scripts you wrote and reviewed.",
141
150
  promptSnippet: "Run a temporary script with daemon + Node access (security-bounded)",
142
151
  promptGuidelines: [
143
- "Write the script with the write tool first; pass its absolute path.",
144
- "Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR — otherwise rejected.",
145
- "Default timeout is 60s; pass timeoutMs to extend (max 600s).",
152
+ "Write the script with the write tool first; pass its absolute path. Use the OS temp directory rather than hardcoding /tmp.",
146
153
  "Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.",
147
154
  "Script MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }. Throw on errors.",
148
155
  ],
@@ -150,11 +157,11 @@ export const runScriptTool = defineBrowserTool({
150
157
  concurrency: "parallel",
151
158
  async handler(args, { client, signal, onUpdate, extensionCtx }): Promise<Result<ToolOk, ToolErr>> {
152
159
  if (!isAbsolute(args.path)) {
153
- return err({ kind: "invalid_state", message: "Script path must be absolute" });
160
+ return err({ kind: "invalid_args", message: "Script path must be absolute" });
154
161
  }
155
162
  if (!isPathAllowed(args.path)) {
156
163
  return err({
157
- kind: "invalid_state",
164
+ kind: "invalid_args",
158
165
  message: `Script path outside allowed directories (allowed: ${allowedRoots().join(", ")})`,
159
166
  });
160
167
  }
@@ -167,9 +174,9 @@ export const runScriptTool = defineBrowserTool({
167
174
  message: `Failed to read script: ${e instanceof Error ? e.message : String(e)}`,
168
175
  });
169
176
  }
170
- if (source.length === 0) return err({ kind: "invalid_state", message: "Script is empty" });
177
+ if (source.length === 0) return err({ kind: "invalid_args", message: "Script is empty" });
171
178
  if (source.length > MAX_SOURCE_BYTES) {
172
- return err({ kind: "invalid_state", message: `Script exceeds ${MAX_SOURCE_BYTES}B size cap` });
179
+ return err({ kind: "invalid_args", message: `Script exceeds ${MAX_SOURCE_BYTES}B size cap` });
173
180
  }
174
181
 
175
182
  let executeFn: (...a: ReadonlyArray<unknown>) => Promise<unknown>;
@@ -181,16 +188,16 @@ export const runScriptTool = defineBrowserTool({
181
188
  );
182
189
  } catch (e) {
183
190
  return err({
184
- kind: "invalid_state",
191
+ kind: "invalid_args",
185
192
  message: `Syntax error: ${e instanceof Error ? e.message : String(e)}`,
186
193
  });
187
194
  }
188
195
 
189
196
  const timeoutMs = args.timeoutMs ?? 60_000;
190
197
  const ac = new AbortController();
191
- const onAbort = (): void => ac.abort();
198
+ const onAbort = (): void => { ac.abort(); };
192
199
  if (signal !== undefined) signal.addEventListener("abort", onAbort, { once: true });
193
- const timeoutTimer = setTimeout(() => ac.abort(), timeoutMs);
200
+ const timeoutTimer = setTimeout(onAbort, timeoutMs);
194
201
 
195
202
  try {
196
203
  const scriptPromise = executeFn(
@@ -206,13 +213,17 @@ export const runScriptTool = defineBrowserTool({
206
213
  if (!isRecord(first)) return;
207
214
  const txt = first["text"];
208
215
  if (typeof txt !== "string") return;
209
- try { onUpdate({ text: txt }); } catch {}
216
+ try {
217
+ onUpdate({ text: txt });
218
+ } catch {
219
+ // Progress is best-effort; a renderer failure must not abort the script.
220
+ }
210
221
  },
211
- extensionCtx ?? { cwd: process.cwd() },
222
+ extensionCtx,
212
223
  console, fetch, JSON, Buffer, setTimeout, clearTimeout,
213
224
  );
214
225
  const abortPromise = new Promise<never>((_, reject) => {
215
- ac.signal.addEventListener("abort", () => reject(new Error("script aborted (timeout or cancellation)")), { once: true });
226
+ ac.signal.addEventListener("abort", () => { reject(new Error("script aborted (timeout or cancellation)")); }, { once: true });
216
227
  });
217
228
  const result = await Promise.race([scriptPromise, abortPromise]);
218
229
  clearTimeout(timeoutTimer);
@@ -230,7 +241,11 @@ export const runScriptTool = defineBrowserTool({
230
241
  }
231
242
  const textOut = content.map((c) => c.text).join("\n");
232
243
  const details = result["details"];
233
- return isRecord(details) ? ok({ text: textOut, details }) : ok({ text: textOut });
244
+ if (details === undefined) return ok({ text: textOut });
245
+ if (!isJsonObject(details)) {
246
+ return err({ kind: "invalid_state", message: "Script details must be a plain JSON object (no Map, function or undefined values)" });
247
+ }
248
+ return ok({ text: textOut, details });
234
249
  } catch (e) {
235
250
  clearTimeout(timeoutTimer);
236
251
  if (signal !== undefined) signal.removeEventListener("abort", onAbort);
@@ -1,14 +1,22 @@
1
1
  import { Type } from "typebox";
2
+ import type { ParamsOf } from "../cdp/commands";
2
3
  import { safeJs } from "../util/js-template";
3
4
  import { type Result, err, ok } from "../util/result";
4
5
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
5
6
  import { cdpCall, evalJs } from "./cdp-call";
6
- import { resolveRefToObjectId } from "./ref-resolve";
7
+ import { runOnElement } from "./element-call";
8
+ import { elementFunctionSource } from "./page/inject";
9
+ import { dispatchKeyEvent, type KeyEventSpec } from "./page/keyboard.page";
10
+
11
+ const DISPATCH_KEY_FN = elementFunctionSource(dispatchKeyEvent);
7
12
 
8
13
  const VIRTUAL_KEY_CODES: Readonly<Record<string, number>> = {
9
14
  Enter: 13, Tab: 9, Backspace: 8, Escape: 27, Delete: 46, " ": 32,
10
15
  ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40,
11
16
  Home: 36, End: 35, PageUp: 33, PageDown: 34,
17
+ Insert: 45, Shift: 16, Control: 17, Alt: 18, Meta: 91, CapsLock: 20, ContextMenu: 93,
18
+ F1: 112, F2: 113, F3: 114, F4: 115, F5: 116, F6: 117,
19
+ F7: 118, F8: 119, F9: 120, F10: 121, F11: 122, F12: 123,
12
20
  };
13
21
 
14
22
  const virtualKeyCode = (key: string): number =>
@@ -31,7 +39,6 @@ export const typeTool = defineBrowserTool({
31
39
  description: "Type text into the currently focused element. Use browser_click first to focus an input field.",
32
40
  promptSnippet: "Type text into the focused element",
33
41
  promptGuidelines: [
34
- "Use browser_type to enter text. Click on an input field with browser_click first to focus it.",
35
42
  "For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.",
36
43
  ],
37
44
  parameters: TypeArgs,
@@ -84,9 +91,7 @@ export const pressKeyTool = defineBrowserTool({
84
91
  promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)",
85
92
  promptGuidelines: [
86
93
  "Use browser_press_key for keyboard shortcuts and navigation keys.",
87
- "Special key names: Enter, Tab, Backspace, Escape, Delete, ArrowLeft, ArrowUp, ArrowRight, ArrowDown, Home, End, PageUp, PageDown.",
88
- "Use Space as ' ' (a single space character).",
89
- "Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
94
+ "browser_press_key modifiers combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
90
95
  ],
91
96
  parameters: PressKeyArgs,
92
97
  concurrency: "serialized",
@@ -95,7 +100,7 @@ export const pressKeyTool = defineBrowserTool({
95
100
  const code = virtualKeyCode(k);
96
101
  const modifiers = args.modifiers ?? 0;
97
102
  const isChar = k.length === 1;
98
- const downParams: Record<string, unknown> = {
103
+ const downParams: ParamsOf<"Input.dispatchKeyEvent"> = {
99
104
  type: "keyDown",
100
105
  key: k,
101
106
  code: k,
@@ -138,11 +143,9 @@ export const dispatchKeyTool = defineBrowserTool({
138
143
  "Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
139
144
  promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector",
140
145
  promptGuidelines: [
141
- "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.",
142
- "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).",
146
+ "Does NOT insert text (use browser_type or browser_press_key for actual typing).",
143
147
  "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
144
148
  "The event carries keyCode/which (e.g. 13 for Enter) for legacy handlers, but is untrusted (isTrusted === false) — a few libraries may still ignore it.",
145
- "eventType defaults to 'keydown'.",
146
149
  ],
147
150
  parameters: DispatchKeyArgs,
148
151
  concurrency: "serialized",
@@ -152,14 +155,9 @@ export const dispatchKeyTool = defineBrowserTool({
152
155
  // Legacy handlers branch on e.keyCode/e.which rather than e.key, so populate both.
153
156
  const code = virtualKeyCode(args.key);
154
157
  if (args.ref !== undefined) {
155
- const objectId = await resolveRefToObjectId(client, args.ref);
156
- if (!objectId.success) return objectId;
157
- const r = await cdpCall(client, "Runtime.callFunctionOn", {
158
- objectId: objectId.data,
159
- functionDeclaration: `function (type, key, keyCode) { this.dispatchEvent(new KeyboardEvent(type, { key, keyCode, which: keyCode, bubbles: true, cancelable: true })); return 1; }`,
160
- arguments: [{ value: eventType }, { value: args.key }, { value: code }],
161
- returnByValue: true,
162
- });
158
+ // Through runOnElement so the resolved RemoteObject is released after the call.
159
+ const spec: KeyEventSpec = { type: eventType, key: args.key, code };
160
+ const r = await runOnElement(client, { ref: args.ref, functionDeclaration: DISPATCH_KEY_FN, args: [spec] });
163
161
  if (!r.success) return r;
164
162
  return ok({
165
163
  text: `Dispatched ${eventType}('${args.key}') on ${target}`,
@@ -167,7 +165,7 @@ export const dispatchKeyTool = defineBrowserTool({
167
165
  });
168
166
  }
169
167
  if (args.selector === undefined) {
170
- return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
168
+ return err({ kind: "invalid_args", message: "Provide either `ref` or `selector`." });
171
169
  }
172
170
  const expr = safeJs`
173
171
  (() => {