pi-feats 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +24 -1
  2. package/THIRD_PARTY_NOTICES.md +12 -0
  3. package/assets/pi-feats-preview.png +0 -0
  4. package/extensions/api-server/application-log-store.ts +31 -0
  5. package/extensions/api-server/application-runtime.ts +80 -18
  6. package/extensions/api-server/application-store.ts +6 -2
  7. package/extensions/api-server/profile-store.ts +47 -20
  8. package/extensions/api-server/server.ts +182 -47
  9. package/extensions/browser-harness/cdp/commands.ts +260 -0
  10. package/extensions/browser-harness/cdp/console-buffer.ts +188 -0
  11. package/extensions/browser-harness/cdp/discovery.ts +198 -0
  12. package/extensions/browser-harness/cdp/errors.ts +23 -0
  13. package/extensions/browser-harness/cdp/events.ts +170 -0
  14. package/extensions/browser-harness/cdp/network-buffer.ts +146 -0
  15. package/extensions/browser-harness/cdp/ownership.ts +73 -0
  16. package/extensions/browser-harness/cdp/record-store.ts +61 -0
  17. package/extensions/browser-harness/cdp/session.ts +403 -0
  18. package/extensions/browser-harness/cdp/steel-transport.ts +151 -0
  19. package/extensions/browser-harness/cdp/target-factory.ts +153 -0
  20. package/extensions/browser-harness/cdp/types.ts +82 -0
  21. package/extensions/browser-harness/client.ts +366 -0
  22. package/extensions/browser-harness/daemon/bridge.ts +531 -0
  23. package/extensions/browser-harness/daemon/index.ts +86 -0
  24. package/extensions/browser-harness/daemon/protocol.ts +110 -0
  25. package/extensions/browser-harness/daemon/queue.ts +85 -0
  26. package/extensions/browser-harness/daemon/server.ts +170 -0
  27. package/extensions/browser-harness/daemon/spawn.ts +124 -0
  28. package/extensions/browser-harness/daemon/transport.ts +178 -0
  29. package/extensions/browser-harness/domains/ax-tree.ts +179 -0
  30. package/extensions/browser-harness/domains/box.ts +51 -0
  31. package/extensions/browser-harness/domains/cdp-call.ts +55 -0
  32. package/extensions/browser-harness/domains/click.ts +97 -0
  33. package/extensions/browser-harness/domains/console.ts +219 -0
  34. package/extensions/browser-harness/domains/dialog.ts +34 -0
  35. package/extensions/browser-harness/domains/drag.ts +67 -0
  36. package/extensions/browser-harness/domains/element-call.ts +117 -0
  37. package/extensions/browser-harness/domains/files.ts +219 -0
  38. package/extensions/browser-harness/domains/fill-engine.ts +80 -0
  39. package/extensions/browser-harness/domains/form-batch.ts +65 -0
  40. package/extensions/browser-harness/domains/form.ts +283 -0
  41. package/extensions/browser-harness/domains/headers.ts +89 -0
  42. package/extensions/browser-harness/domains/history.ts +80 -0
  43. package/extensions/browser-harness/domains/isolated-tab.ts +91 -0
  44. package/extensions/browser-harness/domains/js.ts +241 -0
  45. package/extensions/browser-harness/domains/keyboard.ts +197 -0
  46. package/extensions/browser-harness/domains/navigate.ts +161 -0
  47. package/extensions/browser-harness/domains/network.ts +301 -0
  48. package/extensions/browser-harness/domains/page.ts +154 -0
  49. package/extensions/browser-harness/domains/readpage/read-page.ts +174 -0
  50. package/extensions/browser-harness/domains/readpage/readability.ts +65 -0
  51. package/extensions/browser-harness/domains/ref-resolve.ts +118 -0
  52. package/extensions/browser-harness/domains/render.ts +43 -0
  53. package/extensions/browser-harness/domains/screenshot-capture.ts +80 -0
  54. package/extensions/browser-harness/domains/screenshot.ts +76 -0
  55. package/extensions/browser-harness/domains/scroll.ts +59 -0
  56. package/extensions/browser-harness/domains/search/google-serp.ts +85 -0
  57. package/extensions/browser-harness/domains/search/web-search.ts +144 -0
  58. package/extensions/browser-harness/domains/setup.ts +35 -0
  59. package/extensions/browser-harness/domains/snapshot.ts +256 -0
  60. package/extensions/browser-harness/domains/tabs.ts +202 -0
  61. package/extensions/browser-harness/domains/target.ts +35 -0
  62. package/extensions/browser-harness/domains/viewport.ts +42 -0
  63. package/extensions/browser-harness/index.ts +88 -0
  64. package/extensions/browser-harness/profile/bind.ts +90 -0
  65. package/extensions/browser-harness/profile/browser-process.ts +320 -0
  66. package/extensions/browser-harness/profile/command.ts +189 -0
  67. package/extensions/browser-harness/profile/launch.ts +80 -0
  68. package/extensions/browser-harness/profile/list.ts +131 -0
  69. package/extensions/browser-harness/profile/paths.ts +103 -0
  70. package/extensions/browser-harness/profile/store.ts +84 -0
  71. package/extensions/browser-harness/prompt.ts +16 -0
  72. package/extensions/browser-harness/registry.ts +80 -0
  73. package/extensions/browser-harness/schemas/common.ts +11 -0
  74. package/extensions/browser-harness/schemas/parse.ts +23 -0
  75. package/extensions/browser-harness/setup.ts +47 -0
  76. package/extensions/browser-harness/state.ts +51 -0
  77. package/extensions/browser-harness/steel.ts +215 -0
  78. package/extensions/browser-harness/util/debug.ts +8 -0
  79. package/extensions/browser-harness/util/guards.ts +30 -0
  80. package/extensions/browser-harness/util/js-template.ts +12 -0
  81. package/extensions/browser-harness/util/mutex.ts +20 -0
  82. package/extensions/browser-harness/util/paths.ts +31 -0
  83. package/extensions/browser-harness/util/result.ts +9 -0
  84. package/extensions/browser-harness/util/sharp-shim.ts +42 -0
  85. package/extensions/browser-harness/util/tool.ts +185 -0
  86. package/extensions/browser-harness/util/truncate.ts +41 -0
  87. package/extensions/cli-resources.ts +35 -16
  88. package/extensions/composio/index.ts +163 -0
  89. package/extensions/context-memory/index.ts +58 -0
  90. package/extensions/lib/application-context.ts +22 -0
  91. package/extensions/lib/application-handler-templates.ts +17 -1
  92. package/extensions/lib/context-memory.ts +165 -0
  93. package/extensions/lib/profile-sandbox.ts +3 -0
  94. package/extensions/lib/workflow-harness.ts +238 -0
  95. package/extensions/pi-console-webui/app/[section]/page.tsx +6 -1
  96. package/extensions/pi-console-webui/app/applications/[slug]/page.tsx +15 -0
  97. package/extensions/pi-console-webui/app/layout.tsx +6 -1
  98. package/extensions/pi-console-webui/app/login/page.tsx +5 -9
  99. package/extensions/pi-console-webui/app/not-found.tsx +5 -0
  100. package/extensions/pi-console-webui/app/page.tsx +6 -0
  101. package/extensions/pi-console-webui/app/terminal/page.tsx +3 -0
  102. package/extensions/pi-console-webui/components/application-handler-editor.tsx +4 -4
  103. package/extensions/pi-console-webui/components/application-logs.tsx +12 -8
  104. package/extensions/pi-console-webui/components/application-settings.tsx +5 -1
  105. package/extensions/pi-console-webui/components/chat-workspace.tsx +7 -4
  106. package/extensions/pi-console-webui/components/console-page.tsx +3 -1
  107. package/extensions/pi-console-webui/components/console.tsx +53 -7
  108. package/extensions/pi-console-webui/components/login-form.tsx +14 -0
  109. package/extensions/pi-console-webui/components/pulse-resources.tsx +10 -6
  110. package/extensions/pi-console-webui/components/skill-resources.tsx +3 -3
  111. package/extensions/pi-console-webui/lib/page-metadata.ts +33 -0
  112. package/extensions/profiles.ts +18 -7
  113. package/extensions/pulse/store.ts +5 -2
  114. package/extensions/sequential-workflow.ts +340 -256
  115. package/extensions/skill-sources/store.ts +24 -3
  116. package/licenses/pi-browser-harness-MIT.txt +21 -0
  117. package/package.json +25 -4
  118. package/schemas/sequential-workflow-template.schema.json +62 -0
  119. package/skills/browser-harness/SKILL.md +33 -0
  120. package/skills/composio-connections/SKILL.md +75 -0
  121. package/skills/sequential-workflow/SKILL.md +70 -0
  122. package/skills/sequential-workflow-template/SKILL.md +44 -0
@@ -0,0 +1,241 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import { resolve, isAbsolute } from "node:path";
3
+ import { isPathWithin } from "../util/paths";
4
+ import { tmpdir } from "node:os";
5
+ import { createRequire } from "node:module";
6
+ import { Type } from "typebox";
7
+ import { Markdown, Text } from "@earendil-works/pi-tui";
8
+ import { getMarkdownTheme, keyHint } from "@earendil-works/pi-coding-agent";
9
+ import { type Result, err, ok } from "../util/result";
10
+ import { formatBytes } from "../util/truncate";
11
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
12
+ import { evalJs } from "./cdp-call";
13
+ import { applyTruncation } from "../util/truncate";
14
+ import { asNumber, asRecord, asString, isRecord } from "../util/guards";
15
+
16
+ const COMPACT_PREVIEW_BYTES = 120;
17
+
18
+ // The AsyncFunction constructor is the documented way to compile user source; a plain `unknown` cast would lose the constructor signature, so this typed cast stays.
19
+ const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor as
20
+ new (...args: ReadonlyArray<string>) => (...args: ReadonlyArray<unknown>) => Promise<unknown>;
21
+
22
+ const requireFromHere = createRequire(import.meta.url);
23
+
24
+ const ExecuteJsArgs = Type.Object({
25
+ expression: Type.String({ description: "JavaScript expression. `return X` is auto-wrapped in an IIFE." }),
26
+ targetId: Type.Optional(Type.String({ description: "Optional iframe targetId; default = current page." })),
27
+ });
28
+
29
+ export const executeJsTool = defineBrowserTool({
30
+ name: "browser_execute_js",
31
+ label: "Browser Execute JS",
32
+ description:
33
+ "Surgical reads from the DOM. The cheapest, most precise way to extract a specific element's text, attribute, value, or geometry. Prefer over browser_screenshot for ANY data extraction or coordinate lookup. `return X` is auto-wrapped in an IIFE. Always use safeJs / JSON.stringify when interpolating untrusted strings.",
34
+ promptSnippet: "Execute JS in the page — surgical DOM reads (preferred over screenshot)",
35
+ promptGuidelines: [
36
+ "DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.",
37
+ "For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.",
38
+ "For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.",
39
+ "DO NOT call browser_screenshot to read a value or check element state. Use this tool.",
40
+ "`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.",
41
+ "For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.",
42
+ "Result must be JSON-serializable (Runtime.evaluate returnByValue=true).",
43
+ ],
44
+ parameters: ExecuteJsArgs,
45
+ concurrency: "parallel",
46
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
47
+ const r = await evalJs(client, args.expression, args.targetId);
48
+ if (!r.success) return r;
49
+ const valueStr = r.data === undefined ? "undefined" : JSON.stringify(r.data);
50
+ const truncated = await applyTruncation(valueStr, "js");
51
+ let pretty: string | undefined;
52
+ try {
53
+ const parsed: unknown = JSON.parse(valueStr);
54
+ pretty = JSON.stringify(parsed, null, 2);
55
+ } catch {
56
+ pretty = undefined;
57
+ }
58
+ return ok({
59
+ text: truncated.text,
60
+ details: {
61
+ valueLength: valueStr.length,
62
+ full: valueStr.length > 200_000 ? valueStr.slice(0, 200_000) : valueStr,
63
+ ...(pretty !== undefined ? { pretty: pretty.length > 200_000 ? pretty.slice(0, 200_000) : pretty } : {}),
64
+ ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
65
+ },
66
+ });
67
+ },
68
+
69
+ renderResult(result, expanded, theme) {
70
+ const raw = asRecord(result.details);
71
+ if (raw === undefined) return new Text(theme.fg("error", "execute_js: no details"), 0, 0);
72
+ const details = {
73
+ valueLength: asNumber(raw["valueLength"]),
74
+ full: asString(raw["full"]),
75
+ pretty: asString(raw["pretty"]),
76
+ fullOutputPath: asString(raw["fullOutputPath"]),
77
+ };
78
+
79
+ const len = details.valueLength ?? 0;
80
+ const full = details.full ?? "";
81
+ const isJson = details.pretty !== undefined;
82
+
83
+ if (!expanded) {
84
+ const preview = full.length > COMPACT_PREVIEW_BYTES ? full.slice(0, COMPACT_PREVIEW_BYTES) + "…" : full;
85
+ const md = [
86
+ `**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`,
87
+ "```",
88
+ preview,
89
+ "```",
90
+ keyHint("app.tools.expand", "to expand"),
91
+ ].join("\n");
92
+ return new Markdown(md, 0, 0, getMarkdownTheme());
93
+ }
94
+
95
+ const body = details.pretty ?? full;
96
+ const fence = isJson ? "```json" : "```";
97
+ const tail = details.fullOutputPath
98
+ ? `\n\nFull value at \`${details.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
99
+ : `\n\n${keyHint("app.tools.expand", "to collapse")}`;
100
+ const md = [`**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`, fence, body, "```", tail].join("\n");
101
+ return new Markdown(md, 0, 0, getMarkdownTheme());
102
+ },
103
+ });
104
+
105
+ const RunScriptArgs = Type.Object({
106
+ path: Type.String({ description: "Absolute path to the script file (.js or .mjs)" }),
107
+ params: Type.Optional(
108
+ Type.Object({}, { additionalProperties: true, description: "Args passed to the script as `params`" }),
109
+ ),
110
+ timeoutMs: Type.Optional(
111
+ Type.Integer({
112
+ default: 60_000,
113
+ minimum: 100,
114
+ maximum: 600_000,
115
+ description: "Hard timeout in ms. Default 60s, max 600s.",
116
+ }),
117
+ ),
118
+ });
119
+
120
+ const allowedRoots = (): ReadonlyArray<string> => {
121
+ const env = process.env["BH_SCRIPT_DIR"];
122
+ return [tmpdir(), process.cwd(), ...(env !== undefined ? [env] : [])].map((d) => resolve(d));
123
+ };
124
+
125
+ const isPathAllowed = (p: string): boolean => allowedRoots().some((root) => isPathWithin(root, p));
126
+
127
+ const MAX_SOURCE_BYTES = 1_000_000;
128
+
129
+ type ContentItem = { readonly type: "text"; readonly text: string };
130
+
131
+ const isContentItem = (v: unknown): v is ContentItem =>
132
+ isRecord(v) && v["type"] === "text" && typeof v["text"] === "string";
133
+
134
+ export const runScriptTool = defineBrowserTool({
135
+ name: "browser_run_script",
136
+ label: "Browser Run Script",
137
+ description:
138
+ "Execute a temporary JavaScript script with full Node.js + browser-daemon access. Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR. Mandatory timeout. The script is full RCE on the harness's process — only invoke scripts you wrote and reviewed.",
139
+ promptSnippet: "Run a temporary script with daemon + Node access (security-bounded)",
140
+ promptGuidelines: [
141
+ "Write the script with the write tool first; pass its absolute path.",
142
+ "Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR — otherwise rejected.",
143
+ "Default timeout is 60s; pass timeoutMs to extend (max 600s).",
144
+ "Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.",
145
+ "Script MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }. Throw on errors.",
146
+ ],
147
+ parameters: RunScriptArgs,
148
+ concurrency: "parallel",
149
+ async handler(args, { client, signal, onUpdate, extensionCtx }): Promise<Result<ToolOk, ToolErr>> {
150
+ if (!isAbsolute(args.path)) {
151
+ return err({ kind: "invalid_state", message: "Script path must be absolute" });
152
+ }
153
+ if (!isPathAllowed(args.path)) {
154
+ return err({
155
+ kind: "invalid_state",
156
+ message: `Script path outside allowed directories (allowed: ${allowedRoots().join(", ")})`,
157
+ });
158
+ }
159
+ let source: string;
160
+ try {
161
+ source = await readFile(args.path, "utf8");
162
+ } catch (e) {
163
+ return err({
164
+ kind: "io_error",
165
+ message: `Failed to read script: ${e instanceof Error ? e.message : String(e)}`,
166
+ });
167
+ }
168
+ if (source.length === 0) return err({ kind: "invalid_state", message: "Script is empty" });
169
+ if (source.length > MAX_SOURCE_BYTES) {
170
+ return err({ kind: "invalid_state", message: `Script exceeds ${MAX_SOURCE_BYTES}B size cap` });
171
+ }
172
+
173
+ let executeFn: (...a: ReadonlyArray<unknown>) => Promise<unknown>;
174
+ try {
175
+ executeFn = new AsyncFunction(
176
+ "params", "daemon", "require", "signal", "onUpdate", "ctx",
177
+ "console", "fetch", "JSON", "Buffer", "setTimeout", "clearTimeout",
178
+ `"use strict";\n${source}`,
179
+ );
180
+ } catch (e) {
181
+ return err({
182
+ kind: "invalid_state",
183
+ message: `Syntax error: ${e instanceof Error ? e.message : String(e)}`,
184
+ });
185
+ }
186
+
187
+ const timeoutMs = args.timeoutMs ?? 60_000;
188
+ const ac = new AbortController();
189
+ const onAbort = (): void => ac.abort();
190
+ if (signal !== undefined) signal.addEventListener("abort", onAbort, { once: true });
191
+ const timeoutTimer = setTimeout(() => ac.abort(), timeoutMs);
192
+
193
+ try {
194
+ const scriptPromise = executeFn(
195
+ args.params ?? {},
196
+ client,
197
+ requireFromHere,
198
+ ac.signal,
199
+ (u: unknown) => {
200
+ if (!isRecord(u)) return;
201
+ const content = u["content"];
202
+ if (!Array.isArray(content) || content.length === 0) return;
203
+ const first: unknown = content[0];
204
+ if (!isRecord(first)) return;
205
+ const txt = first["text"];
206
+ if (typeof txt !== "string") return;
207
+ try { onUpdate({ text: txt }); } catch {}
208
+ },
209
+ extensionCtx ?? { cwd: process.cwd() },
210
+ console, fetch, JSON, Buffer, setTimeout, clearTimeout,
211
+ );
212
+ const abortPromise = new Promise<never>((_, reject) => {
213
+ ac.signal.addEventListener("abort", () => reject(new Error("script aborted (timeout or cancellation)")), { once: true });
214
+ });
215
+ const result = await Promise.race([scriptPromise, abortPromise]);
216
+ clearTimeout(timeoutTimer);
217
+ if (signal !== undefined) signal.removeEventListener("abort", onAbort);
218
+
219
+ if (!isRecord(result)) {
220
+ return err({ kind: "invalid_state", message: `Script must return an object; got ${JSON.stringify(result)}` });
221
+ }
222
+ const content = result["content"];
223
+ if (!Array.isArray(content)) {
224
+ return err({ kind: "invalid_state", message: "Script return value must have a content array" });
225
+ }
226
+ if (!content.every(isContentItem)) {
227
+ return err({ kind: "invalid_state", message: "Script content array must contain { type: 'text', text: string } items" });
228
+ }
229
+ const textOut = content.map((c) => c.text).join("\n");
230
+ const details = result["details"];
231
+ return isRecord(details) ? ok({ text: textOut, details }) : ok({ text: textOut });
232
+ } catch (e) {
233
+ clearTimeout(timeoutTimer);
234
+ if (signal !== undefined) signal.removeEventListener("abort", onAbort);
235
+ return err({
236
+ kind: "internal",
237
+ message: `Script execution failed: ${e instanceof Error ? e.message : String(e)}`,
238
+ });
239
+ }
240
+ },
241
+ });
@@ -0,0 +1,197 @@
1
+ import { Type } from "typebox";
2
+ import { safeJs } from "../util/js-template";
3
+ import { type Result, err, ok } from "../util/result";
4
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
5
+ import { cdpCall, evalJs } from "./cdp-call";
6
+ import { resolveRefToObjectId } from "./ref-resolve";
7
+
8
+ const VIRTUAL_KEY_CODES: Readonly<Record<string, number>> = {
9
+ Enter: 13, Tab: 9, Backspace: 8, Escape: 27, Delete: 46, " ": 32,
10
+ ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40,
11
+ Home: 36, End: 35, PageUp: 33, PageDown: 34,
12
+ };
13
+
14
+ const virtualKeyCode = (key: string): number =>
15
+ VIRTUAL_KEY_CODES[key] ?? (key.length === 1 ? key.charCodeAt(0) : 0);
16
+
17
+ const TypeArgs = Type.Object({
18
+ text: Type.String({ description: "Text to type" }),
19
+ });
20
+
21
+ type FocusProbe = { readonly ok: boolean; readonly tag: string | null };
22
+
23
+ const isFocusProbe = (v: unknown): v is FocusProbe =>
24
+ typeof v === "object" && v !== null
25
+ && "ok" in v && typeof v.ok === "boolean"
26
+ && "tag" in v && (typeof v.tag === "string" || v.tag === null);
27
+
28
+ export const typeTool = defineBrowserTool({
29
+ name: "browser_type",
30
+ label: "Browser Type",
31
+ description: "Type text into the currently focused element. Use browser_click first to focus an input field.",
32
+ promptSnippet: "Type text into the focused element",
33
+ promptGuidelines: [
34
+ "Use browser_type to enter text. Click on an input field with browser_click first to focus it.",
35
+ "For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.",
36
+ ],
37
+ parameters: TypeArgs,
38
+ concurrency: "serialized",
39
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
40
+ const focused = await evalJs(client, safeJs`
41
+ (() => {
42
+ const el = document.activeElement;
43
+ const ok = !!el && el !== document.body &&
44
+ (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable);
45
+ return { ok, tag: el ? el.tagName : null };
46
+ })()
47
+ `);
48
+ if (focused.success) {
49
+ const f: FocusProbe = isFocusProbe(focused.data) ? focused.data : { ok: false, tag: null };
50
+ if (!f.ok) {
51
+ return err({
52
+ kind: "invalid_state",
53
+ message:
54
+ "No editable element is focused — typed text would be lost. Click the field with browser_click first, or use browser_fill with a selector.",
55
+ details: { activeElement: f.tag },
56
+ });
57
+ }
58
+ }
59
+ const r = await cdpCall(client, "Input.insertText", { text: args.text });
60
+ if (!r.success) return r;
61
+ return ok({ text: `Typed: "${args.text}"` });
62
+ },
63
+ });
64
+
65
+ const PressKeyArgs = Type.Object({
66
+ key: Type.String({
67
+ description:
68
+ 'Key to press. Special keys: Enter, Tab, Backspace, Escape, Delete, ArrowLeft/Up/Right/Down, Home, End, PageUp, PageDown. Space as " "',
69
+ }),
70
+ modifiers: Type.Optional(
71
+ Type.Integer({
72
+ minimum: 0,
73
+ maximum: 15,
74
+ description: "Modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with OR.",
75
+ }),
76
+ ),
77
+ });
78
+
79
+ export const pressKeyTool = defineBrowserTool({
80
+ name: "browser_press_key",
81
+ label: "Browser Press Key",
82
+ description:
83
+ "Press a keyboard key. Supports special keys (Enter, Tab, Backspace, Escape, Delete, arrows, Home, End, PageUp, PageDown, Space as ' ') and regular characters. Optional modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta/Cmd, 8=Shift.",
84
+ promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)",
85
+ promptGuidelines: [
86
+ "Use browser_press_key for keyboard shortcuts and navigation keys.",
87
+ "Special key names: Enter, Tab, Backspace, Escape, Delete, ArrowLeft, ArrowUp, ArrowRight, ArrowDown, Home, End, PageUp, PageDown.",
88
+ "Use Space as ' ' (a single space character).",
89
+ "Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
90
+ ],
91
+ parameters: PressKeyArgs,
92
+ concurrency: "serialized",
93
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
94
+ const k = args.key;
95
+ const code = virtualKeyCode(k);
96
+ const modifiers = args.modifiers ?? 0;
97
+ const isChar = k.length === 1;
98
+ const downParams: Record<string, unknown> = {
99
+ type: "keyDown",
100
+ key: k,
101
+ code: k,
102
+ modifiers,
103
+ ...(code ? { windowsVirtualKeyCode: code, nativeVirtualKeyCode: code } : {}),
104
+ ...(isChar ? { text: k, unmodifiedText: k } : {}),
105
+ };
106
+ const down = await cdpCall(client, "Input.dispatchKeyEvent", downParams);
107
+ if (!down.success) return down;
108
+ // A printable character needs a `char` event for the page to receive keypress/textInput and actually insert it; Shift (8) still counts as printable.
109
+ const hasCommandModifier = (modifiers & (1 | 2 | 4)) !== 0;
110
+ if (isChar && !hasCommandModifier) {
111
+ const charEv = await cdpCall(client, "Input.dispatchKeyEvent", { type: "char", text: k, key: k });
112
+ if (!charEv.success) return charEv;
113
+ }
114
+ const up = await cdpCall(client, "Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers });
115
+ if (!up.success) return up;
116
+ return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` });
117
+ },
118
+ });
119
+
120
+ const DispatchKeyArgs = Type.Object({
121
+ ref: Type.Optional(
122
+ Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }),
123
+ ),
124
+ selector: Type.Optional(Type.String({ description: "CSS selector of the target element (fallback when no ref)" })),
125
+ key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }),
126
+ eventType: Type.Optional(
127
+ Type.Union(
128
+ [Type.Literal("keydown"), Type.Literal("keyup"), Type.Literal("keypress")],
129
+ { default: "keydown", description: "Event type to dispatch. Default: keydown" },
130
+ ),
131
+ ),
132
+ });
133
+
134
+ export const dispatchKeyTool = defineBrowserTool({
135
+ name: "browser_dispatch_key",
136
+ label: "Browser Dispatch Key",
137
+ description:
138
+ "Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
139
+ promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector",
140
+ promptGuidelines: [
141
+ "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.",
142
+ "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).",
143
+ "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
144
+ "The event carries keyCode/which (e.g. 13 for Enter) for legacy handlers, but is untrusted (isTrusted === false) — a few libraries may still ignore it.",
145
+ "eventType defaults to 'keydown'.",
146
+ ],
147
+ parameters: DispatchKeyArgs,
148
+ concurrency: "serialized",
149
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
150
+ const eventType = args.eventType ?? "keydown";
151
+ const target = args.ref ?? args.selector ?? "";
152
+ // Legacy handlers branch on e.keyCode/e.which rather than e.key, so populate both.
153
+ const code = virtualKeyCode(args.key);
154
+ if (args.ref !== undefined) {
155
+ const objectId = await resolveRefToObjectId(client, args.ref);
156
+ if (!objectId.success) return objectId;
157
+ const r = await cdpCall(client, "Runtime.callFunctionOn", {
158
+ objectId: objectId.data,
159
+ functionDeclaration: `function (type, key, keyCode) { this.dispatchEvent(new KeyboardEvent(type, { key, keyCode, which: keyCode, bubbles: true, cancelable: true })); return 1; }`,
160
+ arguments: [{ value: eventType }, { value: args.key }, { value: code }],
161
+ returnByValue: true,
162
+ });
163
+ if (!r.success) return r;
164
+ return ok({
165
+ text: `Dispatched ${eventType}('${args.key}') on ${target}`,
166
+ details: { matched: 1, ref: args.ref },
167
+ });
168
+ }
169
+ if (args.selector === undefined) {
170
+ return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
171
+ }
172
+ const expr = safeJs`
173
+ (() => {
174
+ const els = document.querySelectorAll(${args.selector});
175
+ if (els.length === 0) return 0;
176
+ for (const el of els) {
177
+ el.dispatchEvent(new KeyboardEvent(${eventType}, { key: ${args.key}, keyCode: ${code}, which: ${code}, bubbles: true, cancelable: true }));
178
+ }
179
+ return els.length;
180
+ })()
181
+ `;
182
+ const r = await evalJs(client, expr);
183
+ if (!r.success) return r;
184
+ const matched = Number(r.data ?? 0);
185
+ if (matched === 0) {
186
+ return err({
187
+ kind: "invalid_state",
188
+ message: `Selector matched 0 elements: ${args.selector}`,
189
+ details: { matched: 0, selector: args.selector },
190
+ });
191
+ }
192
+ return ok({
193
+ text: `Dispatched ${eventType}('${args.key}') on ${matched} element(s)`,
194
+ details: { matched, selector: args.selector },
195
+ });
196
+ },
197
+ });
@@ -0,0 +1,161 @@
1
+ import { Type } from "typebox";
2
+ import { Text } from "@earendil-works/pi-tui";
3
+ import { type Result, err, ok } from "../util/result";
4
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
5
+ import { applyTruncation } from "../util/truncate";
6
+ import { safeJs } from "../util/js-template";
7
+ import { ensureHarnessWindow, openHarnessTab } from "../cdp/target-factory";
8
+ import { cdpCall, cdpCallBrowser, cdpCallOnTarget, evalJs } from "./cdp-call";
9
+
10
+ const NavigateArgs = Type.Object({
11
+ url: Type.String({ description: "Full URL to navigate to (e.g. https://github.com)" }),
12
+ });
13
+
14
+ type NavOutcome =
15
+ | { readonly kind: "in_place"; readonly targetId: string }
16
+ | { readonly kind: "new_tab_created"; readonly targetId: string; readonly reason: "no_tabs" };
17
+
18
+ export const navigateTool = defineBrowserTool({
19
+ name: "browser_navigate",
20
+ label: "Browser Navigate",
21
+ description:
22
+ "Navigate to a URL. Creates a new tab on first call (when no real tabs exist). Otherwise navigates the current tab in place.",
23
+ promptSnippet: "Navigate to a URL",
24
+ promptGuidelines: [
25
+ "Use browser_navigate to go to URLs.",
26
+ "Use browser_wait_for_load after browser_navigate to wait for the page to finish loading.",
27
+ "For extracting data from a page you already navigated to, use browser_execute_js or browser_http_get (faster for APIs).",
28
+ "Note: Google and some sites with strict anti-bot detection may reject CDP navigation. Use browser_http_get for search results.",
29
+ ],
30
+ parameters: NavigateArgs,
31
+ concurrency: "serialized",
32
+ renderCall: (a) => new Text(`🌐 Navigate to ${a.url}`, 0, 0),
33
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
34
+ const tabs = await client.listTabs(false);
35
+ if (!tabs.success) return err({ kind: "cdp_error", message: tabs.error.message });
36
+ let outcome: NavOutcome;
37
+ if (tabs.data.length === 0) {
38
+ const created = await client.newTab(args.url);
39
+ if (!created.success) return err({ kind: "cdp_error", message: created.error.message });
40
+ outcome = { kind: "new_tab_created", targetId: created.data, reason: "no_tabs" };
41
+ } else {
42
+ const nav = await cdpCall(client, "Page.navigate", { url: args.url });
43
+ if (!nav.success) return nav;
44
+ const cur = client.current();
45
+ if (!cur) return err({ kind: "internal", message: "navigate succeeded but no current target tracked" });
46
+ outcome = { kind: "in_place", targetId: cur.targetId };
47
+ }
48
+ const info = await client.pageInfo();
49
+ if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
50
+ if ("dialog" in info.data) {
51
+ return ok({
52
+ text: `Navigated to ${args.url}\nTarget: ${outcome.targetId}\n⚠️ Dialog open: ${info.data.dialog.type} — "${info.data.dialog.message}"\nUse browser_handle_dialog.`,
53
+ details: { outcome, dialog: info.data.dialog },
54
+ });
55
+ }
56
+ const prefix = outcome.kind === "new_tab_created" ? "New tab: " : "";
57
+ return ok({
58
+ text: `${prefix}Navigated to: ${info.data.url}\nTitle: ${info.data.title}\nViewport: ${info.data.width}x${info.data.height}\nTarget: ${outcome.targetId}`,
59
+ details: { outcome, page: info.data },
60
+ });
61
+ },
62
+ });
63
+
64
+ const OpenUrlsArgs = Type.Object({
65
+ urls: Type.Array(Type.String(), { description: "Array of URLs to open in new tabs" }),
66
+ });
67
+
68
+ type TabResult = {
69
+ readonly url: string;
70
+ readonly targetId: string;
71
+ readonly ok: boolean;
72
+ readonly error?: string;
73
+ };
74
+
75
+ export const openUrlsTool = defineBrowserTool({
76
+ name: "browser_open_urls",
77
+ label: "Browser Open URLs",
78
+ description: "Open multiple URLs in parallel new tabs. Returns per-URL outcomes.",
79
+ promptSnippet: "Open multiple URLs in new tabs (parallel)",
80
+ promptGuidelines: [
81
+ "Use after browser_web_search to open citations in parallel.",
82
+ "After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.",
83
+ "Use browser_wait_for_load on a tab before extracting data from SPAs.",
84
+ ],
85
+ parameters: OpenUrlsArgs,
86
+ concurrency: "serialized",
87
+ renderCall: (a) => new Text(`🌐 Opening ${a.urls.length} URL${a.urls.length !== 1 ? "s" : ""}…`, 0, 0),
88
+ async handler(args, { client, onUpdate }): Promise<Result<ToolOk, ToolErr>> {
89
+ const total = args.urls.length;
90
+ // All tabs must land in ONE window: a parallel newWindow:true per url would scatter them across N windows, so the first url creates the window and the rest open as its children.
91
+ const url0 = args.urls[0];
92
+ if (url0 === undefined) return err({ kind: "invalid_state", message: "no URLs provided" });
93
+ const window = await ensureHarnessWindow(client);
94
+ if (!window.success) return err({ kind: "cdp_error", message: window.error.message });
95
+ const seededWindow = window.data.targetId;
96
+ const seedResult: TabResult | undefined = window.data.freshlyCreated
97
+ ? { url: url0, targetId: seededWindow, ok: true }
98
+ : undefined;
99
+ const created = await Promise.all(args.urls.map(async (url, i): Promise<TabResult> => {
100
+ if (i === 0 && seedResult) return seedResult;
101
+ const r = await openHarnessTab(client, seededWindow);
102
+ if (!r.success) return { url, targetId: "", ok: false, error: r.error.message };
103
+ return { url, targetId: r.data, ok: true };
104
+ }));
105
+ let completed = 0;
106
+ const settled = await Promise.all(
107
+ created.filter((t) => t.ok).map(async (tab): Promise<TabResult> => {
108
+ try {
109
+ const attached = await client.session().attach(tab.targetId);
110
+ if (!attached.success) return { ...tab, ok: false, error: attached.error.message };
111
+ const sid = attached.data;
112
+ const enabled = await cdpCallOnTarget(client, "Page.enable", {}, sid);
113
+ if (!enabled.success) return { ...tab, ok: false, error: enabled.error.message };
114
+ const nav = await cdpCallOnTarget(client, "Page.navigate", { url: tab.url }, sid);
115
+ if (!nav.success) return { ...tab, ok: false, error: nav.error.message };
116
+ evalJs(
117
+ client,
118
+ safeJs`if(!document.title.startsWith('🟢'))document.title='🟢 '+document.title`,
119
+ sid
120
+ ).catch(() => {});
121
+ return tab;
122
+ } finally {
123
+ completed++;
124
+ // onUpdate must never escape: a renderer crash would otherwise tear down the tool call.
125
+ try {
126
+ onUpdate({ text: `Opening URLs… ${completed}/${created.filter((t) => t.ok).length} navigated` });
127
+ } catch {}
128
+ }
129
+ }),
130
+ );
131
+ const failures = created.filter((t) => !t.ok);
132
+ const all: ReadonlyArray<TabResult> = [...settled, ...failures];
133
+ const okTabs = all.filter((r) => r.ok);
134
+ const failTabs = all.filter((r) => !r.ok);
135
+ if (okTabs.length > 0) {
136
+ const last = okTabs[okTabs.length - 1];
137
+ if (last) {
138
+ await cdpCallBrowser(client, "Target.activateTarget", { targetId: last.targetId });
139
+ }
140
+ }
141
+ const lines: string[] = [];
142
+ if (okTabs.length) {
143
+ lines.push(`✅ ${okTabs.length}/${total} tabs opened:`, ...okTabs.map((t, i) => ` [${i}] ${t.url} → ${t.targetId}`));
144
+ }
145
+ if (failTabs.length) {
146
+ lines.push(`❌ ${failTabs.length}/${total} failed:`, ...failTabs.map((t) => ` ${t.url}: ${t.error ?? "unknown"}`));
147
+ }
148
+ const truncated = await applyTruncation(lines.join("\n"), "urls");
149
+ if (okTabs.length === 0) {
150
+ return err({
151
+ kind: "cdp_error",
152
+ message: `All ${total} URLs failed`,
153
+ ...(truncated.fullOutputPath ? { details: { tabs: all, fullOutputPath: truncated.fullOutputPath } } : { details: { tabs: all } }),
154
+ });
155
+ }
156
+ return ok({
157
+ text: truncated.text,
158
+ ...(truncated.fullOutputPath ? { details: { tabs: all, fullOutputPath: truncated.fullOutputPath } } : { details: { tabs: all } }),
159
+ });
160
+ },
161
+ });