pi-browser-harness 0.10.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +21 -10
- package/package.json +6 -3
- package/skills/pi-browser-harness/SKILL.md +30 -1
- package/src/cdp/commands.ts +260 -0
- package/src/cdp/console-buffer.ts +10 -85
- package/src/cdp/discovery.ts +31 -71
- package/src/cdp/errors.ts +4 -0
- package/src/cdp/events.ts +170 -0
- package/src/cdp/network-buffer.ts +42 -109
- package/src/cdp/ownership.ts +0 -17
- package/src/cdp/record-store.ts +61 -0
- package/src/cdp/session.ts +205 -193
- package/src/cdp/target-factory.ts +10 -80
- package/src/cdp/types.ts +21 -12
- package/src/client.ts +39 -95
- package/src/daemon/bridge.ts +375 -265
- package/src/daemon/index.ts +0 -28
- package/src/daemon/protocol.ts +6 -68
- package/src/daemon/queue.ts +85 -0
- package/src/daemon/server.ts +24 -99
- package/src/daemon/spawn.ts +90 -82
- package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +24 -52
- package/src/domains/ax-tree.ts +179 -0
- package/src/domains/box.ts +51 -0
- package/src/domains/cdp-call.ts +55 -0
- package/src/domains/click.ts +19 -37
- package/src/domains/console.ts +48 -4
- package/src/domains/dialog.ts +8 -5
- package/src/domains/drag.ts +8 -7
- package/src/domains/element-call.ts +117 -0
- package/src/domains/files.ts +75 -30
- package/src/domains/fill-engine.ts +80 -0
- package/src/domains/form-batch.ts +64 -0
- package/src/domains/form.ts +115 -112
- package/src/domains/headers.ts +89 -0
- package/src/domains/history.ts +16 -12
- package/src/domains/isolated-tab.ts +26 -55
- package/src/domains/js.ts +32 -51
- package/src/domains/keyboard.ts +38 -34
- package/src/domains/navigate.ts +16 -26
- package/src/domains/network.ts +61 -13
- package/src/domains/page.ts +25 -10
- package/src/domains/readpage/read-page.ts +74 -20
- package/src/domains/readpage/readability.ts +0 -24
- package/src/domains/ref-resolve.ts +24 -58
- package/src/domains/render.ts +6 -27
- package/src/domains/screenshot-capture.ts +80 -0
- package/src/domains/screenshot.ts +9 -90
- package/src/domains/scroll.ts +10 -12
- package/src/domains/search/google-serp.ts +0 -29
- package/src/domains/search/web-search.ts +48 -21
- package/src/domains/setup.ts +1 -13
- package/src/domains/snapshot.ts +36 -280
- package/src/domains/tabs.ts +15 -16
- package/src/domains/target.ts +35 -0
- package/src/domains/viewport.ts +5 -4
- package/src/index.ts +23 -69
- package/src/profile/bind.ts +8 -57
- package/src/profile/browser-process.ts +75 -126
- package/src/profile/command.ts +50 -29
- package/src/profile/launch.ts +22 -40
- package/src/profile/list.ts +14 -54
- package/src/profile/paths.ts +2 -65
- package/src/profile/store.ts +35 -56
- package/src/prompt.ts +22 -122
- package/src/registry.ts +33 -33
- package/src/schemas/parse.ts +23 -0
- package/src/setup.ts +0 -25
- package/src/state.ts +48 -18
- package/src/util/debug.ts +8 -0
- package/src/util/guards.ts +30 -0
- package/src/util/js-template.ts +1 -16
- package/src/util/mutex.ts +0 -5
- package/src/util/paths.ts +11 -1
- package/src/util/result.ts +0 -6
- package/src/util/sharp-shim.ts +12 -19
- package/src/util/tool.ts +17 -42
- package/src/util/truncate.ts +6 -0
- package/tsconfig.json +1 -16
- package/skills/deep-research/SKILL.md +0 -66
- package/src/cdp/event-queue.ts +0 -128
- package/src/cdp/transport.ts +0 -156
- package/src/cdp/window-binding.ts +0 -15
- package/src/deep-research.ts +0 -41
- package/src/domains/forms.ts +0 -355
- package/src/domains/readpage/capture.ts +0 -46
- package/src/domains/search/extract.ts +0 -44
- package/src/profile/picker.ts +0 -73
- package/src/util/keycodes.ts +0 -10
- package/src/util/time.ts +0 -29
|
@@ -1,22 +1,11 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Isolated-attach tab lifecycle (ADR-0002 v1). Opens a harness-owned tab,
|
|
3
|
-
* attaches to it by its own CDP sessionId, and drives it via callOnTarget /
|
|
4
|
-
* evaluateJs(expr, sid) WITHOUT ever touching the global "current" tab — so a
|
|
5
|
-
* search or page-read never disturbs the tab the user is looking at, even when
|
|
6
|
-
* several subagents run concurrently.
|
|
7
|
-
*
|
|
8
|
-
* Mirrors the create/attach sequence in browser_open_urls
|
|
9
|
-
* (src/domains/navigate.ts:84-139). Every page evaluation is bounded by a
|
|
10
|
-
* timeout, so a JS dialog on the (uninstrumented) isolated tab surfaces as a
|
|
11
|
-
* typed timeout rather than hanging — see the dialog-hang caveat in plan.md.
|
|
12
|
-
*/
|
|
13
1
|
import type { BrowserClient } from "../client";
|
|
14
2
|
import { type Result, err, ok } from "../util/result";
|
|
15
3
|
import { safeJs } from "../util/js-template";
|
|
16
4
|
import { ensureHarnessWindow, openHarnessTab } from "../cdp/target-factory";
|
|
5
|
+
import { evaluateJson } from "../cdp/session";
|
|
17
6
|
import type { ToolErr } from "../util/tool";
|
|
7
|
+
import { cdpCallBrowser, cdpCallOnTarget, cdpErrToToolErr, evalJs } from "./cdp-call";
|
|
18
8
|
|
|
19
|
-
/** A tab attached by its own sessionId, ready to drive via callOnTarget. */
|
|
20
9
|
export type IsolatedTab = {
|
|
21
10
|
readonly targetId: string;
|
|
22
11
|
readonly sessionId: string;
|
|
@@ -25,56 +14,46 @@ export type IsolatedTab = {
|
|
|
25
14
|
const EVAL_TIMEOUT_MS = 15_000;
|
|
26
15
|
const READY_POLL_INTERVAL_MS = 50;
|
|
27
16
|
|
|
28
|
-
const toToolErr = (message: string, kind: ToolErr["kind"] = "cdp_error"): ToolErr => ({ kind, message });
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Open a harness-owned tab and attach to it, returning its targetId + private
|
|
32
|
-
* sessionId. Does not navigate. The caller MUST `closeIsolatedTab` when done.
|
|
33
|
-
*/
|
|
34
17
|
export const openIsolatedTab = async (client: BrowserClient): Promise<Result<IsolatedTab, ToolErr>> => {
|
|
35
|
-
// Routed through the target factory so an isolated tab lands in the pinned
|
|
36
|
-
// profile like every other harness tab — otherwise a search or page read
|
|
37
|
-
// would run with a different profile's cookies than the visible tabs.
|
|
18
|
+
// Routed through the target factory so an isolated tab lands in the pinned profile, not another profile's cookies.
|
|
38
19
|
const window = await ensureHarnessWindow(client);
|
|
39
|
-
if (!window.success) return err(
|
|
20
|
+
if (!window.success) return err(cdpErrToToolErr(window.error, "Target.createTarget"));
|
|
40
21
|
let targetId: string;
|
|
41
22
|
if (window.data.freshlyCreated) {
|
|
42
23
|
targetId = window.data.targetId;
|
|
43
24
|
} else {
|
|
44
25
|
const opened = await openHarnessTab(client, window.data.targetId);
|
|
45
|
-
if (!opened.success) return err(
|
|
26
|
+
if (!opened.success) return err(cdpErrToToolErr(opened.error, "Target.createTarget"));
|
|
46
27
|
targetId = opened.data;
|
|
47
28
|
}
|
|
48
29
|
|
|
49
|
-
const attached = await client
|
|
30
|
+
const attached = await cdpCallBrowser(client, "Target.attachToTarget", { targetId, flatten: true });
|
|
50
31
|
if (!attached.success) {
|
|
51
32
|
await client.closeTab(targetId);
|
|
52
|
-
return
|
|
33
|
+
return attached;
|
|
53
34
|
}
|
|
54
|
-
const { sessionId } = attached.data
|
|
35
|
+
const { sessionId } = attached.data;
|
|
55
36
|
|
|
56
|
-
const enabled = await client
|
|
37
|
+
const enabled = await cdpCallOnTarget(client, "Page.enable", {}, sessionId);
|
|
57
38
|
if (!enabled.success) {
|
|
58
39
|
await client.closeTab(targetId);
|
|
59
|
-
return
|
|
40
|
+
return enabled;
|
|
60
41
|
}
|
|
61
42
|
return ok({ targetId, sessionId });
|
|
62
43
|
};
|
|
63
44
|
|
|
64
|
-
/** Navigate an isolated tab to a URL (does not wait for load). */
|
|
65
45
|
export const navigateIsolatedTab = async (
|
|
66
46
|
client: BrowserClient,
|
|
67
47
|
tab: IsolatedTab,
|
|
68
48
|
url: string,
|
|
69
49
|
): Promise<Result<void, ToolErr>> => {
|
|
70
|
-
const nav = await client
|
|
50
|
+
const nav = await cdpCallOnTarget(client, "Page.navigate", { url }, tab.sessionId, {
|
|
71
51
|
timeoutMs: EVAL_TIMEOUT_MS,
|
|
72
52
|
});
|
|
73
|
-
if (!nav.success) return
|
|
53
|
+
if (!nav.success) return nav;
|
|
74
54
|
return ok(undefined);
|
|
75
55
|
};
|
|
76
56
|
|
|
77
|
-
/** Poll the isolated tab until document.readyState === 'complete' or timeout. */
|
|
78
57
|
export const waitForIsolatedLoad = async (
|
|
79
58
|
client: BrowserClient,
|
|
80
59
|
tab: IsolatedTab,
|
|
@@ -83,38 +62,30 @@ export const waitForIsolatedLoad = async (
|
|
|
83
62
|
): Promise<Result<void, ToolErr>> => {
|
|
84
63
|
const start = Date.now();
|
|
85
64
|
while (Date.now() - start < timeoutMs) {
|
|
86
|
-
if (signal?.aborted) return err(
|
|
87
|
-
const r = await client
|
|
65
|
+
if (signal?.aborted) return err({ kind: "internal", message: "aborted" });
|
|
66
|
+
const r = await evalJs(client, safeJs`document.readyState`, tab.sessionId);
|
|
88
67
|
if (r.success && r.data === "complete") return ok(undefined);
|
|
89
68
|
await new Promise((res) => setTimeout(res, READY_POLL_INTERVAL_MS));
|
|
90
69
|
}
|
|
91
|
-
return err(
|
|
70
|
+
return err({ kind: "timeout", message: `page did not finish loading in ${Math.round(timeoutMs / 1000)}s` });
|
|
92
71
|
};
|
|
93
72
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
*/
|
|
98
|
-
export const evalInIsolatedTab = async (
|
|
73
|
+
export const isJsonText = (v: unknown): v is string => typeof v === "string";
|
|
74
|
+
|
|
75
|
+
export const evalInIsolatedTab = async <T>(
|
|
99
76
|
client: BrowserClient,
|
|
100
77
|
tab: IsolatedTab,
|
|
101
78
|
expression: string,
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
);
|
|
109
|
-
|
|
110
|
-
const data = r.data as { result?: { value?: unknown }; exceptionDetails?: unknown };
|
|
111
|
-
if (data.exceptionDetails) {
|
|
112
|
-
return err(toToolErr(`page evaluation failed: ${JSON.stringify(data.exceptionDetails)}`));
|
|
113
|
-
}
|
|
114
|
-
return ok(data.result?.value);
|
|
79
|
+
check: (v: unknown) => v is T,
|
|
80
|
+
): Promise<Result<T, ToolErr>> => {
|
|
81
|
+
const r = await evaluateJson(client.session(), expression, check, {
|
|
82
|
+
sessionId: tab.sessionId,
|
|
83
|
+
timeoutMs: EVAL_TIMEOUT_MS,
|
|
84
|
+
});
|
|
85
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, "Runtime.evaluate"));
|
|
86
|
+
return ok(r.data);
|
|
115
87
|
};
|
|
116
88
|
|
|
117
|
-
/** Close an isolated tab and drop its ownership. Best-effort; never throws. */
|
|
118
89
|
export const closeIsolatedTab = async (client: BrowserClient, tab: IsolatedTab): Promise<void> => {
|
|
119
90
|
await client.closeTab(tab.targetId).catch(() => {});
|
|
120
91
|
};
|
package/src/domains/js.ts
CHANGED
|
@@ -1,33 +1,24 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
2
|
import { resolve, isAbsolute } from "node:path";
|
|
3
|
+
import { isPathWithin } from "../util/paths";
|
|
3
4
|
import { tmpdir } from "node:os";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import { Type } from "typebox";
|
|
6
7
|
import { Markdown, Text } from "@mariozechner/pi-tui";
|
|
7
8
|
import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
|
|
8
9
|
import { type Result, err, ok } from "../util/result";
|
|
10
|
+
import { formatBytes } from "../util/truncate";
|
|
9
11
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
12
|
+
import { evalJs } from "./cdp-call";
|
|
10
13
|
import { applyTruncation } from "../util/truncate";
|
|
14
|
+
import { asNumber, asRecord, asString, isRecord } from "../util/guards";
|
|
11
15
|
|
|
12
16
|
const COMPACT_PREVIEW_BYTES = 120;
|
|
13
17
|
|
|
14
|
-
|
|
15
|
-
if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`;
|
|
16
|
-
if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`;
|
|
17
|
-
return `${n} B`;
|
|
18
|
-
};
|
|
19
|
-
|
|
20
|
-
// AsyncFunction constructor — the documented Node.js mechanism for compiling
|
|
21
|
-
// arbitrary user-supplied source. Equivalent to `new Function` but produces an
|
|
22
|
-
// async function we can `await`. The full RCE risk is contained at the source
|
|
23
|
-
// boundary: see the path allowlist + timeout below. A plain `unknown` cast
|
|
24
|
-
// would lose the constructor signature; we keep this single typed cast.
|
|
18
|
+
// The AsyncFunction constructor is the documented way to compile user source; a plain `unknown` cast would lose the constructor signature, so this typed cast stays.
|
|
25
19
|
const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor as
|
|
26
20
|
new (...args: ReadonlyArray<string>) => (...args: ReadonlyArray<unknown>) => Promise<unknown>;
|
|
27
21
|
|
|
28
|
-
// ESNext module — bare `require` is not in scope. createRequire provides the
|
|
29
|
-
// CJS resolver anchored to this file's URL so scripts can require() builtins
|
|
30
|
-
// and installed CJS packages.
|
|
31
22
|
const requireFromHere = createRequire(import.meta.url);
|
|
32
23
|
|
|
33
24
|
const ExecuteJsArgs = Type.Object({
|
|
@@ -45,22 +36,19 @@ export const executeJsTool = defineBrowserTool({
|
|
|
45
36
|
"DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.",
|
|
46
37
|
"For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.",
|
|
47
38
|
"For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.",
|
|
48
|
-
"DO NOT call browser_screenshot to read a value or check element state. Use this tool.",
|
|
49
|
-
"`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.",
|
|
50
39
|
"For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.",
|
|
51
40
|
"Result must be JSON-serializable (Runtime.evaluate returnByValue=true).",
|
|
52
41
|
],
|
|
53
42
|
parameters: ExecuteJsArgs,
|
|
43
|
+
concurrency: "parallel",
|
|
54
44
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
55
|
-
const r = await client
|
|
56
|
-
if (!r.success) return
|
|
45
|
+
const r = await evalJs(client, args.expression, args.targetId);
|
|
46
|
+
if (!r.success) return r;
|
|
57
47
|
const valueStr = r.data === undefined ? "undefined" : JSON.stringify(r.data);
|
|
58
48
|
const truncated = await applyTruncation(valueStr, "js");
|
|
59
|
-
// If the value is JSON, pretty-print for the expanded TUI view. Falls back
|
|
60
|
-
// to the raw string if parse fails (e.g. value is "undefined" sentinel).
|
|
61
49
|
let pretty: string | undefined;
|
|
62
50
|
try {
|
|
63
|
-
const parsed = JSON.parse(valueStr);
|
|
51
|
+
const parsed: unknown = JSON.parse(valueStr);
|
|
64
52
|
pretty = JSON.stringify(parsed, null, 2);
|
|
65
53
|
} catch {
|
|
66
54
|
pretty = undefined;
|
|
@@ -69,8 +57,6 @@ export const executeJsTool = defineBrowserTool({
|
|
|
69
57
|
text: truncated.text,
|
|
70
58
|
details: {
|
|
71
59
|
valueLength: valueStr.length,
|
|
72
|
-
// Full value for the expanded renderer. Capped at 200 KB so a giant
|
|
73
|
-
// dump doesn't bloat the conversation transcript on disk.
|
|
74
60
|
full: valueStr.length > 200_000 ? valueStr.slice(0, 200_000) : valueStr,
|
|
75
61
|
...(pretty !== undefined ? { pretty: pretty.length > 200_000 ? pretty.slice(0, 200_000) : pretty } : {}),
|
|
76
62
|
...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
|
|
@@ -79,10 +65,14 @@ export const executeJsTool = defineBrowserTool({
|
|
|
79
65
|
},
|
|
80
66
|
|
|
81
67
|
renderResult(result, expanded, theme) {
|
|
82
|
-
const
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
68
|
+
const raw = asRecord(result.details);
|
|
69
|
+
if (raw === undefined) return new Text(theme.fg("error", "execute_js: no details"), 0, 0);
|
|
70
|
+
const details = {
|
|
71
|
+
valueLength: asNumber(raw["valueLength"]),
|
|
72
|
+
full: asString(raw["full"]),
|
|
73
|
+
pretty: asString(raw["pretty"]),
|
|
74
|
+
fullOutputPath: asString(raw["fullOutputPath"]),
|
|
75
|
+
};
|
|
86
76
|
|
|
87
77
|
const len = details.valueLength ?? 0;
|
|
88
78
|
const full = details.full ?? "";
|
|
@@ -130,19 +120,14 @@ const allowedRoots = (): ReadonlyArray<string> => {
|
|
|
130
120
|
return [tmpdir(), process.cwd(), ...(env !== undefined ? [env] : [])].map((d) => resolve(d));
|
|
131
121
|
};
|
|
132
122
|
|
|
133
|
-
const isPathAllowed = (p: string): boolean =>
|
|
134
|
-
const abs = resolve(p);
|
|
135
|
-
return allowedRoots().some((root) => abs === root || abs.startsWith(`${root}/`));
|
|
136
|
-
};
|
|
123
|
+
const isPathAllowed = (p: string): boolean => allowedRoots().some((root) => isPathWithin(root, p));
|
|
137
124
|
|
|
138
125
|
const MAX_SOURCE_BYTES = 1_000_000;
|
|
139
126
|
|
|
140
127
|
type ContentItem = { readonly type: "text"; readonly text: string };
|
|
141
128
|
|
|
142
129
|
const isContentItem = (v: unknown): v is ContentItem =>
|
|
143
|
-
|
|
144
|
-
&& (v as Readonly<Record<string, unknown>>)["type"] === "text"
|
|
145
|
-
&& typeof (v as Readonly<Record<string, unknown>>)["text"] === "string";
|
|
130
|
+
isRecord(v) && v["type"] === "text" && typeof v["text"] === "string";
|
|
146
131
|
|
|
147
132
|
export const runScriptTool = defineBrowserTool({
|
|
148
133
|
name: "browser_run_script",
|
|
@@ -151,13 +136,12 @@ export const runScriptTool = defineBrowserTool({
|
|
|
151
136
|
"Execute a temporary JavaScript script with full Node.js + browser-daemon access. Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR. Mandatory timeout. The script is full RCE on the harness's process — only invoke scripts you wrote and reviewed.",
|
|
152
137
|
promptSnippet: "Run a temporary script with daemon + Node access (security-bounded)",
|
|
153
138
|
promptGuidelines: [
|
|
154
|
-
"Write the script with the write tool first; pass its absolute path.",
|
|
155
|
-
"Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR — otherwise rejected.",
|
|
156
|
-
"Default timeout is 60s; pass timeoutMs to extend (max 600s).",
|
|
139
|
+
"Write the script with the write tool first; pass its absolute path. Use the OS temp directory rather than hardcoding /tmp.",
|
|
157
140
|
"Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.",
|
|
158
141
|
"Script MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }. Throw on errors.",
|
|
159
142
|
],
|
|
160
143
|
parameters: RunScriptArgs,
|
|
144
|
+
concurrency: "parallel",
|
|
161
145
|
async handler(args, { client, signal, onUpdate, extensionCtx }): Promise<Result<ToolOk, ToolErr>> {
|
|
162
146
|
if (!isAbsolute(args.path)) {
|
|
163
147
|
return err({ kind: "invalid_state", message: "Script path must be absolute" });
|
|
@@ -205,18 +189,18 @@ export const runScriptTool = defineBrowserTool({
|
|
|
205
189
|
try {
|
|
206
190
|
const scriptPromise = executeFn(
|
|
207
191
|
args.params ?? {},
|
|
208
|
-
client,
|
|
192
|
+
client,
|
|
209
193
|
requireFromHere,
|
|
210
194
|
ac.signal,
|
|
211
195
|
(u: unknown) => {
|
|
212
|
-
if (
|
|
213
|
-
const content =
|
|
196
|
+
if (!isRecord(u)) return;
|
|
197
|
+
const content = u["content"];
|
|
214
198
|
if (!Array.isArray(content) || content.length === 0) return;
|
|
215
|
-
const first = content[0];
|
|
216
|
-
if (
|
|
217
|
-
const txt =
|
|
199
|
+
const first: unknown = content[0];
|
|
200
|
+
if (!isRecord(first)) return;
|
|
201
|
+
const txt = first["text"];
|
|
218
202
|
if (typeof txt !== "string") return;
|
|
219
|
-
try { onUpdate({ text: txt }); } catch {
|
|
203
|
+
try { onUpdate({ text: txt }); } catch {}
|
|
220
204
|
},
|
|
221
205
|
extensionCtx ?? { cwd: process.cwd() },
|
|
222
206
|
console, fetch, JSON, Buffer, setTimeout, clearTimeout,
|
|
@@ -228,10 +212,10 @@ export const runScriptTool = defineBrowserTool({
|
|
|
228
212
|
clearTimeout(timeoutTimer);
|
|
229
213
|
if (signal !== undefined) signal.removeEventListener("abort", onAbort);
|
|
230
214
|
|
|
231
|
-
if (
|
|
215
|
+
if (!isRecord(result)) {
|
|
232
216
|
return err({ kind: "invalid_state", message: `Script must return an object; got ${JSON.stringify(result)}` });
|
|
233
217
|
}
|
|
234
|
-
const content =
|
|
218
|
+
const content = result["content"];
|
|
235
219
|
if (!Array.isArray(content)) {
|
|
236
220
|
return err({ kind: "invalid_state", message: "Script return value must have a content array" });
|
|
237
221
|
}
|
|
@@ -239,11 +223,8 @@ export const runScriptTool = defineBrowserTool({
|
|
|
239
223
|
return err({ kind: "invalid_state", message: "Script content array must contain { type: 'text', text: string } items" });
|
|
240
224
|
}
|
|
241
225
|
const textOut = content.map((c) => c.text).join("\n");
|
|
242
|
-
const details =
|
|
243
|
-
|
|
244
|
-
return isPlainObject
|
|
245
|
-
? ok({ text: textOut, details: details as Readonly<Record<string, unknown>> })
|
|
246
|
-
: ok({ text: textOut });
|
|
226
|
+
const details = result["details"];
|
|
227
|
+
return isRecord(details) ? ok({ text: textOut, details }) : ok({ text: textOut });
|
|
247
228
|
} catch (e) {
|
|
248
229
|
clearTimeout(timeoutTimer);
|
|
249
230
|
if (signal !== undefined) signal.removeEventListener("abort", onAbort);
|
package/src/domains/keyboard.ts
CHANGED
|
@@ -1,30 +1,42 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
2
|
import { safeJs } from "../util/js-template";
|
|
3
|
-
import { virtualKeyCode } from "../util/keycodes";
|
|
4
3
|
import { type Result, err, ok } from "../util/result";
|
|
5
4
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
5
|
+
import { cdpCall, evalJs } from "./cdp-call";
|
|
6
6
|
import { resolveRefToObjectId } from "./ref-resolve";
|
|
7
7
|
|
|
8
|
+
const VIRTUAL_KEY_CODES: Readonly<Record<string, number>> = {
|
|
9
|
+
Enter: 13, Tab: 9, Backspace: 8, Escape: 27, Delete: 46, " ": 32,
|
|
10
|
+
ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40,
|
|
11
|
+
Home: 36, End: 35, PageUp: 33, PageDown: 34,
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const virtualKeyCode = (key: string): number =>
|
|
15
|
+
VIRTUAL_KEY_CODES[key] ?? (key.length === 1 ? key.charCodeAt(0) : 0);
|
|
16
|
+
|
|
8
17
|
const TypeArgs = Type.Object({
|
|
9
18
|
text: Type.String({ description: "Text to type" }),
|
|
10
19
|
});
|
|
11
20
|
|
|
21
|
+
type FocusProbe = { readonly ok: boolean; readonly tag: string | null };
|
|
22
|
+
|
|
23
|
+
const isFocusProbe = (v: unknown): v is FocusProbe =>
|
|
24
|
+
typeof v === "object" && v !== null
|
|
25
|
+
&& "ok" in v && typeof v.ok === "boolean"
|
|
26
|
+
&& "tag" in v && (typeof v.tag === "string" || v.tag === null);
|
|
27
|
+
|
|
12
28
|
export const typeTool = defineBrowserTool({
|
|
13
29
|
name: "browser_type",
|
|
14
30
|
label: "Browser Type",
|
|
15
31
|
description: "Type text into the currently focused element. Use browser_click first to focus an input field.",
|
|
16
32
|
promptSnippet: "Type text into the focused element",
|
|
17
33
|
promptGuidelines: [
|
|
18
|
-
"Use browser_type to enter text. Click on an input field with browser_click first to focus it.",
|
|
19
34
|
"For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.",
|
|
20
35
|
],
|
|
21
36
|
parameters: TypeArgs,
|
|
37
|
+
concurrency: "serialized",
|
|
22
38
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
23
|
-
|
|
24
|
-
// whether an editable element is focused, so without this check typed text
|
|
25
|
-
// can vanish into the void with no error. Fail loudly with an actionable
|
|
26
|
-
// message instead. IIFE starts with "(" so evaluateJs returns it directly.
|
|
27
|
-
const focused = await client.evaluateJs(safeJs`
|
|
39
|
+
const focused = await evalJs(client, safeJs`
|
|
28
40
|
(() => {
|
|
29
41
|
const el = document.activeElement;
|
|
30
42
|
const ok = !!el && el !== document.body &&
|
|
@@ -33,7 +45,7 @@ export const typeTool = defineBrowserTool({
|
|
|
33
45
|
})()
|
|
34
46
|
`);
|
|
35
47
|
if (focused.success) {
|
|
36
|
-
const f = focused.data
|
|
48
|
+
const f: FocusProbe = isFocusProbe(focused.data) ? focused.data : { ok: false, tag: null };
|
|
37
49
|
if (!f.ok) {
|
|
38
50
|
return err({
|
|
39
51
|
kind: "invalid_state",
|
|
@@ -43,8 +55,8 @@ export const typeTool = defineBrowserTool({
|
|
|
43
55
|
});
|
|
44
56
|
}
|
|
45
57
|
}
|
|
46
|
-
const r = await client
|
|
47
|
-
if (!r.success) return
|
|
58
|
+
const r = await cdpCall(client, "Input.insertText", { text: args.text });
|
|
59
|
+
if (!r.success) return r;
|
|
48
60
|
return ok({ text: `Typed: "${args.text}"` });
|
|
49
61
|
},
|
|
50
62
|
});
|
|
@@ -71,11 +83,10 @@ export const pressKeyTool = defineBrowserTool({
|
|
|
71
83
|
promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)",
|
|
72
84
|
promptGuidelines: [
|
|
73
85
|
"Use browser_press_key for keyboard shortcuts and navigation keys.",
|
|
74
|
-
"
|
|
75
|
-
"Use Space as ' ' (a single space character).",
|
|
76
|
-
"Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
|
|
86
|
+
"browser_press_key modifiers combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
|
|
77
87
|
],
|
|
78
88
|
parameters: PressKeyArgs,
|
|
89
|
+
concurrency: "serialized",
|
|
79
90
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
80
91
|
const k = args.key;
|
|
81
92
|
const code = virtualKeyCode(k);
|
|
@@ -89,18 +100,16 @@ export const pressKeyTool = defineBrowserTool({
|
|
|
89
100
|
...(code ? { windowsVirtualKeyCode: code, nativeVirtualKeyCode: code } : {}),
|
|
90
101
|
...(isChar ? { text: k, unmodifiedText: k } : {}),
|
|
91
102
|
};
|
|
92
|
-
const down = await client
|
|
93
|
-
if (!down.success) return
|
|
94
|
-
//
|
|
95
|
-
// emit a `char` event so the page receives keypress/textInput and the
|
|
96
|
-
// character is actually inserted. Shift (8) is allowed (capitals/symbols).
|
|
103
|
+
const down = await cdpCall(client, "Input.dispatchKeyEvent", downParams);
|
|
104
|
+
if (!down.success) return down;
|
|
105
|
+
// A printable character needs a `char` event for the page to receive keypress/textInput and actually insert it; Shift (8) still counts as printable.
|
|
97
106
|
const hasCommandModifier = (modifiers & (1 | 2 | 4)) !== 0;
|
|
98
107
|
if (isChar && !hasCommandModifier) {
|
|
99
|
-
const charEv = await client
|
|
100
|
-
if (!charEv.success) return
|
|
108
|
+
const charEv = await cdpCall(client, "Input.dispatchKeyEvent", { type: "char", text: k, key: k });
|
|
109
|
+
if (!charEv.success) return charEv;
|
|
101
110
|
}
|
|
102
|
-
const up = await client
|
|
103
|
-
if (!up.success) return
|
|
111
|
+
const up = await cdpCall(client, "Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers });
|
|
112
|
+
if (!up.success) return up;
|
|
104
113
|
return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` });
|
|
105
114
|
},
|
|
106
115
|
});
|
|
@@ -126,32 +135,27 @@ export const dispatchKeyTool = defineBrowserTool({
|
|
|
126
135
|
"Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
|
|
127
136
|
promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector",
|
|
128
137
|
promptGuidelines: [
|
|
129
|
-
"
|
|
130
|
-
"Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).",
|
|
138
|
+
"Does NOT insert text (use browser_type or browser_press_key for actual typing).",
|
|
131
139
|
"Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
|
|
132
140
|
"The event carries keyCode/which (e.g. 13 for Enter) for legacy handlers, but is untrusted (isTrusted === false) — a few libraries may still ignore it.",
|
|
133
|
-
"eventType defaults to 'keydown'.",
|
|
134
141
|
],
|
|
135
142
|
parameters: DispatchKeyArgs,
|
|
143
|
+
concurrency: "serialized",
|
|
136
144
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
137
145
|
const eventType = args.eventType ?? "keydown";
|
|
138
146
|
const target = args.ref ?? args.selector ?? "";
|
|
139
|
-
// Legacy handlers
|
|
140
|
-
// Enter) rather than e.key, so populate both. virtualKeyCode maps named
|
|
141
|
-
// keys and single chars; 0 for anything unknown.
|
|
147
|
+
// Legacy handlers branch on e.keyCode/e.which rather than e.key, so populate both.
|
|
142
148
|
const code = virtualKeyCode(args.key);
|
|
143
|
-
// Ref path: dispatch on the single resolved node. Selector path: dispatch on
|
|
144
|
-
// all matches (preserves the original multi-match behavior).
|
|
145
149
|
if (args.ref !== undefined) {
|
|
146
150
|
const objectId = await resolveRefToObjectId(client, args.ref);
|
|
147
151
|
if (!objectId.success) return objectId;
|
|
148
|
-
const r = await client
|
|
152
|
+
const r = await cdpCall(client, "Runtime.callFunctionOn", {
|
|
149
153
|
objectId: objectId.data,
|
|
150
154
|
functionDeclaration: `function (type, key, keyCode) { this.dispatchEvent(new KeyboardEvent(type, { key, keyCode, which: keyCode, bubbles: true, cancelable: true })); return 1; }`,
|
|
151
155
|
arguments: [{ value: eventType }, { value: args.key }, { value: code }],
|
|
152
156
|
returnByValue: true,
|
|
153
157
|
});
|
|
154
|
-
if (!r.success) return
|
|
158
|
+
if (!r.success) return r;
|
|
155
159
|
return ok({
|
|
156
160
|
text: `Dispatched ${eventType}('${args.key}') on ${target}`,
|
|
157
161
|
details: { matched: 1, ref: args.ref },
|
|
@@ -170,8 +174,8 @@ export const dispatchKeyTool = defineBrowserTool({
|
|
|
170
174
|
return els.length;
|
|
171
175
|
})()
|
|
172
176
|
`;
|
|
173
|
-
const r = await client
|
|
174
|
-
if (!r.success) return
|
|
177
|
+
const r = await evalJs(client, expr);
|
|
178
|
+
if (!r.success) return r;
|
|
175
179
|
const matched = Number(r.data ?? 0);
|
|
176
180
|
if (matched === 0) {
|
|
177
181
|
return err({
|
package/src/domains/navigate.ts
CHANGED
|
@@ -5,12 +5,13 @@ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
|
5
5
|
import { applyTruncation } from "../util/truncate";
|
|
6
6
|
import { safeJs } from "../util/js-template";
|
|
7
7
|
import { ensureHarnessWindow, openHarnessTab } from "../cdp/target-factory";
|
|
8
|
+
import { cdpCall, cdpCallBrowser, cdpCallOnTarget, evalJs } from "./cdp-call";
|
|
8
9
|
|
|
9
10
|
const NavigateArgs = Type.Object({
|
|
10
11
|
url: Type.String({ description: "Full URL to navigate to (e.g. https://github.com)" }),
|
|
11
12
|
});
|
|
12
13
|
|
|
13
|
-
|
|
14
|
+
type NavOutcome =
|
|
14
15
|
| { readonly kind: "in_place"; readonly targetId: string }
|
|
15
16
|
| { readonly kind: "new_tab_created"; readonly targetId: string; readonly reason: "no_tabs" };
|
|
16
17
|
|
|
@@ -21,12 +22,11 @@ export const navigateTool = defineBrowserTool({
|
|
|
21
22
|
"Navigate to a URL. Creates a new tab on first call (when no real tabs exist). Otherwise navigates the current tab in place.",
|
|
22
23
|
promptSnippet: "Navigate to a URL",
|
|
23
24
|
promptGuidelines: [
|
|
24
|
-
"Use browser_navigate to go to URLs.",
|
|
25
|
-
"Use browser_wait_for_load after browser_navigate to wait for the page to finish loading.",
|
|
26
25
|
"For extracting data from a page you already navigated to, use browser_execute_js or browser_http_get (faster for APIs).",
|
|
27
26
|
"Note: Google and some sites with strict anti-bot detection may reject CDP navigation. Use browser_http_get for search results.",
|
|
28
27
|
],
|
|
29
28
|
parameters: NavigateArgs,
|
|
29
|
+
concurrency: "serialized",
|
|
30
30
|
renderCall: (a) => new Text(`🌐 Navigate to ${a.url}`, 0, 0),
|
|
31
31
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
32
32
|
const tabs = await client.listTabs(false);
|
|
@@ -37,8 +37,8 @@ export const navigateTool = defineBrowserTool({
|
|
|
37
37
|
if (!created.success) return err({ kind: "cdp_error", message: created.error.message });
|
|
38
38
|
outcome = { kind: "new_tab_created", targetId: created.data, reason: "no_tabs" };
|
|
39
39
|
} else {
|
|
40
|
-
const nav = await client
|
|
41
|
-
if (!nav.success) return
|
|
40
|
+
const nav = await cdpCall(client, "Page.navigate", { url: args.url });
|
|
41
|
+
if (!nav.success) return nav;
|
|
42
42
|
const cur = client.current();
|
|
43
43
|
if (!cur) return err({ kind: "internal", message: "navigate succeeded but no current target tracked" });
|
|
44
44
|
outcome = { kind: "in_place", targetId: cur.targetId };
|
|
@@ -78,59 +78,50 @@ export const openUrlsTool = defineBrowserTool({
|
|
|
78
78
|
promptGuidelines: [
|
|
79
79
|
"Use after browser_web_search to open citations in parallel.",
|
|
80
80
|
"After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.",
|
|
81
|
-
"Use browser_wait_for_load on a tab before extracting data from SPAs.",
|
|
82
81
|
],
|
|
83
82
|
parameters: OpenUrlsArgs,
|
|
83
|
+
concurrency: "serialized",
|
|
84
84
|
renderCall: (a) => new Text(`🌐 Opening ${a.urls.length} URL${a.urls.length !== 1 ? "s" : ""}…`, 0, 0),
|
|
85
85
|
async handler(args, { client, onUpdate }): Promise<Result<ToolOk, ToolErr>> {
|
|
86
86
|
const total = args.urls.length;
|
|
87
|
-
//
|
|
88
|
-
// already exists, open every url as its child in parallel. If it doesn't
|
|
89
|
-
// (fresh/closed), create the FIRST url as a new dedicated window (binding
|
|
90
|
-
// its windowId) and open the rest as children of it — a parallel
|
|
91
|
-
// newWindow:true per url would otherwise scatter tabs across N windows.
|
|
87
|
+
// All tabs must land in ONE window: a parallel newWindow:true per url would scatter them across N windows, so the first url creates the window and the rest open as its children.
|
|
92
88
|
const url0 = args.urls[0];
|
|
93
89
|
if (url0 === undefined) return err({ kind: "invalid_state", message: "no URLs provided" });
|
|
94
90
|
const window = await ensureHarnessWindow(client);
|
|
95
91
|
if (!window.success) return err({ kind: "cdp_error", message: window.error.message });
|
|
96
92
|
const seededWindow = window.data.targetId;
|
|
97
|
-
// A freshly created window's seed tab serves as the first URL's tab.
|
|
98
93
|
const seedResult: TabResult | undefined = window.data.freshlyCreated
|
|
99
94
|
? { url: url0, targetId: seededWindow, ok: true }
|
|
100
95
|
: undefined;
|
|
101
96
|
const created = await Promise.all(args.urls.map(async (url, i): Promise<TabResult> => {
|
|
102
|
-
// Reuse the seed tab (created above) for the first url instead of duplicating it.
|
|
103
97
|
if (i === 0 && seedResult) return seedResult;
|
|
104
98
|
const r = await openHarnessTab(client, seededWindow);
|
|
105
99
|
if (!r.success) return { url, targetId: "", ok: false, error: r.error.message };
|
|
106
100
|
return { url, targetId: r.data, ok: true };
|
|
107
101
|
}));
|
|
108
|
-
// Phase 2: attach + navigate each created tab in parallel.
|
|
109
102
|
let completed = 0;
|
|
110
103
|
const settled = await Promise.all(
|
|
111
104
|
created.filter((t) => t.ok).map(async (tab): Promise<TabResult> => {
|
|
112
105
|
try {
|
|
113
|
-
const attached = await client.session().
|
|
106
|
+
const attached = await client.session().attach(tab.targetId);
|
|
114
107
|
if (!attached.success) return { ...tab, ok: false, error: attached.error.message };
|
|
115
|
-
const sid =
|
|
116
|
-
const enabled = await client
|
|
108
|
+
const sid = attached.data;
|
|
109
|
+
const enabled = await cdpCallOnTarget(client, "Page.enable", {}, sid);
|
|
117
110
|
if (!enabled.success) return { ...tab, ok: false, error: enabled.error.message };
|
|
118
|
-
const nav = await client
|
|
111
|
+
const nav = await cdpCallOnTarget(client, "Page.navigate", { url: tab.url }, sid);
|
|
119
112
|
if (!nav.success) return { ...tab, ok: false, error: nav.error.message };
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
client.evaluateJs(
|
|
113
|
+
evalJs(
|
|
114
|
+
client,
|
|
123
115
|
safeJs`if(!document.title.startsWith('🟢'))document.title='🟢 '+document.title`,
|
|
124
116
|
sid
|
|
125
117
|
).catch(() => {});
|
|
126
118
|
return tab;
|
|
127
119
|
} finally {
|
|
128
120
|
completed++;
|
|
129
|
-
// onUpdate must never escape: a renderer crash would otherwise
|
|
130
|
-
// tear down the tool call.
|
|
121
|
+
// onUpdate must never escape: a renderer crash would otherwise tear down the tool call.
|
|
131
122
|
try {
|
|
132
123
|
onUpdate({ text: `Opening URLs… ${completed}/${created.filter((t) => t.ok).length} navigated` });
|
|
133
|
-
} catch {
|
|
124
|
+
} catch {}
|
|
134
125
|
}
|
|
135
126
|
}),
|
|
136
127
|
);
|
|
@@ -138,11 +129,10 @@ export const openUrlsTool = defineBrowserTool({
|
|
|
138
129
|
const all: ReadonlyArray<TabResult> = [...settled, ...failures];
|
|
139
130
|
const okTabs = all.filter((r) => r.ok);
|
|
140
131
|
const failTabs = all.filter((r) => !r.ok);
|
|
141
|
-
// Phase 3: activate the last successfully-opened tab so the user sees something.
|
|
142
132
|
if (okTabs.length > 0) {
|
|
143
133
|
const last = okTabs[okTabs.length - 1];
|
|
144
134
|
if (last) {
|
|
145
|
-
await client
|
|
135
|
+
await cdpCallBrowser(client, "Target.activateTarget", { targetId: last.targetId });
|
|
146
136
|
}
|
|
147
137
|
}
|
|
148
138
|
const lines: string[] = [];
|