pi-browser-harness 0.10.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +21 -10
- package/package.json +6 -3
- package/skills/pi-browser-harness/SKILL.md +30 -1
- package/src/cdp/commands.ts +260 -0
- package/src/cdp/console-buffer.ts +10 -85
- package/src/cdp/discovery.ts +31 -71
- package/src/cdp/errors.ts +4 -0
- package/src/cdp/events.ts +170 -0
- package/src/cdp/network-buffer.ts +42 -109
- package/src/cdp/ownership.ts +0 -17
- package/src/cdp/record-store.ts +61 -0
- package/src/cdp/session.ts +205 -193
- package/src/cdp/target-factory.ts +10 -80
- package/src/cdp/types.ts +21 -12
- package/src/client.ts +39 -95
- package/src/daemon/bridge.ts +375 -265
- package/src/daemon/index.ts +0 -28
- package/src/daemon/protocol.ts +6 -68
- package/src/daemon/queue.ts +85 -0
- package/src/daemon/server.ts +24 -99
- package/src/daemon/spawn.ts +90 -82
- package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +24 -52
- package/src/domains/ax-tree.ts +179 -0
- package/src/domains/box.ts +51 -0
- package/src/domains/cdp-call.ts +55 -0
- package/src/domains/click.ts +19 -37
- package/src/domains/console.ts +48 -4
- package/src/domains/dialog.ts +8 -5
- package/src/domains/drag.ts +8 -7
- package/src/domains/element-call.ts +117 -0
- package/src/domains/files.ts +75 -30
- package/src/domains/fill-engine.ts +80 -0
- package/src/domains/form-batch.ts +64 -0
- package/src/domains/form.ts +115 -112
- package/src/domains/headers.ts +89 -0
- package/src/domains/history.ts +16 -12
- package/src/domains/isolated-tab.ts +26 -55
- package/src/domains/js.ts +32 -51
- package/src/domains/keyboard.ts +38 -34
- package/src/domains/navigate.ts +16 -26
- package/src/domains/network.ts +61 -13
- package/src/domains/page.ts +25 -10
- package/src/domains/readpage/read-page.ts +74 -20
- package/src/domains/readpage/readability.ts +0 -24
- package/src/domains/ref-resolve.ts +24 -58
- package/src/domains/render.ts +6 -27
- package/src/domains/screenshot-capture.ts +80 -0
- package/src/domains/screenshot.ts +9 -90
- package/src/domains/scroll.ts +10 -12
- package/src/domains/search/google-serp.ts +0 -29
- package/src/domains/search/web-search.ts +48 -21
- package/src/domains/setup.ts +1 -13
- package/src/domains/snapshot.ts +36 -280
- package/src/domains/tabs.ts +15 -16
- package/src/domains/target.ts +35 -0
- package/src/domains/viewport.ts +5 -4
- package/src/index.ts +23 -69
- package/src/profile/bind.ts +8 -57
- package/src/profile/browser-process.ts +75 -126
- package/src/profile/command.ts +50 -29
- package/src/profile/launch.ts +22 -40
- package/src/profile/list.ts +14 -54
- package/src/profile/paths.ts +2 -65
- package/src/profile/store.ts +35 -56
- package/src/prompt.ts +22 -122
- package/src/registry.ts +33 -33
- package/src/schemas/parse.ts +23 -0
- package/src/setup.ts +0 -25
- package/src/state.ts +48 -18
- package/src/util/debug.ts +8 -0
- package/src/util/guards.ts +30 -0
- package/src/util/js-template.ts +1 -16
- package/src/util/mutex.ts +0 -5
- package/src/util/paths.ts +11 -1
- package/src/util/result.ts +0 -6
- package/src/util/sharp-shim.ts +12 -19
- package/src/util/tool.ts +17 -42
- package/src/util/truncate.ts +6 -0
- package/tsconfig.json +1 -16
- package/skills/deep-research/SKILL.md +0 -66
- package/src/cdp/event-queue.ts +0 -128
- package/src/cdp/transport.ts +0 -156
- package/src/cdp/window-binding.ts +0 -15
- package/src/deep-research.ts +0 -41
- package/src/domains/forms.ts +0 -355
- package/src/domains/readpage/capture.ts +0 -46
- package/src/domains/search/extract.ts +0 -44
- package/src/profile/picker.ts +0 -73
- package/src/util/keycodes.ts +0 -10
- package/src/util/time.ts +0 -29
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { BrowserClient } from "../client";
|
|
2
|
+
import type { CdpMethod, ParamsOf, ResultOf } from "../cdp/commands";
|
|
3
|
+
import type { CdpError } from "../cdp/errors";
|
|
4
|
+
import { type Result, err } from "../util/result";
|
|
5
|
+
import type { ToolErr } from "../util/tool";
|
|
6
|
+
|
|
7
|
+
export const cdpErrToToolErr = (e: CdpError, method: string): ToolErr => ({
|
|
8
|
+
kind: e.kind === "timeout" ? "timeout" : "cdp_error",
|
|
9
|
+
message: e.message,
|
|
10
|
+
details: { method },
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
export const cdpCall = async <M extends CdpMethod>(
|
|
14
|
+
client: BrowserClient,
|
|
15
|
+
method: M,
|
|
16
|
+
params: ParamsOf<M>,
|
|
17
|
+
opts?: { timeoutMs?: number },
|
|
18
|
+
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
19
|
+
const r = await client.session().call(method, params, opts);
|
|
20
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
21
|
+
return r;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export const cdpCallOnTarget = async <M extends CdpMethod>(
|
|
25
|
+
client: BrowserClient,
|
|
26
|
+
method: M,
|
|
27
|
+
params: ParamsOf<M>,
|
|
28
|
+
sessionId: string,
|
|
29
|
+
opts?: { timeoutMs?: number },
|
|
30
|
+
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
31
|
+
const r = await client.session().callOnTarget(method, params, sessionId, opts);
|
|
32
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
33
|
+
return r;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export const cdpCallBrowser = async <M extends CdpMethod>(
|
|
37
|
+
client: BrowserClient,
|
|
38
|
+
method: M,
|
|
39
|
+
params?: ParamsOf<M>,
|
|
40
|
+
opts?: { timeoutMs?: number },
|
|
41
|
+
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
42
|
+
const r = await client.session().callBrowser(method, params, opts);
|
|
43
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
44
|
+
return r;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export const evalJs = async (
|
|
48
|
+
client: BrowserClient,
|
|
49
|
+
expression: string,
|
|
50
|
+
sessionId?: string,
|
|
51
|
+
): Promise<Result<unknown, ToolErr>> => {
|
|
52
|
+
const r = await client.evaluateJs(expression, sessionId);
|
|
53
|
+
if (!r.success) return err(cdpErrToToolErr(r.error, "Runtime.evaluate"));
|
|
54
|
+
return r;
|
|
55
|
+
};
|
package/src/domains/click.ts
CHANGED
|
@@ -2,10 +2,13 @@ import { Type } from "typebox";
|
|
|
2
2
|
import { Text } from "@mariozechner/pi-tui";
|
|
3
3
|
import type { BrowserClient } from "../client";
|
|
4
4
|
import { Coords, MouseButton } from "../schemas/common";
|
|
5
|
-
import { type Result,
|
|
5
|
+
import { type Result, ok } from "../util/result";
|
|
6
|
+
import { debugClicksEnabled } from "../util/debug";
|
|
6
7
|
import { defineBrowserTool, type ToolErr } from "../util/tool";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
8
|
+
import { cdpCall } from "./cdp-call";
|
|
9
|
+
import { captureWithCrosshair } from "./screenshot-capture";
|
|
10
|
+
import { interactiveDiff } from "./ref-resolve";
|
|
11
|
+
import { resolveTarget } from "./target";
|
|
9
12
|
|
|
10
13
|
const ClickArgs = Type.Object({
|
|
11
14
|
ref: Type.Optional(
|
|
@@ -34,21 +37,17 @@ const dispatchClick = async (
|
|
|
34
37
|
button: "left" | "right" | "middle",
|
|
35
38
|
count: number,
|
|
36
39
|
): Promise<Result<void, ToolErr>> => {
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
type: "mouseMoved", x, y,
|
|
42
|
-
});
|
|
43
|
-
if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message });
|
|
44
|
-
const pressed = await client.session().call("Input.dispatchMouseEvent", {
|
|
40
|
+
// Some focus/hover handlers and React synthetic events only fire when a mousemove precedes the press, so without it a click lands without focusing.
|
|
41
|
+
const moved = await cdpCall(client, "Input.dispatchMouseEvent", { type: "mouseMoved", x, y });
|
|
42
|
+
if (!moved.success) return moved;
|
|
43
|
+
const pressed = await cdpCall(client, "Input.dispatchMouseEvent", {
|
|
45
44
|
type: "mousePressed", x, y, button, clickCount: count,
|
|
46
45
|
});
|
|
47
|
-
if (!pressed.success) return
|
|
48
|
-
const released = await client
|
|
46
|
+
if (!pressed.success) return pressed;
|
|
47
|
+
const released = await cdpCall(client, "Input.dispatchMouseEvent", {
|
|
49
48
|
type: "mouseReleased", x, y, button, clickCount: count,
|
|
50
49
|
});
|
|
51
|
-
if (!released.success) return
|
|
50
|
+
if (!released.success) return released;
|
|
52
51
|
return ok(undefined);
|
|
53
52
|
};
|
|
54
53
|
|
|
@@ -59,41 +58,24 @@ export const clickTool = defineBrowserTool({
|
|
|
59
58
|
"Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.",
|
|
60
59
|
promptSnippet: "Click an element by ref (preferred) or pixel coordinates",
|
|
61
60
|
promptGuidelines: [
|
|
62
|
-
"
|
|
63
|
-
"Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.",
|
|
64
|
-
"A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
|
|
65
|
-
"After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
|
|
66
|
-
"Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
|
|
67
|
-
"If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.",
|
|
61
|
+
"If a click doesn't register, run pi with --browser-debug-clicks (or set BH_DEBUG_CLICKS=1) to get annotated screenshots. For React/Vue components ignoring clicks, try browser_dispatch_key.",
|
|
68
62
|
],
|
|
69
63
|
parameters: ClickArgs,
|
|
64
|
+
concurrency: "serialized",
|
|
70
65
|
renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0),
|
|
71
66
|
async handler(args, { client }) {
|
|
72
67
|
const button = args.button ?? "left";
|
|
73
68
|
const count = args.count ?? 1;
|
|
74
69
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
let y: number;
|
|
79
|
-
if (args.ref !== undefined) {
|
|
80
|
-
const box = await resolveRefToBox(client, args.ref);
|
|
81
|
-
if (!box.success) return box;
|
|
82
|
-
x = box.data.cx;
|
|
83
|
-
y = box.data.cy;
|
|
84
|
-
} else if (args.x !== undefined && args.y !== undefined) {
|
|
85
|
-
x = args.x;
|
|
86
|
-
y = args.y;
|
|
87
|
-
} else {
|
|
88
|
-
return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." });
|
|
89
|
-
}
|
|
70
|
+
const resolved = await resolveTarget(client, args);
|
|
71
|
+
if (!resolved.success) return resolved;
|
|
72
|
+
const { x, y, label: target } = resolved.data;
|
|
90
73
|
|
|
91
74
|
const clicked = await dispatchClick(client, x, y, button, count);
|
|
92
75
|
if (!clicked.success) return clicked;
|
|
93
76
|
|
|
94
|
-
const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`;
|
|
95
77
|
const diff = await interactiveDiff(client);
|
|
96
|
-
if (
|
|
78
|
+
if (debugClicksEnabled()) {
|
|
97
79
|
const debug = await captureWithCrosshair(client, { x, y });
|
|
98
80
|
if (debug.success) {
|
|
99
81
|
return ok({
|
package/src/domains/console.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { type Result, ok } from "../util/result";
|
|
|
5
5
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
6
6
|
import { applyTruncation } from "../util/truncate";
|
|
7
7
|
import type { ConsoleRecord, ConsoleLevel } from "../cdp/console-buffer";
|
|
8
|
+
import { asArrayOf, asBoolean, asNumber, asRecord, asString } from "../util/guards";
|
|
8
9
|
|
|
9
10
|
const ConsoleArgs = Type.Object({
|
|
10
11
|
levels: Type.Optional(
|
|
@@ -106,6 +107,50 @@ type ConsoleDetails = {
|
|
|
106
107
|
nextCursor?: number;
|
|
107
108
|
};
|
|
108
109
|
|
|
110
|
+
const CONSOLE_LEVELS: ReadonlyArray<ConsoleLevel> = ["log", "info", "warn", "error", "debug"];
|
|
111
|
+
|
|
112
|
+
const asConsoleLevel = (v: unknown): ConsoleLevel | undefined =>
|
|
113
|
+
CONSOLE_LEVELS.find((l) => l === v);
|
|
114
|
+
|
|
115
|
+
const asConsoleRecord = (v: unknown): ConsoleRecord | undefined => {
|
|
116
|
+
const o = asRecord(v);
|
|
117
|
+
if (o === undefined) return undefined;
|
|
118
|
+
const level = asConsoleLevel(o["level"]);
|
|
119
|
+
const text = asString(o["text"]);
|
|
120
|
+
if (level === undefined || text === undefined) return undefined;
|
|
121
|
+
const url = asString(o["url"]);
|
|
122
|
+
const lineNumber = asNumber(o["lineNumber"]);
|
|
123
|
+
const stackTrace = asString(o["stackTrace"]);
|
|
124
|
+
return {
|
|
125
|
+
seq: asNumber(o["seq"]) ?? 0,
|
|
126
|
+
timestampMs: asNumber(o["timestampMs"]) ?? 0,
|
|
127
|
+
level,
|
|
128
|
+
text,
|
|
129
|
+
source: o["source"] === "log-entry" ? "log-entry" : "console-api",
|
|
130
|
+
...(url !== undefined ? { url } : {}),
|
|
131
|
+
...(lineNumber !== undefined ? { lineNumber } : {}),
|
|
132
|
+
...(stackTrace !== undefined ? { stackTrace } : {}),
|
|
133
|
+
};
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
const asConsoleDetails = (v: unknown): ConsoleDetails | undefined => {
|
|
137
|
+
const o = asRecord(v);
|
|
138
|
+
if (o === undefined) return undefined;
|
|
139
|
+
const records = asArrayOf(o["records"], asConsoleRecord);
|
|
140
|
+
if (records === undefined) return undefined;
|
|
141
|
+
const fullOutputPath = asString(o["fullOutputPath"]);
|
|
142
|
+
const nextCursor = asNumber(o["nextCursor"]);
|
|
143
|
+
return {
|
|
144
|
+
total: asNumber(o["total"]) ?? records.length,
|
|
145
|
+
returned: asNumber(o["returned"]) ?? records.length,
|
|
146
|
+
bufferOverflowed: asBoolean(o["bufferOverflowed"]) ?? false,
|
|
147
|
+
truncated: asBoolean(o["truncated"]) ?? false,
|
|
148
|
+
records,
|
|
149
|
+
...(fullOutputPath !== undefined ? { fullOutputPath } : {}),
|
|
150
|
+
...(nextCursor !== undefined ? { nextCursor } : {}),
|
|
151
|
+
};
|
|
152
|
+
};
|
|
153
|
+
|
|
109
154
|
export const consoleTool = defineBrowserTool({
|
|
110
155
|
name: "browser_console",
|
|
111
156
|
label: "Browser Console",
|
|
@@ -113,12 +158,11 @@ export const consoleTool = defineBrowserTool({
|
|
|
113
158
|
"Read JS errors and console messages from the active tab. Diagnostic — use when an action looks broken or silent.",
|
|
114
159
|
promptSnippet: "Read JS errors / console output on the current tab",
|
|
115
160
|
promptGuidelines: [
|
|
116
|
-
"Use after a click, submit, or navigate produced no visible change — errors often explain it.",
|
|
117
|
-
"Pass `sinceSeq` from the previous call's `nextCursor` to see only new messages after an action.",
|
|
118
161
|
"Buffer is reset on tab switch (capacity 500 records). bufferOverflowed:true means older entries were dropped.",
|
|
119
162
|
"Don't use as a default observation tool — prefer browser_snapshot for page structure.",
|
|
120
163
|
],
|
|
121
164
|
parameters: ConsoleArgs,
|
|
165
|
+
concurrency: "parallel",
|
|
122
166
|
|
|
123
167
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
124
168
|
const session = client.session();
|
|
@@ -136,7 +180,7 @@ export const consoleTool = defineBrowserTool({
|
|
|
136
180
|
const text = renderConsoleMarkdown(drained.records, {
|
|
137
181
|
total: drained.total,
|
|
138
182
|
bufferOverflowed: drained.bufferOverflowed,
|
|
139
|
-
expanded: true,
|
|
183
|
+
expanded: true,
|
|
140
184
|
nextCursor,
|
|
141
185
|
});
|
|
142
186
|
const trunc = await applyTruncation(text, "console");
|
|
@@ -155,7 +199,7 @@ export const consoleTool = defineBrowserTool({
|
|
|
155
199
|
},
|
|
156
200
|
|
|
157
201
|
renderResult(result, expanded, theme) {
|
|
158
|
-
const details = result.details
|
|
202
|
+
const details = asConsoleDetails(result.details);
|
|
159
203
|
if (!details) return new Text(theme.fg("error", "console: no details"), 0, 0);
|
|
160
204
|
|
|
161
205
|
const md = renderConsoleMarkdown(details.records, {
|
package/src/domains/dialog.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
-
import { type Result,
|
|
2
|
+
import { type Result, ok } from "../util/result";
|
|
3
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
import { cdpCall } from "./cdp-call";
|
|
4
5
|
|
|
5
6
|
const HandleDialogArgs = Type.Object({
|
|
6
7
|
accept: Type.Boolean({ description: "true = accept, false = dismiss" }),
|
|
@@ -18,11 +19,13 @@ export const handleDialogTool = defineBrowserTool({
|
|
|
18
19
|
"For prompt() dialogs, supply promptText with the value to submit.",
|
|
19
20
|
],
|
|
20
21
|
parameters: HandleDialogArgs,
|
|
22
|
+
concurrency: "serialized",
|
|
21
23
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
const r = await cdpCall(client, "Page.handleJavaScriptDialog", {
|
|
25
|
+
accept: args.accept,
|
|
26
|
+
...(args.promptText !== undefined ? { promptText: args.promptText } : {}),
|
|
27
|
+
});
|
|
28
|
+
if (!r.success) return r;
|
|
26
29
|
return ok({
|
|
27
30
|
text: `Dialog ${args.accept ? "accepted" : "dismissed"}`,
|
|
28
31
|
details: { accept: args.accept },
|
package/src/domains/drag.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
-
import { type Result,
|
|
2
|
+
import { type Result, ok } from "../util/result";
|
|
3
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
+
import { cdpCall } from "./cdp-call";
|
|
4
5
|
|
|
5
6
|
const DragArgs = Type.Object({
|
|
6
7
|
startX: Type.Number(),
|
|
@@ -24,6 +25,7 @@ export const dragAndDropTool = defineBrowserTool({
|
|
|
24
25
|
"dataTransfer is an optional MIME→data map (e.g., { 'text/plain': 'hello' }).",
|
|
25
26
|
],
|
|
26
27
|
parameters: DragArgs,
|
|
28
|
+
concurrency: "serialized",
|
|
27
29
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
28
30
|
const data = args.dataTransfer
|
|
29
31
|
? {
|
|
@@ -34,12 +36,11 @@ export const dragAndDropTool = defineBrowserTool({
|
|
|
34
36
|
dragOperationsMask: 1,
|
|
35
37
|
}
|
|
36
38
|
: { items: [], dragOperationsMask: 1 };
|
|
37
|
-
|
|
38
|
-
const calls: ReadonlyArray<
|
|
39
|
+
type InputCall = readonly ["Input.dispatchMouseEvent" | "Input.dispatchDragEvent", Record<string, unknown>];
|
|
40
|
+
const calls: ReadonlyArray<InputCall> = [
|
|
39
41
|
["Input.dispatchMouseEvent", { type: "mousePressed", x: args.startX, y: args.startY, button: "left", clickCount: 1 }],
|
|
40
42
|
["Input.dispatchDragEvent", { type: "dragEnter", x: args.startX, y: args.startY, data, modifiers: 0 }],
|
|
41
|
-
|
|
42
|
-
...Array.from({ length: 5 }, (_, i): readonly [string, Record<string, unknown>] => {
|
|
43
|
+
...Array.from({ length: 5 }, (_, i): InputCall => {
|
|
43
44
|
const t = (i + 1) / 5;
|
|
44
45
|
return [
|
|
45
46
|
"Input.dispatchDragEvent",
|
|
@@ -56,8 +57,8 @@ export const dragAndDropTool = defineBrowserTool({
|
|
|
56
57
|
["Input.dispatchMouseEvent", { type: "mouseReleased", x: args.endX, y: args.endY, button: "left", clickCount: 1 }],
|
|
57
58
|
];
|
|
58
59
|
for (const [method, params] of calls) {
|
|
59
|
-
const r = await
|
|
60
|
-
if (!r.success) return
|
|
60
|
+
const r = await cdpCall(client, method, params);
|
|
61
|
+
if (!r.success) return r;
|
|
61
62
|
}
|
|
62
63
|
return ok({
|
|
63
64
|
text: `Dragged from (${args.startX},${args.startY}) to (${args.endX},${args.endY})`,
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import type { BrowserClient } from "../client";
|
|
2
|
+
import { safeJs } from "../util/js-template";
|
|
3
|
+
import { type Result, err, ok } from "../util/result";
|
|
4
|
+
import type { ToolErr } from "../util/tool";
|
|
5
|
+
import { cdpCall, evalJs } from "./cdp-call";
|
|
6
|
+
import { resolveRefToObjectId } from "./ref-resolve";
|
|
7
|
+
|
|
8
|
+
export type PageResult = {
|
|
9
|
+
readonly ok: boolean;
|
|
10
|
+
readonly reason?: string;
|
|
11
|
+
readonly kind?: string;
|
|
12
|
+
readonly value?: unknown;
|
|
13
|
+
readonly text?: string;
|
|
14
|
+
readonly checked?: boolean;
|
|
15
|
+
readonly changed?: boolean;
|
|
16
|
+
readonly tag?: string;
|
|
17
|
+
readonly options?: ReadonlyArray<{ value: string; text: string }>;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const isOption = (v: unknown): v is { readonly value: string; readonly text: string } =>
|
|
21
|
+
typeof v === "object" && v !== null
|
|
22
|
+
&& "value" in v && typeof v.value === "string"
|
|
23
|
+
&& "text" in v && typeof v.text === "string";
|
|
24
|
+
|
|
25
|
+
export const isPageResult = (v: unknown): v is PageResult => {
|
|
26
|
+
if (typeof v !== "object" || v === null) return false;
|
|
27
|
+
if (!("ok" in v) || typeof v.ok !== "boolean") return false;
|
|
28
|
+
if ("reason" in v && typeof v.reason !== "string") return false;
|
|
29
|
+
if ("kind" in v && typeof v.kind !== "string") return false;
|
|
30
|
+
if ("text" in v && typeof v.text !== "string") return false;
|
|
31
|
+
if ("checked" in v && typeof v.checked !== "boolean") return false;
|
|
32
|
+
if ("changed" in v && typeof v.changed !== "boolean") return false;
|
|
33
|
+
if ("tag" in v && typeof v.tag !== "string") return false;
|
|
34
|
+
if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false;
|
|
35
|
+
return true;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export const runOnElement = async (
|
|
39
|
+
client: BrowserClient,
|
|
40
|
+
opts: {
|
|
41
|
+
ref?: string | undefined;
|
|
42
|
+
selector?: string | undefined;
|
|
43
|
+
fnBody: string;
|
|
44
|
+
arg: unknown;
|
|
45
|
+
notFound?: unknown;
|
|
46
|
+
},
|
|
47
|
+
): Promise<Result<unknown, ToolErr>> => {
|
|
48
|
+
if (opts.ref !== undefined) {
|
|
49
|
+
const objectId = await resolveRefToObjectId(client, opts.ref);
|
|
50
|
+
if (!objectId.success) return objectId;
|
|
51
|
+
const r = await cdpCall(client, "Runtime.callFunctionOn", {
|
|
52
|
+
objectId: objectId.data,
|
|
53
|
+
functionDeclaration: `function (arg) { ${opts.fnBody} }`,
|
|
54
|
+
arguments: [{ value: opts.arg }],
|
|
55
|
+
returnByValue: true,
|
|
56
|
+
});
|
|
57
|
+
if (!r.success) return r;
|
|
58
|
+
if (r.data.exceptionDetails !== undefined) {
|
|
59
|
+
return err({
|
|
60
|
+
kind: "cdp_error",
|
|
61
|
+
message: `page function threw: ${r.data.exceptionDetails.text}`,
|
|
62
|
+
details: { ref: opts.ref },
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
return ok(r.data.result.value);
|
|
66
|
+
}
|
|
67
|
+
if (opts.selector === undefined) {
|
|
68
|
+
return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
|
|
69
|
+
}
|
|
70
|
+
// fnBody is harness-authored, not user input; only the selector and value are interpolated, and those go through safeJs.
|
|
71
|
+
const prelude = safeJs`
|
|
72
|
+
(() => {
|
|
73
|
+
const el = document.querySelector(${opts.selector});
|
|
74
|
+
if (!el) return ${opts.notFound ?? { status: "not_found" }};
|
|
75
|
+
const __arg = ${opts.arg};
|
|
76
|
+
return (function (arg) {`;
|
|
77
|
+
const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`;
|
|
78
|
+
const r = await evalJs(client, expr);
|
|
79
|
+
if (!r.success) return r;
|
|
80
|
+
return ok(r.data);
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
export const resolveAndCall = async (
|
|
84
|
+
client: BrowserClient,
|
|
85
|
+
ref: string,
|
|
86
|
+
functionDeclaration: string,
|
|
87
|
+
args: ReadonlyArray<unknown>,
|
|
88
|
+
): Promise<Result<PageResult, ToolErr>> => {
|
|
89
|
+
const objectId = await resolveRefToObjectId(client, ref);
|
|
90
|
+
if (!objectId.success) return objectId;
|
|
91
|
+
const handle = objectId.data;
|
|
92
|
+
try {
|
|
93
|
+
const called = await cdpCall(client, "Runtime.callFunctionOn", {
|
|
94
|
+
objectId: handle,
|
|
95
|
+
functionDeclaration,
|
|
96
|
+
arguments: args.map((v) => ({ value: v })),
|
|
97
|
+
returnByValue: true,
|
|
98
|
+
awaitPromise: true,
|
|
99
|
+
});
|
|
100
|
+
if (!called.success) return err({ ...called.error, details: { ref } });
|
|
101
|
+
if (called.data.exceptionDetails !== undefined) {
|
|
102
|
+
return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } });
|
|
103
|
+
}
|
|
104
|
+
const value = called.data.result.value;
|
|
105
|
+
if (!isPageResult(value)) {
|
|
106
|
+
return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
|
|
107
|
+
}
|
|
108
|
+
return ok(value);
|
|
109
|
+
} finally {
|
|
110
|
+
await cdpCall(client, "Runtime.releaseObject", { objectId: handle });
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
export const detailsOf = (ref: string, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
|
|
115
|
+
const { ok: _ok, ...rest } = res;
|
|
116
|
+
return { ref, ...rest, ...(extra ?? {}) };
|
|
117
|
+
};
|
package/src/domains/files.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { type Result, err, ok } from "../util/result";
|
|
|
6
6
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
7
7
|
import { safeJs } from "../util/js-template";
|
|
8
8
|
import { pdfPath } from "../util/paths";
|
|
9
|
+
import { cdpCall, cdpCallBrowser, evalJs } from "./cdp-call";
|
|
9
10
|
import { resolveRefToBackendId } from "./ref-resolve";
|
|
10
11
|
|
|
11
12
|
const UploadArgs = Type.Object({
|
|
@@ -16,15 +17,13 @@ const UploadArgs = Type.Object({
|
|
|
16
17
|
filePath: Type.String({ description: "Absolute path to the file to upload" }),
|
|
17
18
|
});
|
|
18
19
|
|
|
19
|
-
// Ref path: set files directly on the resolved backendNodeId via CDP — simplest
|
|
20
|
-
// and most reliable, no selector needed.
|
|
21
20
|
const tryRefUpload = async (
|
|
22
21
|
client: BrowserClient,
|
|
23
22
|
backendId: number,
|
|
24
23
|
filePath: string,
|
|
25
24
|
): Promise<Result<void, ToolErr>> => {
|
|
26
|
-
const set = await client
|
|
27
|
-
if (!set.success) return
|
|
25
|
+
const set = await cdpCall(client, "DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId });
|
|
26
|
+
if (!set.success) return set;
|
|
28
27
|
return ok(undefined);
|
|
29
28
|
};
|
|
30
29
|
|
|
@@ -45,19 +44,16 @@ const tryCdpUpload = async (
|
|
|
45
44
|
selector: string,
|
|
46
45
|
filePath: string,
|
|
47
46
|
): Promise<Result<void, ToolErr>> => {
|
|
48
|
-
const doc = await client
|
|
49
|
-
if (!doc.success) return
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const
|
|
53
|
-
if (!q.success) return err({ kind: "cdp_error", message: q.error.message });
|
|
54
|
-
// CDP boundary cast: DOM.querySelector returns { nodeId: number }
|
|
55
|
-
const nodeId = (q.data as { nodeId: number }).nodeId;
|
|
47
|
+
const doc = await cdpCall(client, "DOM.getDocument", { depth: -1 });
|
|
48
|
+
if (!doc.success) return doc;
|
|
49
|
+
const q = await cdpCall(client, "DOM.querySelector", { nodeId: doc.data.root.nodeId, selector });
|
|
50
|
+
if (!q.success) return q;
|
|
51
|
+
const nodeId = q.data.nodeId;
|
|
56
52
|
if (!nodeId) return err({ kind: "invalid_state", message: `Selector matched 0 file inputs: ${selector}` });
|
|
57
|
-
const set = await client
|
|
58
|
-
if (!set.success) return
|
|
59
|
-
const verify = await client
|
|
60
|
-
if (!verify.success) return
|
|
53
|
+
const set = await cdpCall(client, "DOM.setFileInputFiles", { files: [filePath], nodeId });
|
|
54
|
+
if (!set.success) return set;
|
|
55
|
+
const verify = await evalJs(client, safeJs`document.querySelector(${selector})?.files?.length || 0`);
|
|
56
|
+
if (!verify.success) return verify;
|
|
61
57
|
if (Number(verify.data ?? 0) === 0) {
|
|
62
58
|
return err({ kind: "invalid_state", message: "CDP upload reported success but file count is 0" });
|
|
63
59
|
}
|
|
@@ -96,8 +92,8 @@ const jsFallbackUpload = async (
|
|
|
96
92
|
return input.files.length;
|
|
97
93
|
})()
|
|
98
94
|
`;
|
|
99
|
-
const r = await client
|
|
100
|
-
if (!r.success) return
|
|
95
|
+
const r = await evalJs(client, expr);
|
|
96
|
+
if (!r.success) return r;
|
|
101
97
|
if (!Number(r.data ?? 0)) return err({ kind: "invalid_state", message: "JS fallback set 0 files" });
|
|
102
98
|
return ok(undefined);
|
|
103
99
|
};
|
|
@@ -108,15 +104,15 @@ export const uploadFileTool = defineBrowserTool({
|
|
|
108
104
|
description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.",
|
|
109
105
|
promptSnippet: "Upload a file to a file input by ref (preferred) or selector",
|
|
110
106
|
promptGuidelines: [
|
|
111
|
-
"
|
|
107
|
+
"File inputs are often hidden; browser_snapshot still surfaces them.",
|
|
112
108
|
"File path must be absolute and readable.",
|
|
113
109
|
],
|
|
114
110
|
parameters: UploadArgs,
|
|
111
|
+
concurrency: "serialized",
|
|
115
112
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
116
113
|
const readable = await verifyReadable(args.filePath);
|
|
117
114
|
if (!readable.success) return readable;
|
|
118
115
|
|
|
119
|
-
// Ref path: resolve to backendNodeId and set files directly via CDP.
|
|
120
116
|
if (args.ref !== undefined) {
|
|
121
117
|
const backendId = resolveRefToBackendId(client, args.ref);
|
|
122
118
|
if (!backendId.success) return backendId;
|
|
@@ -149,21 +145,70 @@ export const uploadFileTool = defineBrowserTool({
|
|
|
149
145
|
},
|
|
150
146
|
});
|
|
151
147
|
|
|
148
|
+
// Chrome stores the override in its default browser context, so it outlives this client and
|
|
149
|
+
// silently redirects the user's own downloads. Track it, so session teardown can undo it.
|
|
150
|
+
// The flag only records what THIS process set: the daemon is shared by every pi process on the
|
|
151
|
+
// machine and the CDP primitive is last-writer-wins, so another session may have reset or
|
|
152
|
+
// re-pointed it since. A crashed or killed session never reaches teardown, so the override can
|
|
153
|
+
// also outlive the process that set it.
|
|
154
|
+
let activeDownloadOverride: string | undefined;
|
|
155
|
+
|
|
156
|
+
// Shutdown must not block on a browser that is already gone: a pending restore against a
|
|
157
|
+
// disconnected daemon would otherwise trigger the one-shot reconnect (and Chrome's consent
|
|
158
|
+
// prompt) that 0.11.1 confined to explicit tool calls, and wait up to the connect timeout.
|
|
159
|
+
const SHUTDOWN_RESTORE_TIMEOUT_MS = 2_000;
|
|
160
|
+
|
|
161
|
+
export const restoreDownloadBehavior = async (client: BrowserClient): Promise<Result<void, ToolErr>> => {
|
|
162
|
+
const pending = activeDownloadOverride;
|
|
163
|
+
if (pending === undefined) return ok(undefined);
|
|
164
|
+
if (!client.status().alive) {
|
|
165
|
+
return err({ kind: "not_connected", message: `browser not connected; downloads may keep going to ${pending} until Chrome restarts` });
|
|
166
|
+
}
|
|
167
|
+
const r = await cdpCallBrowser(client, "Browser.setDownloadBehavior", { behavior: "default" }, { timeoutMs: SHUTDOWN_RESTORE_TIMEOUT_MS });
|
|
168
|
+
if (!r.success) {
|
|
169
|
+
return err({ ...r.error, message: `${r.error.message} (downloads may keep going to ${pending})` });
|
|
170
|
+
}
|
|
171
|
+
activeDownloadOverride = undefined;
|
|
172
|
+
return ok(undefined);
|
|
173
|
+
};
|
|
174
|
+
|
|
152
175
|
const DownloadArgs = Type.Object({
|
|
153
|
-
downloadPath: Type.
|
|
176
|
+
downloadPath: Type.Optional(
|
|
177
|
+
Type.String({ description: "Absolute path to a writable directory where downloads should be saved" }),
|
|
178
|
+
),
|
|
179
|
+
restore: Type.Optional(
|
|
180
|
+
Type.Boolean({
|
|
181
|
+
description:
|
|
182
|
+
"Undo a previous browser_download: reset Chrome's download behavior back to the user's own default.",
|
|
183
|
+
}),
|
|
184
|
+
),
|
|
154
185
|
});
|
|
155
186
|
|
|
156
187
|
export const downloadTool = defineBrowserTool({
|
|
157
188
|
name: "browser_download",
|
|
158
189
|
label: "Browser Download",
|
|
159
|
-
description:
|
|
190
|
+
description:
|
|
191
|
+
"Configure Chrome's download behavior: set the save directory and disable the save-as prompt, or restore the user's default. Restored automatically when this session shuts down cleanly.",
|
|
160
192
|
promptSnippet: "Configure download directory",
|
|
161
193
|
promptGuidelines: [
|
|
162
194
|
"Pass an absolute path to an existing writable directory.",
|
|
163
|
-
"
|
|
195
|
+
"The override applies to the user's own downloads too, not just yours, and only a clean session shutdown undoes it. Call browser_download({restore:true}) as soon as you are done.",
|
|
164
196
|
],
|
|
165
197
|
parameters: DownloadArgs,
|
|
198
|
+
concurrency: "serialized",
|
|
166
199
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
200
|
+
if (args.restore === true) {
|
|
201
|
+
if (args.downloadPath !== undefined) {
|
|
202
|
+
return err({ kind: "invalid_state", message: "Pass either downloadPath or restore, not both" });
|
|
203
|
+
}
|
|
204
|
+
const r = await cdpCallBrowser(client, "Browser.setDownloadBehavior", { behavior: "default" });
|
|
205
|
+
if (!r.success) return r;
|
|
206
|
+
activeDownloadOverride = undefined;
|
|
207
|
+
return ok({ text: "Downloads restored to Chrome's default location", details: { restored: true } });
|
|
208
|
+
}
|
|
209
|
+
if (args.downloadPath === undefined) {
|
|
210
|
+
return err({ kind: "invalid_state", message: "Provide downloadPath to set a directory, or restore:true to undo" });
|
|
211
|
+
}
|
|
167
212
|
try {
|
|
168
213
|
await mkdir(args.downloadPath, { recursive: true });
|
|
169
214
|
} catch (e) {
|
|
@@ -184,12 +229,13 @@ export const downloadTool = defineBrowserTool({
|
|
|
184
229
|
message: `Download path unusable: ${e instanceof Error ? e.message : String(e)}`,
|
|
185
230
|
});
|
|
186
231
|
}
|
|
187
|
-
const r = await client
|
|
232
|
+
const r = await cdpCallBrowser(client, "Browser.setDownloadBehavior", {
|
|
188
233
|
behavior: "allow",
|
|
189
234
|
downloadPath: args.downloadPath,
|
|
190
235
|
eventsEnabled: true,
|
|
191
236
|
});
|
|
192
|
-
if (!r.success) return
|
|
237
|
+
if (!r.success) return r;
|
|
238
|
+
activeDownloadOverride = args.downloadPath;
|
|
193
239
|
return ok({
|
|
194
240
|
text: `Downloads will save to: ${args.downloadPath}`,
|
|
195
241
|
details: { downloadPath: args.downloadPath },
|
|
@@ -208,16 +254,15 @@ export const printToPdfTool = defineBrowserTool({
|
|
|
208
254
|
promptSnippet: "Print the current page to PDF",
|
|
209
255
|
promptGuidelines: ["Default output path is in tmpdir; pass outputPath to control."],
|
|
210
256
|
parameters: PrintPdfArgs,
|
|
257
|
+
concurrency: "serialized",
|
|
211
258
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
212
|
-
const r = await client
|
|
259
|
+
const r = await cdpCall(client, "Page.printToPDF", {
|
|
213
260
|
printBackground: true,
|
|
214
261
|
preferCSSPageSize: true,
|
|
215
262
|
});
|
|
216
|
-
if (!r.success) return
|
|
217
|
-
// CDP boundary cast: Page.printToPDF returns { data: string } (base64-encoded PDF)
|
|
218
|
-
const data = (r.data as { data: string }).data;
|
|
263
|
+
if (!r.success) return r;
|
|
219
264
|
const path = args.outputPath ?? pdfPath(client.namespace);
|
|
220
|
-
await writeFile(path, Buffer.from(data, "base64"));
|
|
265
|
+
await writeFile(path, Buffer.from(r.data.data, "base64"));
|
|
221
266
|
return ok({ text: `PDF saved: ${path}`, details: { path } });
|
|
222
267
|
},
|
|
223
268
|
});
|