pi-browser-harness 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/README.md +102 -1
- package/package.json +13 -7
- package/skills/pi-browser-harness/SKILL.md +84 -0
- package/src/cdp/commands.ts +204 -14
- package/src/cdp/console-buffer.ts +37 -23
- package/src/cdp/discovery.ts +72 -37
- package/src/cdp/events.ts +8 -3
- package/src/cdp/network-buffer.ts +38 -32
- package/src/cdp/ownership.ts +22 -4
- package/src/cdp/record-store.ts +2 -2
- package/src/cdp/session.ts +320 -217
- package/src/cdp/target-factory.ts +34 -15
- package/src/cdp/types.ts +17 -1
- package/src/client.ts +76 -43
- package/src/daemon/bridge.ts +115 -191
- package/src/daemon/connection.ts +323 -0
- package/src/daemon/index.ts +77 -48
- package/src/daemon/lines.ts +40 -0
- package/src/daemon/main.ts +12 -0
- package/src/daemon/protocol.ts +55 -25
- package/src/daemon/server.ts +100 -63
- package/src/daemon/socket.ts +10 -0
- package/src/daemon/spawn.ts +84 -27
- package/src/daemon/transport.ts +88 -52
- package/src/domains/ax-tree.ts +8 -8
- package/src/domains/cdp-call.ts +14 -7
- package/src/domains/click.ts +2 -7
- package/src/domains/concurrency.ts +50 -0
- package/src/domains/console.ts +20 -66
- package/src/domains/drag.ts +26 -28
- package/src/domains/element-call.ts +102 -68
- package/src/domains/files.ts +64 -11
- package/src/domains/form-batch.ts +13 -11
- package/src/domains/form.ts +79 -116
- package/src/domains/headers.ts +89 -0
- package/src/domains/history.ts +7 -7
- package/src/domains/io.ts +35 -0
- package/src/domains/isolated-tab.ts +6 -12
- package/src/domains/js.ts +68 -53
- package/src/domains/keyboard.ts +16 -18
- package/src/domains/navigate.ts +102 -69
- package/src/domains/network.ts +56 -98
- package/src/domains/page/fill.page.ts +139 -0
- package/src/domains/page/form.page.ts +61 -0
- package/src/domains/page/inject.ts +37 -0
- package/src/domains/page/keyboard.page.ts +16 -0
- package/src/domains/page/live-value.page.ts +10 -0
- package/src/domains/page/read-page.page.ts +51 -0
- package/src/domains/page/serp.page.ts +39 -0
- package/src/domains/page.ts +19 -58
- package/src/domains/poll.ts +42 -0
- package/src/domains/readpage/read-page.ts +48 -54
- package/src/domains/ref-resolve.ts +43 -29
- package/src/domains/ref-signature.ts +11 -0
- package/src/domains/render.ts +12 -23
- package/src/domains/screenshot-capture.ts +4 -2
- package/src/domains/screenshot.ts +13 -14
- package/src/domains/scroll.ts +4 -5
- package/src/domains/search/google-serp.ts +1 -1
- package/src/domains/search/web-search.ts +38 -34
- package/src/domains/setup.ts +0 -2
- package/src/domains/snapshot.ts +72 -77
- package/src/domains/tab-resolve.ts +36 -0
- package/src/domains/tabs.ts +19 -78
- package/src/domains/target.ts +1 -1
- package/src/domains/viewport.ts +0 -1
- package/src/index.ts +100 -39
- package/src/profile/bind.ts +45 -14
- package/src/profile/browser-process.ts +50 -12
- package/src/profile/command.ts +20 -12
- package/src/profile/launch.ts +13 -9
- package/src/profile/list.ts +9 -13
- package/src/profile/store.ts +2 -1
- package/src/prompt.ts +42 -117
- package/src/registry.ts +68 -4
- package/src/schemas/parse.ts +2 -2
- package/src/setup.ts +28 -19
- package/src/state.ts +33 -2
- package/src/util/debug.ts +7 -1
- package/src/util/guards.ts +3 -0
- package/src/util/mutex.ts +1 -1
- package/src/util/paths.ts +12 -1
- package/src/util/result.ts +8 -0
- package/src/util/sharp-shim.ts +18 -11
- package/src/util/sleep.ts +18 -0
- package/src/util/tool.ts +163 -114
- package/src/util/truncate.ts +1 -1
- package/tsconfig.json +2 -1
- package/src/domains/fill-engine.ts +0 -80
package/src/domains/ax-tree.ts
CHANGED
|
@@ -2,6 +2,8 @@ import type { AxNodeResult } from "../cdp/commands";
|
|
|
2
2
|
import type { BrowserClient } from "../client";
|
|
3
3
|
import type { Box } from "./box";
|
|
4
4
|
import { cdpCall } from "./cdp-call";
|
|
5
|
+
import { elementFunctionSource } from "./page/inject";
|
|
6
|
+
import { liveValue as liveValueInPage } from "./page/live-value.page";
|
|
5
7
|
|
|
6
8
|
export type RawAxNode = AxNodeResult;
|
|
7
9
|
type AxValue = NonNullable<RawAxNode["role"]>;
|
|
@@ -138,8 +140,9 @@ export const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
|
|
|
138
140
|
|
|
139
141
|
export const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
|
|
140
142
|
nodes.map((n) => {
|
|
141
|
-
const
|
|
142
|
-
|
|
143
|
+
const stripped: SlimNode = { ...n, children: stripInternals(n.children) };
|
|
144
|
+
delete stripped._backendId;
|
|
145
|
+
return stripped;
|
|
143
146
|
});
|
|
144
147
|
|
|
145
148
|
export const collectInteractiveTargets = (
|
|
@@ -158,19 +161,16 @@ export const collectInteractiveTargets = (
|
|
|
158
161
|
return out;
|
|
159
162
|
};
|
|
160
163
|
|
|
164
|
+
const LIVE_VALUE_FN = elementFunctionSource(liveValueInPage);
|
|
165
|
+
|
|
161
166
|
export const liveValue = async (client: BrowserClient, backendId: number): Promise<string | undefined> => {
|
|
162
167
|
const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId });
|
|
163
168
|
if (!resolved.success) return undefined;
|
|
164
169
|
const objectId = resolved.data.object.objectId;
|
|
165
170
|
if (!objectId) return undefined;
|
|
166
|
-
const fn = `function () {
|
|
167
|
-
if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
|
|
168
|
-
if (typeof this.value === "string") return this.value;
|
|
169
|
-
return null;
|
|
170
|
-
}`;
|
|
171
171
|
const r = await cdpCall(client, "Runtime.callFunctionOn", {
|
|
172
172
|
objectId,
|
|
173
|
-
functionDeclaration:
|
|
173
|
+
functionDeclaration: LIVE_VALUE_FN,
|
|
174
174
|
returnByValue: true,
|
|
175
175
|
});
|
|
176
176
|
if (!r.success) return undefined;
|
package/src/domains/cdp-call.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BrowserClient } from "../client";
|
|
2
2
|
import type { CdpMethod, ParamsOf, ResultOf } from "../cdp/commands";
|
|
3
3
|
import type { CdpError } from "../cdp/errors";
|
|
4
|
+
import type { CallArgs, CallOpts } from "../cdp/session";
|
|
4
5
|
import { type Result, err } from "../util/result";
|
|
5
6
|
import type { ToolErr } from "../util/tool";
|
|
6
7
|
|
|
@@ -10,13 +11,20 @@ export const cdpErrToToolErr = (e: CdpError, method: string): ToolErr => ({
|
|
|
10
11
|
details: { method },
|
|
11
12
|
});
|
|
12
13
|
|
|
14
|
+
// Maps a BrowserClient-level Result (listTabs, newTab, pageInfo, …) to the tool error shape without losing the
|
|
15
|
+
// timeout kind or the CDP method the error carries. `fallbackMethod` names the operation when the error has none.
|
|
16
|
+
export const fromClient = <T>(r: Result<T, CdpError>, fallbackMethod = "client"): Result<T, ToolErr> => {
|
|
17
|
+
if (r.success) return r;
|
|
18
|
+
return err(cdpErrToToolErr(r.error, r.error.method ?? fallbackMethod));
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
// Params are required only when the method has a required key, mirroring CdpSession.call.
|
|
13
22
|
export const cdpCall = async <M extends CdpMethod>(
|
|
14
23
|
client: BrowserClient,
|
|
15
24
|
method: M,
|
|
16
|
-
|
|
17
|
-
opts?: { timeoutMs?: number },
|
|
25
|
+
...args: CallArgs<M>
|
|
18
26
|
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
19
|
-
const r = await client.session().call(method,
|
|
27
|
+
const r = await client.session().call(method, ...args);
|
|
20
28
|
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
21
29
|
return r;
|
|
22
30
|
};
|
|
@@ -26,7 +34,7 @@ export const cdpCallOnTarget = async <M extends CdpMethod>(
|
|
|
26
34
|
method: M,
|
|
27
35
|
params: ParamsOf<M>,
|
|
28
36
|
sessionId: string,
|
|
29
|
-
opts?:
|
|
37
|
+
opts?: CallOpts,
|
|
30
38
|
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
31
39
|
const r = await client.session().callOnTarget(method, params, sessionId, opts);
|
|
32
40
|
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
@@ -36,10 +44,9 @@ export const cdpCallOnTarget = async <M extends CdpMethod>(
|
|
|
36
44
|
export const cdpCallBrowser = async <M extends CdpMethod>(
|
|
37
45
|
client: BrowserClient,
|
|
38
46
|
method: M,
|
|
39
|
-
|
|
40
|
-
opts?: { timeoutMs?: number },
|
|
47
|
+
...args: CallArgs<M>
|
|
41
48
|
): Promise<Result<ResultOf<M>, ToolErr>> => {
|
|
42
|
-
const r = await client.session().callBrowser(method,
|
|
49
|
+
const r = await client.session().callBrowser(method, ...args);
|
|
43
50
|
if (!r.success) return err(cdpErrToToolErr(r.error, method));
|
|
44
51
|
return r;
|
|
45
52
|
};
|
package/src/domains/click.ts
CHANGED
|
@@ -58,11 +58,6 @@ export const clickTool = defineBrowserTool({
|
|
|
58
58
|
"Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.",
|
|
59
59
|
promptSnippet: "Click an element by ref (preferred) or pixel coordinates",
|
|
60
60
|
promptGuidelines: [
|
|
61
|
-
"PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.",
|
|
62
|
-
"Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.",
|
|
63
|
-
"A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
|
|
64
|
-
"After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
|
|
65
|
-
"Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
|
|
66
61
|
"If a click doesn't register, run pi with --browser-debug-clicks (or set BH_DEBUG_CLICKS=1) to get annotated screenshots. For React/Vue components ignoring clicks, try browser_dispatch_key.",
|
|
67
62
|
],
|
|
68
63
|
parameters: ClickArgs,
|
|
@@ -85,13 +80,13 @@ export const clickTool = defineBrowserTool({
|
|
|
85
80
|
if (debug.success) {
|
|
86
81
|
return ok({
|
|
87
82
|
text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`,
|
|
88
|
-
details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref },
|
|
83
|
+
details: { debugScreenshotPath: debug.data.path, x, y, ...(args.ref !== undefined ? { ref: args.ref } : {}) },
|
|
89
84
|
});
|
|
90
85
|
}
|
|
91
86
|
}
|
|
92
87
|
return ok({
|
|
93
88
|
text: `Clicked at ${target}${diff}`,
|
|
94
|
-
details: { x, y, ref: args.ref },
|
|
89
|
+
details: { x, y, ...(args.ref !== undefined ? { ref: args.ref } : {}) },
|
|
95
90
|
});
|
|
96
91
|
},
|
|
97
92
|
});
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// A bounded fan-out runner: at most `limit` tasks in flight, nothing started once `signal` aborts, and a task
|
|
2
|
+
// still running at abort time is abandoned (its slot resolves to undefined) so the caller returns on the deadline.
|
|
3
|
+
export const runLimited = async <T, R>(
|
|
4
|
+
items: ReadonlyArray<T>,
|
|
5
|
+
opts: { readonly limit: number; readonly signal: AbortSignal },
|
|
6
|
+
fn: (item: T, signal: AbortSignal) => Promise<R>,
|
|
7
|
+
): Promise<ReadonlyArray<R | undefined>> => {
|
|
8
|
+
const out: Array<R | undefined> = items.map(() => undefined);
|
|
9
|
+
const { signal } = opts;
|
|
10
|
+
const aborted = new Promise<undefined>((resolve) => {
|
|
11
|
+
if (signal.aborted) resolve(undefined);
|
|
12
|
+
else signal.addEventListener("abort", () => { resolve(undefined); }, { once: true });
|
|
13
|
+
});
|
|
14
|
+
const queue = items.entries();
|
|
15
|
+
const worker = async (): Promise<void> => {
|
|
16
|
+
for (;;) {
|
|
17
|
+
const next = queue.next();
|
|
18
|
+
if (next.done) return;
|
|
19
|
+
const [i, item] = next.value;
|
|
20
|
+
if (signal.aborted) continue;
|
|
21
|
+
try {
|
|
22
|
+
out[i] = await Promise.race([fn(item, signal), aborted]);
|
|
23
|
+
} catch {
|
|
24
|
+
out[i] = undefined;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
const workers = Array.from({ length: Math.max(1, Math.min(opts.limit, items.length)) }, worker);
|
|
29
|
+
await Promise.all(workers);
|
|
30
|
+
return out;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
// A signal that fires after `ms`, or earlier if `parent` aborts; `clear` releases the timer once the work is done.
|
|
34
|
+
export const deadlineSignal = (
|
|
35
|
+
ms: number,
|
|
36
|
+
parent?: AbortSignal,
|
|
37
|
+
): { readonly signal: AbortSignal; readonly clear: () => void } => {
|
|
38
|
+
const ac = new AbortController();
|
|
39
|
+
const onParentAbort = (): void => { ac.abort(); };
|
|
40
|
+
const timer = setTimeout(() => { ac.abort(); }, ms);
|
|
41
|
+
if (parent?.aborted) ac.abort();
|
|
42
|
+
else parent?.addEventListener("abort", onParentAbort, { once: true });
|
|
43
|
+
return {
|
|
44
|
+
signal: ac.signal,
|
|
45
|
+
clear: () => {
|
|
46
|
+
clearTimeout(timer);
|
|
47
|
+
parent?.removeEventListener("abort", onParentAbort);
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
};
|
package/src/domains/console.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
-
import { Markdown
|
|
2
|
+
import { Markdown } from "@mariozechner/pi-tui";
|
|
3
3
|
import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
|
|
4
4
|
import { type Result, ok } from "../util/result";
|
|
5
|
-
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
5
|
+
import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
6
6
|
import { applyTruncation } from "../util/truncate";
|
|
7
|
-
import type
|
|
8
|
-
import { asArrayOf, asBoolean, asNumber, asRecord, asString } from "../util/guards";
|
|
7
|
+
import { type ConsoleRecord, type ConsoleLevel, ConsoleRecordSchema } from "../cdp/console-buffer";
|
|
9
8
|
|
|
10
9
|
const ConsoleArgs = Type.Object({
|
|
11
10
|
levels: Type.Optional(
|
|
@@ -42,7 +41,7 @@ const levelTag = (lvl: ConsoleLevel): string => {
|
|
|
42
41
|
case "warn": return "[warn] ";
|
|
43
42
|
case "info": return "[info] ";
|
|
44
43
|
case "debug": return "[debug]";
|
|
45
|
-
|
|
44
|
+
case "log": return "[log] ";
|
|
46
45
|
}
|
|
47
46
|
};
|
|
48
47
|
|
|
@@ -97,59 +96,17 @@ const renderConsoleMarkdown = (
|
|
|
97
96
|
return `${headerLines.join("\n")}\n\n${rows}${more}${cursor}`;
|
|
98
97
|
};
|
|
99
98
|
|
|
100
|
-
|
|
101
|
-
total:
|
|
102
|
-
returned:
|
|
103
|
-
bufferOverflowed:
|
|
104
|
-
truncated:
|
|
105
|
-
fullOutputPath
|
|
106
|
-
records:
|
|
107
|
-
nextCursor
|
|
108
|
-
};
|
|
109
|
-
|
|
110
|
-
const CONSOLE_LEVELS: ReadonlyArray<ConsoleLevel> = ["log", "info", "warn", "error", "debug"];
|
|
111
|
-
|
|
112
|
-
const asConsoleLevel = (v: unknown): ConsoleLevel | undefined =>
|
|
113
|
-
CONSOLE_LEVELS.find((l) => l === v);
|
|
114
|
-
|
|
115
|
-
const asConsoleRecord = (v: unknown): ConsoleRecord | undefined => {
|
|
116
|
-
const o = asRecord(v);
|
|
117
|
-
if (o === undefined) return undefined;
|
|
118
|
-
const level = asConsoleLevel(o["level"]);
|
|
119
|
-
const text = asString(o["text"]);
|
|
120
|
-
if (level === undefined || text === undefined) return undefined;
|
|
121
|
-
const url = asString(o["url"]);
|
|
122
|
-
const lineNumber = asNumber(o["lineNumber"]);
|
|
123
|
-
const stackTrace = asString(o["stackTrace"]);
|
|
124
|
-
return {
|
|
125
|
-
seq: asNumber(o["seq"]) ?? 0,
|
|
126
|
-
timestampMs: asNumber(o["timestampMs"]) ?? 0,
|
|
127
|
-
level,
|
|
128
|
-
text,
|
|
129
|
-
source: o["source"] === "log-entry" ? "log-entry" : "console-api",
|
|
130
|
-
...(url !== undefined ? { url } : {}),
|
|
131
|
-
...(lineNumber !== undefined ? { lineNumber } : {}),
|
|
132
|
-
...(stackTrace !== undefined ? { stackTrace } : {}),
|
|
133
|
-
};
|
|
134
|
-
};
|
|
99
|
+
const ConsoleDetailsSchema = Type.Object({
|
|
100
|
+
total: Type.Number(),
|
|
101
|
+
returned: Type.Number(),
|
|
102
|
+
bufferOverflowed: Type.Boolean(),
|
|
103
|
+
truncated: Type.Boolean(),
|
|
104
|
+
fullOutputPath: Type.Optional(Type.String()),
|
|
105
|
+
records: Type.Array(ConsoleRecordSchema),
|
|
106
|
+
nextCursor: Type.Optional(Type.Number()),
|
|
107
|
+
});
|
|
135
108
|
|
|
136
|
-
|
|
137
|
-
const o = asRecord(v);
|
|
138
|
-
if (o === undefined) return undefined;
|
|
139
|
-
const records = asArrayOf(o["records"], asConsoleRecord);
|
|
140
|
-
if (records === undefined) return undefined;
|
|
141
|
-
const fullOutputPath = asString(o["fullOutputPath"]);
|
|
142
|
-
const nextCursor = asNumber(o["nextCursor"]);
|
|
143
|
-
return {
|
|
144
|
-
total: asNumber(o["total"]) ?? records.length,
|
|
145
|
-
returned: asNumber(o["returned"]) ?? records.length,
|
|
146
|
-
bufferOverflowed: asBoolean(o["bufferOverflowed"]) ?? false,
|
|
147
|
-
truncated: asBoolean(o["truncated"]) ?? false,
|
|
148
|
-
records,
|
|
149
|
-
...(fullOutputPath !== undefined ? { fullOutputPath } : {}),
|
|
150
|
-
...(nextCursor !== undefined ? { nextCursor } : {}),
|
|
151
|
-
};
|
|
152
|
-
};
|
|
109
|
+
export type ConsoleDetails = DetailsOf<typeof ConsoleDetailsSchema>;
|
|
153
110
|
|
|
154
111
|
export const consoleTool = defineBrowserTool({
|
|
155
112
|
name: "browser_console",
|
|
@@ -158,15 +115,14 @@ export const consoleTool = defineBrowserTool({
|
|
|
158
115
|
"Read JS errors and console messages from the active tab. Diagnostic — use when an action looks broken or silent.",
|
|
159
116
|
promptSnippet: "Read JS errors / console output on the current tab",
|
|
160
117
|
promptGuidelines: [
|
|
161
|
-
"Use after a click, submit, or navigate produced no visible change — errors often explain it.",
|
|
162
|
-
"Pass `sinceSeq` from the previous call's `nextCursor` to see only new messages after an action.",
|
|
163
118
|
"Buffer is reset on tab switch (capacity 500 records). bufferOverflowed:true means older entries were dropped.",
|
|
164
119
|
"Don't use as a default observation tool — prefer browser_snapshot for page structure.",
|
|
165
120
|
],
|
|
166
121
|
parameters: ConsoleArgs,
|
|
122
|
+
details: ConsoleDetailsSchema,
|
|
167
123
|
concurrency: "parallel",
|
|
168
124
|
|
|
169
|
-
async handler(args, { client }): Promise<Result<ToolOk
|
|
125
|
+
async handler(args, { client }): Promise<Result<ToolOk<ConsoleDetails>, ToolErr>> {
|
|
170
126
|
const session = client.session();
|
|
171
127
|
const drained = session.drainConsoleBuffer({
|
|
172
128
|
...(args.levels !== undefined ? { levels: args.levels } : {}),
|
|
@@ -194,16 +150,14 @@ export const consoleTool = defineBrowserTool({
|
|
|
194
150
|
truncated: trunc.wasTruncated,
|
|
195
151
|
records: drained.records,
|
|
196
152
|
...(nextCursor !== undefined ? { nextCursor } : {}),
|
|
153
|
+
...(trunc.fullOutputPath !== undefined ? { fullOutputPath: trunc.fullOutputPath } : {}),
|
|
197
154
|
};
|
|
198
|
-
if (trunc.fullOutputPath !== undefined) details.fullOutputPath = trunc.fullOutputPath;
|
|
199
155
|
|
|
200
156
|
return ok({ text: trunc.text, details });
|
|
201
157
|
},
|
|
202
158
|
|
|
203
|
-
renderResult(result, expanded
|
|
204
|
-
const details =
|
|
205
|
-
if (!details) return new Text(theme.fg("error", "console: no details"), 0, 0);
|
|
206
|
-
|
|
159
|
+
renderResult(result, expanded) {
|
|
160
|
+
const details = result.details;
|
|
207
161
|
const md = renderConsoleMarkdown(details.records, {
|
|
208
162
|
total: details.total,
|
|
209
163
|
bufferOverflowed: details.bufferOverflowed,
|
|
@@ -211,7 +165,7 @@ export const consoleTool = defineBrowserTool({
|
|
|
211
165
|
nextCursor: details.nextCursor,
|
|
212
166
|
});
|
|
213
167
|
const tail = expanded
|
|
214
|
-
? `\n\n${keyHint("app.tools.expand", "to collapse")}${details.fullOutputPath ? ` · full payload at \`${details.fullOutputPath}\`` : ""}`
|
|
168
|
+
? `\n\n${keyHint("app.tools.expand", "to collapse")}${details.fullOutputPath !== undefined ? ` · full payload at \`${details.fullOutputPath}\`` : ""}`
|
|
215
169
|
: `\n\n${keyHint("app.tools.expand", "to expand")}`;
|
|
216
170
|
|
|
217
171
|
return new Markdown(md + tail, 0, 0, getMarkdownTheme());
|
package/src/domains/drag.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
+
import type { ParamsOf } from "../cdp/commands";
|
|
2
3
|
import { type Result, ok } from "../util/result";
|
|
3
4
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
5
|
import { cdpCall } from "./cdp-call";
|
|
@@ -27,37 +28,34 @@ export const dragAndDropTool = defineBrowserTool({
|
|
|
27
28
|
parameters: DragArgs,
|
|
28
29
|
concurrency: "serialized",
|
|
29
30
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
30
|
-
const data =
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
...Array.from({ length: 5 }, (_, i)
|
|
31
|
+
const data: ParamsOf<"Input.dispatchDragEvent">["data"] = {
|
|
32
|
+
items: Object.entries(args.dataTransfer ?? {}).map(([mimeType, payload]) => ({
|
|
33
|
+
mimeType,
|
|
34
|
+
data: Buffer.from(payload).toString("base64"),
|
|
35
|
+
})),
|
|
36
|
+
dragOperationsMask: 1,
|
|
37
|
+
};
|
|
38
|
+
// Each step is a thunk so the whole sequence is typed against the real Input schemas and still runs in order.
|
|
39
|
+
const mouse = (p: ParamsOf<"Input.dispatchMouseEvent">) => () => cdpCall(client, "Input.dispatchMouseEvent", p);
|
|
40
|
+
const drag = (p: ParamsOf<"Input.dispatchDragEvent">) => () => cdpCall(client, "Input.dispatchDragEvent", p);
|
|
41
|
+
const steps = [
|
|
42
|
+
mouse({ type: "mousePressed", x: args.startX, y: args.startY, button: "left", clickCount: 1 }),
|
|
43
|
+
drag({ type: "dragEnter", x: args.startX, y: args.startY, data, modifiers: 0 }),
|
|
44
|
+
...Array.from({ length: 5 }, (_, i) => {
|
|
44
45
|
const t = (i + 1) / 5;
|
|
45
|
-
return
|
|
46
|
-
"
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
modifiers: 0,
|
|
53
|
-
},
|
|
54
|
-
];
|
|
46
|
+
return drag({
|
|
47
|
+
type: "dragOver",
|
|
48
|
+
x: Math.round(args.startX + (args.endX - args.startX) * t),
|
|
49
|
+
y: Math.round(args.startY + (args.endY - args.startY) * t),
|
|
50
|
+
data,
|
|
51
|
+
modifiers: 0,
|
|
52
|
+
});
|
|
55
53
|
}),
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
drag({ type: "drop", x: args.endX, y: args.endY, data, modifiers: 0 }),
|
|
55
|
+
mouse({ type: "mouseReleased", x: args.endX, y: args.endY, button: "left", clickCount: 1 }),
|
|
58
56
|
];
|
|
59
|
-
for (const
|
|
60
|
-
const r = await
|
|
57
|
+
for (const step of steps) {
|
|
58
|
+
const r = await step();
|
|
61
59
|
if (!r.success) return r;
|
|
62
60
|
}
|
|
63
61
|
return ok({
|
|
@@ -1,82 +1,120 @@
|
|
|
1
|
+
import { isRecord } from "../util/guards";
|
|
1
2
|
import type { BrowserClient } from "../client";
|
|
2
3
|
import { safeJs } from "../util/js-template";
|
|
3
4
|
import { type Result, err, ok } from "../util/result";
|
|
4
|
-
import type { ToolErr } from "../util/tool";
|
|
5
|
+
import type { JsonObject, ToolErr } from "../util/tool";
|
|
5
6
|
import { cdpCall, evalJs } from "./cdp-call";
|
|
7
|
+
import type { PageOption, PageResult } from "./page/fill.page";
|
|
6
8
|
import { resolveRefToObjectId } from "./ref-resolve";
|
|
7
9
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
readonly kind?: string;
|
|
12
|
-
readonly value?: unknown;
|
|
13
|
-
readonly text?: string;
|
|
14
|
-
readonly checked?: boolean;
|
|
15
|
-
readonly changed?: boolean;
|
|
16
|
-
readonly tag?: string;
|
|
17
|
-
readonly options?: ReadonlyArray<{ value: string; text: string }>;
|
|
18
|
-
};
|
|
10
|
+
// PageResult is declared once, in the page module that produces it (page/fill.page.ts), so the guard below is checked
|
|
11
|
+
// against the same declaration the page function returns.
|
|
12
|
+
export type { PageOption, PageResult };
|
|
19
13
|
|
|
20
|
-
const isOption = (v: unknown): v is
|
|
14
|
+
const isOption = (v: unknown): v is PageOption =>
|
|
21
15
|
typeof v === "object" && v !== null
|
|
22
16
|
&& "value" in v && typeof v.value === "string"
|
|
23
17
|
&& "text" in v && typeof v.text === "string";
|
|
24
18
|
|
|
19
|
+
const hasTag = (v: object): v is { readonly tag: string } => "tag" in v && typeof v.tag === "string";
|
|
20
|
+
|
|
25
21
|
export const isPageResult = (v: unknown): v is PageResult => {
|
|
26
22
|
if (typeof v !== "object" || v === null) return false;
|
|
27
23
|
if (!("ok" in v) || typeof v.ok !== "boolean") return false;
|
|
28
|
-
if (
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return
|
|
24
|
+
if (!v.ok) {
|
|
25
|
+
if (!("reason" in v) || typeof v.reason !== "string") return false;
|
|
26
|
+
if ("tag" in v && typeof v.tag !== "string") return false;
|
|
27
|
+
if ("code" in v && v.code !== "select_rejected" && v.code !== "no_match") return false;
|
|
28
|
+
if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false;
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
if (!("kind" in v) || typeof v.kind !== "string" || !hasTag(v)) return false;
|
|
32
|
+
switch (v.kind) {
|
|
33
|
+
case "select":
|
|
34
|
+
return "value" in v && typeof v.value === "string" && "text" in v && typeof v.text === "string";
|
|
35
|
+
case "checkbox":
|
|
36
|
+
case "radio":
|
|
37
|
+
return "checked" in v && typeof v.checked === "boolean" && "changed" in v && typeof v.changed === "boolean";
|
|
38
|
+
case "input":
|
|
39
|
+
case "textarea":
|
|
40
|
+
case "contenteditable":
|
|
41
|
+
return "value" in v && typeof v.value === "string";
|
|
42
|
+
default:
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
// The one not-found shape for element lookups: the page never reports it, the harness does.
|
|
48
|
+
export type ElementOutcome =
|
|
49
|
+
| { readonly found: false }
|
|
50
|
+
| { readonly found: true; readonly value: unknown };
|
|
51
|
+
|
|
52
|
+
const isElementOutcome = (v: unknown): v is ElementOutcome => {
|
|
53
|
+
if (!isRecord(v)) return false;
|
|
54
|
+
if (v["found"] === false) return true;
|
|
55
|
+
return v["found"] === true && "value" in v;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const callOnObject = async (
|
|
59
|
+
client: BrowserClient,
|
|
60
|
+
objectId: string,
|
|
61
|
+
functionDeclaration: string,
|
|
62
|
+
args: ReadonlyArray<unknown>,
|
|
63
|
+
ref: string,
|
|
64
|
+
): Promise<Result<unknown, ToolErr>> => {
|
|
65
|
+
try {
|
|
66
|
+
const called = await cdpCall(client, "Runtime.callFunctionOn", {
|
|
67
|
+
objectId,
|
|
68
|
+
functionDeclaration,
|
|
69
|
+
arguments: args.map((v) => ({ value: v })),
|
|
70
|
+
returnByValue: true,
|
|
71
|
+
awaitPromise: true,
|
|
72
|
+
});
|
|
73
|
+
if (!called.success) return err({ ...called.error, details: { ...(called.error.details ?? {}), ref } });
|
|
74
|
+
if (called.data.exceptionDetails !== undefined) {
|
|
75
|
+
return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } });
|
|
76
|
+
}
|
|
77
|
+
return ok(called.data.result.value);
|
|
78
|
+
} finally {
|
|
79
|
+
await cdpCall(client, "Runtime.releaseObject", { objectId });
|
|
80
|
+
}
|
|
36
81
|
};
|
|
37
82
|
|
|
83
|
+
// Runs an element function (its declaration from page/inject.ts elementFunctionSource) on the element a ref or
|
|
84
|
+
// selector names. A ref goes through Runtime.callFunctionOn; a selector is looked up in one Runtime.evaluate round trip.
|
|
38
85
|
export const runOnElement = async (
|
|
39
86
|
client: BrowserClient,
|
|
40
87
|
opts: {
|
|
41
88
|
ref?: string | undefined;
|
|
42
89
|
selector?: string | undefined;
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
notFound?: unknown;
|
|
90
|
+
functionDeclaration: string;
|
|
91
|
+
args: ReadonlyArray<unknown>;
|
|
46
92
|
},
|
|
47
|
-
): Promise<Result<
|
|
93
|
+
): Promise<Result<ElementOutcome, ToolErr>> => {
|
|
48
94
|
if (opts.ref !== undefined) {
|
|
49
95
|
const objectId = await resolveRefToObjectId(client, opts.ref);
|
|
50
96
|
if (!objectId.success) return objectId;
|
|
51
|
-
const r = await
|
|
52
|
-
objectId: objectId.data,
|
|
53
|
-
functionDeclaration: `function (arg) { ${opts.fnBody} }`,
|
|
54
|
-
arguments: [{ value: opts.arg }],
|
|
55
|
-
returnByValue: true,
|
|
56
|
-
});
|
|
97
|
+
const r = await callOnObject(client, objectId.data, opts.functionDeclaration, opts.args, opts.ref);
|
|
57
98
|
if (!r.success) return r;
|
|
58
|
-
|
|
59
|
-
return err({
|
|
60
|
-
kind: "cdp_error",
|
|
61
|
-
message: `page function threw: ${r.data.exceptionDetails.text}`,
|
|
62
|
-
details: { ref: opts.ref },
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
return ok(r.data.result.value);
|
|
99
|
+
return ok({ found: true, value: r.data });
|
|
66
100
|
}
|
|
67
101
|
if (opts.selector === undefined) {
|
|
68
|
-
return err({ kind: "
|
|
102
|
+
return err({ kind: "invalid_args", message: "Provide either `ref` or `selector`." });
|
|
69
103
|
}
|
|
70
|
-
//
|
|
104
|
+
// The function source is harness-authored, not user input; only the selector and the arguments are interpolated,
|
|
105
|
+
// and those go through safeJs.
|
|
71
106
|
const prelude = safeJs`
|
|
72
107
|
(() => {
|
|
73
108
|
const el = document.querySelector(${opts.selector});
|
|
74
|
-
if (!el) return
|
|
75
|
-
const
|
|
76
|
-
return
|
|
77
|
-
const expr =
|
|
109
|
+
if (!el) return { found: false };
|
|
110
|
+
const __args = ${opts.args};
|
|
111
|
+
return { found: true, value: (`;
|
|
112
|
+
const expr = prelude + opts.functionDeclaration + ").apply(el, __args) }; })()";
|
|
78
113
|
const r = await evalJs(client, expr);
|
|
79
114
|
if (!r.success) return r;
|
|
115
|
+
if (!isElementOutcome(r.data)) {
|
|
116
|
+
return err({ kind: "internal", message: "unexpected page result shape from element lookup", details: { selector: opts.selector } });
|
|
117
|
+
}
|
|
80
118
|
return ok(r.data);
|
|
81
119
|
};
|
|
82
120
|
|
|
@@ -88,30 +126,26 @@ export const resolveAndCall = async (
|
|
|
88
126
|
): Promise<Result<PageResult, ToolErr>> => {
|
|
89
127
|
const objectId = await resolveRefToObjectId(client, ref);
|
|
90
128
|
if (!objectId.success) return objectId;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
functionDeclaration,
|
|
96
|
-
arguments: args.map((v) => ({ value: v })),
|
|
97
|
-
returnByValue: true,
|
|
98
|
-
awaitPromise: true,
|
|
99
|
-
});
|
|
100
|
-
if (!called.success) return err({ ...called.error, details: { ref } });
|
|
101
|
-
if (called.data.exceptionDetails !== undefined) {
|
|
102
|
-
return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } });
|
|
103
|
-
}
|
|
104
|
-
const value = called.data.result.value;
|
|
105
|
-
if (!isPageResult(value)) {
|
|
106
|
-
return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
|
|
107
|
-
}
|
|
108
|
-
return ok(value);
|
|
109
|
-
} finally {
|
|
110
|
-
await cdpCall(client, "Runtime.releaseObject", { objectId: handle });
|
|
129
|
+
const r = await callOnObject(client, objectId.data, functionDeclaration, args, ref);
|
|
130
|
+
if (!r.success) return r;
|
|
131
|
+
if (!isPageResult(r.data)) {
|
|
132
|
+
return err({ kind: "internal", message: "unexpected page result shape", details: { ref } });
|
|
111
133
|
}
|
|
134
|
+
return ok(r.data);
|
|
112
135
|
};
|
|
113
136
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
137
|
+
// The page result minus its `ok` flag (the caller has already branched on it), as JSON for a tool's details.
|
|
138
|
+
export const detailsOf = (ref: string, res: PageResult): JsonObject => {
|
|
139
|
+
if (!res.ok) {
|
|
140
|
+
return {
|
|
141
|
+
ref,
|
|
142
|
+
reason: res.reason,
|
|
143
|
+
...(res.tag !== undefined ? { tag: res.tag } : {}),
|
|
144
|
+
...(res.code !== undefined ? { code: res.code } : {}),
|
|
145
|
+
...(res.options !== undefined ? { options: res.options } : {}),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
if ("checked" in res) return { ref, kind: res.kind, checked: res.checked, changed: res.changed, tag: res.tag };
|
|
149
|
+
if (res.kind === "select") return { ref, kind: res.kind, value: res.value, text: res.text, tag: res.tag };
|
|
150
|
+
return { ref, kind: res.kind, value: res.value, tag: res.tag };
|
|
117
151
|
};
|