pi-browser-harness 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +86 -0
- package/package.json +12 -6
- package/skills/pi-browser-harness/SKILL.md +57 -2
- package/src/cdp/commands.ts +200 -14
- package/src/cdp/console-buffer.ts +37 -23
- package/src/cdp/discovery.ts +57 -25
- package/src/cdp/events.ts +8 -3
- package/src/cdp/network-buffer.ts +38 -32
- package/src/cdp/ownership.ts +22 -4
- package/src/cdp/record-store.ts +2 -2
- package/src/cdp/session.ts +320 -217
- package/src/cdp/target-factory.ts +34 -15
- package/src/cdp/types.ts +17 -1
- package/src/client.ts +76 -43
- package/src/daemon/bridge.ts +110 -349
- package/src/daemon/connection.ts +323 -0
- package/src/daemon/index.ts +77 -48
- package/src/daemon/lines.ts +40 -0
- package/src/daemon/main.ts +12 -0
- package/src/daemon/protocol.ts +55 -25
- package/src/daemon/server.ts +100 -63
- package/src/daemon/socket.ts +10 -0
- package/src/daemon/spawn.ts +21 -22
- package/src/daemon/transport.ts +80 -49
- package/src/domains/ax-tree.ts +8 -8
- package/src/domains/cdp-call.ts +14 -7
- package/src/domains/click.ts +2 -2
- package/src/domains/concurrency.ts +50 -0
- package/src/domains/console.ts +20 -64
- package/src/domains/drag.ts +26 -28
- package/src/domains/element-call.ts +102 -68
- package/src/domains/files.ts +13 -9
- package/src/domains/form-batch.ts +13 -10
- package/src/domains/form.ts +78 -107
- package/src/domains/headers.ts +2 -2
- package/src/domains/history.ts +4 -4
- package/src/domains/io.ts +35 -0
- package/src/domains/isolated-tab.ts +6 -12
- package/src/domains/js.ts +65 -44
- package/src/domains/keyboard.ts +14 -11
- package/src/domains/navigate.ts +102 -66
- package/src/domains/network.ts +55 -95
- package/src/domains/page/fill.page.ts +139 -0
- package/src/domains/page/form.page.ts +61 -0
- package/src/domains/page/inject.ts +37 -0
- package/src/domains/page/keyboard.page.ts +16 -0
- package/src/domains/page/live-value.page.ts +10 -0
- package/src/domains/page/read-page.page.ts +51 -0
- package/src/domains/page/serp.page.ts +39 -0
- package/src/domains/page.ts +18 -51
- package/src/domains/poll.ts +42 -0
- package/src/domains/readpage/read-page.ts +48 -53
- package/src/domains/ref-resolve.ts +43 -29
- package/src/domains/ref-signature.ts +11 -0
- package/src/domains/render.ts +12 -23
- package/src/domains/screenshot-capture.ts +4 -2
- package/src/domains/screenshot.ts +13 -10
- package/src/domains/scroll.ts +4 -4
- package/src/domains/search/google-serp.ts +1 -1
- package/src/domains/search/web-search.ts +38 -33
- package/src/domains/snapshot.ts +71 -73
- package/src/domains/tab-resolve.ts +36 -0
- package/src/domains/tabs.ts +18 -69
- package/src/domains/target.ts +1 -1
- package/src/index.ts +90 -33
- package/src/profile/bind.ts +45 -14
- package/src/profile/browser-process.ts +50 -12
- package/src/profile/command.ts +20 -12
- package/src/profile/launch.ts +13 -9
- package/src/profile/list.ts +9 -13
- package/src/profile/store.ts +2 -1
- package/src/prompt.ts +32 -7
- package/src/registry.ts +65 -4
- package/src/schemas/parse.ts +2 -2
- package/src/setup.ts +28 -19
- package/src/state.ts +2 -1
- package/src/util/debug.ts +7 -1
- package/src/util/guards.ts +3 -0
- package/src/util/mutex.ts +1 -1
- package/src/util/paths.ts +3 -2
- package/src/util/result.ts +8 -0
- package/src/util/sharp-shim.ts +18 -11
- package/src/util/sleep.ts +18 -0
- package/src/util/tool.ts +163 -114
- package/src/util/truncate.ts +1 -1
- package/tsconfig.json +2 -1
- package/src/domains/fill-engine.ts +0 -80
package/src/domains/page.ts
CHANGED
|
@@ -1,25 +1,9 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
2
|
import { type Result, err, ok } from "../util/result";
|
|
3
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
-
import {
|
|
4
|
+
import { fromClient } from "./cdp-call";
|
|
5
5
|
import { safeJs } from "../util/js-template";
|
|
6
|
-
|
|
7
|
-
const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>
|
|
8
|
-
new Promise((resolve, reject) => {
|
|
9
|
-
if (signal?.aborted) {
|
|
10
|
-
reject(signal.reason ?? new Error("aborted"));
|
|
11
|
-
return;
|
|
12
|
-
}
|
|
13
|
-
const onAbort = (): void => {
|
|
14
|
-
clearTimeout(timer);
|
|
15
|
-
reject(signal?.reason ?? new Error("aborted"));
|
|
16
|
-
};
|
|
17
|
-
const timer = setTimeout(() => {
|
|
18
|
-
signal?.removeEventListener("abort", onAbort);
|
|
19
|
-
resolve();
|
|
20
|
-
}, ms);
|
|
21
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
22
|
-
});
|
|
6
|
+
import { pollUntil, READY_STATE_COMPLETE, sleep } from "./poll";
|
|
23
7
|
|
|
24
8
|
export const pageInfoTool = defineBrowserTool({
|
|
25
9
|
name: "browser_page_info",
|
|
@@ -33,9 +17,9 @@ export const pageInfoTool = defineBrowserTool({
|
|
|
33
17
|
parameters: Type.Object({}),
|
|
34
18
|
concurrency: "parallel",
|
|
35
19
|
async handler(_args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
36
|
-
const r = await client.pageInfo();
|
|
37
|
-
if (!r.success) return
|
|
38
|
-
if ("dialog"
|
|
20
|
+
const r = fromClient(await client.pageInfo(), "pageInfo");
|
|
21
|
+
if (!r.success) return r;
|
|
22
|
+
if (r.data.kind === "dialog") {
|
|
39
23
|
const d = r.data.dialog;
|
|
40
24
|
return ok({
|
|
41
25
|
text: `⚠️ DIALOG OPEN: ${d.type}\nMessage: ${d.message}${d.defaultPrompt ? `\nDefault: ${d.defaultPrompt}` : ""}\n\nUse browser_handle_dialog to accept or dismiss.`,
|
|
@@ -63,12 +47,9 @@ export const waitTool = defineBrowserTool({
|
|
|
63
47
|
parameters: WaitArgs,
|
|
64
48
|
concurrency: "parallel",
|
|
65
49
|
async handler(args, { signal }): Promise<Result<ToolOk, ToolErr>> {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
} catch (e) {
|
|
70
|
-
return err({ kind: "internal", message: e instanceof Error ? e.message : String(e) });
|
|
71
|
-
}
|
|
50
|
+
await sleep(Math.round(args.seconds * 1000), signal);
|
|
51
|
+
if (signal?.aborted) return err({ kind: "timeout", message: `Aborted before ${args.seconds}s elapsed` });
|
|
52
|
+
return ok({ text: `Waited ${args.seconds}s` });
|
|
72
53
|
},
|
|
73
54
|
});
|
|
74
55
|
|
|
@@ -89,17 +70,10 @@ export const waitForLoadTool = defineBrowserTool({
|
|
|
89
70
|
concurrency: "serialized",
|
|
90
71
|
async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
|
|
91
72
|
const timeoutMs = (args.timeout ?? 15) * 1000;
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
if (r.success && r.data === "complete") {
|
|
97
|
-
const ms = Date.now() - start;
|
|
98
|
-
return ok({ text: `Page loaded in ${Math.round(ms / 100) / 10}s`, details: { ms } });
|
|
99
|
-
}
|
|
100
|
-
await new Promise((res) => setTimeout(res, 50));
|
|
101
|
-
}
|
|
102
|
-
return err({ kind: "timeout", message: `Page did not finish loading in ${args.timeout ?? 15}s` });
|
|
73
|
+
const r = await pollUntil(client, READY_STATE_COMPLETE, { timeoutMs, signal, what: "page load" });
|
|
74
|
+
if (!r.success) return r;
|
|
75
|
+
const ms = r.data.elapsedMs;
|
|
76
|
+
return ok({ text: `Page loaded in ${Math.round(ms / 100) / 10}s`, details: { ms } });
|
|
103
77
|
},
|
|
104
78
|
});
|
|
105
79
|
|
|
@@ -123,7 +97,7 @@ export const waitForTool = defineBrowserTool({
|
|
|
123
97
|
concurrency: "parallel",
|
|
124
98
|
async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
|
|
125
99
|
if (args.selector === undefined && args.text === undefined) {
|
|
126
|
-
return err({ kind: "
|
|
100
|
+
return err({ kind: "invalid_args", message: "Provide a selector or text to wait for." });
|
|
127
101
|
}
|
|
128
102
|
const gone = args.gone ?? false;
|
|
129
103
|
const timeoutMs = (args.timeout ?? 5) * 1000;
|
|
@@ -131,18 +105,11 @@ export const waitForTool = defineBrowserTool({
|
|
|
131
105
|
args.selector !== undefined
|
|
132
106
|
? safeJs`(() => { const found = !!document.querySelector(${args.selector}); return ${gone} ? !found : found; })()`
|
|
133
107
|
: safeJs`(() => { const t = document.body ? document.body.innerText : ""; return t.includes(${args.text}); })()`;
|
|
134
|
-
const start = Date.now();
|
|
135
|
-
while (Date.now() - start < timeoutMs) {
|
|
136
|
-
if (signal?.aborted) return err({ kind: "internal", message: "aborted" });
|
|
137
|
-
const r = await evalJs(client, expr);
|
|
138
|
-
if (r.success && r.data === true) {
|
|
139
|
-
const ms = Date.now() - start;
|
|
140
|
-
const what = args.selector !== undefined ? `${args.selector}${gone ? " gone" : ""}` : `text "${args.text}"`;
|
|
141
|
-
return ok({ text: `Condition met (${what}) in ${Math.round(ms / 100) / 10}s`, details: { ms } });
|
|
142
|
-
}
|
|
143
|
-
await sleep(50, signal);
|
|
144
|
-
}
|
|
145
108
|
const what = args.selector !== undefined ? `${args.selector}${gone ? " to disappear" : ""}` : `text "${args.text}"`;
|
|
146
|
-
|
|
109
|
+
const r = await pollUntil(client, expr, { timeoutMs, signal, what });
|
|
110
|
+
if (!r.success) return r;
|
|
111
|
+
const ms = r.data.elapsedMs;
|
|
112
|
+
const met = args.selector !== undefined ? `${args.selector}${gone ? " gone" : ""}` : `text "${args.text}"`;
|
|
113
|
+
return ok({ text: `Condition met (${met}) in ${Math.round(ms / 100) / 10}s`, details: { ms } });
|
|
147
114
|
},
|
|
148
115
|
});
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { sleep } from "../util/sleep";
|
|
2
|
+
import type { BrowserClient } from "../client";
|
|
3
|
+
import { type Result, err, ok } from "../util/result";
|
|
4
|
+
import type { ToolErr } from "../util/tool";
|
|
5
|
+
import { evalJs } from "./cdp-call";
|
|
6
|
+
|
|
7
|
+
export type PollOptions = {
|
|
8
|
+
readonly timeoutMs: number;
|
|
9
|
+
readonly signal?: AbortSignal | undefined;
|
|
10
|
+
readonly sessionId?: string | undefined;
|
|
11
|
+
readonly intervalMs?: number | undefined;
|
|
12
|
+
// Names the condition in the timeout message, e.g. "page load".
|
|
13
|
+
readonly what?: string | undefined;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
const DEFAULT_INTERVAL_MS = 50;
|
|
17
|
+
|
|
18
|
+
export { sleep } from "../util/sleep";
|
|
19
|
+
|
|
20
|
+
// Evaluates `expression` in the page until it yields `true`. A deadline or an abort both surface as kind "timeout";
|
|
21
|
+
// evaluation errors (e.g. mid-navigation) are retried until the deadline.
|
|
22
|
+
export const pollUntil = async (
|
|
23
|
+
client: BrowserClient,
|
|
24
|
+
expression: string,
|
|
25
|
+
opts: PollOptions,
|
|
26
|
+
): Promise<Result<{ readonly elapsedMs: number }, ToolErr>> => {
|
|
27
|
+
const start = Date.now();
|
|
28
|
+
const interval = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
|
|
29
|
+
const what = opts.what ?? "condition";
|
|
30
|
+
const seconds = Math.round(opts.timeoutMs / 100) / 10;
|
|
31
|
+
while (Date.now() - start < opts.timeoutMs) {
|
|
32
|
+
if (opts.signal?.aborted) {
|
|
33
|
+
return err({ kind: "timeout", message: `Aborted while waiting for ${what}` });
|
|
34
|
+
}
|
|
35
|
+
const r = await evalJs(client, expression, opts.sessionId);
|
|
36
|
+
if (r.success && r.data === true) return ok({ elapsedMs: Date.now() - start });
|
|
37
|
+
await sleep(interval, opts.signal);
|
|
38
|
+
}
|
|
39
|
+
return err({ kind: "timeout", message: `Timed out after ${seconds}s waiting for ${what}` });
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export const READY_STATE_COMPLETE = `document.readyState === "complete"`;
|
|
@@ -1,10 +1,15 @@
|
|
|
1
|
+
import { borrowSession } from "../../cdp/target-factory";
|
|
1
2
|
import { Type } from "typebox";
|
|
2
3
|
import { Compile } from "typebox/compile";
|
|
3
4
|
import { parseJson } from "../../schemas/parse";
|
|
4
5
|
import { type Result, err, ok } from "../../util/result";
|
|
5
6
|
import { applyTruncation } from "../../util/truncate";
|
|
6
|
-
import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
|
|
7
|
-
import {
|
|
7
|
+
import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
|
|
8
|
+
import type { BrowserClient } from "../../client";
|
|
9
|
+
import { evalJs, fromClient } from "../cdp-call";
|
|
10
|
+
import { pageCallExpression } from "../page/inject";
|
|
11
|
+
import { capturePage } from "../page/read-page.page";
|
|
12
|
+
import type * as captured from "../page/read-page.page";
|
|
8
13
|
import {
|
|
9
14
|
closeIsolatedTab,
|
|
10
15
|
evalInIsolatedTab,
|
|
@@ -14,7 +19,7 @@ import {
|
|
|
14
19
|
openIsolatedTab,
|
|
15
20
|
waitForIsolatedLoad,
|
|
16
21
|
} from "../isolated-tab";
|
|
17
|
-
import { renderExpandableText } from "../render";
|
|
22
|
+
import { ExpandableTextSchema, renderExpandableText } from "../render";
|
|
18
23
|
import { extractReadable, type PageCapture, type ReadablePage } from "./readability";
|
|
19
24
|
|
|
20
25
|
const LOAD_TIMEOUT_MS = 15_000;
|
|
@@ -48,42 +53,28 @@ const PageCaptureSchema = Type.Object({
|
|
|
48
53
|
|
|
49
54
|
const pageCaptureValidator = Compile(PageCaptureSchema);
|
|
50
55
|
|
|
56
|
+
const ReadPageDetailsSchema = Type.Object({
|
|
57
|
+
title: Type.String(),
|
|
58
|
+
url: Type.String(),
|
|
59
|
+
wordCount: Type.Number(),
|
|
60
|
+
render: ExpandableTextSchema,
|
|
61
|
+
fullOutputPath: Type.Optional(Type.String()),
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
export type ReadPageDetails = DetailsOf<typeof ReadPageDetailsSchema>;
|
|
65
|
+
|
|
51
66
|
const BODY_TEXT_LIMIT = 20_000;
|
|
52
67
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
for (let n = el; n; n = n.parentElement) {
|
|
58
|
-
if (BOILERPLATE.has(n.tagName)) return true;
|
|
59
|
-
const role = n.getAttribute && n.getAttribute('role');
|
|
60
|
-
if (role === 'navigation' || role === 'banner' || role === 'contentinfo') return true;
|
|
61
|
-
}
|
|
62
|
-
return false;
|
|
63
|
-
};
|
|
64
|
-
const kindOf = (tag) => {
|
|
65
|
-
if (/^H[1-6]$/.test(tag)) return 'heading';
|
|
66
|
-
if (tag === 'LI') return 'listitem';
|
|
67
|
-
if (tag === 'BLOCKQUOTE') return 'blockquote';
|
|
68
|
-
if (tag === 'P') return 'paragraph';
|
|
69
|
-
return 'other';
|
|
70
|
-
};
|
|
71
|
-
const selector = 'p, h1, h2, h3, h4, h5, h6, li, blockquote';
|
|
72
|
-
const root = document.querySelector('article, main, [role=main]') || document.body;
|
|
73
|
-
const blocks = [];
|
|
74
|
-
for (const el of root.querySelectorAll(selector)) {
|
|
75
|
-
const text = (el.innerText || '').replace(/\\s+/g, ' ').trim();
|
|
76
|
-
if (!text) continue;
|
|
77
|
-
let linkTextLength = 0;
|
|
78
|
-
for (const a of el.querySelectorAll('a')) linkTextLength += (a.innerText || '').length;
|
|
79
|
-
blocks.push({ kind: kindOf(el.tagName), text, linkTextLength, inBoilerplate: inBoilerplate(el) });
|
|
80
|
-
}
|
|
81
|
-
const bodyText = (document.body.innerText || '').replace(/\\n{3,}/g, '\\n\\n').slice(0, ${BODY_TEXT_LIMIT});
|
|
82
|
-
return JSON.stringify({ url: location.href, title: document.title, blocks, bodyText });
|
|
83
|
-
})()
|
|
84
|
-
`;
|
|
68
|
+
// The page function declares the capture shape it produces; this pins it to the shape the readability pass consumes,
|
|
69
|
+
// so the two cannot drift apart silently (the runtime validator above still guards the actual wire value).
|
|
70
|
+
type Pinned<T extends PageCapture> = T;
|
|
71
|
+
export type CaptureFromPage = Pinned<captured.PageCapture>;
|
|
85
72
|
|
|
86
|
-
|
|
73
|
+
// Built once: stringifying the page function per call was the only cost of the thunk.
|
|
74
|
+
const PAGE_CAPTURE_EXPR = pageCallExpression(capturePage, BODY_TEXT_LIMIT);
|
|
75
|
+
|
|
76
|
+
// Accepts the capture either as JSON text (isolated tab) or as the already-decoded value (owned tab).
|
|
77
|
+
export const toPageCapture = (raw: unknown): PageCapture | undefined => {
|
|
87
78
|
if (typeof raw === "string") {
|
|
88
79
|
const parsed = parseJson(raw, pageCaptureValidator);
|
|
89
80
|
return parsed.success ? parsed.data : undefined;
|
|
@@ -91,7 +82,7 @@ const toPageCapture = (raw: unknown): PageCapture | undefined => {
|
|
|
91
82
|
return pageCaptureValidator.Check(raw) ? raw : undefined;
|
|
92
83
|
};
|
|
93
84
|
|
|
94
|
-
const captureToResult = async (raw: unknown): Promise<Result<ToolOk
|
|
85
|
+
const captureToResult = async (raw: unknown): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> => {
|
|
95
86
|
const parsed = toPageCapture(raw);
|
|
96
87
|
if (parsed === undefined) return err({ kind: "internal", message: "page capture returned an unexpected shape" });
|
|
97
88
|
const page: ReadablePage = extractReadable(parsed);
|
|
@@ -115,10 +106,10 @@ const captureToResult = async (raw: unknown): Promise<Result<ToolOk, ToolErr>> =
|
|
|
115
106
|
};
|
|
116
107
|
|
|
117
108
|
const readOpenedUrl = async (
|
|
118
|
-
client:
|
|
109
|
+
client: BrowserClient,
|
|
119
110
|
url: string,
|
|
120
111
|
signal: AbortSignal | undefined,
|
|
121
|
-
): Promise<Result<ToolOk
|
|
112
|
+
): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> => {
|
|
122
113
|
const opened = await openIsolatedTab(client);
|
|
123
114
|
if (!opened.success) return err(opened.error);
|
|
124
115
|
const tab: IsolatedTab = opened.data;
|
|
@@ -127,27 +118,30 @@ const readOpenedUrl = async (
|
|
|
127
118
|
if (!navigated.success) return err(navigated.error);
|
|
128
119
|
const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
|
|
129
120
|
if (!loaded.success) return err(loaded.error);
|
|
130
|
-
const captured = await evalInIsolatedTab(client, tab,
|
|
121
|
+
const captured = await evalInIsolatedTab(client, tab, PAGE_CAPTURE_EXPR, isJsonText);
|
|
131
122
|
if (!captured.success) return err(captured.error);
|
|
132
|
-
return captureToResult(captured.data);
|
|
123
|
+
return await captureToResult(captured.data);
|
|
133
124
|
} finally {
|
|
134
125
|
await closeIsolatedTab(client, tab);
|
|
135
126
|
}
|
|
136
127
|
};
|
|
137
128
|
|
|
138
129
|
const readOwnedTab = async (
|
|
139
|
-
client:
|
|
130
|
+
client: BrowserClient,
|
|
140
131
|
targetId: string,
|
|
141
|
-
): Promise<Result<ToolOk
|
|
132
|
+
): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> => {
|
|
142
133
|
if (!client.owns(targetId)) {
|
|
143
134
|
return err({ kind: "invalid_state", message: `Tab ${targetId} is not owned by this session.` });
|
|
144
135
|
}
|
|
145
|
-
const
|
|
146
|
-
if (!
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
136
|
+
const borrowed = await borrowSession(client, targetId);
|
|
137
|
+
if (!borrowed.success) return fromClient(borrowed, "Target.attachToTarget");
|
|
138
|
+
try {
|
|
139
|
+
const captured = await evalJs(client, PAGE_CAPTURE_EXPR, borrowed.data.sessionId);
|
|
140
|
+
if (!captured.success) return captured;
|
|
141
|
+
return await captureToResult(captured.data);
|
|
142
|
+
} finally {
|
|
143
|
+
await borrowed.data.release();
|
|
144
|
+
}
|
|
151
145
|
};
|
|
152
146
|
|
|
153
147
|
export const readPageTool = defineBrowserTool({
|
|
@@ -161,13 +155,14 @@ export const readPageTool = defineBrowserTool({
|
|
|
161
155
|
"Boilerplate-heavy or structure-less pages fall back to bounded body text rather than erroring.",
|
|
162
156
|
],
|
|
163
157
|
parameters: ReadPageArgs,
|
|
158
|
+
details: ReadPageDetailsSchema,
|
|
164
159
|
concurrency: "parallel",
|
|
165
|
-
async handler(args, { client, signal }): Promise<Result<ToolOk
|
|
160
|
+
async handler(args, { client, signal }): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> {
|
|
166
161
|
if (args.url !== undefined && args.url.length > 0) return readOpenedUrl(client, args.url, signal);
|
|
167
162
|
if (args.targetId !== undefined && args.targetId.length > 0) return readOwnedTab(client, args.targetId);
|
|
168
|
-
return err({ kind: "
|
|
163
|
+
return err({ kind: "invalid_args", message: "provide either url or targetId" });
|
|
169
164
|
},
|
|
170
|
-
renderResult(result, expanded
|
|
171
|
-
return renderExpandableText(
|
|
165
|
+
renderResult(result, expanded) {
|
|
166
|
+
return renderExpandableText(result.details.render, expanded);
|
|
172
167
|
},
|
|
173
168
|
});
|
|
@@ -3,7 +3,8 @@ import { type Result, err, ok } from "../util/result";
|
|
|
3
3
|
import type { ToolErr } from "../util/tool";
|
|
4
4
|
import { type Box, boxOf } from "./box";
|
|
5
5
|
import { cdpCall } from "./cdp-call";
|
|
6
|
-
import { buildTree, collectInteractiveTargets
|
|
6
|
+
import { buildTree, collectInteractiveTargets } from "./ax-tree";
|
|
7
|
+
import { type RefSignature, signatureOf } from "./ref-signature";
|
|
7
8
|
|
|
8
9
|
const staleErr = (ref: string): ToolErr => ({
|
|
9
10
|
kind: "invalid_state",
|
|
@@ -43,17 +44,20 @@ export const resolveRefToBox = async (
|
|
|
43
44
|
return ok(box.data);
|
|
44
45
|
};
|
|
45
46
|
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
export type RefRefresh = {
|
|
48
|
+
readonly prev: ReadonlyMap<string, RefSignature>;
|
|
49
|
+
readonly cur: ReadonlyMap<string, RefSignature>;
|
|
50
|
+
};
|
|
48
51
|
|
|
49
|
-
|
|
52
|
+
// Re-reads the AX tree and republishes the session's ref map: an element seen before keeps its ref, a new one gets
|
|
53
|
+
// the next unused index. Returns the signatures before and after so the caller can diff them.
|
|
54
|
+
export const refreshRefs = async (client: BrowserClient): Promise<Result<RefRefresh, ToolErr>> => {
|
|
50
55
|
const session = client.session();
|
|
51
56
|
const prev = session.refSignatures();
|
|
52
57
|
|
|
53
58
|
const axRes = await cdpCall(client, "Accessibility.getFullAXTree", {});
|
|
54
|
-
if (!axRes.success) return
|
|
55
|
-
const
|
|
56
|
-
const slim = buildTree(rawNodes, { interestingOnly: true, maxNodes: 1000 });
|
|
59
|
+
if (!axRes.success) return axRes;
|
|
60
|
+
const slim = buildTree(axRes.data.nodes, { interestingOnly: true, maxNodes: 1000 });
|
|
57
61
|
const targets = collectInteractiveTargets(slim);
|
|
58
62
|
|
|
59
63
|
const priorRefs = session.refMappings();
|
|
@@ -65,7 +69,7 @@ export const interactiveDiff = async (client: BrowserClient): Promise<string> =>
|
|
|
65
69
|
if (digits !== undefined) nextIndex = Math.max(nextIndex, Number(digits) + 1);
|
|
66
70
|
}
|
|
67
71
|
const refMap = new Map<string, number>();
|
|
68
|
-
const
|
|
72
|
+
const cur = new Map<string, RefSignature>();
|
|
69
73
|
const taken = new Set<string>();
|
|
70
74
|
for (const { node, backendId } of targets) {
|
|
71
75
|
const prior = refByBackendId.get(backendId);
|
|
@@ -73,38 +77,40 @@ export const interactiveDiff = async (client: BrowserClient): Promise<string> =>
|
|
|
73
77
|
taken.add(ref);
|
|
74
78
|
node.ref = ref;
|
|
75
79
|
refMap.set(ref, backendId);
|
|
76
|
-
|
|
80
|
+
cur.set(ref, signatureOf(node));
|
|
77
81
|
}
|
|
78
|
-
session.setRefMap(refMap,
|
|
82
|
+
session.setRefMap(refMap, cur);
|
|
83
|
+
return ok({ prev, cur });
|
|
84
|
+
};
|
|
79
85
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
86
|
+
const identityKey = (s: RefSignature): string => JSON.stringify([s.role, s.name]);
|
|
87
|
+
const describe = (s: RefSignature): string => `${s.role} "${s.name}"`;
|
|
88
|
+
|
|
89
|
+
// Pure: renders what changed between two signature sets, keyed by role+name. Empty string when nothing changed.
|
|
90
|
+
export const diffSignatures = (
|
|
91
|
+
prev: ReadonlyMap<string, RefSignature>,
|
|
92
|
+
cur: ReadonlyMap<string, RefSignature>,
|
|
93
|
+
): string => {
|
|
94
|
+
const prevByKey = new Map<string, RefSignature>();
|
|
95
|
+
for (const sig of prev.values()) prevByKey.set(identityKey(sig), sig);
|
|
96
|
+
const curByKey = new Map<string, { readonly ref: string; readonly sig: RefSignature }>();
|
|
97
|
+
for (const [ref, sig] of cur) curByKey.set(identityKey(sig), { ref, sig });
|
|
89
98
|
|
|
90
99
|
const appeared: string[] = [];
|
|
91
100
|
const changed: string[] = [];
|
|
92
101
|
for (const [key, { ref, sig }] of curByKey) {
|
|
93
102
|
const before = prevByKey.get(key);
|
|
94
103
|
if (before === undefined) {
|
|
95
|
-
appeared.push(` *[${ref}] ${
|
|
96
|
-
} else if (before !== sig) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
const sb = sig.split("|")[3] ?? "";
|
|
101
|
-
const detail = va !== vb ? `value ${JSON.stringify(va)} → ${JSON.stringify(vb)}` : `state ${JSON.stringify(sa)} → ${JSON.stringify(sb)}`;
|
|
102
|
-
changed.push(` [${ref}] ${key.replace("|", ' "')}": ${detail}`);
|
|
104
|
+
appeared.push(` *[${ref}] ${describe(sig)}`);
|
|
105
|
+
} else if (before.value !== sig.value) {
|
|
106
|
+
changed.push(` [${ref}] ${describe(sig)}: value ${JSON.stringify(before.value)} → ${JSON.stringify(sig.value)}`);
|
|
107
|
+
} else if (before.state !== sig.state) {
|
|
108
|
+
changed.push(` [${ref}] ${describe(sig)}: state ${JSON.stringify(before.state)} → ${JSON.stringify(sig.state)}`);
|
|
103
109
|
}
|
|
104
110
|
}
|
|
105
111
|
const removed: string[] = [];
|
|
106
|
-
for (const key of prevByKey
|
|
107
|
-
if (!curByKey.has(key)) removed.push(` ${
|
|
112
|
+
for (const [key, sig] of prevByKey) {
|
|
113
|
+
if (!curByKey.has(key)) removed.push(` ${describe(sig)}`);
|
|
108
114
|
}
|
|
109
115
|
|
|
110
116
|
if (appeared.length === 0 && changed.length === 0 && removed.length === 0) return "";
|
|
@@ -116,3 +122,11 @@ export const interactiveDiff = async (client: BrowserClient): Promise<string> =>
|
|
|
116
122
|
if (overflow > 0) lines.push(` …and ${overflow} more (browser_snapshot for full state)`);
|
|
117
123
|
return lines.join("\n");
|
|
118
124
|
};
|
|
125
|
+
|
|
126
|
+
// Convenience for action tools: refresh refs and append a diff to the action's text. A failed refresh is reported
|
|
127
|
+
// inline rather than failing the action that already happened.
|
|
128
|
+
export const interactiveDiff = async (client: BrowserClient): Promise<string> => {
|
|
129
|
+
const refreshed = await refreshRefs(client);
|
|
130
|
+
if (!refreshed.success) return `\n(page diff unavailable: ${refreshed.error.message})`;
|
|
131
|
+
return diffSignatures(refreshed.data.prev, refreshed.data.cur);
|
|
132
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RefSignature } from "../cdp/types";
|
|
2
|
+
import type { SlimNode } from "./ax-tree";
|
|
3
|
+
|
|
4
|
+
export type { RefSignature };
|
|
5
|
+
|
|
6
|
+
export const signatureOf = (n: SlimNode): RefSignature => ({
|
|
7
|
+
role: n.role,
|
|
8
|
+
name: n.name ?? "",
|
|
9
|
+
value: n.value ?? "",
|
|
10
|
+
state: n.state ?? "",
|
|
11
|
+
});
|
package/src/domains/render.ts
CHANGED
|
@@ -1,32 +1,21 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { Markdown } from "@mariozechner/pi-tui";
|
|
2
3
|
import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
|
|
3
4
|
import type { Component } from "@mariozechner/pi-tui";
|
|
4
|
-
import type {
|
|
5
|
-
import { isRecord } from "../util/guards";
|
|
5
|
+
import type { DetailsOf } from "../util/tool";
|
|
6
6
|
|
|
7
7
|
const COMPACT_PREVIEW_LINES = 6;
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const isExpandableText = (value: unknown): value is ExpandableText =>
|
|
16
|
-
isRecord(value) && typeof value["summary"] === "string" && typeof value["body"] === "string";
|
|
17
|
-
|
|
18
|
-
const extractRender = (details: unknown): unknown =>
|
|
19
|
-
isRecord(details) ? details["render"] : undefined;
|
|
9
|
+
// A tool whose details embed this under `render` gets a compact/expanded text view for free.
|
|
10
|
+
export const ExpandableTextSchema = Type.Object({
|
|
11
|
+
summary: Type.String(),
|
|
12
|
+
body: Type.String(),
|
|
13
|
+
fullOutputPath: Type.Optional(Type.String()),
|
|
14
|
+
});
|
|
20
15
|
|
|
21
|
-
export
|
|
22
|
-
label: string,
|
|
23
|
-
result: { readonly details?: unknown },
|
|
24
|
-
expanded: boolean,
|
|
25
|
-
theme: Theme,
|
|
26
|
-
): Component => {
|
|
27
|
-
const render = extractRender(result.details);
|
|
28
|
-
if (!isExpandableText(render)) return new Text(theme.fg("error", `${label}: no details`), 0, 0);
|
|
16
|
+
export type ExpandableText = DetailsOf<typeof ExpandableTextSchema>;
|
|
29
17
|
|
|
18
|
+
export const renderExpandableText = (render: ExpandableText, expanded: boolean): Component => {
|
|
30
19
|
if (!expanded) {
|
|
31
20
|
const lines = render.body.split("\n");
|
|
32
21
|
const preview = lines.slice(0, COMPACT_PREVIEW_LINES).join("\n");
|
|
@@ -35,7 +24,7 @@ export const renderExpandableText = (
|
|
|
35
24
|
return new Markdown(md, 0, 0, getMarkdownTheme());
|
|
36
25
|
}
|
|
37
26
|
|
|
38
|
-
const tail = render.fullOutputPath
|
|
27
|
+
const tail = render.fullOutputPath !== undefined
|
|
39
28
|
? `\n\nFull output at \`${render.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
|
|
40
29
|
: `\n\n${keyHint("app.tools.expand", "to collapse")}`;
|
|
41
30
|
const md = `**${render.summary}**\n\n${render.body}${tail}`;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { rename
|
|
1
|
+
import { rename } from "node:fs/promises";
|
|
2
2
|
import type { BrowserClient } from "../client";
|
|
3
3
|
import { type Result, ok } from "../util/result";
|
|
4
4
|
import type { ToolErr } from "../util/tool";
|
|
@@ -6,6 +6,7 @@ import { screenshotPath } from "../util/paths";
|
|
|
6
6
|
import { loadSharp } from "../util/sharp-shim";
|
|
7
7
|
import { safeJs } from "../util/js-template";
|
|
8
8
|
import { cdpCall, evalJs } from "./cdp-call";
|
|
9
|
+
import { writeFileResult } from "./io";
|
|
9
10
|
|
|
10
11
|
export type CaptureArgs = {
|
|
11
12
|
readonly fullPage?: boolean;
|
|
@@ -26,7 +27,8 @@ export const captureBase = async (
|
|
|
26
27
|
...(format === "jpeg" ? { quality } : {}),
|
|
27
28
|
});
|
|
28
29
|
if (!r.success) return r;
|
|
29
|
-
await
|
|
30
|
+
const written = await writeFileResult(path, Buffer.from(r.data.data, "base64"));
|
|
31
|
+
if (!written.success) return written;
|
|
30
32
|
return ok({ path, format });
|
|
31
33
|
};
|
|
32
34
|
|
|
@@ -2,8 +2,7 @@ import { readFileSync } from "node:fs";
|
|
|
2
2
|
import { Type } from "typebox";
|
|
3
3
|
import { Image, type ImageTheme, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
|
|
4
4
|
import { type Result, ok } from "../util/result";
|
|
5
|
-
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
6
|
-
import { asRecord, asString } from "../util/guards";
|
|
5
|
+
import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
7
6
|
import { captureBase, resizeIfNeeded } from "./screenshot-capture";
|
|
8
7
|
|
|
9
8
|
const ScreenshotArgs = Type.Object({
|
|
@@ -13,6 +12,14 @@ const ScreenshotArgs = Type.Object({
|
|
|
13
12
|
maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })),
|
|
14
13
|
});
|
|
15
14
|
|
|
15
|
+
const ScreenshotDetailsSchema = Type.Object({
|
|
16
|
+
path: Type.String(),
|
|
17
|
+
format: Type.Union([Type.Literal("png"), Type.Literal("jpeg")]),
|
|
18
|
+
attached: Type.Boolean(),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
export type ScreenshotDetails = DetailsOf<typeof ScreenshotDetailsSchema>;
|
|
22
|
+
|
|
16
23
|
export const screenshotTool = defineBrowserTool({
|
|
17
24
|
name: "browser_screenshot",
|
|
18
25
|
label: "Browser Screenshot",
|
|
@@ -23,8 +30,9 @@ export const screenshotTool = defineBrowserTool({
|
|
|
23
30
|
"format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.",
|
|
24
31
|
],
|
|
25
32
|
parameters: ScreenshotArgs,
|
|
33
|
+
details: ScreenshotDetailsSchema,
|
|
26
34
|
concurrency: "parallel",
|
|
27
|
-
async handler(args, { client }): Promise<Result<ToolOk
|
|
35
|
+
async handler(args, { client }): Promise<Result<ToolOk<ScreenshotDetails>, ToolErr>> {
|
|
28
36
|
const cap = await captureBase(client, args);
|
|
29
37
|
if (!cap.success) return cap;
|
|
30
38
|
let note = "";
|
|
@@ -39,12 +47,7 @@ export const screenshotTool = defineBrowserTool({
|
|
|
39
47
|
},
|
|
40
48
|
|
|
41
49
|
renderResult(result, _expanded, theme) {
|
|
42
|
-
const
|
|
43
|
-
const filePath = raw === undefined ? undefined : asString(raw["path"]);
|
|
44
|
-
if (!filePath) {
|
|
45
|
-
return new Text(theme.fg("error", "Screenshot path missing"), 0, 0);
|
|
46
|
-
}
|
|
47
|
-
const format = raw === undefined ? undefined : asString(raw["format"]);
|
|
50
|
+
const { path: filePath, format } = result.details;
|
|
48
51
|
try {
|
|
49
52
|
const buf = readFileSync(filePath);
|
|
50
53
|
const b64 = buf.toString("base64");
|
|
@@ -59,7 +62,7 @@ export const screenshotTool = defineBrowserTool({
|
|
|
59
62
|
});
|
|
60
63
|
// Image's text fallback does not respect the rendered width, so an unwrapped long path can blow past the terminal width and crash the host TUI.
|
|
61
64
|
return {
|
|
62
|
-
invalidate: () => image.invalidate(),
|
|
65
|
+
invalidate: () => { image.invalidate(); },
|
|
63
66
|
render: (width: number) =>
|
|
64
67
|
image.render(width).map((line) =>
|
|
65
68
|
visibleWidth(line) > width ? truncateToWidth(line, width) : line,
|
package/src/domains/scroll.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
2
|
import { type Result, err, ok } from "../util/result";
|
|
3
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
-
import { cdpCall } from "./cdp-call";
|
|
4
|
+
import { cdpCall, fromClient } from "./cdp-call";
|
|
5
5
|
|
|
6
6
|
const ScrollArgs = Type.Object({
|
|
7
7
|
x: Type.Optional(Type.Number({ description: "X coordinate where to scroll. Default: viewport center" })),
|
|
@@ -25,9 +25,9 @@ export const scrollTool = defineBrowserTool({
|
|
|
25
25
|
parameters: ScrollArgs,
|
|
26
26
|
concurrency: "serialized",
|
|
27
27
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
28
|
-
const info = await client.pageInfo();
|
|
29
|
-
if (!info.success) return
|
|
30
|
-
if ("dialog"
|
|
28
|
+
const info = fromClient(await client.pageInfo(), "pageInfo");
|
|
29
|
+
if (!info.success) return info;
|
|
30
|
+
if (info.data.kind === "dialog") {
|
|
31
31
|
return err({
|
|
32
32
|
kind: "invalid_state",
|
|
33
33
|
message: `Dialog open: ${info.data.dialog.type} — ${info.data.dialog.message}`,
|
|
@@ -29,7 +29,7 @@ const CAPTCHA_MARKERS: ReadonlyArray<string> = [
|
|
|
29
29
|
];
|
|
30
30
|
|
|
31
31
|
const unwrapRedirect = (href: string): string => {
|
|
32
|
-
const wrapper =
|
|
32
|
+
const wrapper = /^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i.exec(href);
|
|
33
33
|
if (!wrapper) return href;
|
|
34
34
|
const query = href.slice(href.indexOf("?") + 1);
|
|
35
35
|
const target = new URLSearchParams(query).get("q");
|