pi-browser-harness 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +86 -0
- package/package.json +12 -6
- package/skills/pi-browser-harness/SKILL.md +57 -2
- package/src/cdp/commands.ts +200 -14
- package/src/cdp/console-buffer.ts +37 -23
- package/src/cdp/discovery.ts +57 -25
- package/src/cdp/events.ts +8 -3
- package/src/cdp/network-buffer.ts +38 -32
- package/src/cdp/ownership.ts +22 -4
- package/src/cdp/record-store.ts +2 -2
- package/src/cdp/session.ts +320 -217
- package/src/cdp/target-factory.ts +34 -15
- package/src/cdp/types.ts +17 -1
- package/src/client.ts +76 -43
- package/src/daemon/bridge.ts +110 -349
- package/src/daemon/connection.ts +323 -0
- package/src/daemon/index.ts +77 -48
- package/src/daemon/lines.ts +40 -0
- package/src/daemon/main.ts +12 -0
- package/src/daemon/protocol.ts +55 -25
- package/src/daemon/server.ts +100 -63
- package/src/daemon/socket.ts +10 -0
- package/src/daemon/spawn.ts +21 -22
- package/src/daemon/transport.ts +80 -49
- package/src/domains/ax-tree.ts +8 -8
- package/src/domains/cdp-call.ts +14 -7
- package/src/domains/click.ts +2 -2
- package/src/domains/concurrency.ts +50 -0
- package/src/domains/console.ts +20 -64
- package/src/domains/drag.ts +26 -28
- package/src/domains/element-call.ts +102 -68
- package/src/domains/files.ts +13 -9
- package/src/domains/form-batch.ts +13 -10
- package/src/domains/form.ts +78 -107
- package/src/domains/headers.ts +2 -2
- package/src/domains/history.ts +4 -4
- package/src/domains/io.ts +35 -0
- package/src/domains/isolated-tab.ts +6 -12
- package/src/domains/js.ts +65 -44
- package/src/domains/keyboard.ts +14 -11
- package/src/domains/navigate.ts +102 -66
- package/src/domains/network.ts +55 -95
- package/src/domains/page/fill.page.ts +139 -0
- package/src/domains/page/form.page.ts +61 -0
- package/src/domains/page/inject.ts +37 -0
- package/src/domains/page/keyboard.page.ts +16 -0
- package/src/domains/page/live-value.page.ts +10 -0
- package/src/domains/page/read-page.page.ts +51 -0
- package/src/domains/page/serp.page.ts +39 -0
- package/src/domains/page.ts +18 -51
- package/src/domains/poll.ts +42 -0
- package/src/domains/readpage/read-page.ts +48 -53
- package/src/domains/ref-resolve.ts +43 -29
- package/src/domains/ref-signature.ts +11 -0
- package/src/domains/render.ts +12 -23
- package/src/domains/screenshot-capture.ts +4 -2
- package/src/domains/screenshot.ts +13 -10
- package/src/domains/scroll.ts +4 -4
- package/src/domains/search/google-serp.ts +1 -1
- package/src/domains/search/web-search.ts +38 -33
- package/src/domains/snapshot.ts +71 -73
- package/src/domains/tab-resolve.ts +36 -0
- package/src/domains/tabs.ts +18 -69
- package/src/domains/target.ts +1 -1
- package/src/index.ts +90 -33
- package/src/profile/bind.ts +45 -14
- package/src/profile/browser-process.ts +50 -12
- package/src/profile/command.ts +20 -12
- package/src/profile/launch.ts +13 -9
- package/src/profile/list.ts +9 -13
- package/src/profile/store.ts +2 -1
- package/src/prompt.ts +32 -7
- package/src/registry.ts +65 -4
- package/src/schemas/parse.ts +2 -2
- package/src/setup.ts +28 -19
- package/src/state.ts +2 -1
- package/src/util/debug.ts +7 -1
- package/src/util/guards.ts +3 -0
- package/src/util/mutex.ts +1 -1
- package/src/util/paths.ts +3 -2
- package/src/util/result.ts +8 -0
- package/src/util/sharp-shim.ts +18 -11
- package/src/util/sleep.ts +18 -0
- package/src/util/tool.ts +163 -114
- package/src/util/truncate.ts +1 -1
- package/tsconfig.json +2 -1
- package/src/domains/fill-engine.ts +0 -80
|
@@ -3,7 +3,7 @@ import { Compile } from "typebox/compile";
|
|
|
3
3
|
import { parseJson } from "../../schemas/parse";
|
|
4
4
|
import { type Result, err, ok } from "../../util/result";
|
|
5
5
|
import { applyTruncation } from "../../util/truncate";
|
|
6
|
-
import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
|
|
6
|
+
import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
|
|
7
7
|
import {
|
|
8
8
|
closeIsolatedTab,
|
|
9
9
|
evalInIsolatedTab,
|
|
@@ -12,7 +12,10 @@ import {
|
|
|
12
12
|
openIsolatedTab,
|
|
13
13
|
waitForIsolatedLoad,
|
|
14
14
|
} from "../isolated-tab";
|
|
15
|
-
import {
|
|
15
|
+
import { pageCallExpression } from "../page/inject";
|
|
16
|
+
import { extractSerp } from "../page/serp.page";
|
|
17
|
+
import type * as extracted from "../page/serp.page";
|
|
18
|
+
import { ExpandableTextSchema, renderExpandableText } from "../render";
|
|
16
19
|
import { classifySerp, parseGoogleSerp, type SearchResult, type SerpExtraction } from "./google-serp";
|
|
17
20
|
|
|
18
21
|
const LOAD_TIMEOUT_MS = 15_000;
|
|
@@ -26,32 +29,13 @@ const WebSearchArgs = Type.Object({
|
|
|
26
29
|
|
|
27
30
|
const PAGE_TEXT_LIMIT = 4000;
|
|
28
31
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const a = h3.closest('a[href]');
|
|
37
|
-
if (!a) continue;
|
|
38
|
-
const href = a.href;
|
|
39
|
-
if (!href || seen.has(href)) continue;
|
|
40
|
-
seen.add(href);
|
|
41
|
-
const container =
|
|
42
|
-
a.closest('div.g') ||
|
|
43
|
-
a.closest('div[data-hveid]') ||
|
|
44
|
-
a.parentElement?.parentElement ||
|
|
45
|
-
a.parentElement;
|
|
46
|
-
const containerText = container ? container.innerText || '' : '';
|
|
47
|
-
const heading = h3.innerText || h3.textContent || '';
|
|
48
|
-
const snippet = containerText.replace(heading, '').replace(/\\s+/g, ' ').trim().slice(0, 500);
|
|
49
|
-
anchors.push({ href, heading, snippet });
|
|
50
|
-
}
|
|
51
|
-
const pageText = (document.body.innerText || '').slice(0, ${PAGE_TEXT_LIMIT});
|
|
52
|
-
return JSON.stringify({ anchors, pageText });
|
|
53
|
-
})()
|
|
54
|
-
`;
|
|
32
|
+
// The page function declares the extraction shape it produces; this pins it to the shape the SERP parser consumes,
|
|
33
|
+
// so the two cannot drift apart silently (the runtime validator below still guards the actual wire value).
|
|
34
|
+
type Pinned<T extends SerpExtraction> = T;
|
|
35
|
+
export type ExtractionFromPage = Pinned<extracted.SerpExtraction>;
|
|
36
|
+
|
|
37
|
+
// Built once: stringifying the page function per call was the only cost of the thunk.
|
|
38
|
+
const SERP_EXTRACTION_EXPR = pageCallExpression(extractSerp, PAGE_TEXT_LIMIT);
|
|
55
39
|
|
|
56
40
|
const buildSerpUrl = (query: string, limit: number): string =>
|
|
57
41
|
`https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
|
|
@@ -65,6 +49,26 @@ const SerpExtractionSchema = Type.Object({
|
|
|
65
49
|
|
|
66
50
|
const serpExtractionValidator = Compile(SerpExtractionSchema);
|
|
67
51
|
|
|
52
|
+
export const parseSerpExtraction = (raw: string): Result<SerpExtraction, string> =>
|
|
53
|
+
parseJson(raw, serpExtractionValidator);
|
|
54
|
+
|
|
55
|
+
const SearchResultSchema = Type.Object({
|
|
56
|
+
title: Type.String(),
|
|
57
|
+
url: Type.String(),
|
|
58
|
+
snippet: Type.String(),
|
|
59
|
+
rank: Type.Number(),
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
const WebSearchDetailsSchema = Type.Object({
|
|
63
|
+
results: Type.Array(SearchResultSchema),
|
|
64
|
+
engine: Type.Literal("google"),
|
|
65
|
+
query: Type.String(),
|
|
66
|
+
render: ExpandableTextSchema,
|
|
67
|
+
fullOutputPath: Type.Optional(Type.String()),
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
export type WebSearchDetails = DetailsOf<typeof WebSearchDetailsSchema>;
|
|
71
|
+
|
|
68
72
|
const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
|
|
69
73
|
const lines = results.map((r) => `${r.rank}. ${r.title}\n ${r.url}\n ${r.snippet}`);
|
|
70
74
|
return `${results.length} result(s) for "${query}":\n\n${lines.join("\n\n")}`;
|
|
@@ -82,8 +86,9 @@ export const webSearchTool = defineBrowserTool({
|
|
|
82
86
|
"Runs in its own isolated tab and never disturbs the user's current tab.",
|
|
83
87
|
],
|
|
84
88
|
parameters: WebSearchArgs,
|
|
89
|
+
details: WebSearchDetailsSchema,
|
|
85
90
|
concurrency: "serialized",
|
|
86
|
-
async handler(args, { client, signal }): Promise<Result<ToolOk
|
|
91
|
+
async handler(args, { client, signal }): Promise<Result<ToolOk<WebSearchDetails>, ToolErr>> {
|
|
87
92
|
const limit = args.limit ?? 10;
|
|
88
93
|
const serpUrl = buildSerpUrl(args.query, limit);
|
|
89
94
|
|
|
@@ -98,10 +103,10 @@ export const webSearchTool = defineBrowserTool({
|
|
|
98
103
|
const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
|
|
99
104
|
if (!loaded.success) return err(loaded.error);
|
|
100
105
|
|
|
101
|
-
const extracted = await evalInIsolatedTab(client, tab,
|
|
106
|
+
const extracted = await evalInIsolatedTab(client, tab, SERP_EXTRACTION_EXPR, isJsonText);
|
|
102
107
|
if (!extracted.success) return err(extracted.error);
|
|
103
108
|
|
|
104
|
-
const parsedJson =
|
|
109
|
+
const parsedJson = parseSerpExtraction(extracted.data);
|
|
105
110
|
if (!parsedJson.success) {
|
|
106
111
|
return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
|
|
107
112
|
}
|
|
@@ -137,7 +142,7 @@ export const webSearchTool = defineBrowserTool({
|
|
|
137
142
|
await closeIsolatedTab(client, tab);
|
|
138
143
|
}
|
|
139
144
|
},
|
|
140
|
-
renderResult(result, expanded
|
|
141
|
-
return renderExpandableText(
|
|
145
|
+
renderResult(result, expanded) {
|
|
146
|
+
return renderExpandableText(result.details.render, expanded);
|
|
142
147
|
},
|
|
143
148
|
});
|
package/src/domains/snapshot.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { readFileSync
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
2
|
import { Type } from "typebox";
|
|
3
|
-
import { Container, Image, type ImageTheme, Markdown,
|
|
3
|
+
import { Container, Image, type ImageTheme, Markdown, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
|
|
4
4
|
import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
|
|
5
|
+
import type { BrowserClient } from "../client";
|
|
5
6
|
import { boxOf } from "./box";
|
|
6
|
-
import { cdpCall } from "./cdp-call";
|
|
7
|
+
import { cdpCall, fromClient } from "./cdp-call";
|
|
8
|
+
import { deadlineSignal, runLimited } from "./concurrency";
|
|
9
|
+
import { writeFileResult } from "./io";
|
|
10
|
+
import { type RefSignature, signatureOf } from "./ref-signature";
|
|
7
11
|
import {
|
|
8
12
|
buildTree,
|
|
9
13
|
collectInteractiveTargets,
|
|
@@ -14,11 +18,10 @@ import {
|
|
|
14
18
|
stripInternals,
|
|
15
19
|
VALUE_ROLES,
|
|
16
20
|
} from "./ax-tree";
|
|
17
|
-
import { type Result,
|
|
18
|
-
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
21
|
+
import { type Result, ok } from "../util/result";
|
|
22
|
+
import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
19
23
|
import { applyTruncation } from "../util/truncate";
|
|
20
24
|
import { screenshotPath } from "../util/paths";
|
|
21
|
-
import { asBoolean, asNumber, asRecord, asString } from "../util/guards";
|
|
22
25
|
|
|
23
26
|
const SnapshotArgs = Type.Object({
|
|
24
27
|
includeScreenshot: Type.Optional(
|
|
@@ -82,13 +85,28 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
|
|
|
82
85
|
return lines.join("\n");
|
|
83
86
|
};
|
|
84
87
|
|
|
85
|
-
|
|
86
|
-
nodeCount:
|
|
87
|
-
truncated:
|
|
88
|
-
fullOutputPath
|
|
89
|
-
screenshotPath
|
|
90
|
-
url:
|
|
91
|
-
title:
|
|
88
|
+
const SnapshotDetailsSchema = Type.Object({
|
|
89
|
+
nodeCount: Type.Number(),
|
|
90
|
+
truncated: Type.Boolean(),
|
|
91
|
+
fullOutputPath: Type.Optional(Type.String()),
|
|
92
|
+
screenshotPath: Type.Optional(Type.String()),
|
|
93
|
+
url: Type.String(),
|
|
94
|
+
title: Type.String(),
|
|
95
|
+
summary: Type.String(),
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
export type SnapshotDetails = DetailsOf<typeof SnapshotDetailsSchema>;
|
|
99
|
+
|
|
100
|
+
const BOX_BUDGET_MS = 1_500;
|
|
101
|
+
const BOX_CONCURRENCY = 8;
|
|
102
|
+
|
|
103
|
+
// Best-effort: a capture or write failure just drops the screenshot from the result.
|
|
104
|
+
const captureJpeg = async (client: BrowserClient): Promise<string | undefined> => {
|
|
105
|
+
const shot = await cdpCall(client, "Page.captureScreenshot", { format: "jpeg", quality: 80 });
|
|
106
|
+
if (!shot.success) return undefined;
|
|
107
|
+
const path = screenshotPath(client.namespace, "jpeg");
|
|
108
|
+
const written = await writeFileResult(path, Buffer.from(shot.data.data, "base64"));
|
|
109
|
+
return written.success ? path : undefined;
|
|
92
110
|
};
|
|
93
111
|
|
|
94
112
|
export const snapshotTool = defineBrowserTool({
|
|
@@ -103,19 +121,20 @@ export const snapshotTool = defineBrowserTool({
|
|
|
103
121
|
"format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.",
|
|
104
122
|
],
|
|
105
123
|
parameters: SnapshotArgs,
|
|
124
|
+
details: SnapshotDetailsSchema,
|
|
106
125
|
concurrency: "parallel",
|
|
107
126
|
|
|
108
|
-
async handler(args, { client }): Promise<Result<ToolOk
|
|
127
|
+
async handler(args, { client, signal }): Promise<Result<ToolOk<SnapshotDetails>, ToolErr>> {
|
|
109
128
|
const session = client.session();
|
|
110
129
|
|
|
111
130
|
const axRes = await cdpCall(client, "Accessibility.getFullAXTree", {});
|
|
112
131
|
if (!axRes.success) return axRes;
|
|
113
132
|
const rawNodes = axRes.data.nodes;
|
|
114
133
|
|
|
115
|
-
const piRes = await client.pageInfo();
|
|
116
|
-
if (!piRes.success) return
|
|
117
|
-
const pageUrl = "dialog"
|
|
118
|
-
const pageTitle = "dialog"
|
|
134
|
+
const piRes = fromClient(await client.pageInfo(), "pageInfo");
|
|
135
|
+
if (!piRes.success) return piRes;
|
|
136
|
+
const pageUrl = piRes.data.kind === "dialog" ? "" : piRes.data.url;
|
|
137
|
+
const pageTitle = piRes.data.kind === "dialog" ? "" : piRes.data.title;
|
|
119
138
|
|
|
120
139
|
const slim = buildTree(rawNodes, {
|
|
121
140
|
interestingOnly: args.interestingOnly ?? true,
|
|
@@ -132,101 +151,80 @@ export const snapshotTool = defineBrowserTool({
|
|
|
132
151
|
});
|
|
133
152
|
|
|
134
153
|
if (targets.length > 0) {
|
|
135
|
-
|
|
136
|
-
const
|
|
137
|
-
await
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}),
|
|
148
|
-
);
|
|
149
|
-
clearTimeout(timer);
|
|
154
|
+
// Geometry and live values are best-effort decoration: whatever lands inside the budget is kept, the rest is omitted.
|
|
155
|
+
const deadline = deadlineSignal(BOX_BUDGET_MS, signal);
|
|
156
|
+
await runLimited(targets, { limit: BOX_CONCURRENCY, signal: deadline.signal }, async ({ node, backendId }, sig) => {
|
|
157
|
+
const box = await boxOf(client, backendId);
|
|
158
|
+
if (!box.success || sig.aborted) return;
|
|
159
|
+
if (box.data.x < 0 || box.data.y < 0) return;
|
|
160
|
+
node.box = box.data;
|
|
161
|
+
if (!VALUE_ROLES.has(node.role)) return;
|
|
162
|
+
const live = await liveValue(client, backendId);
|
|
163
|
+
if (live !== undefined) node.value = live;
|
|
164
|
+
});
|
|
165
|
+
deadline.clear();
|
|
150
166
|
}
|
|
151
167
|
|
|
152
|
-
const refSig = new Map<string,
|
|
168
|
+
const refSig = new Map<string, RefSignature>();
|
|
153
169
|
for (const { node } of targets) {
|
|
154
170
|
if (node.ref === undefined) continue;
|
|
155
|
-
refSig.set(node.ref,
|
|
171
|
+
refSig.set(node.ref, signatureOf(node));
|
|
156
172
|
}
|
|
157
173
|
session.setRefMap(refMap, refSig);
|
|
158
174
|
|
|
159
|
-
const
|
|
160
|
-
const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2);
|
|
175
|
+
const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripInternals(slim), null, 2);
|
|
161
176
|
const trunc = await applyTruncation(text, "snapshot");
|
|
162
177
|
|
|
163
|
-
|
|
164
|
-
if (args.includeScreenshot) {
|
|
165
|
-
const shot = await cdpCall(client, "Page.captureScreenshot", { format: "jpeg", quality: 80 });
|
|
166
|
-
if (shot.success) {
|
|
167
|
-
shotPath = screenshotPath(client.namespace, "jpeg");
|
|
168
|
-
writeFileSync(shotPath, Buffer.from(shot.data.data, "base64"));
|
|
169
|
-
}
|
|
170
|
-
}
|
|
178
|
+
const shotPath = args.includeScreenshot ? await captureJpeg(client) : undefined;
|
|
171
179
|
|
|
172
180
|
const details: SnapshotDetails = {
|
|
173
181
|
nodeCount: countNodes(slim),
|
|
174
182
|
truncated: trunc.wasTruncated,
|
|
175
183
|
url: pageUrl,
|
|
176
184
|
title: pageTitle,
|
|
185
|
+
summary: summarize(slim),
|
|
186
|
+
...(trunc.fullOutputPath !== undefined ? { fullOutputPath: trunc.fullOutputPath } : {}),
|
|
187
|
+
...(shotPath !== undefined ? { screenshotPath: shotPath } : {}),
|
|
177
188
|
};
|
|
178
|
-
if (trunc.fullOutputPath !== undefined) details.fullOutputPath = trunc.fullOutputPath;
|
|
179
|
-
if (shotPath !== undefined) details.screenshotPath = shotPath;
|
|
180
189
|
|
|
181
|
-
return ok({ text: trunc.text, details
|
|
190
|
+
return ok({ text: trunc.text, details });
|
|
182
191
|
},
|
|
183
192
|
|
|
184
193
|
renderResult(result, expanded, theme) {
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
const
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
fullOutputPath: asString(raw["fullOutputPath"]),
|
|
191
|
-
screenshotPath: asString(raw["screenshotPath"]),
|
|
192
|
-
url: asString(raw["url"]),
|
|
193
|
-
title: asString(raw["title"]),
|
|
194
|
-
summary: asString(raw["summary"]),
|
|
195
|
-
};
|
|
196
|
-
|
|
197
|
-
const titleLine = details.title ? ` "${details.title}"` : "";
|
|
198
|
-
const summary = details.summary ?? "";
|
|
199
|
-
const screenshotNote = details.screenshotPath ? "screenshot attached" : "screenshot omitted";
|
|
200
|
-
const truncNote = details.truncated && details.fullOutputPath ? `\n\nFull tree at \`${details.fullOutputPath}\`` : "";
|
|
194
|
+
const details = result.details;
|
|
195
|
+
const titleLine = details.title !== "" ? ` "${details.title}"` : "";
|
|
196
|
+
const urlLabel = details.url !== "" ? details.url : "(no url)";
|
|
197
|
+
const screenshotNote = details.screenshotPath !== undefined ? "screenshot attached" : "screenshot omitted";
|
|
198
|
+
const truncNote = details.truncated && details.fullOutputPath !== undefined ? `\n\nFull tree at \`${details.fullOutputPath}\`` : "";
|
|
201
199
|
|
|
202
200
|
if (!expanded) {
|
|
203
201
|
const md = [
|
|
204
|
-
`**AX tree:** ${details.nodeCount} nodes · \`${
|
|
205
|
-
|
|
206
|
-
summary ? ` • ${summary}` : "",
|
|
202
|
+
`**AX tree:** ${details.nodeCount} nodes · \`${urlLabel}\``,
|
|
203
|
+
titleLine !== "" ? ` ${titleLine}` : "",
|
|
204
|
+
details.summary !== "" ? ` • ${details.summary}` : "",
|
|
207
205
|
` ${keyHint("app.tools.expand", "to expand")} · ${screenshotNote}`,
|
|
208
206
|
].filter((l) => l !== "").join("\n");
|
|
209
207
|
return new Markdown(md, 0, 0, getMarkdownTheme());
|
|
210
208
|
}
|
|
211
209
|
|
|
212
210
|
const content = result.content[0];
|
|
213
|
-
const treeText = content
|
|
211
|
+
const treeText = content?.type === "text" ? content.text : "";
|
|
214
212
|
|
|
215
213
|
const headerMd = [
|
|
216
|
-
`**AX tree:** ${details.nodeCount} nodes · \`${
|
|
217
|
-
titleLine ? ` ${titleLine}` : "",
|
|
214
|
+
`**AX tree:** ${details.nodeCount} nodes · \`${urlLabel}\``,
|
|
215
|
+
titleLine !== "" ? ` ${titleLine}` : "",
|
|
218
216
|
"",
|
|
219
217
|
"```",
|
|
220
218
|
treeText,
|
|
221
219
|
"```",
|
|
222
220
|
truncNote,
|
|
223
221
|
"",
|
|
224
|
-
|
|
222
|
+
keyHint("app.tools.expand", "to collapse"),
|
|
225
223
|
].filter((l) => l !== "").join("\n");
|
|
226
224
|
|
|
227
225
|
const treeBlock = new Markdown(headerMd, 0, 0, getMarkdownTheme());
|
|
228
226
|
|
|
229
|
-
if (
|
|
227
|
+
if (details.screenshotPath === undefined) return treeBlock;
|
|
230
228
|
|
|
231
229
|
try {
|
|
232
230
|
const buf = readFileSync(details.screenshotPath);
|
|
@@ -238,7 +236,7 @@ export const snapshotTool = defineBrowserTool({
|
|
|
238
236
|
filename: details.screenshotPath,
|
|
239
237
|
});
|
|
240
238
|
const safeImage = {
|
|
241
|
-
invalidate: () => image.invalidate(),
|
|
239
|
+
invalidate: () => { image.invalidate(); },
|
|
242
240
|
render: (width: number) =>
|
|
243
241
|
image.render(width).map((line) => (visibleWidth(line) > width ? truncateToWidth(line, width) : line)),
|
|
244
242
|
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { BrowserClient } from "../client";
|
|
2
|
+
import { type Result, err, ok } from "../util/result";
|
|
3
|
+
import type { ToolErr } from "../util/tool";
|
|
4
|
+
|
|
5
|
+
const HEX_PREFIX = /^[0-9A-Fa-f]{8,}$/;
|
|
6
|
+
|
|
7
|
+
export const ownershipDeniedMessage = (id: string, namespace: string): string =>
|
|
8
|
+
`Tab ${id} is not owned by this harness session (namespace: ${namespace}). ` +
|
|
9
|
+
`Use browser_list_tabs with scope:'all' to see it, or browser_new_tab to open one this session can control.`;
|
|
10
|
+
|
|
11
|
+
// Resolves a user-supplied target id against the tabs this session owns, and only those:
|
|
12
|
+
// - an exact owned id resolves to itself;
|
|
13
|
+
// - a hex prefix of 8+ characters resolves when it matches exactly one owned tab;
|
|
14
|
+
// - a prefix matching several owned tabs is invalid_args;
|
|
15
|
+
// - anything else (unowned id, prefix matching nothing owned) is invalid_state with the ownership message.
|
|
16
|
+
// Ownership is read from the registry, so no CDP round trip is made.
|
|
17
|
+
export const resolveOwnedTargetId = (client: BrowserClient, input: string): Result<string, ToolErr> => {
|
|
18
|
+
if (client.owns(input)) return ok(input);
|
|
19
|
+
if (HEX_PREFIX.test(input)) {
|
|
20
|
+
const matches = client.ownership().list().filter((id) => id.startsWith(input));
|
|
21
|
+
const first = matches[0];
|
|
22
|
+
if (matches.length === 1 && first !== undefined) return ok(first);
|
|
23
|
+
if (matches.length > 1) {
|
|
24
|
+
return err({
|
|
25
|
+
kind: "invalid_args",
|
|
26
|
+
message: `Ambiguous prefix "${input}" matches ${matches.length} owned tabs: ${matches.join(", ")}`,
|
|
27
|
+
details: { matches },
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return err({
|
|
32
|
+
kind: "invalid_state",
|
|
33
|
+
message: ownershipDeniedMessage(input, client.namespace),
|
|
34
|
+
details: { targetId: input, namespace: client.namespace },
|
|
35
|
+
});
|
|
36
|
+
};
|
package/src/domains/tabs.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
2
|
import { type Result, err, ok } from "../util/result";
|
|
3
3
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
4
|
-
import { cdpCallBrowser } from "./cdp-call";
|
|
4
|
+
import { cdpCallBrowser, fromClient } from "./cdp-call";
|
|
5
|
+
import { resolveOwnedTargetId } from "./tab-resolve";
|
|
5
6
|
|
|
6
7
|
const ListTabsArgs = Type.Object({
|
|
7
8
|
includeInternal: Type.Optional(Type.Boolean({ default: true, description: "Include chrome:// pages" })),
|
|
@@ -24,8 +25,8 @@ export const listTabsTool = defineBrowserTool({
|
|
|
24
25
|
parameters: ListTabsArgs,
|
|
25
26
|
concurrency: "parallel",
|
|
26
27
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
27
|
-
const r = await client.listTabs(args.includeInternal ?? true);
|
|
28
|
-
if (!r.success) return
|
|
28
|
+
const r = fromClient(await client.listTabs(args.includeInternal ?? true), "Target.getTargets");
|
|
29
|
+
if (!r.success) return r;
|
|
29
30
|
const scope = args.scope ?? "owned";
|
|
30
31
|
const filtered = scope === "owned" ? r.data.filter((t) => t.owned) : r.data;
|
|
31
32
|
const lines = filtered.map((t, i) => {
|
|
@@ -68,10 +69,6 @@ const SwitchTabArgs = Type.Object({
|
|
|
68
69
|
targetId: Type.String({ description: "Target ID from browser_list_tabs" }),
|
|
69
70
|
});
|
|
70
71
|
|
|
71
|
-
const ownershipDeniedMessage = (id: string, namespace: string): string =>
|
|
72
|
-
`Tab ${id} is not owned by this harness session (namespace: ${namespace}). ` +
|
|
73
|
-
`Use browser_list_tabs with scope:'all' to see it, or browser_new_tab to open one this session can control.`;
|
|
74
|
-
|
|
75
72
|
export const switchTabTool = defineBrowserTool({
|
|
76
73
|
name: "browser_switch_tab",
|
|
77
74
|
label: "Browser Switch Tab",
|
|
@@ -79,44 +76,16 @@ export const switchTabTool = defineBrowserTool({
|
|
|
79
76
|
promptSnippet: "Switch tabs",
|
|
80
77
|
promptGuidelines: [
|
|
81
78
|
"Get a targetId via browser_list_tabs first.",
|
|
82
|
-
"Accepts exact targetId or a unique prefix
|
|
79
|
+
"Accepts exact targetId or a unique prefix (>=8 hex characters) of an owned tab.",
|
|
83
80
|
],
|
|
84
81
|
parameters: SwitchTabArgs,
|
|
85
82
|
concurrency: "serialized",
|
|
86
83
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
87
|
-
|
|
88
|
-
if (!
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
if (!tabs.success) return err({ kind: "cdp_error", message: tabs.error.message });
|
|
93
|
-
const matches = tabs.data.filter((t) => t.targetId.startsWith(args.targetId));
|
|
94
|
-
if (matches.length === 0) {
|
|
95
|
-
return err({ kind: "cdp_error", message: `No tab found with prefix "${args.targetId}"` });
|
|
96
|
-
}
|
|
97
|
-
if (matches.length > 1) {
|
|
98
|
-
const ids = matches.map((t) => t.targetId).join(", ");
|
|
99
|
-
return err({
|
|
100
|
-
kind: "invalid_state",
|
|
101
|
-
message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} tabs: ${ids}`,
|
|
102
|
-
details: { matches: matches.map((t) => ({ targetId: t.targetId, url: t.url })) },
|
|
103
|
-
});
|
|
104
|
-
}
|
|
105
|
-
const first = matches[0];
|
|
106
|
-
if (first === undefined) return err({ kind: "invalid_state", message: "no matching tab" });
|
|
107
|
-
resolved = first.targetId;
|
|
108
|
-
}
|
|
109
|
-
if (!client.owns(resolved)) {
|
|
110
|
-
return err({
|
|
111
|
-
kind: "invalid_state",
|
|
112
|
-
message: ownershipDeniedMessage(resolved, client.namespace),
|
|
113
|
-
details: { targetId: resolved, namespace: client.namespace },
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
const r = await client.switchTab(resolved);
|
|
119
|
-
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
84
|
+
const target = resolveOwnedTargetId(client, args.targetId);
|
|
85
|
+
if (!target.success) return target;
|
|
86
|
+
const resolved = target.data;
|
|
87
|
+
const r = fromClient(await client.switchTab(resolved), "Target.attachToTarget");
|
|
88
|
+
if (!r.success) return r;
|
|
120
89
|
const note = resolved === args.targetId ? "" : ` (resolved from prefix "${args.targetId}")`;
|
|
121
90
|
return ok({
|
|
122
91
|
text: `Switched to ${resolved}${note}`,
|
|
@@ -138,8 +107,8 @@ export const newTabTool = defineBrowserTool({
|
|
|
138
107
|
parameters: NewTabArgs,
|
|
139
108
|
concurrency: "serialized",
|
|
140
109
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
141
|
-
const r = await client.newTab(args.url);
|
|
142
|
-
if (!r.success) return
|
|
110
|
+
const r = fromClient(await client.newTab(args.url), "Target.createTarget");
|
|
111
|
+
if (!r.success) return r;
|
|
143
112
|
return ok({
|
|
144
113
|
text: `New tab: ${r.data}${args.url ? ` (${args.url})` : ""}`,
|
|
145
114
|
details: args.url !== undefined
|
|
@@ -159,36 +128,16 @@ export const closeTabTool = defineBrowserTool({
|
|
|
159
128
|
description: "Close a tab the harness owns. Refuses tabs not opened by this session.",
|
|
160
129
|
promptSnippet: "Close an owned tab",
|
|
161
130
|
promptGuidelines: [
|
|
162
|
-
"
|
|
131
|
+
"Same targetId rules as browser_switch_tab: exact id or a unique prefix (>=8 hex) of an owned tab.",
|
|
163
132
|
],
|
|
164
133
|
parameters: CloseTabArgs,
|
|
165
134
|
concurrency: "serialized",
|
|
166
135
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
167
|
-
|
|
168
|
-
if (!
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
const matches = owned.filter((id) => id.startsWith(args.targetId));
|
|
173
|
-
const first = matches[0];
|
|
174
|
-
if (matches.length === 1 && first !== undefined) resolved = first;
|
|
175
|
-
else if (matches.length > 1) {
|
|
176
|
-
return err({
|
|
177
|
-
kind: "invalid_state",
|
|
178
|
-
message: `Ambiguous prefix "${args.targetId}" matches ${matches.length} owned tabs: ${matches.join(", ")}`,
|
|
179
|
-
});
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
if (!client.owns(resolved)) {
|
|
183
|
-
return err({
|
|
184
|
-
kind: "invalid_state",
|
|
185
|
-
message: ownershipDeniedMessage(resolved, client.namespace),
|
|
186
|
-
details: { targetId: resolved, namespace: client.namespace },
|
|
187
|
-
});
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
const r = await client.closeTab(resolved);
|
|
191
|
-
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
|
|
136
|
+
const target = resolveOwnedTargetId(client, args.targetId);
|
|
137
|
+
if (!target.success) return target;
|
|
138
|
+
const resolved = target.data;
|
|
139
|
+
const r = fromClient(await client.closeTab(resolved), "Target.closeTarget");
|
|
140
|
+
if (!r.success) return r;
|
|
192
141
|
return ok({ text: `Closed tab ${resolved}`, details: { targetId: resolved } });
|
|
193
142
|
},
|
|
194
143
|
});
|
package/src/domains/target.ts
CHANGED
|
@@ -31,5 +31,5 @@ export const resolveTarget = async (
|
|
|
31
31
|
if (args.x !== undefined && args.y !== undefined) {
|
|
32
32
|
return ok({ x: args.x, y: args.y, label: `(${args.x}, ${args.y})` });
|
|
33
33
|
}
|
|
34
|
-
return err({ kind: "
|
|
34
|
+
return err({ kind: "invalid_args", message: "Provide either `ref` or both `x` and `y`." });
|
|
35
35
|
};
|