pi-browser-harness 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +5 -9
- package/package.json +5 -2
- package/skills/pi-browser-harness/SKILL.md +1 -1
- package/src/cdp/commands.ts +256 -0
- package/src/cdp/console-buffer.ts +10 -85
- package/src/cdp/discovery.ts +28 -71
- package/src/cdp/errors.ts +4 -0
- package/src/cdp/events.ts +170 -0
- package/src/cdp/network-buffer.ts +42 -109
- package/src/cdp/ownership.ts +0 -17
- package/src/cdp/record-store.ts +61 -0
- package/src/cdp/session.ts +205 -193
- package/src/cdp/target-factory.ts +10 -80
- package/src/cdp/types.ts +21 -12
- package/src/client.ts +39 -95
- package/src/daemon/bridge.ts +150 -203
- package/src/daemon/index.ts +0 -28
- package/src/daemon/protocol.ts +6 -68
- package/src/daemon/queue.ts +85 -0
- package/src/daemon/server.ts +24 -99
- package/src/daemon/spawn.ts +18 -68
- package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +18 -51
- package/src/domains/ax-tree.ts +179 -0
- package/src/domains/box.ts +51 -0
- package/src/domains/cdp-call.ts +55 -0
- package/src/domains/click.ts +19 -32
- package/src/domains/console.ts +48 -2
- package/src/domains/dialog.ts +8 -5
- package/src/domains/drag.ts +8 -7
- package/src/domains/element-call.ts +117 -0
- package/src/domains/files.ts +22 -26
- package/src/domains/fill-engine.ts +80 -0
- package/src/domains/form-batch.ts +65 -0
- package/src/domains/form.ts +116 -105
- package/src/domains/history.ts +13 -9
- package/src/domains/isolated-tab.ts +26 -55
- package/src/domains/js.ts +29 -42
- package/src/domains/keyboard.ts +36 -27
- package/src/domains/navigate.ts +16 -23
- package/src/domains/network.ts +60 -10
- package/src/domains/page.ts +24 -3
- package/src/domains/readpage/read-page.ts +74 -19
- package/src/domains/readpage/readability.ts +0 -24
- package/src/domains/ref-resolve.ts +24 -58
- package/src/domains/render.ts +6 -27
- package/src/domains/screenshot-capture.ts +80 -0
- package/src/domains/screenshot.ts +9 -86
- package/src/domains/scroll.ts +10 -11
- package/src/domains/search/google-serp.ts +0 -29
- package/src/domains/search/web-search.ts +48 -20
- package/src/domains/setup.ts +1 -11
- package/src/domains/snapshot.ts +35 -276
- package/src/domains/tabs.ts +14 -7
- package/src/domains/target.ts +35 -0
- package/src/domains/viewport.ts +5 -3
- package/src/index.ts +8 -58
- package/src/profile/bind.ts +8 -57
- package/src/profile/browser-process.ts +75 -126
- package/src/profile/command.ts +50 -29
- package/src/profile/launch.ts +22 -40
- package/src/profile/list.ts +14 -54
- package/src/profile/paths.ts +2 -65
- package/src/profile/store.ts +35 -56
- package/src/registry.ts +30 -33
- package/src/schemas/parse.ts +23 -0
- package/src/setup.ts +0 -25
- package/src/state.ts +16 -16
- package/src/util/debug.ts +8 -0
- package/src/util/guards.ts +30 -0
- package/src/util/js-template.ts +1 -16
- package/src/util/mutex.ts +0 -5
- package/src/util/result.ts +0 -6
- package/src/util/sharp-shim.ts +12 -19
- package/src/util/tool.ts +17 -42
- package/src/util/truncate.ts +6 -0
- package/tsconfig.json +1 -16
- package/skills/deep-research/SKILL.md +0 -66
- package/src/cdp/event-queue.ts +0 -128
- package/src/cdp/transport.ts +0 -156
- package/src/cdp/window-binding.ts +0 -15
- package/src/deep-research.ts +0 -41
- package/src/domains/forms.ts +0 -355
- package/src/domains/readpage/capture.ts +0 -46
- package/src/domains/search/extract.ts +0 -44
- package/src/profile/picker.ts +0 -73
- package/src/util/keycodes.ts +0 -10
- package/src/util/time.ts +0 -29
|
@@ -1,27 +1,14 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pure Google SERP parsing — no CDP, no browser. Everything here is a pure
|
|
3
|
-
* function over data captured from the page, so it is unit-testable against
|
|
4
|
-
* saved fixtures (see test/deep-research/serp-parser-test.ts).
|
|
5
|
-
*
|
|
6
|
-
* ADR-0001: results are extracted by structure/semantics (anchors carrying an
|
|
7
|
-
* <h3> heading in the main results column), never by brittle Google CSS class
|
|
8
|
-
* names, and Google redirect wrappers are unwound to the real target URL.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
/** One raw anchor captured in-page: a link with a heading and nearby snippet. */
|
|
12
1
|
export type SerpAnchor = {
|
|
13
2
|
readonly href: string;
|
|
14
3
|
readonly heading: string;
|
|
15
4
|
readonly snippet: string;
|
|
16
5
|
};
|
|
17
6
|
|
|
18
|
-
/** The JSON payload the in-page extraction expression returns. */
|
|
19
7
|
export type SerpExtraction = {
|
|
20
8
|
readonly anchors: ReadonlyArray<SerpAnchor>;
|
|
21
9
|
readonly pageText: string;
|
|
22
10
|
};
|
|
23
11
|
|
|
24
|
-
/** A parsed, ranked search result — the tool's public output shape. */
|
|
25
12
|
export type SearchResult = {
|
|
26
13
|
readonly title: string;
|
|
27
14
|
readonly url: string;
|
|
@@ -29,7 +16,6 @@ export type SearchResult = {
|
|
|
29
16
|
readonly rank: number;
|
|
30
17
|
};
|
|
31
18
|
|
|
32
|
-
/** Why a SERP yielded no usable results, or that it did. */
|
|
33
19
|
export type SerpVerdict = "ok" | "captcha" | "no_results";
|
|
34
20
|
|
|
35
21
|
const CAPTCHA_MARKERS: ReadonlyArray<string> = [
|
|
@@ -42,10 +28,6 @@ const CAPTCHA_MARKERS: ReadonlyArray<string> = [
|
|
|
42
28
|
"our systems have detected unusual traffic",
|
|
43
29
|
];
|
|
44
30
|
|
|
45
|
-
/**
|
|
46
|
-
* Decode a Google redirect wrapper (`/url?q=<real>&sa=...`) to its target.
|
|
47
|
-
* Returns the input unchanged when it is not a wrapper.
|
|
48
|
-
*/
|
|
49
31
|
const unwrapRedirect = (href: string): string => {
|
|
50
32
|
const wrapper = href.match(/^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i);
|
|
51
33
|
if (!wrapper) return href;
|
|
@@ -66,7 +48,6 @@ const isExternalResult = (url: string): boolean => {
|
|
|
66
48
|
}
|
|
67
49
|
};
|
|
68
50
|
|
|
69
|
-
/** Normalized key for dedupe: origin + pathname, trailing slash and hash removed. */
|
|
70
51
|
const dedupeKey = (url: string): string => {
|
|
71
52
|
try {
|
|
72
53
|
const u = new URL(url);
|
|
@@ -76,11 +57,6 @@ const dedupeKey = (url: string): string => {
|
|
|
76
57
|
}
|
|
77
58
|
};
|
|
78
59
|
|
|
79
|
-
/**
|
|
80
|
-
* Parse captured anchors into ranked results: unwrap redirects, keep only
|
|
81
|
-
* external http(s) links, dedupe by normalized url (first wins), rank by input
|
|
82
|
-
* order, cap to `limit`.
|
|
83
|
-
*/
|
|
84
60
|
export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: number): ReadonlyArray<SearchResult> => {
|
|
85
61
|
const seen = new Set<string>();
|
|
86
62
|
const results: SearchResult[] = [];
|
|
@@ -101,11 +77,6 @@ export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: numbe
|
|
|
101
77
|
return results;
|
|
102
78
|
};
|
|
103
79
|
|
|
104
|
-
/**
|
|
105
|
-
* Decide whether the SERP is usable, blocked by a CAPTCHA, or genuinely empty.
|
|
106
|
-
* `resultCount` is the number of parsed results; `pageText` is a bounded slice
|
|
107
|
-
* of the page's visible text.
|
|
108
|
-
*/
|
|
109
80
|
export const classifySerp = (pageText: string, resultCount: number): SerpVerdict => {
|
|
110
81
|
if (resultCount > 0) return "ok";
|
|
111
82
|
const lower = pageText.toLowerCase();
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import { Type } from "typebox";
|
|
2
|
+
import { Compile } from "typebox/compile";
|
|
3
|
+
import { parseJson } from "../../schemas/parse";
|
|
2
4
|
import { type Result, err, ok } from "../../util/result";
|
|
3
5
|
import { applyTruncation } from "../../util/truncate";
|
|
4
6
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
|
|
5
7
|
import {
|
|
6
8
|
closeIsolatedTab,
|
|
7
9
|
evalInIsolatedTab,
|
|
10
|
+
isJsonText,
|
|
8
11
|
navigateIsolatedTab,
|
|
9
12
|
openIsolatedTab,
|
|
10
13
|
waitForIsolatedLoad,
|
|
11
14
|
} from "../isolated-tab";
|
|
12
15
|
import { renderExpandableText } from "../render";
|
|
13
|
-
import { buildSerpExtractionExpr } from "./extract";
|
|
14
16
|
import { classifySerp, parseGoogleSerp, type SearchResult, type SerpExtraction } from "./google-serp";
|
|
15
17
|
|
|
16
18
|
const LOAD_TIMEOUT_MS = 15_000;
|
|
@@ -22,14 +24,46 @@ const WebSearchArgs = Type.Object({
|
|
|
22
24
|
),
|
|
23
25
|
});
|
|
24
26
|
|
|
27
|
+
const PAGE_TEXT_LIMIT = 4000;
|
|
28
|
+
|
|
29
|
+
const buildSerpExtractionExpr = (): string => `
|
|
30
|
+
(() => {
|
|
31
|
+
const main = document.querySelector('#search, #rso, #main') || document.body;
|
|
32
|
+
const headings = Array.from(main.querySelectorAll('a h3'));
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
const anchors = [];
|
|
35
|
+
for (const h3 of headings) {
|
|
36
|
+
const a = h3.closest('a[href]');
|
|
37
|
+
if (!a) continue;
|
|
38
|
+
const href = a.href;
|
|
39
|
+
if (!href || seen.has(href)) continue;
|
|
40
|
+
seen.add(href);
|
|
41
|
+
const container =
|
|
42
|
+
a.closest('div.g') ||
|
|
43
|
+
a.closest('div[data-hveid]') ||
|
|
44
|
+
a.parentElement?.parentElement ||
|
|
45
|
+
a.parentElement;
|
|
46
|
+
const containerText = container ? container.innerText || '' : '';
|
|
47
|
+
const heading = h3.innerText || h3.textContent || '';
|
|
48
|
+
const snippet = containerText.replace(heading, '').replace(/\\s+/g, ' ').trim().slice(0, 500);
|
|
49
|
+
anchors.push({ href, heading, snippet });
|
|
50
|
+
}
|
|
51
|
+
const pageText = (document.body.innerText || '').slice(0, ${PAGE_TEXT_LIMIT});
|
|
52
|
+
return JSON.stringify({ anchors, pageText });
|
|
53
|
+
})()
|
|
54
|
+
`;
|
|
55
|
+
|
|
25
56
|
const buildSerpUrl = (query: string, limit: number): string =>
|
|
26
57
|
`https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
|
|
27
58
|
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
59
|
+
const SerpExtractionSchema = Type.Object({
|
|
60
|
+
anchors: Type.Array(
|
|
61
|
+
Type.Object({ href: Type.String(), heading: Type.String(), snippet: Type.String() }),
|
|
62
|
+
),
|
|
63
|
+
pageText: Type.String(),
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const serpExtractionValidator = Compile(SerpExtractionSchema);
|
|
33
67
|
|
|
34
68
|
const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
|
|
35
69
|
const lines = results.map((r) => `${r.rank}. ${r.title}\n ${r.url}\n ${r.snippet}`);
|
|
@@ -49,6 +83,7 @@ export const webSearchTool = defineBrowserTool({
|
|
|
49
83
|
"Runs in its own isolated tab and never disturbs the user's current tab.",
|
|
50
84
|
],
|
|
51
85
|
parameters: WebSearchArgs,
|
|
86
|
+
concurrency: "serialized",
|
|
52
87
|
async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
|
|
53
88
|
const limit = args.limit ?? 10;
|
|
54
89
|
const serpUrl = buildSerpUrl(args.query, limit);
|
|
@@ -64,16 +99,17 @@ export const webSearchTool = defineBrowserTool({
|
|
|
64
99
|
const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
|
|
65
100
|
if (!loaded.success) return err(loaded.error);
|
|
66
101
|
|
|
67
|
-
const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr());
|
|
102
|
+
const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr(), isJsonText);
|
|
68
103
|
if (!extracted.success) return err(extracted.error);
|
|
69
104
|
|
|
70
|
-
const parsedJson =
|
|
71
|
-
if (!
|
|
72
|
-
return err({ kind: "internal", message:
|
|
105
|
+
const parsedJson = parseJson(extracted.data, serpExtractionValidator);
|
|
106
|
+
if (!parsedJson.success) {
|
|
107
|
+
return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
|
|
73
108
|
}
|
|
109
|
+
const extraction: SerpExtraction = parsedJson.data;
|
|
74
110
|
|
|
75
|
-
const results = parseGoogleSerp(
|
|
76
|
-
const verdict = classifySerp(
|
|
111
|
+
const results = parseGoogleSerp(extraction.anchors, limit);
|
|
112
|
+
const verdict = classifySerp(extraction.pageText, results.length);
|
|
77
113
|
if (verdict !== "ok") {
|
|
78
114
|
return err({
|
|
79
115
|
kind: "invalid_state",
|
|
@@ -106,11 +142,3 @@ export const webSearchTool = defineBrowserTool({
|
|
|
106
142
|
return renderExpandableText("web_search", result, expanded, theme);
|
|
107
143
|
},
|
|
108
144
|
});
|
|
109
|
-
|
|
110
|
-
const safeParse = (text: string): unknown => {
|
|
111
|
-
try {
|
|
112
|
-
return JSON.parse(text) as unknown;
|
|
113
|
-
} catch {
|
|
114
|
-
return undefined;
|
|
115
|
-
}
|
|
116
|
-
};
|
package/src/domains/setup.ts
CHANGED
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* browser_setup tool — agent-callable browser initialization.
|
|
3
|
-
*
|
|
4
|
-
* The agent can call this directly when browser tools fail with
|
|
5
|
-
* "not initialized" instead of asking the user. Idempotent: safe
|
|
6
|
-
* to call even when already connected.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
1
|
import { Type } from "typebox";
|
|
10
2
|
import { Text } from "@mariozechner/pi-tui";
|
|
11
3
|
import type { Result } from "../util/result";
|
|
@@ -28,12 +20,10 @@ export const setupTool = defineBrowserTool({
|
|
|
28
20
|
"After browser_setup succeeds, retry the browser tool that failed.",
|
|
29
21
|
],
|
|
30
22
|
parameters: SetupArgs,
|
|
31
|
-
|
|
23
|
+
concurrency: "parallel",
|
|
32
24
|
ensureAlive: false,
|
|
33
25
|
renderCall: () => new Text("🔧 Initializing browser...", 0, 0),
|
|
34
26
|
async handler(_args, { client, extensionCtx }): Promise<Result<ToolOk, ToolErr>> {
|
|
35
|
-
// extensionCtx carries the UI, so the agent calling browser_setup gets the
|
|
36
|
-
// same first-run profile picker the user gets from /browser-setup.
|
|
37
27
|
const result = await performSetup(client, extensionCtx);
|
|
38
28
|
if (result.success) {
|
|
39
29
|
return ok({ text: result.data });
|
package/src/domains/snapshot.ts
CHANGED
|
@@ -2,11 +2,23 @@ import { readFileSync, writeFileSync } from "node:fs";
|
|
|
2
2
|
import { Type } from "typebox";
|
|
3
3
|
import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
|
|
4
4
|
import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
|
|
5
|
-
import
|
|
5
|
+
import { boxOf } from "./box";
|
|
6
|
+
import { cdpCall } from "./cdp-call";
|
|
7
|
+
import {
|
|
8
|
+
buildTree,
|
|
9
|
+
collectInteractiveTargets,
|
|
10
|
+
countNodes,
|
|
11
|
+
INTERACTIVE_ROLES,
|
|
12
|
+
liveValue,
|
|
13
|
+
type SlimNode,
|
|
14
|
+
stripInternals,
|
|
15
|
+
VALUE_ROLES,
|
|
16
|
+
} from "./ax-tree";
|
|
6
17
|
import { type Result, err, ok } from "../util/result";
|
|
7
18
|
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
|
|
8
19
|
import { applyTruncation } from "../util/truncate";
|
|
9
20
|
import { screenshotPath } from "../util/paths";
|
|
21
|
+
import { asBoolean, asNumber, asRecord, asString } from "../util/guards";
|
|
10
22
|
|
|
11
23
|
const SnapshotArgs = Type.Object({
|
|
12
24
|
includeScreenshot: Type.Optional(
|
|
@@ -26,168 +38,6 @@ const SnapshotArgs = Type.Object({
|
|
|
26
38
|
),
|
|
27
39
|
});
|
|
28
40
|
|
|
29
|
-
// Raw shapes from CDP Accessibility.getFullAXTree. Cast at this boundary only.
|
|
30
|
-
type AxValue = { value?: unknown };
|
|
31
|
-
export type RawAxNode = {
|
|
32
|
-
nodeId: string;
|
|
33
|
-
parentId?: string;
|
|
34
|
-
childIds?: ReadonlyArray<string>;
|
|
35
|
-
ignored?: boolean;
|
|
36
|
-
role?: AxValue;
|
|
37
|
-
name?: AxValue;
|
|
38
|
-
value?: AxValue;
|
|
39
|
-
description?: AxValue;
|
|
40
|
-
properties?: ReadonlyArray<{ name?: string; value?: AxValue }>;
|
|
41
|
-
backendDOMNodeId?: number;
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number };
|
|
45
|
-
|
|
46
|
-
export type SlimNode = {
|
|
47
|
-
role: string;
|
|
48
|
-
name?: string;
|
|
49
|
-
value?: string;
|
|
50
|
-
description?: string;
|
|
51
|
-
state?: string;
|
|
52
|
-
children: SlimNode[];
|
|
53
|
-
/** Internal: kept for the post-build box fetch. Stripped before returning to the LLM. */
|
|
54
|
-
_backendId?: number;
|
|
55
|
-
/** Stable handle (e.g. "e12") for interaction tools. Survives re-renders via backendNodeId. */
|
|
56
|
-
ref?: string;
|
|
57
|
-
/** Click target — center of the element's bounding box, in viewport CSS pixels. */
|
|
58
|
-
box?: Box;
|
|
59
|
-
};
|
|
60
|
-
|
|
61
|
-
const INTERACTIVE_ROLES = new Set([
|
|
62
|
-
"button",
|
|
63
|
-
"link",
|
|
64
|
-
"textbox",
|
|
65
|
-
"checkbox",
|
|
66
|
-
"radio",
|
|
67
|
-
"combobox",
|
|
68
|
-
"menuitem",
|
|
69
|
-
"tab",
|
|
70
|
-
"switch",
|
|
71
|
-
"slider",
|
|
72
|
-
"searchbox",
|
|
73
|
-
"spinbutton",
|
|
74
|
-
]);
|
|
75
|
-
|
|
76
|
-
// Roles whose live DOM value/checked state should override the (often stale)
|
|
77
|
-
// AX-tree value. The AX `value` is unreliable for freshly-typed or controlled
|
|
78
|
-
// inputs, so for these we read the real DOM property at snapshot time.
|
|
79
|
-
const VALUE_ROLES = new Set([
|
|
80
|
-
"textbox",
|
|
81
|
-
"searchbox",
|
|
82
|
-
"spinbutton",
|
|
83
|
-
"combobox",
|
|
84
|
-
"checkbox",
|
|
85
|
-
"radio",
|
|
86
|
-
"switch",
|
|
87
|
-
"slider",
|
|
88
|
-
]);
|
|
89
|
-
|
|
90
|
-
const stringOf = (v: AxValue | undefined): string | undefined => {
|
|
91
|
-
const x = v?.value;
|
|
92
|
-
if (typeof x === "string" && x !== "") return x;
|
|
93
|
-
if (typeof x === "number") return String(x);
|
|
94
|
-
return undefined;
|
|
95
|
-
};
|
|
96
|
-
|
|
97
|
-
const collectState = (props: RawAxNode["properties"]): string | undefined => {
|
|
98
|
-
if (!props || props.length === 0) return undefined;
|
|
99
|
-
const flags: string[] = [];
|
|
100
|
-
for (const p of props) {
|
|
101
|
-
const name = p.name;
|
|
102
|
-
if (!name) continue;
|
|
103
|
-
const raw = p.value?.value;
|
|
104
|
-
// Boolean flags worth surfacing in the outline.
|
|
105
|
-
if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
|
|
106
|
-
flags.push(name);
|
|
107
|
-
}
|
|
108
|
-
if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
|
|
109
|
-
}
|
|
110
|
-
return flags.length > 0 ? flags.join(", ") : undefined;
|
|
111
|
-
};
|
|
112
|
-
|
|
113
|
-
export const buildTree = (
|
|
114
|
-
rawNodes: ReadonlyArray<RawAxNode>,
|
|
115
|
-
opts: { interestingOnly: boolean; maxNodes: number },
|
|
116
|
-
): SlimNode[] => {
|
|
117
|
-
// Index for parent/child lookups.
|
|
118
|
-
const byId = new Map<string, RawAxNode>();
|
|
119
|
-
for (const n of rawNodes) byId.set(n.nodeId, n);
|
|
120
|
-
|
|
121
|
-
// CDP doesn't mark a single root explicitly; the root nodes are the ones
|
|
122
|
-
// whose parentId isn't in the index (typically just one — RootWebArea).
|
|
123
|
-
const rootIds: string[] = [];
|
|
124
|
-
for (const n of rawNodes) {
|
|
125
|
-
if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
let budget = opts.maxNodes;
|
|
129
|
-
|
|
130
|
-
const slim = (node: RawAxNode): SlimNode | undefined => {
|
|
131
|
-
if (budget <= 0) return undefined;
|
|
132
|
-
if (opts.interestingOnly && node.ignored) {
|
|
133
|
-
// Skip the ignored node but recurse through its children — Chrome often
|
|
134
|
-
// ignores wrapper divs but the meaningful descendants are still there.
|
|
135
|
-
const out: SlimNode[] = [];
|
|
136
|
-
for (const cid of node.childIds ?? []) {
|
|
137
|
-
const child = byId.get(cid);
|
|
138
|
-
if (!child) continue;
|
|
139
|
-
const slimChild = slim(child);
|
|
140
|
-
if (slimChild) out.push(slimChild);
|
|
141
|
-
}
|
|
142
|
-
// Hoist children into the parent. We return undefined; the parent's
|
|
143
|
-
// recursion will see them by re-walking — but that double-counts. Instead
|
|
144
|
-
// we handle hoisting at the parent level. So here just signal "skip me"
|
|
145
|
-
// by returning undefined; the parent will hoist by checking ignored flag.
|
|
146
|
-
return out.length > 0 ? { role: "_hoist", children: out } : undefined;
|
|
147
|
-
}
|
|
148
|
-
budget--;
|
|
149
|
-
const role = stringOf(node.role) ?? "unknown";
|
|
150
|
-
const name = stringOf(node.name);
|
|
151
|
-
const value = stringOf(node.value);
|
|
152
|
-
const description = stringOf(node.description);
|
|
153
|
-
const state = collectState(node.properties);
|
|
154
|
-
const children: SlimNode[] = [];
|
|
155
|
-
for (const cid of node.childIds ?? []) {
|
|
156
|
-
const child = byId.get(cid);
|
|
157
|
-
if (!child) continue;
|
|
158
|
-
const slimChild = slim(child);
|
|
159
|
-
if (!slimChild) continue;
|
|
160
|
-
// Hoist any synthetic _hoist nodes' children into this level.
|
|
161
|
-
if (slimChild.role === "_hoist") children.push(...slimChild.children);
|
|
162
|
-
else children.push(slimChild);
|
|
163
|
-
}
|
|
164
|
-
const out: SlimNode = { role, children };
|
|
165
|
-
if (name !== undefined) out.name = name;
|
|
166
|
-
if (value !== undefined) out.value = value;
|
|
167
|
-
if (description !== undefined) out.description = description;
|
|
168
|
-
if (state !== undefined) out.state = state;
|
|
169
|
-
if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
|
|
170
|
-
return out;
|
|
171
|
-
};
|
|
172
|
-
|
|
173
|
-
const result: SlimNode[] = [];
|
|
174
|
-
for (const rid of rootIds) {
|
|
175
|
-
const root = byId.get(rid);
|
|
176
|
-
if (!root) continue;
|
|
177
|
-
const s = slim(root);
|
|
178
|
-
if (!s) continue;
|
|
179
|
-
if (s.role === "_hoist") result.push(...s.children);
|
|
180
|
-
else result.push(s);
|
|
181
|
-
}
|
|
182
|
-
return result;
|
|
183
|
-
};
|
|
184
|
-
|
|
185
|
-
const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
|
|
186
|
-
let n = 0;
|
|
187
|
-
for (const node of nodes) n += 1 + countNodes(node.children);
|
|
188
|
-
return n;
|
|
189
|
-
};
|
|
190
|
-
|
|
191
41
|
const summarize = (nodes: ReadonlyArray<SlimNode>): string => {
|
|
192
42
|
const counts = new Map<string, number>();
|
|
193
43
|
const walk = (ns: ReadonlyArray<SlimNode>): void => {
|
|
@@ -197,7 +47,6 @@ const summarize = (nodes: ReadonlyArray<SlimNode>): string => {
|
|
|
197
47
|
}
|
|
198
48
|
};
|
|
199
49
|
walk(nodes);
|
|
200
|
-
// Group landmarks (roles that are top-level structural).
|
|
201
50
|
const landmarkRoles = new Set(["banner", "navigation", "main", "complementary", "contentinfo", "search", "form", "region"]);
|
|
202
51
|
let landmarks = 0;
|
|
203
52
|
for (const [role, c] of counts) if (landmarkRoles.has(role)) landmarks += c;
|
|
@@ -221,8 +70,6 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
|
|
|
221
70
|
if (n.name) line += ` "${n.name}"`;
|
|
222
71
|
if (n.value && n.value !== n.name) line += ` = ${JSON.stringify(n.value)}`;
|
|
223
72
|
if (n.state) line += ` (${n.state})`;
|
|
224
|
-
// Surface the stable ref + click coordinates for interactive nodes. Agents
|
|
225
|
-
// pass `ref` to interaction tools (survives re-renders); @(x,y) is a fallback.
|
|
226
73
|
if (n.ref) line += ` [${n.ref}]`;
|
|
227
74
|
if (n.box && INTERACTIVE_ROLES.has(n.role)) {
|
|
228
75
|
line += ` @(${n.box.cx},${n.box.cy})`;
|
|
@@ -235,61 +82,6 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
|
|
|
235
82
|
return lines.join("\n");
|
|
236
83
|
};
|
|
237
84
|
|
|
238
|
-
/** Strip internal _backendId from a slim tree before returning to callers. */
|
|
239
|
-
const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
|
|
240
|
-
nodes.map((n) => {
|
|
241
|
-
const { _backendId: _bid, ...rest } = n;
|
|
242
|
-
return { ...rest, children: stripInternals(n.children) };
|
|
243
|
-
});
|
|
244
|
-
|
|
245
|
-
/**
|
|
246
|
-
* Walk all slim nodes and collect (slimNode, backendId) pairs for interactive
|
|
247
|
-
* roles. Caller fetches DOM.getBoxModel for each and writes box back into the
|
|
248
|
-
* slim node by reference.
|
|
249
|
-
*/
|
|
250
|
-
export const collectInteractiveTargets = (
|
|
251
|
-
nodes: ReadonlyArray<SlimNode>,
|
|
252
|
-
): Array<{ node: SlimNode; backendId: number }> => {
|
|
253
|
-
const out: Array<{ node: SlimNode; backendId: number }> = [];
|
|
254
|
-
const walk = (ns: ReadonlyArray<SlimNode>): void => {
|
|
255
|
-
for (const n of ns) {
|
|
256
|
-
if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
|
|
257
|
-
out.push({ node: n, backendId: n._backendId });
|
|
258
|
-
}
|
|
259
|
-
walk(n.children);
|
|
260
|
-
}
|
|
261
|
-
};
|
|
262
|
-
walk(nodes);
|
|
263
|
-
return out;
|
|
264
|
-
};
|
|
265
|
-
|
|
266
|
-
/**
|
|
267
|
-
* Read the live DOM value for a form control by backend node id. Resolves the
|
|
268
|
-
* node to a JS object then reads the property that matters for its type:
|
|
269
|
-
* `.value` for inputs/selects/textareas, `.checked` (→ "checked"/"unchecked")
|
|
270
|
-
* for checkboxes/radios. Returns undefined on any failure so the caller falls
|
|
271
|
-
* back to the AX value. Mirrors the box-fetch error handling: never throws.
|
|
272
|
-
*/
|
|
273
|
-
const liveValue = async (session: CdpSession, backendId: number): Promise<string | undefined> => {
|
|
274
|
-
const resolved = await session.call("DOM.resolveNode", { backendNodeId: backendId });
|
|
275
|
-
if (!resolved.success) return undefined;
|
|
276
|
-
const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId;
|
|
277
|
-
if (!objectId) return undefined;
|
|
278
|
-
const fn = `function () {
|
|
279
|
-
if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
|
|
280
|
-
if (typeof this.value === "string") return this.value;
|
|
281
|
-
return null;
|
|
282
|
-
}`;
|
|
283
|
-
const r = await session.call("Runtime.callFunctionOn", {
|
|
284
|
-
objectId,
|
|
285
|
-
functionDeclaration: fn,
|
|
286
|
-
returnByValue: true,
|
|
287
|
-
});
|
|
288
|
-
if (!r.success) return undefined;
|
|
289
|
-
const value = (r.data as { result?: { value?: unknown } }).result?.value;
|
|
290
|
-
return typeof value === "string" ? value : undefined;
|
|
291
|
-
};
|
|
292
|
-
|
|
293
85
|
type SnapshotDetails = {
|
|
294
86
|
nodeCount: number;
|
|
295
87
|
truncated: boolean;
|
|
@@ -314,37 +106,27 @@ export const snapshotTool = defineBrowserTool({
|
|
|
314
106
|
"format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.",
|
|
315
107
|
],
|
|
316
108
|
parameters: SnapshotArgs,
|
|
109
|
+
concurrency: "parallel",
|
|
317
110
|
|
|
318
111
|
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
|
|
319
112
|
const session = client.session();
|
|
320
113
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
const rawNodes = (axRes.data as { nodes: ReadonlyArray<RawAxNode> }).nodes;
|
|
114
|
+
const axRes = await cdpCall(client, "Accessibility.getFullAXTree", {});
|
|
115
|
+
if (!axRes.success) return axRes;
|
|
116
|
+
const rawNodes = axRes.data.nodes;
|
|
325
117
|
|
|
326
|
-
// 2. Page url/title
|
|
327
118
|
const piRes = await client.pageInfo();
|
|
328
119
|
if (!piRes.success) return err({ kind: "cdp_error", message: piRes.error.message });
|
|
329
120
|
const pageUrl = "dialog" in piRes.data ? "" : piRes.data.url;
|
|
330
121
|
const pageTitle = "dialog" in piRes.data ? "" : piRes.data.title;
|
|
331
122
|
|
|
332
|
-
// 3. Slim + format
|
|
333
123
|
const slim = buildTree(rawNodes, {
|
|
334
124
|
interestingOnly: args.interestingOnly ?? true,
|
|
335
125
|
maxNodes: args.maxNodes ?? 1000,
|
|
336
126
|
});
|
|
337
127
|
|
|
338
|
-
// 4. Fetch bounding boxes for interactive nodes so the agent can click
|
|
339
|
-
// without a screenshot round-trip. Capped budget so a slow page can't
|
|
340
|
-
// wedge the call.
|
|
341
128
|
const targets = collectInteractiveTargets(slim);
|
|
342
129
|
|
|
343
|
-
// Assign stable refs (e1, e2, …) to every interactive target. Interaction
|
|
344
|
-
// tools resolve these refs to backendNodeIds at call time, so they stay
|
|
345
|
-
// valid across re-renders. The ref → backendNodeId map (and the per-ref
|
|
346
|
-
// signature baseline for the post-mutation diff) is published below, after
|
|
347
|
-
// the box/live-value fetch so the signature reflects the live DOM value.
|
|
348
130
|
const refMap = new Map<string, number>();
|
|
349
131
|
targets.forEach(({ node, backendId }, i) => {
|
|
350
132
|
const ref = `e${i + 1}`;
|
|
@@ -358,42 +140,18 @@ export const snapshotTool = defineBrowserTool({
|
|
|
358
140
|
await Promise.allSettled(
|
|
359
141
|
targets.map(async ({ node, backendId }) => {
|
|
360
142
|
if (ac.signal.aborted) return;
|
|
361
|
-
const
|
|
362
|
-
if (!
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
const content = data.model?.content;
|
|
366
|
-
if (!content || content.length < 8) return;
|
|
367
|
-
// content is [x1,y1, x2,y1, x2,y2, x1,y2] in viewport CSS pixels.
|
|
368
|
-
const x = content[0]!;
|
|
369
|
-
const y = content[1]!;
|
|
370
|
-
const width = data.model!.width ?? 0;
|
|
371
|
-
const height = data.model!.height ?? 0;
|
|
372
|
-
// Skip zero-size and off-viewport boxes — they can't be clicked.
|
|
373
|
-
if (width <= 0 || height <= 0 || x < 0 || y < 0) return;
|
|
374
|
-
node.box = {
|
|
375
|
-
x: Math.round(x),
|
|
376
|
-
y: Math.round(y),
|
|
377
|
-
width: Math.round(width),
|
|
378
|
-
height: Math.round(height),
|
|
379
|
-
cx: Math.round(x + width / 2),
|
|
380
|
-
cy: Math.round(y + height / 2),
|
|
381
|
-
};
|
|
382
|
-
// Override the AX value with the live DOM property for form controls.
|
|
383
|
-
// The AX `value` is stale for freshly-typed / controlled inputs, so the
|
|
384
|
-
// agent can't trust it as a read-back signal. Reading .value/.checked
|
|
385
|
-
// from the actual element fixes that.
|
|
143
|
+
const box = await boxOf(client, backendId);
|
|
144
|
+
if (!box.success) return;
|
|
145
|
+
if (box.data.x < 0 || box.data.y < 0) return;
|
|
146
|
+
node.box = box.data;
|
|
386
147
|
if (ac.signal.aborted || !VALUE_ROLES.has(node.role)) return;
|
|
387
|
-
const live = await liveValue(
|
|
148
|
+
const live = await liveValue(client, backendId);
|
|
388
149
|
if (live !== undefined) node.value = live;
|
|
389
150
|
}),
|
|
390
151
|
);
|
|
391
152
|
clearTimeout(timer);
|
|
392
153
|
}
|
|
393
154
|
|
|
394
|
-
// Publish refs to the active tab. The signature (role|name|value|state) is
|
|
395
|
-
// the baseline the post-mutation auto-diff compares against, so it must use
|
|
396
|
-
// the live values resolved above.
|
|
397
155
|
const refSig = new Map<string, string>();
|
|
398
156
|
for (const { node } of targets) {
|
|
399
157
|
if (node.ref === undefined) continue;
|
|
@@ -405,14 +163,12 @@ export const snapshotTool = defineBrowserTool({
|
|
|
405
163
|
const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2);
|
|
406
164
|
const trunc = await applyTruncation(text, "snapshot");
|
|
407
165
|
|
|
408
|
-
// 4. Optional screenshot — JPEG q=80 to match screenshotTool's default.
|
|
409
166
|
let shotPath: string | undefined;
|
|
410
167
|
if (args.includeScreenshot) {
|
|
411
|
-
const shot = await
|
|
168
|
+
const shot = await cdpCall(client, "Page.captureScreenshot", { format: "jpeg", quality: 80 });
|
|
412
169
|
if (shot.success) {
|
|
413
|
-
const data = (shot.data as { data: string }).data;
|
|
414
170
|
shotPath = screenshotPath(client.namespace, "jpeg");
|
|
415
|
-
writeFileSync(shotPath, Buffer.from(data, "base64"));
|
|
171
|
+
writeFileSync(shotPath, Buffer.from(shot.data.data, "base64"));
|
|
416
172
|
}
|
|
417
173
|
}
|
|
418
174
|
|
|
@@ -429,10 +185,17 @@ export const snapshotTool = defineBrowserTool({
|
|
|
429
185
|
},
|
|
430
186
|
|
|
431
187
|
renderResult(result, expanded, theme) {
|
|
432
|
-
const
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
188
|
+
const raw = asRecord(result.details);
|
|
189
|
+
if (raw === undefined) return new Text(theme.fg("error", "snapshot: no details"), 0, 0);
|
|
190
|
+
const details = {
|
|
191
|
+
nodeCount: asNumber(raw["nodeCount"]),
|
|
192
|
+
truncated: asBoolean(raw["truncated"]),
|
|
193
|
+
fullOutputPath: asString(raw["fullOutputPath"]),
|
|
194
|
+
screenshotPath: asString(raw["screenshotPath"]),
|
|
195
|
+
url: asString(raw["url"]),
|
|
196
|
+
title: asString(raw["title"]),
|
|
197
|
+
summary: asString(raw["summary"]),
|
|
198
|
+
};
|
|
436
199
|
|
|
437
200
|
const titleLine = details.title ? ` "${details.title}"` : "";
|
|
438
201
|
const summary = details.summary ?? "";
|
|
@@ -449,9 +212,6 @@ export const snapshotTool = defineBrowserTool({
|
|
|
449
212
|
return new Markdown(md, 0, 0, getMarkdownTheme());
|
|
450
213
|
}
|
|
451
214
|
|
|
452
|
-
// Expanded view: read truncated text from result.content (the spilled file
|
|
453
|
-
// is too large to inline) — the same `text` we returned from handler is in
|
|
454
|
-
// result.content[0]. Fall back to a placeholder if shape is unexpected.
|
|
455
215
|
const content = result.content[0];
|
|
456
216
|
const treeText = content && content.type === "text" ? content.text : "";
|
|
457
217
|
|
|
@@ -471,7 +231,6 @@ export const snapshotTool = defineBrowserTool({
|
|
|
471
231
|
|
|
472
232
|
if (!details.screenshotPath) return treeBlock;
|
|
473
233
|
|
|
474
|
-
// Mirror screenshotTool's width-clamp wrapper for the inline image.
|
|
475
234
|
try {
|
|
476
235
|
const buf = readFileSync(details.screenshotPath);
|
|
477
236
|
const b64 = buf.toString("base64");
|