pi-browser-harness 0.10.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +5 -9
  3. package/package.json +5 -2
  4. package/skills/pi-browser-harness/SKILL.md +1 -1
  5. package/src/cdp/commands.ts +256 -0
  6. package/src/cdp/console-buffer.ts +10 -85
  7. package/src/cdp/discovery.ts +28 -71
  8. package/src/cdp/errors.ts +4 -0
  9. package/src/cdp/events.ts +170 -0
  10. package/src/cdp/network-buffer.ts +42 -109
  11. package/src/cdp/ownership.ts +0 -17
  12. package/src/cdp/record-store.ts +61 -0
  13. package/src/cdp/session.ts +205 -193
  14. package/src/cdp/target-factory.ts +10 -80
  15. package/src/cdp/types.ts +21 -12
  16. package/src/client.ts +39 -95
  17. package/src/daemon/bridge.ts +150 -203
  18. package/src/daemon/index.ts +0 -28
  19. package/src/daemon/protocol.ts +6 -68
  20. package/src/daemon/queue.ts +85 -0
  21. package/src/daemon/server.ts +24 -99
  22. package/src/daemon/spawn.ts +18 -68
  23. package/src/{cdp/daemon-transport.ts → daemon/transport.ts} +18 -51
  24. package/src/domains/ax-tree.ts +179 -0
  25. package/src/domains/box.ts +51 -0
  26. package/src/domains/cdp-call.ts +55 -0
  27. package/src/domains/click.ts +19 -32
  28. package/src/domains/console.ts +48 -2
  29. package/src/domains/dialog.ts +8 -5
  30. package/src/domains/drag.ts +8 -7
  31. package/src/domains/element-call.ts +117 -0
  32. package/src/domains/files.ts +22 -26
  33. package/src/domains/fill-engine.ts +80 -0
  34. package/src/domains/form-batch.ts +65 -0
  35. package/src/domains/form.ts +116 -105
  36. package/src/domains/history.ts +13 -9
  37. package/src/domains/isolated-tab.ts +26 -55
  38. package/src/domains/js.ts +29 -42
  39. package/src/domains/keyboard.ts +36 -27
  40. package/src/domains/navigate.ts +16 -23
  41. package/src/domains/network.ts +60 -10
  42. package/src/domains/page.ts +24 -3
  43. package/src/domains/readpage/read-page.ts +74 -19
  44. package/src/domains/readpage/readability.ts +0 -24
  45. package/src/domains/ref-resolve.ts +24 -58
  46. package/src/domains/render.ts +6 -27
  47. package/src/domains/screenshot-capture.ts +80 -0
  48. package/src/domains/screenshot.ts +9 -86
  49. package/src/domains/scroll.ts +10 -11
  50. package/src/domains/search/google-serp.ts +0 -29
  51. package/src/domains/search/web-search.ts +48 -20
  52. package/src/domains/setup.ts +1 -11
  53. package/src/domains/snapshot.ts +35 -276
  54. package/src/domains/tabs.ts +14 -7
  55. package/src/domains/target.ts +35 -0
  56. package/src/domains/viewport.ts +5 -3
  57. package/src/index.ts +8 -58
  58. package/src/profile/bind.ts +8 -57
  59. package/src/profile/browser-process.ts +75 -126
  60. package/src/profile/command.ts +50 -29
  61. package/src/profile/launch.ts +22 -40
  62. package/src/profile/list.ts +14 -54
  63. package/src/profile/paths.ts +2 -65
  64. package/src/profile/store.ts +35 -56
  65. package/src/registry.ts +30 -33
  66. package/src/schemas/parse.ts +23 -0
  67. package/src/setup.ts +0 -25
  68. package/src/state.ts +16 -16
  69. package/src/util/debug.ts +8 -0
  70. package/src/util/guards.ts +30 -0
  71. package/src/util/js-template.ts +1 -16
  72. package/src/util/mutex.ts +0 -5
  73. package/src/util/result.ts +0 -6
  74. package/src/util/sharp-shim.ts +12 -19
  75. package/src/util/tool.ts +17 -42
  76. package/src/util/truncate.ts +6 -0
  77. package/tsconfig.json +1 -16
  78. package/skills/deep-research/SKILL.md +0 -66
  79. package/src/cdp/event-queue.ts +0 -128
  80. package/src/cdp/transport.ts +0 -156
  81. package/src/cdp/window-binding.ts +0 -15
  82. package/src/deep-research.ts +0 -41
  83. package/src/domains/forms.ts +0 -355
  84. package/src/domains/readpage/capture.ts +0 -46
  85. package/src/domains/search/extract.ts +0 -44
  86. package/src/profile/picker.ts +0 -73
  87. package/src/util/keycodes.ts +0 -10
  88. package/src/util/time.ts +0 -29
@@ -1,27 +1,14 @@
1
- /**
2
- * Pure Google SERP parsing — no CDP, no browser. Everything here is a pure
3
- * function over data captured from the page, so it is unit-testable against
4
- * saved fixtures (see test/deep-research/serp-parser-test.ts).
5
- *
6
- * ADR-0001: results are extracted by structure/semantics (anchors carrying an
7
- * <h3> heading in the main results column), never by brittle Google CSS class
8
- * names, and Google redirect wrappers are unwound to the real target URL.
9
- */
10
-
11
- /** One raw anchor captured in-page: a link with a heading and nearby snippet. */
12
1
  export type SerpAnchor = {
13
2
  readonly href: string;
14
3
  readonly heading: string;
15
4
  readonly snippet: string;
16
5
  };
17
6
 
18
- /** The JSON payload the in-page extraction expression returns. */
19
7
  export type SerpExtraction = {
20
8
  readonly anchors: ReadonlyArray<SerpAnchor>;
21
9
  readonly pageText: string;
22
10
  };
23
11
 
24
- /** A parsed, ranked search result — the tool's public output shape. */
25
12
  export type SearchResult = {
26
13
  readonly title: string;
27
14
  readonly url: string;
@@ -29,7 +16,6 @@ export type SearchResult = {
29
16
  readonly rank: number;
30
17
  };
31
18
 
32
- /** Why a SERP yielded no usable results, or that it did. */
33
19
  export type SerpVerdict = "ok" | "captcha" | "no_results";
34
20
 
35
21
  const CAPTCHA_MARKERS: ReadonlyArray<string> = [
@@ -42,10 +28,6 @@ const CAPTCHA_MARKERS: ReadonlyArray<string> = [
42
28
  "our systems have detected unusual traffic",
43
29
  ];
44
30
 
45
- /**
46
- * Decode a Google redirect wrapper (`/url?q=<real>&sa=...`) to its target.
47
- * Returns the input unchanged when it is not a wrapper.
48
- */
49
31
  const unwrapRedirect = (href: string): string => {
50
32
  const wrapper = href.match(/^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i);
51
33
  if (!wrapper) return href;
@@ -66,7 +48,6 @@ const isExternalResult = (url: string): boolean => {
66
48
  }
67
49
  };
68
50
 
69
- /** Normalized key for dedupe: origin + pathname, trailing slash and hash removed. */
70
51
  const dedupeKey = (url: string): string => {
71
52
  try {
72
53
  const u = new URL(url);
@@ -76,11 +57,6 @@ const dedupeKey = (url: string): string => {
76
57
  }
77
58
  };
78
59
 
79
- /**
80
- * Parse captured anchors into ranked results: unwrap redirects, keep only
81
- * external http(s) links, dedupe by normalized url (first wins), rank by input
82
- * order, cap to `limit`.
83
- */
84
60
  export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: number): ReadonlyArray<SearchResult> => {
85
61
  const seen = new Set<string>();
86
62
  const results: SearchResult[] = [];
@@ -101,11 +77,6 @@ export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: numbe
101
77
  return results;
102
78
  };
103
79
 
104
- /**
105
- * Decide whether the SERP is usable, blocked by a CAPTCHA, or genuinely empty.
106
- * `resultCount` is the number of parsed results; `pageText` is a bounded slice
107
- * of the page's visible text.
108
- */
109
80
  export const classifySerp = (pageText: string, resultCount: number): SerpVerdict => {
110
81
  if (resultCount > 0) return "ok";
111
82
  const lower = pageText.toLowerCase();
@@ -1,16 +1,18 @@
1
1
  import { Type } from "typebox";
2
+ import { Compile } from "typebox/compile";
3
+ import { parseJson } from "../../schemas/parse";
2
4
  import { type Result, err, ok } from "../../util/result";
3
5
  import { applyTruncation } from "../../util/truncate";
4
6
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
5
7
  import {
6
8
  closeIsolatedTab,
7
9
  evalInIsolatedTab,
10
+ isJsonText,
8
11
  navigateIsolatedTab,
9
12
  openIsolatedTab,
10
13
  waitForIsolatedLoad,
11
14
  } from "../isolated-tab";
12
15
  import { renderExpandableText } from "../render";
13
- import { buildSerpExtractionExpr } from "./extract";
14
16
  import { classifySerp, parseGoogleSerp, type SearchResult, type SerpExtraction } from "./google-serp";
15
17
 
16
18
  const LOAD_TIMEOUT_MS = 15_000;
@@ -22,14 +24,46 @@ const WebSearchArgs = Type.Object({
22
24
  ),
23
25
  });
24
26
 
27
+ const PAGE_TEXT_LIMIT = 4000;
28
+
29
+ const buildSerpExtractionExpr = (): string => `
30
+ (() => {
31
+ const main = document.querySelector('#search, #rso, #main') || document.body;
32
+ const headings = Array.from(main.querySelectorAll('a h3'));
33
+ const seen = new Set();
34
+ const anchors = [];
35
+ for (const h3 of headings) {
36
+ const a = h3.closest('a[href]');
37
+ if (!a) continue;
38
+ const href = a.href;
39
+ if (!href || seen.has(href)) continue;
40
+ seen.add(href);
41
+ const container =
42
+ a.closest('div.g') ||
43
+ a.closest('div[data-hveid]') ||
44
+ a.parentElement?.parentElement ||
45
+ a.parentElement;
46
+ const containerText = container ? container.innerText || '' : '';
47
+ const heading = h3.innerText || h3.textContent || '';
48
+ const snippet = containerText.replace(heading, '').replace(/\\s+/g, ' ').trim().slice(0, 500);
49
+ anchors.push({ href, heading, snippet });
50
+ }
51
+ const pageText = (document.body.innerText || '').slice(0, ${PAGE_TEXT_LIMIT});
52
+ return JSON.stringify({ anchors, pageText });
53
+ })()
54
+ `;
55
+
25
56
  const buildSerpUrl = (query: string, limit: number): string =>
26
57
  `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
27
58
 
28
- const isSerpExtraction = (value: unknown): value is SerpExtraction => {
29
- if (typeof value !== "object" || value === null) return false;
30
- const v = value as Record<string, unknown>;
31
- return Array.isArray(v["anchors"]) && typeof v["pageText"] === "string";
32
- };
59
+ const SerpExtractionSchema = Type.Object({
60
+ anchors: Type.Array(
61
+ Type.Object({ href: Type.String(), heading: Type.String(), snippet: Type.String() }),
62
+ ),
63
+ pageText: Type.String(),
64
+ });
65
+
66
+ const serpExtractionValidator = Compile(SerpExtractionSchema);
33
67
 
34
68
  const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
35
69
  const lines = results.map((r) => `${r.rank}. ${r.title}\n ${r.url}\n ${r.snippet}`);
@@ -49,6 +83,7 @@ export const webSearchTool = defineBrowserTool({
49
83
  "Runs in its own isolated tab and never disturbs the user's current tab.",
50
84
  ],
51
85
  parameters: WebSearchArgs,
86
+ concurrency: "serialized",
52
87
  async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
53
88
  const limit = args.limit ?? 10;
54
89
  const serpUrl = buildSerpUrl(args.query, limit);
@@ -64,16 +99,17 @@ export const webSearchTool = defineBrowserTool({
64
99
  const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
65
100
  if (!loaded.success) return err(loaded.error);
66
101
 
67
- const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr());
102
+ const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr(), isJsonText);
68
103
  if (!extracted.success) return err(extracted.error);
69
104
 
70
- const parsedJson = typeof extracted.data === "string" ? safeParse(extracted.data) : extracted.data;
71
- if (!isSerpExtraction(parsedJson)) {
72
- return err({ kind: "internal", message: "SERP extraction returned an unexpected shape" });
105
+ const parsedJson = parseJson(extracted.data, serpExtractionValidator);
106
+ if (!parsedJson.success) {
107
+ return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
73
108
  }
109
+ const extraction: SerpExtraction = parsedJson.data;
74
110
 
75
- const results = parseGoogleSerp(parsedJson.anchors, limit);
76
- const verdict = classifySerp(parsedJson.pageText, results.length);
111
+ const results = parseGoogleSerp(extraction.anchors, limit);
112
+ const verdict = classifySerp(extraction.pageText, results.length);
77
113
  if (verdict !== "ok") {
78
114
  return err({
79
115
  kind: "invalid_state",
@@ -106,11 +142,3 @@ export const webSearchTool = defineBrowserTool({
106
142
  return renderExpandableText("web_search", result, expanded, theme);
107
143
  },
108
144
  });
109
-
110
- const safeParse = (text: string): unknown => {
111
- try {
112
- return JSON.parse(text) as unknown;
113
- } catch {
114
- return undefined;
115
- }
116
- };
@@ -1,11 +1,3 @@
1
- /**
2
- * browser_setup tool — agent-callable browser initialization.
3
- *
4
- * The agent can call this directly when browser tools fail with
5
- * "not initialized" instead of asking the user. Idempotent: safe
6
- * to call even when already connected.
7
- */
8
-
9
1
  import { Type } from "typebox";
10
2
  import { Text } from "@mariozechner/pi-tui";
11
3
  import type { Result } from "../util/result";
@@ -28,12 +20,10 @@ export const setupTool = defineBrowserTool({
28
20
  "After browser_setup succeeds, retry the browser tool that failed.",
29
21
  ],
30
22
  parameters: SetupArgs,
31
- // ensureAlive:false — this tool IS the setup; it must bypass the socket guard
23
+ concurrency: "parallel",
32
24
  ensureAlive: false,
33
25
  renderCall: () => new Text("🔧 Initializing browser...", 0, 0),
34
26
  async handler(_args, { client, extensionCtx }): Promise<Result<ToolOk, ToolErr>> {
35
- // extensionCtx carries the UI, so the agent calling browser_setup gets the
36
- // same first-run profile picker the user gets from /browser-setup.
37
27
  const result = await performSetup(client, extensionCtx);
38
28
  if (result.success) {
39
29
  return ok({ text: result.data });
@@ -2,11 +2,23 @@ import { readFileSync, writeFileSync } from "node:fs";
2
2
  import { Type } from "typebox";
3
3
  import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
4
4
  import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
5
- import type { CdpSession } from "../cdp/session";
5
+ import { boxOf } from "./box";
6
+ import { cdpCall } from "./cdp-call";
7
+ import {
8
+ buildTree,
9
+ collectInteractiveTargets,
10
+ countNodes,
11
+ INTERACTIVE_ROLES,
12
+ liveValue,
13
+ type SlimNode,
14
+ stripInternals,
15
+ VALUE_ROLES,
16
+ } from "./ax-tree";
6
17
  import { type Result, err, ok } from "../util/result";
7
18
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
8
19
  import { applyTruncation } from "../util/truncate";
9
20
  import { screenshotPath } from "../util/paths";
21
+ import { asBoolean, asNumber, asRecord, asString } from "../util/guards";
10
22
 
11
23
  const SnapshotArgs = Type.Object({
12
24
  includeScreenshot: Type.Optional(
@@ -26,168 +38,6 @@ const SnapshotArgs = Type.Object({
26
38
  ),
27
39
  });
28
40
 
29
- // Raw shapes from CDP Accessibility.getFullAXTree. Cast at this boundary only.
30
- type AxValue = { value?: unknown };
31
- export type RawAxNode = {
32
- nodeId: string;
33
- parentId?: string;
34
- childIds?: ReadonlyArray<string>;
35
- ignored?: boolean;
36
- role?: AxValue;
37
- name?: AxValue;
38
- value?: AxValue;
39
- description?: AxValue;
40
- properties?: ReadonlyArray<{ name?: string; value?: AxValue }>;
41
- backendDOMNodeId?: number;
42
- };
43
-
44
- type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number };
45
-
46
- export type SlimNode = {
47
- role: string;
48
- name?: string;
49
- value?: string;
50
- description?: string;
51
- state?: string;
52
- children: SlimNode[];
53
- /** Internal: kept for the post-build box fetch. Stripped before returning to the LLM. */
54
- _backendId?: number;
55
- /** Stable handle (e.g. "e12") for interaction tools. Survives re-renders via backendNodeId. */
56
- ref?: string;
57
- /** Click target — center of the element's bounding box, in viewport CSS pixels. */
58
- box?: Box;
59
- };
60
-
61
- const INTERACTIVE_ROLES = new Set([
62
- "button",
63
- "link",
64
- "textbox",
65
- "checkbox",
66
- "radio",
67
- "combobox",
68
- "menuitem",
69
- "tab",
70
- "switch",
71
- "slider",
72
- "searchbox",
73
- "spinbutton",
74
- ]);
75
-
76
- // Roles whose live DOM value/checked state should override the (often stale)
77
- // AX-tree value. The AX `value` is unreliable for freshly-typed or controlled
78
- // inputs, so for these we read the real DOM property at snapshot time.
79
- const VALUE_ROLES = new Set([
80
- "textbox",
81
- "searchbox",
82
- "spinbutton",
83
- "combobox",
84
- "checkbox",
85
- "radio",
86
- "switch",
87
- "slider",
88
- ]);
89
-
90
- const stringOf = (v: AxValue | undefined): string | undefined => {
91
- const x = v?.value;
92
- if (typeof x === "string" && x !== "") return x;
93
- if (typeof x === "number") return String(x);
94
- return undefined;
95
- };
96
-
97
- const collectState = (props: RawAxNode["properties"]): string | undefined => {
98
- if (!props || props.length === 0) return undefined;
99
- const flags: string[] = [];
100
- for (const p of props) {
101
- const name = p.name;
102
- if (!name) continue;
103
- const raw = p.value?.value;
104
- // Boolean flags worth surfacing in the outline.
105
- if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
106
- flags.push(name);
107
- }
108
- if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
109
- }
110
- return flags.length > 0 ? flags.join(", ") : undefined;
111
- };
112
-
113
- export const buildTree = (
114
- rawNodes: ReadonlyArray<RawAxNode>,
115
- opts: { interestingOnly: boolean; maxNodes: number },
116
- ): SlimNode[] => {
117
- // Index for parent/child lookups.
118
- const byId = new Map<string, RawAxNode>();
119
- for (const n of rawNodes) byId.set(n.nodeId, n);
120
-
121
- // CDP doesn't mark a single root explicitly; the root nodes are the ones
122
- // whose parentId isn't in the index (typically just one — RootWebArea).
123
- const rootIds: string[] = [];
124
- for (const n of rawNodes) {
125
- if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
126
- }
127
-
128
- let budget = opts.maxNodes;
129
-
130
- const slim = (node: RawAxNode): SlimNode | undefined => {
131
- if (budget <= 0) return undefined;
132
- if (opts.interestingOnly && node.ignored) {
133
- // Skip the ignored node but recurse through its children — Chrome often
134
- // ignores wrapper divs but the meaningful descendants are still there.
135
- const out: SlimNode[] = [];
136
- for (const cid of node.childIds ?? []) {
137
- const child = byId.get(cid);
138
- if (!child) continue;
139
- const slimChild = slim(child);
140
- if (slimChild) out.push(slimChild);
141
- }
142
- // Hoist children into the parent. We return undefined; the parent's
143
- // recursion will see them by re-walking — but that double-counts. Instead
144
- // we handle hoisting at the parent level. So here just signal "skip me"
145
- // by returning undefined; the parent will hoist by checking ignored flag.
146
- return out.length > 0 ? { role: "_hoist", children: out } : undefined;
147
- }
148
- budget--;
149
- const role = stringOf(node.role) ?? "unknown";
150
- const name = stringOf(node.name);
151
- const value = stringOf(node.value);
152
- const description = stringOf(node.description);
153
- const state = collectState(node.properties);
154
- const children: SlimNode[] = [];
155
- for (const cid of node.childIds ?? []) {
156
- const child = byId.get(cid);
157
- if (!child) continue;
158
- const slimChild = slim(child);
159
- if (!slimChild) continue;
160
- // Hoist any synthetic _hoist nodes' children into this level.
161
- if (slimChild.role === "_hoist") children.push(...slimChild.children);
162
- else children.push(slimChild);
163
- }
164
- const out: SlimNode = { role, children };
165
- if (name !== undefined) out.name = name;
166
- if (value !== undefined) out.value = value;
167
- if (description !== undefined) out.description = description;
168
- if (state !== undefined) out.state = state;
169
- if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
170
- return out;
171
- };
172
-
173
- const result: SlimNode[] = [];
174
- for (const rid of rootIds) {
175
- const root = byId.get(rid);
176
- if (!root) continue;
177
- const s = slim(root);
178
- if (!s) continue;
179
- if (s.role === "_hoist") result.push(...s.children);
180
- else result.push(s);
181
- }
182
- return result;
183
- };
184
-
185
- const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
186
- let n = 0;
187
- for (const node of nodes) n += 1 + countNodes(node.children);
188
- return n;
189
- };
190
-
191
41
  const summarize = (nodes: ReadonlyArray<SlimNode>): string => {
192
42
  const counts = new Map<string, number>();
193
43
  const walk = (ns: ReadonlyArray<SlimNode>): void => {
@@ -197,7 +47,6 @@ const summarize = (nodes: ReadonlyArray<SlimNode>): string => {
197
47
  }
198
48
  };
199
49
  walk(nodes);
200
- // Group landmarks (roles that are top-level structural).
201
50
  const landmarkRoles = new Set(["banner", "navigation", "main", "complementary", "contentinfo", "search", "form", "region"]);
202
51
  let landmarks = 0;
203
52
  for (const [role, c] of counts) if (landmarkRoles.has(role)) landmarks += c;
@@ -221,8 +70,6 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
221
70
  if (n.name) line += ` "${n.name}"`;
222
71
  if (n.value && n.value !== n.name) line += ` = ${JSON.stringify(n.value)}`;
223
72
  if (n.state) line += ` (${n.state})`;
224
- // Surface the stable ref + click coordinates for interactive nodes. Agents
225
- // pass `ref` to interaction tools (survives re-renders); @(x,y) is a fallback.
226
73
  if (n.ref) line += ` [${n.ref}]`;
227
74
  if (n.box && INTERACTIVE_ROLES.has(n.role)) {
228
75
  line += ` @(${n.box.cx},${n.box.cy})`;
@@ -235,61 +82,6 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
235
82
  return lines.join("\n");
236
83
  };
237
84
 
238
- /** Strip internal _backendId from a slim tree before returning to callers. */
239
- const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
240
- nodes.map((n) => {
241
- const { _backendId: _bid, ...rest } = n;
242
- return { ...rest, children: stripInternals(n.children) };
243
- });
244
-
245
- /**
246
- * Walk all slim nodes and collect (slimNode, backendId) pairs for interactive
247
- * roles. Caller fetches DOM.getBoxModel for each and writes box back into the
248
- * slim node by reference.
249
- */
250
- export const collectInteractiveTargets = (
251
- nodes: ReadonlyArray<SlimNode>,
252
- ): Array<{ node: SlimNode; backendId: number }> => {
253
- const out: Array<{ node: SlimNode; backendId: number }> = [];
254
- const walk = (ns: ReadonlyArray<SlimNode>): void => {
255
- for (const n of ns) {
256
- if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
257
- out.push({ node: n, backendId: n._backendId });
258
- }
259
- walk(n.children);
260
- }
261
- };
262
- walk(nodes);
263
- return out;
264
- };
265
-
266
- /**
267
- * Read the live DOM value for a form control by backend node id. Resolves the
268
- * node to a JS object then reads the property that matters for its type:
269
- * `.value` for inputs/selects/textareas, `.checked` (→ "checked"/"unchecked")
270
- * for checkboxes/radios. Returns undefined on any failure so the caller falls
271
- * back to the AX value. Mirrors the box-fetch error handling: never throws.
272
- */
273
- const liveValue = async (session: CdpSession, backendId: number): Promise<string | undefined> => {
274
- const resolved = await session.call("DOM.resolveNode", { backendNodeId: backendId });
275
- if (!resolved.success) return undefined;
276
- const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId;
277
- if (!objectId) return undefined;
278
- const fn = `function () {
279
- if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
280
- if (typeof this.value === "string") return this.value;
281
- return null;
282
- }`;
283
- const r = await session.call("Runtime.callFunctionOn", {
284
- objectId,
285
- functionDeclaration: fn,
286
- returnByValue: true,
287
- });
288
- if (!r.success) return undefined;
289
- const value = (r.data as { result?: { value?: unknown } }).result?.value;
290
- return typeof value === "string" ? value : undefined;
291
- };
292
-
293
85
  type SnapshotDetails = {
294
86
  nodeCount: number;
295
87
  truncated: boolean;
@@ -314,37 +106,27 @@ export const snapshotTool = defineBrowserTool({
314
106
  "format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.",
315
107
  ],
316
108
  parameters: SnapshotArgs,
109
+ concurrency: "parallel",
317
110
 
318
111
  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
319
112
  const session = client.session();
320
113
 
321
- // 1. AX tree
322
- const axRes = await session.call("Accessibility.getFullAXTree", {});
323
- if (!axRes.success) return err({ kind: "cdp_error", message: axRes.error.message });
324
- const rawNodes = (axRes.data as { nodes: ReadonlyArray<RawAxNode> }).nodes;
114
+ const axRes = await cdpCall(client, "Accessibility.getFullAXTree", {});
115
+ if (!axRes.success) return axRes;
116
+ const rawNodes = axRes.data.nodes;
325
117
 
326
- // 2. Page url/title
327
118
  const piRes = await client.pageInfo();
328
119
  if (!piRes.success) return err({ kind: "cdp_error", message: piRes.error.message });
329
120
  const pageUrl = "dialog" in piRes.data ? "" : piRes.data.url;
330
121
  const pageTitle = "dialog" in piRes.data ? "" : piRes.data.title;
331
122
 
332
- // 3. Slim + format
333
123
  const slim = buildTree(rawNodes, {
334
124
  interestingOnly: args.interestingOnly ?? true,
335
125
  maxNodes: args.maxNodes ?? 1000,
336
126
  });
337
127
 
338
- // 4. Fetch bounding boxes for interactive nodes so the agent can click
339
- // without a screenshot round-trip. Capped budget so a slow page can't
340
- // wedge the call.
341
128
  const targets = collectInteractiveTargets(slim);
342
129
 
343
- // Assign stable refs (e1, e2, …) to every interactive target. Interaction
344
- // tools resolve these refs to backendNodeIds at call time, so they stay
345
- // valid across re-renders. The ref → backendNodeId map (and the per-ref
346
- // signature baseline for the post-mutation diff) is published below, after
347
- // the box/live-value fetch so the signature reflects the live DOM value.
348
130
  const refMap = new Map<string, number>();
349
131
  targets.forEach(({ node, backendId }, i) => {
350
132
  const ref = `e${i + 1}`;
@@ -358,42 +140,18 @@ export const snapshotTool = defineBrowserTool({
358
140
  await Promise.allSettled(
359
141
  targets.map(async ({ node, backendId }) => {
360
142
  if (ac.signal.aborted) return;
361
- const r = await session.call("DOM.getBoxModel", { backendNodeId: backendId });
362
- if (!r.success) return;
363
- // CDP boundary cast: DOM.getBoxModel
364
- const data = r.data as { model?: { content?: ReadonlyArray<number>; width?: number; height?: number } };
365
- const content = data.model?.content;
366
- if (!content || content.length < 8) return;
367
- // content is [x1,y1, x2,y1, x2,y2, x1,y2] in viewport CSS pixels.
368
- const x = content[0]!;
369
- const y = content[1]!;
370
- const width = data.model!.width ?? 0;
371
- const height = data.model!.height ?? 0;
372
- // Skip zero-size and off-viewport boxes — they can't be clicked.
373
- if (width <= 0 || height <= 0 || x < 0 || y < 0) return;
374
- node.box = {
375
- x: Math.round(x),
376
- y: Math.round(y),
377
- width: Math.round(width),
378
- height: Math.round(height),
379
- cx: Math.round(x + width / 2),
380
- cy: Math.round(y + height / 2),
381
- };
382
- // Override the AX value with the live DOM property for form controls.
383
- // The AX `value` is stale for freshly-typed / controlled inputs, so the
384
- // agent can't trust it as a read-back signal. Reading .value/.checked
385
- // from the actual element fixes that.
143
+ const box = await boxOf(client, backendId);
144
+ if (!box.success) return;
145
+ if (box.data.x < 0 || box.data.y < 0) return;
146
+ node.box = box.data;
386
147
  if (ac.signal.aborted || !VALUE_ROLES.has(node.role)) return;
387
- const live = await liveValue(session, backendId);
148
+ const live = await liveValue(client, backendId);
388
149
  if (live !== undefined) node.value = live;
389
150
  }),
390
151
  );
391
152
  clearTimeout(timer);
392
153
  }
393
154
 
394
- // Publish refs to the active tab. The signature (role|name|value|state) is
395
- // the baseline the post-mutation auto-diff compares against, so it must use
396
- // the live values resolved above.
397
155
  const refSig = new Map<string, string>();
398
156
  for (const { node } of targets) {
399
157
  if (node.ref === undefined) continue;
@@ -405,14 +163,12 @@ export const snapshotTool = defineBrowserTool({
405
163
  const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2);
406
164
  const trunc = await applyTruncation(text, "snapshot");
407
165
 
408
- // 4. Optional screenshot — JPEG q=80 to match screenshotTool's default.
409
166
  let shotPath: string | undefined;
410
167
  if (args.includeScreenshot) {
411
- const shot = await session.call("Page.captureScreenshot", { format: "jpeg", quality: 80 });
168
+ const shot = await cdpCall(client, "Page.captureScreenshot", { format: "jpeg", quality: 80 });
412
169
  if (shot.success) {
413
- const data = (shot.data as { data: string }).data;
414
170
  shotPath = screenshotPath(client.namespace, "jpeg");
415
- writeFileSync(shotPath, Buffer.from(data, "base64"));
171
+ writeFileSync(shotPath, Buffer.from(shot.data.data, "base64"));
416
172
  }
417
173
  }
418
174
 
@@ -429,10 +185,17 @@ export const snapshotTool = defineBrowserTool({
429
185
  },
430
186
 
431
187
  renderResult(result, expanded, theme) {
432
- const details = result.details as
433
- | (SnapshotDetails & { summary?: string; ok?: boolean })
434
- | undefined;
435
- if (!details) return new Text(theme.fg("error", "snapshot: no details"), 0, 0);
188
+ const raw = asRecord(result.details);
189
+ if (raw === undefined) return new Text(theme.fg("error", "snapshot: no details"), 0, 0);
190
+ const details = {
191
+ nodeCount: asNumber(raw["nodeCount"]),
192
+ truncated: asBoolean(raw["truncated"]),
193
+ fullOutputPath: asString(raw["fullOutputPath"]),
194
+ screenshotPath: asString(raw["screenshotPath"]),
195
+ url: asString(raw["url"]),
196
+ title: asString(raw["title"]),
197
+ summary: asString(raw["summary"]),
198
+ };
436
199
 
437
200
  const titleLine = details.title ? ` "${details.title}"` : "";
438
201
  const summary = details.summary ?? "";
@@ -449,9 +212,6 @@ export const snapshotTool = defineBrowserTool({
449
212
  return new Markdown(md, 0, 0, getMarkdownTheme());
450
213
  }
451
214
 
452
- // Expanded view: read truncated text from result.content (the spilled file
453
- // is too large to inline) — the same `text` we returned from handler is in
454
- // result.content[0]. Fall back to a placeholder if shape is unexpected.
455
215
  const content = result.content[0];
456
216
  const treeText = content && content.type === "text" ? content.text : "";
457
217
 
@@ -471,7 +231,6 @@ export const snapshotTool = defineBrowserTool({
471
231
 
472
232
  if (!details.screenshotPath) return treeBlock;
473
233
 
474
- // Mirror screenshotTool's width-clamp wrapper for the inline image.
475
234
  try {
476
235
  const buf = readFileSync(details.screenshotPath);
477
236
  const b64 = buf.toString("base64");