pi-browser-harness 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/README.md +86 -0
  3. package/package.json +12 -6
  4. package/skills/pi-browser-harness/SKILL.md +57 -2
  5. package/src/cdp/commands.ts +200 -14
  6. package/src/cdp/console-buffer.ts +37 -23
  7. package/src/cdp/discovery.ts +57 -25
  8. package/src/cdp/events.ts +8 -3
  9. package/src/cdp/network-buffer.ts +38 -32
  10. package/src/cdp/ownership.ts +22 -4
  11. package/src/cdp/record-store.ts +2 -2
  12. package/src/cdp/session.ts +320 -217
  13. package/src/cdp/target-factory.ts +34 -15
  14. package/src/cdp/types.ts +17 -1
  15. package/src/client.ts +76 -43
  16. package/src/daemon/bridge.ts +110 -349
  17. package/src/daemon/connection.ts +323 -0
  18. package/src/daemon/index.ts +77 -48
  19. package/src/daemon/lines.ts +40 -0
  20. package/src/daemon/main.ts +12 -0
  21. package/src/daemon/protocol.ts +55 -25
  22. package/src/daemon/server.ts +100 -63
  23. package/src/daemon/socket.ts +10 -0
  24. package/src/daemon/spawn.ts +21 -22
  25. package/src/daemon/transport.ts +80 -49
  26. package/src/domains/ax-tree.ts +8 -8
  27. package/src/domains/cdp-call.ts +14 -7
  28. package/src/domains/click.ts +2 -2
  29. package/src/domains/concurrency.ts +50 -0
  30. package/src/domains/console.ts +20 -64
  31. package/src/domains/drag.ts +26 -28
  32. package/src/domains/element-call.ts +102 -68
  33. package/src/domains/files.ts +13 -9
  34. package/src/domains/form-batch.ts +13 -10
  35. package/src/domains/form.ts +78 -107
  36. package/src/domains/headers.ts +2 -2
  37. package/src/domains/history.ts +4 -4
  38. package/src/domains/io.ts +35 -0
  39. package/src/domains/isolated-tab.ts +6 -12
  40. package/src/domains/js.ts +65 -44
  41. package/src/domains/keyboard.ts +14 -11
  42. package/src/domains/navigate.ts +102 -66
  43. package/src/domains/network.ts +55 -95
  44. package/src/domains/page/fill.page.ts +139 -0
  45. package/src/domains/page/form.page.ts +61 -0
  46. package/src/domains/page/inject.ts +37 -0
  47. package/src/domains/page/keyboard.page.ts +16 -0
  48. package/src/domains/page/live-value.page.ts +10 -0
  49. package/src/domains/page/read-page.page.ts +51 -0
  50. package/src/domains/page/serp.page.ts +39 -0
  51. package/src/domains/page.ts +18 -51
  52. package/src/domains/poll.ts +42 -0
  53. package/src/domains/readpage/read-page.ts +48 -53
  54. package/src/domains/ref-resolve.ts +43 -29
  55. package/src/domains/ref-signature.ts +11 -0
  56. package/src/domains/render.ts +12 -23
  57. package/src/domains/screenshot-capture.ts +4 -2
  58. package/src/domains/screenshot.ts +13 -10
  59. package/src/domains/scroll.ts +4 -4
  60. package/src/domains/search/google-serp.ts +1 -1
  61. package/src/domains/search/web-search.ts +38 -33
  62. package/src/domains/snapshot.ts +71 -73
  63. package/src/domains/tab-resolve.ts +36 -0
  64. package/src/domains/tabs.ts +18 -69
  65. package/src/domains/target.ts +1 -1
  66. package/src/index.ts +90 -33
  67. package/src/profile/bind.ts +45 -14
  68. package/src/profile/browser-process.ts +50 -12
  69. package/src/profile/command.ts +20 -12
  70. package/src/profile/launch.ts +13 -9
  71. package/src/profile/list.ts +9 -13
  72. package/src/profile/store.ts +2 -1
  73. package/src/prompt.ts +32 -7
  74. package/src/registry.ts +65 -4
  75. package/src/schemas/parse.ts +2 -2
  76. package/src/setup.ts +28 -19
  77. package/src/state.ts +2 -1
  78. package/src/util/debug.ts +7 -1
  79. package/src/util/guards.ts +3 -0
  80. package/src/util/mutex.ts +1 -1
  81. package/src/util/paths.ts +3 -2
  82. package/src/util/result.ts +8 -0
  83. package/src/util/sharp-shim.ts +18 -11
  84. package/src/util/sleep.ts +18 -0
  85. package/src/util/tool.ts +163 -114
  86. package/src/util/truncate.ts +1 -1
  87. package/tsconfig.json +2 -1
  88. package/src/domains/fill-engine.ts +0 -80
@@ -1,25 +1,9 @@
1
1
  import { Type } from "typebox";
2
2
  import { type Result, err, ok } from "../util/result";
3
3
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
4
- import { evalJs } from "./cdp-call";
4
+ import { fromClient } from "./cdp-call";
5
5
  import { safeJs } from "../util/js-template";
6
-
7
- const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>
8
- new Promise((resolve, reject) => {
9
- if (signal?.aborted) {
10
- reject(signal.reason ?? new Error("aborted"));
11
- return;
12
- }
13
- const onAbort = (): void => {
14
- clearTimeout(timer);
15
- reject(signal?.reason ?? new Error("aborted"));
16
- };
17
- const timer = setTimeout(() => {
18
- signal?.removeEventListener("abort", onAbort);
19
- resolve();
20
- }, ms);
21
- signal?.addEventListener("abort", onAbort, { once: true });
22
- });
6
+ import { pollUntil, READY_STATE_COMPLETE, sleep } from "./poll";
23
7
 
24
8
  export const pageInfoTool = defineBrowserTool({
25
9
  name: "browser_page_info",
@@ -33,9 +17,9 @@ export const pageInfoTool = defineBrowserTool({
33
17
  parameters: Type.Object({}),
34
18
  concurrency: "parallel",
35
19
  async handler(_args, { client }): Promise<Result<ToolOk, ToolErr>> {
36
- const r = await client.pageInfo();
37
- if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
38
- if ("dialog" in r.data) {
20
+ const r = fromClient(await client.pageInfo(), "pageInfo");
21
+ if (!r.success) return r;
22
+ if (r.data.kind === "dialog") {
39
23
  const d = r.data.dialog;
40
24
  return ok({
41
25
  text: `⚠️ DIALOG OPEN: ${d.type}\nMessage: ${d.message}${d.defaultPrompt ? `\nDefault: ${d.defaultPrompt}` : ""}\n\nUse browser_handle_dialog to accept or dismiss.`,
@@ -63,12 +47,9 @@ export const waitTool = defineBrowserTool({
63
47
  parameters: WaitArgs,
64
48
  concurrency: "parallel",
65
49
  async handler(args, { signal }): Promise<Result<ToolOk, ToolErr>> {
66
- try {
67
- await sleep(Math.round(args.seconds * 1000), signal);
68
- return ok({ text: `Waited ${args.seconds}s` });
69
- } catch (e) {
70
- return err({ kind: "internal", message: e instanceof Error ? e.message : String(e) });
71
- }
50
+ await sleep(Math.round(args.seconds * 1000), signal);
51
+ if (signal?.aborted) return err({ kind: "timeout", message: `Aborted before ${args.seconds}s elapsed` });
52
+ return ok({ text: `Waited ${args.seconds}s` });
72
53
  },
73
54
  });
74
55
 
@@ -89,17 +70,10 @@ export const waitForLoadTool = defineBrowserTool({
89
70
  concurrency: "serialized",
90
71
  async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
91
72
  const timeoutMs = (args.timeout ?? 15) * 1000;
92
- const start = Date.now();
93
- while (Date.now() - start < timeoutMs) {
94
- if (signal?.aborted) return err({ kind: "internal", message: "aborted" });
95
- const r = await evalJs(client, safeJs`document.readyState`);
96
- if (r.success && r.data === "complete") {
97
- const ms = Date.now() - start;
98
- return ok({ text: `Page loaded in ${Math.round(ms / 100) / 10}s`, details: { ms } });
99
- }
100
- await new Promise((res) => setTimeout(res, 50));
101
- }
102
- return err({ kind: "timeout", message: `Page did not finish loading in ${args.timeout ?? 15}s` });
73
+ const r = await pollUntil(client, READY_STATE_COMPLETE, { timeoutMs, signal, what: "page load" });
74
+ if (!r.success) return r;
75
+ const ms = r.data.elapsedMs;
76
+ return ok({ text: `Page loaded in ${Math.round(ms / 100) / 10}s`, details: { ms } });
103
77
  },
104
78
  });
105
79
 
@@ -123,7 +97,7 @@ export const waitForTool = defineBrowserTool({
123
97
  concurrency: "parallel",
124
98
  async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
125
99
  if (args.selector === undefined && args.text === undefined) {
126
- return err({ kind: "invalid_state", message: "Provide a selector or text to wait for." });
100
+ return err({ kind: "invalid_args", message: "Provide a selector or text to wait for." });
127
101
  }
128
102
  const gone = args.gone ?? false;
129
103
  const timeoutMs = (args.timeout ?? 5) * 1000;
@@ -131,18 +105,11 @@ export const waitForTool = defineBrowserTool({
131
105
  args.selector !== undefined
132
106
  ? safeJs`(() => { const found = !!document.querySelector(${args.selector}); return ${gone} ? !found : found; })()`
133
107
  : safeJs`(() => { const t = document.body ? document.body.innerText : ""; return t.includes(${args.text}); })()`;
134
- const start = Date.now();
135
- while (Date.now() - start < timeoutMs) {
136
- if (signal?.aborted) return err({ kind: "internal", message: "aborted" });
137
- const r = await evalJs(client, expr);
138
- if (r.success && r.data === true) {
139
- const ms = Date.now() - start;
140
- const what = args.selector !== undefined ? `${args.selector}${gone ? " gone" : ""}` : `text "${args.text}"`;
141
- return ok({ text: `Condition met (${what}) in ${Math.round(ms / 100) / 10}s`, details: { ms } });
142
- }
143
- await sleep(50, signal);
144
- }
145
108
  const what = args.selector !== undefined ? `${args.selector}${gone ? " to disappear" : ""}` : `text "${args.text}"`;
146
- return err({ kind: "timeout", message: `Timed out after ${args.timeout ?? 5}s waiting for ${what}` });
109
+ const r = await pollUntil(client, expr, { timeoutMs, signal, what });
110
+ if (!r.success) return r;
111
+ const ms = r.data.elapsedMs;
112
+ const met = args.selector !== undefined ? `${args.selector}${gone ? " gone" : ""}` : `text "${args.text}"`;
113
+ return ok({ text: `Condition met (${met}) in ${Math.round(ms / 100) / 10}s`, details: { ms } });
147
114
  },
148
115
  });
@@ -0,0 +1,42 @@
1
+ import { sleep } from "../util/sleep";
2
+ import type { BrowserClient } from "../client";
3
+ import { type Result, err, ok } from "../util/result";
4
+ import type { ToolErr } from "../util/tool";
5
+ import { evalJs } from "./cdp-call";
6
+
7
+ export type PollOptions = {
8
+ readonly timeoutMs: number;
9
+ readonly signal?: AbortSignal | undefined;
10
+ readonly sessionId?: string | undefined;
11
+ readonly intervalMs?: number | undefined;
12
+ // Names the condition in the timeout message, e.g. "page load".
13
+ readonly what?: string | undefined;
14
+ };
15
+
16
+ const DEFAULT_INTERVAL_MS = 50;
17
+
18
+ export { sleep } from "../util/sleep";
19
+
20
+ // Evaluates `expression` in the page until it yields `true`. A deadline or an abort both surface as kind "timeout";
21
+ // evaluation errors (e.g. mid-navigation) are retried until the deadline.
22
+ export const pollUntil = async (
23
+ client: BrowserClient,
24
+ expression: string,
25
+ opts: PollOptions,
26
+ ): Promise<Result<{ readonly elapsedMs: number }, ToolErr>> => {
27
+ const start = Date.now();
28
+ const interval = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
29
+ const what = opts.what ?? "condition";
30
+ const seconds = Math.round(opts.timeoutMs / 100) / 10;
31
+ while (Date.now() - start < opts.timeoutMs) {
32
+ if (opts.signal?.aborted) {
33
+ return err({ kind: "timeout", message: `Aborted while waiting for ${what}` });
34
+ }
35
+ const r = await evalJs(client, expression, opts.sessionId);
36
+ if (r.success && r.data === true) return ok({ elapsedMs: Date.now() - start });
37
+ await sleep(interval, opts.signal);
38
+ }
39
+ return err({ kind: "timeout", message: `Timed out after ${seconds}s waiting for ${what}` });
40
+ };
41
+
42
+ export const READY_STATE_COMPLETE = `document.readyState === "complete"`;
@@ -1,10 +1,15 @@
1
+ import { borrowSession } from "../../cdp/target-factory";
1
2
  import { Type } from "typebox";
2
3
  import { Compile } from "typebox/compile";
3
4
  import { parseJson } from "../../schemas/parse";
4
5
  import { type Result, err, ok } from "../../util/result";
5
6
  import { applyTruncation } from "../../util/truncate";
6
- import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
7
- import { cdpCallBrowser, evalJs } from "../cdp-call";
7
+ import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
8
+ import type { BrowserClient } from "../../client";
9
+ import { evalJs, fromClient } from "../cdp-call";
10
+ import { pageCallExpression } from "../page/inject";
11
+ import { capturePage } from "../page/read-page.page";
12
+ import type * as captured from "../page/read-page.page";
8
13
  import {
9
14
  closeIsolatedTab,
10
15
  evalInIsolatedTab,
@@ -14,7 +19,7 @@ import {
14
19
  openIsolatedTab,
15
20
  waitForIsolatedLoad,
16
21
  } from "../isolated-tab";
17
- import { renderExpandableText } from "../render";
22
+ import { ExpandableTextSchema, renderExpandableText } from "../render";
18
23
  import { extractReadable, type PageCapture, type ReadablePage } from "./readability";
19
24
 
20
25
  const LOAD_TIMEOUT_MS = 15_000;
@@ -48,42 +53,28 @@ const PageCaptureSchema = Type.Object({
48
53
 
49
54
  const pageCaptureValidator = Compile(PageCaptureSchema);
50
55
 
56
+ const ReadPageDetailsSchema = Type.Object({
57
+ title: Type.String(),
58
+ url: Type.String(),
59
+ wordCount: Type.Number(),
60
+ render: ExpandableTextSchema,
61
+ fullOutputPath: Type.Optional(Type.String()),
62
+ });
63
+
64
+ export type ReadPageDetails = DetailsOf<typeof ReadPageDetailsSchema>;
65
+
51
66
  const BODY_TEXT_LIMIT = 20_000;
52
67
 
53
- const buildPageCaptureExpr = (): string => `
54
- (() => {
55
- const BOILERPLATE = new Set(['NAV', 'HEADER', 'FOOTER', 'ASIDE']);
56
- const inBoilerplate = (el) => {
57
- for (let n = el; n; n = n.parentElement) {
58
- if (BOILERPLATE.has(n.tagName)) return true;
59
- const role = n.getAttribute && n.getAttribute('role');
60
- if (role === 'navigation' || role === 'banner' || role === 'contentinfo') return true;
61
- }
62
- return false;
63
- };
64
- const kindOf = (tag) => {
65
- if (/^H[1-6]$/.test(tag)) return 'heading';
66
- if (tag === 'LI') return 'listitem';
67
- if (tag === 'BLOCKQUOTE') return 'blockquote';
68
- if (tag === 'P') return 'paragraph';
69
- return 'other';
70
- };
71
- const selector = 'p, h1, h2, h3, h4, h5, h6, li, blockquote';
72
- const root = document.querySelector('article, main, [role=main]') || document.body;
73
- const blocks = [];
74
- for (const el of root.querySelectorAll(selector)) {
75
- const text = (el.innerText || '').replace(/\\s+/g, ' ').trim();
76
- if (!text) continue;
77
- let linkTextLength = 0;
78
- for (const a of el.querySelectorAll('a')) linkTextLength += (a.innerText || '').length;
79
- blocks.push({ kind: kindOf(el.tagName), text, linkTextLength, inBoilerplate: inBoilerplate(el) });
80
- }
81
- const bodyText = (document.body.innerText || '').replace(/\\n{3,}/g, '\\n\\n').slice(0, ${BODY_TEXT_LIMIT});
82
- return JSON.stringify({ url: location.href, title: document.title, blocks, bodyText });
83
- })()
84
- `;
68
+ // The page function declares the capture shape it produces; this pins it to the shape the readability pass consumes,
69
+ // so the two cannot drift apart silently (the runtime validator above still guards the actual wire value).
70
+ type Pinned<T extends PageCapture> = T;
71
+ export type CaptureFromPage = Pinned<captured.PageCapture>;
85
72
 
86
- const toPageCapture = (raw: unknown): PageCapture | undefined => {
73
+ // Built once: stringifying the page function per call was the only cost of the thunk.
74
+ const PAGE_CAPTURE_EXPR = pageCallExpression(capturePage, BODY_TEXT_LIMIT);
75
+
76
+ // Accepts the capture either as JSON text (isolated tab) or as the already-decoded value (owned tab).
77
+ export const toPageCapture = (raw: unknown): PageCapture | undefined => {
87
78
  if (typeof raw === "string") {
88
79
  const parsed = parseJson(raw, pageCaptureValidator);
89
80
  return parsed.success ? parsed.data : undefined;
@@ -91,7 +82,7 @@ const toPageCapture = (raw: unknown): PageCapture | undefined => {
91
82
  return pageCaptureValidator.Check(raw) ? raw : undefined;
92
83
  };
93
84
 
94
- const captureToResult = async (raw: unknown): Promise<Result<ToolOk, ToolErr>> => {
85
+ const captureToResult = async (raw: unknown): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> => {
95
86
  const parsed = toPageCapture(raw);
96
87
  if (parsed === undefined) return err({ kind: "internal", message: "page capture returned an unexpected shape" });
97
88
  const page: ReadablePage = extractReadable(parsed);
@@ -115,10 +106,10 @@ const captureToResult = async (raw: unknown): Promise<Result<ToolOk, ToolErr>> =
115
106
  };
116
107
 
117
108
  const readOpenedUrl = async (
118
- client: Parameters<typeof openIsolatedTab>[0],
109
+ client: BrowserClient,
119
110
  url: string,
120
111
  signal: AbortSignal | undefined,
121
- ): Promise<Result<ToolOk, ToolErr>> => {
112
+ ): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> => {
122
113
  const opened = await openIsolatedTab(client);
123
114
  if (!opened.success) return err(opened.error);
124
115
  const tab: IsolatedTab = opened.data;
@@ -127,27 +118,30 @@ const readOpenedUrl = async (
127
118
  if (!navigated.success) return err(navigated.error);
128
119
  const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
129
120
  if (!loaded.success) return err(loaded.error);
130
- const captured = await evalInIsolatedTab(client, tab, buildPageCaptureExpr(), isJsonText);
121
+ const captured = await evalInIsolatedTab(client, tab, PAGE_CAPTURE_EXPR, isJsonText);
131
122
  if (!captured.success) return err(captured.error);
132
- return captureToResult(captured.data);
123
+ return await captureToResult(captured.data);
133
124
  } finally {
134
125
  await closeIsolatedTab(client, tab);
135
126
  }
136
127
  };
137
128
 
138
129
  const readOwnedTab = async (
139
- client: Parameters<typeof openIsolatedTab>[0],
130
+ client: BrowserClient,
140
131
  targetId: string,
141
- ): Promise<Result<ToolOk, ToolErr>> => {
132
+ ): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> => {
142
133
  if (!client.owns(targetId)) {
143
134
  return err({ kind: "invalid_state", message: `Tab ${targetId} is not owned by this session.` });
144
135
  }
145
- const attached = await cdpCallBrowser(client, "Target.attachToTarget", { targetId, flatten: true });
146
- if (!attached.success) return attached;
147
- const sessionId = attached.data.sessionId;
148
- const captured = await evalJs(client, buildPageCaptureExpr(), sessionId);
149
- if (!captured.success) return captured;
150
- return captureToResult(captured.data);
136
+ const borrowed = await borrowSession(client, targetId);
137
+ if (!borrowed.success) return fromClient(borrowed, "Target.attachToTarget");
138
+ try {
139
+ const captured = await evalJs(client, PAGE_CAPTURE_EXPR, borrowed.data.sessionId);
140
+ if (!captured.success) return captured;
141
+ return await captureToResult(captured.data);
142
+ } finally {
143
+ await borrowed.data.release();
144
+ }
151
145
  };
152
146
 
153
147
  export const readPageTool = defineBrowserTool({
@@ -161,13 +155,14 @@ export const readPageTool = defineBrowserTool({
161
155
  "Boilerplate-heavy or structure-less pages fall back to bounded body text rather than erroring.",
162
156
  ],
163
157
  parameters: ReadPageArgs,
158
+ details: ReadPageDetailsSchema,
164
159
  concurrency: "parallel",
165
- async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
160
+ async handler(args, { client, signal }): Promise<Result<ToolOk<ReadPageDetails>, ToolErr>> {
166
161
  if (args.url !== undefined && args.url.length > 0) return readOpenedUrl(client, args.url, signal);
167
162
  if (args.targetId !== undefined && args.targetId.length > 0) return readOwnedTab(client, args.targetId);
168
- return err({ kind: "invalid_state", message: "provide either url or targetId" });
163
+ return err({ kind: "invalid_args", message: "provide either url or targetId" });
169
164
  },
170
- renderResult(result, expanded, theme) {
171
- return renderExpandableText("read_page", result, expanded, theme);
165
+ renderResult(result, expanded) {
166
+ return renderExpandableText(result.details.render, expanded);
172
167
  },
173
168
  });
@@ -3,7 +3,8 @@ import { type Result, err, ok } from "../util/result";
3
3
  import type { ToolErr } from "../util/tool";
4
4
  import { type Box, boxOf } from "./box";
5
5
  import { cdpCall } from "./cdp-call";
6
- import { buildTree, collectInteractiveTargets, type SlimNode } from "./ax-tree";
6
+ import { buildTree, collectInteractiveTargets } from "./ax-tree";
7
+ import { type RefSignature, signatureOf } from "./ref-signature";
7
8
 
8
9
  const staleErr = (ref: string): ToolErr => ({
9
10
  kind: "invalid_state",
@@ -43,17 +44,20 @@ export const resolveRefToBox = async (
43
44
  return ok(box.data);
44
45
  };
45
46
 
46
- const sigOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}|${n.value ?? ""}|${n.state ?? ""}`;
47
- const keyOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}`;
47
+ export type RefRefresh = {
48
+ readonly prev: ReadonlyMap<string, RefSignature>;
49
+ readonly cur: ReadonlyMap<string, RefSignature>;
50
+ };
48
51
 
49
- export const interactiveDiff = async (client: BrowserClient): Promise<string> => {
52
+ // Re-reads the AX tree and republishes the session's ref map: an element seen before keeps its ref, a new one gets
53
+ // the next unused index. Returns the signatures before and after so the caller can diff them.
54
+ export const refreshRefs = async (client: BrowserClient): Promise<Result<RefRefresh, ToolErr>> => {
50
55
  const session = client.session();
51
56
  const prev = session.refSignatures();
52
57
 
53
58
  const axRes = await cdpCall(client, "Accessibility.getFullAXTree", {});
54
- if (!axRes.success) return "";
55
- const rawNodes = axRes.data.nodes;
56
- const slim = buildTree(rawNodes, { interestingOnly: true, maxNodes: 1000 });
59
+ if (!axRes.success) return axRes;
60
+ const slim = buildTree(axRes.data.nodes, { interestingOnly: true, maxNodes: 1000 });
57
61
  const targets = collectInteractiveTargets(slim);
58
62
 
59
63
  const priorRefs = session.refMappings();
@@ -65,7 +69,7 @@ export const interactiveDiff = async (client: BrowserClient): Promise<string> =>
65
69
  if (digits !== undefined) nextIndex = Math.max(nextIndex, Number(digits) + 1);
66
70
  }
67
71
  const refMap = new Map<string, number>();
68
- const refSig = new Map<string, string>();
72
+ const cur = new Map<string, RefSignature>();
69
73
  const taken = new Set<string>();
70
74
  for (const { node, backendId } of targets) {
71
75
  const prior = refByBackendId.get(backendId);
@@ -73,38 +77,40 @@ export const interactiveDiff = async (client: BrowserClient): Promise<string> =>
73
77
  taken.add(ref);
74
78
  node.ref = ref;
75
79
  refMap.set(ref, backendId);
76
- refSig.set(ref, sigOf(node));
80
+ cur.set(ref, signatureOf(node));
77
81
  }
78
- session.setRefMap(refMap, refSig);
82
+ session.setRefMap(refMap, cur);
83
+ return ok({ prev, cur });
84
+ };
79
85
 
80
- const prevByKey = new Map<string, string>();
81
- for (const sig of prev.values()) {
82
- const parts = sig.split("|");
83
- prevByKey.set(`${parts[0]}|${parts[1]}`, sig);
84
- }
85
- const curByKey = new Map<string, { ref: string; sig: string }>();
86
- for (const { node } of targets) {
87
- if (node.ref) curByKey.set(keyOf(node), { ref: node.ref, sig: sigOf(node) });
88
- }
86
+ const identityKey = (s: RefSignature): string => JSON.stringify([s.role, s.name]);
87
+ const describe = (s: RefSignature): string => `${s.role} "${s.name}"`;
88
+
89
+ // Pure: renders what changed between two signature sets, keyed by role+name. Empty string when nothing changed.
90
+ export const diffSignatures = (
91
+ prev: ReadonlyMap<string, RefSignature>,
92
+ cur: ReadonlyMap<string, RefSignature>,
93
+ ): string => {
94
+ const prevByKey = new Map<string, RefSignature>();
95
+ for (const sig of prev.values()) prevByKey.set(identityKey(sig), sig);
96
+ const curByKey = new Map<string, { readonly ref: string; readonly sig: RefSignature }>();
97
+ for (const [ref, sig] of cur) curByKey.set(identityKey(sig), { ref, sig });
89
98
 
90
99
  const appeared: string[] = [];
91
100
  const changed: string[] = [];
92
101
  for (const [key, { ref, sig }] of curByKey) {
93
102
  const before = prevByKey.get(key);
94
103
  if (before === undefined) {
95
- appeared.push(` *[${ref}] ${key.replace("|", ' "')}"`);
96
- } else if (before !== sig) {
97
- const va = before.split("|")[2] ?? "";
98
- const vb = sig.split("|")[2] ?? "";
99
- const sa = before.split("|")[3] ?? "";
100
- const sb = sig.split("|")[3] ?? "";
101
- const detail = va !== vb ? `value ${JSON.stringify(va)} → ${JSON.stringify(vb)}` : `state ${JSON.stringify(sa)} → ${JSON.stringify(sb)}`;
102
- changed.push(` [${ref}] ${key.replace("|", ' "')}": ${detail}`);
104
+ appeared.push(` *[${ref}] ${describe(sig)}`);
105
+ } else if (before.value !== sig.value) {
106
+ changed.push(` [${ref}] ${describe(sig)}: value ${JSON.stringify(before.value)} → ${JSON.stringify(sig.value)}`);
107
+ } else if (before.state !== sig.state) {
108
+ changed.push(` [${ref}] ${describe(sig)}: state ${JSON.stringify(before.state)} → ${JSON.stringify(sig.state)}`);
103
109
  }
104
110
  }
105
111
  const removed: string[] = [];
106
- for (const key of prevByKey.keys()) {
107
- if (!curByKey.has(key)) removed.push(` ${key.replace("|", ' "')}"`);
112
+ for (const [key, sig] of prevByKey) {
113
+ if (!curByKey.has(key)) removed.push(` ${describe(sig)}`);
108
114
  }
109
115
 
110
116
  if (appeared.length === 0 && changed.length === 0 && removed.length === 0) return "";
@@ -116,3 +122,11 @@ export const interactiveDiff = async (client: BrowserClient): Promise<string> =>
116
122
  if (overflow > 0) lines.push(` …and ${overflow} more (browser_snapshot for full state)`);
117
123
  return lines.join("\n");
118
124
  };
125
+
126
+ // Convenience for action tools: refresh refs and append a diff to the action's text. A failed refresh is reported
127
+ // inline rather than failing the action that already happened.
128
+ export const interactiveDiff = async (client: BrowserClient): Promise<string> => {
129
+ const refreshed = await refreshRefs(client);
130
+ if (!refreshed.success) return `\n(page diff unavailable: ${refreshed.error.message})`;
131
+ return diffSignatures(refreshed.data.prev, refreshed.data.cur);
132
+ };
@@ -0,0 +1,11 @@
1
+ import type { RefSignature } from "../cdp/types";
2
+ import type { SlimNode } from "./ax-tree";
3
+
4
+ export type { RefSignature };
5
+
6
+ export const signatureOf = (n: SlimNode): RefSignature => ({
7
+ role: n.role,
8
+ name: n.name ?? "",
9
+ value: n.value ?? "",
10
+ state: n.state ?? "",
11
+ });
@@ -1,32 +1,21 @@
1
- import { Markdown, Text } from "@mariozechner/pi-tui";
1
+ import { Type } from "typebox";
2
+ import { Markdown } from "@mariozechner/pi-tui";
2
3
  import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
3
4
  import type { Component } from "@mariozechner/pi-tui";
4
- import type { Theme } from "@mariozechner/pi-coding-agent";
5
- import { isRecord } from "../util/guards";
5
+ import type { DetailsOf } from "../util/tool";
6
6
 
7
7
  const COMPACT_PREVIEW_LINES = 6;
8
8
 
9
- type ExpandableText = {
10
- readonly summary: string;
11
- readonly body: string;
12
- readonly fullOutputPath?: string;
13
- };
14
-
15
- const isExpandableText = (value: unknown): value is ExpandableText =>
16
- isRecord(value) && typeof value["summary"] === "string" && typeof value["body"] === "string";
17
-
18
- const extractRender = (details: unknown): unknown =>
19
- isRecord(details) ? details["render"] : undefined;
9
+ // A tool whose details embed this under `render` gets a compact/expanded text view for free.
10
+ export const ExpandableTextSchema = Type.Object({
11
+ summary: Type.String(),
12
+ body: Type.String(),
13
+ fullOutputPath: Type.Optional(Type.String()),
14
+ });
20
15
 
21
- export const renderExpandableText = (
22
- label: string,
23
- result: { readonly details?: unknown },
24
- expanded: boolean,
25
- theme: Theme,
26
- ): Component => {
27
- const render = extractRender(result.details);
28
- if (!isExpandableText(render)) return new Text(theme.fg("error", `${label}: no details`), 0, 0);
16
+ export type ExpandableText = DetailsOf<typeof ExpandableTextSchema>;
29
17
 
18
+ export const renderExpandableText = (render: ExpandableText, expanded: boolean): Component => {
30
19
  if (!expanded) {
31
20
  const lines = render.body.split("\n");
32
21
  const preview = lines.slice(0, COMPACT_PREVIEW_LINES).join("\n");
@@ -35,7 +24,7 @@ export const renderExpandableText = (
35
24
  return new Markdown(md, 0, 0, getMarkdownTheme());
36
25
  }
37
26
 
38
- const tail = render.fullOutputPath
27
+ const tail = render.fullOutputPath !== undefined
39
28
  ? `\n\nFull output at \`${render.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
40
29
  : `\n\n${keyHint("app.tools.expand", "to collapse")}`;
41
30
  const md = `**${render.summary}**\n\n${render.body}${tail}`;
@@ -1,4 +1,4 @@
1
- import { rename, writeFile } from "node:fs/promises";
1
+ import { rename } from "node:fs/promises";
2
2
  import type { BrowserClient } from "../client";
3
3
  import { type Result, ok } from "../util/result";
4
4
  import type { ToolErr } from "../util/tool";
@@ -6,6 +6,7 @@ import { screenshotPath } from "../util/paths";
6
6
  import { loadSharp } from "../util/sharp-shim";
7
7
  import { safeJs } from "../util/js-template";
8
8
  import { cdpCall, evalJs } from "./cdp-call";
9
+ import { writeFileResult } from "./io";
9
10
 
10
11
  export type CaptureArgs = {
11
12
  readonly fullPage?: boolean;
@@ -26,7 +27,8 @@ export const captureBase = async (
26
27
  ...(format === "jpeg" ? { quality } : {}),
27
28
  });
28
29
  if (!r.success) return r;
29
- await writeFile(path, Buffer.from(r.data.data, "base64"));
30
+ const written = await writeFileResult(path, Buffer.from(r.data.data, "base64"));
31
+ if (!written.success) return written;
30
32
  return ok({ path, format });
31
33
  };
32
34
 
@@ -2,8 +2,7 @@ import { readFileSync } from "node:fs";
2
2
  import { Type } from "typebox";
3
3
  import { Image, type ImageTheme, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
4
4
  import { type Result, ok } from "../util/result";
5
- import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
6
- import { asRecord, asString } from "../util/guards";
5
+ import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
7
6
  import { captureBase, resizeIfNeeded } from "./screenshot-capture";
8
7
 
9
8
  const ScreenshotArgs = Type.Object({
@@ -13,6 +12,14 @@ const ScreenshotArgs = Type.Object({
13
12
  maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })),
14
13
  });
15
14
 
15
+ const ScreenshotDetailsSchema = Type.Object({
16
+ path: Type.String(),
17
+ format: Type.Union([Type.Literal("png"), Type.Literal("jpeg")]),
18
+ attached: Type.Boolean(),
19
+ });
20
+
21
+ export type ScreenshotDetails = DetailsOf<typeof ScreenshotDetailsSchema>;
22
+
16
23
  export const screenshotTool = defineBrowserTool({
17
24
  name: "browser_screenshot",
18
25
  label: "Browser Screenshot",
@@ -23,8 +30,9 @@ export const screenshotTool = defineBrowserTool({
23
30
  "format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.",
24
31
  ],
25
32
  parameters: ScreenshotArgs,
33
+ details: ScreenshotDetailsSchema,
26
34
  concurrency: "parallel",
27
- async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
35
+ async handler(args, { client }): Promise<Result<ToolOk<ScreenshotDetails>, ToolErr>> {
28
36
  const cap = await captureBase(client, args);
29
37
  if (!cap.success) return cap;
30
38
  let note = "";
@@ -39,12 +47,7 @@ export const screenshotTool = defineBrowserTool({
39
47
  },
40
48
 
41
49
  renderResult(result, _expanded, theme) {
42
- const raw = asRecord(result.details);
43
- const filePath = raw === undefined ? undefined : asString(raw["path"]);
44
- if (!filePath) {
45
- return new Text(theme.fg("error", "Screenshot path missing"), 0, 0);
46
- }
47
- const format = raw === undefined ? undefined : asString(raw["format"]);
50
+ const { path: filePath, format } = result.details;
48
51
  try {
49
52
  const buf = readFileSync(filePath);
50
53
  const b64 = buf.toString("base64");
@@ -59,7 +62,7 @@ export const screenshotTool = defineBrowserTool({
59
62
  });
60
63
  // Image's text fallback does not respect the rendered width, so an unwrapped long path can blow past the terminal width and crash the host TUI.
61
64
  return {
62
- invalidate: () => image.invalidate(),
65
+ invalidate: () => { image.invalidate(); },
63
66
  render: (width: number) =>
64
67
  image.render(width).map((line) =>
65
68
  visibleWidth(line) > width ? truncateToWidth(line, width) : line,
@@ -1,7 +1,7 @@
1
1
  import { Type } from "typebox";
2
2
  import { type Result, err, ok } from "../util/result";
3
3
  import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
4
- import { cdpCall } from "./cdp-call";
4
+ import { cdpCall, fromClient } from "./cdp-call";
5
5
 
6
6
  const ScrollArgs = Type.Object({
7
7
  x: Type.Optional(Type.Number({ description: "X coordinate where to scroll. Default: viewport center" })),
@@ -25,9 +25,9 @@ export const scrollTool = defineBrowserTool({
25
25
  parameters: ScrollArgs,
26
26
  concurrency: "serialized",
27
27
  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
28
- const info = await client.pageInfo();
29
- if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
30
- if ("dialog" in info.data) {
28
+ const info = fromClient(await client.pageInfo(), "pageInfo");
29
+ if (!info.success) return info;
30
+ if (info.data.kind === "dialog") {
31
31
  return err({
32
32
  kind: "invalid_state",
33
33
  message: `Dialog open: ${info.data.dialog.type} — ${info.data.dialog.message}`,
@@ -29,7 +29,7 @@ const CAPTCHA_MARKERS: ReadonlyArray<string> = [
29
29
  ];
30
30
 
31
31
  const unwrapRedirect = (href: string): string => {
32
- const wrapper = href.match(/^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i);
32
+ const wrapper = /^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i.exec(href);
33
33
  if (!wrapper) return href;
34
34
  const query = href.slice(href.indexOf("?") + 1);
35
35
  const target = new URLSearchParams(query).get("q");