pi-agent-browser-native 0.2.32 → 0.2.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/README.md +61 -20
  3. package/docs/ARCHITECTURE.md +9 -2
  4. package/docs/COMMAND_REFERENCE.md +45 -14
  5. package/docs/ELECTRON.md +23 -4
  6. package/docs/RELEASE.md +15 -5
  7. package/docs/REQUIREMENTS.md +1 -1
  8. package/docs/SUPPORT_MATRIX.md +36 -22
  9. package/docs/TOOL_CONTRACT.md +90 -31
  10. package/extensions/agent-browser/index.ts +407 -4373
  11. package/extensions/agent-browser/lib/input-modes/electron.ts +170 -0
  12. package/extensions/agent-browser/lib/input-modes/job.ts +265 -0
  13. package/extensions/agent-browser/lib/input-modes/lookups.ts +447 -0
  14. package/extensions/agent-browser/lib/input-modes/params.ts +188 -0
  15. package/extensions/agent-browser/lib/input-modes/semantic-action.ts +107 -0
  16. package/extensions/agent-browser/lib/input-modes/shared.ts +46 -0
  17. package/extensions/agent-browser/lib/input-modes/types.ts +221 -0
  18. package/extensions/agent-browser/lib/input-modes.ts +44 -0
  19. package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts +762 -0
  20. package/extensions/agent-browser/lib/orchestration/browser-run/final-result.ts +450 -0
  21. package/extensions/agent-browser/lib/orchestration/browser-run/index.ts +46 -0
  22. package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts +736 -0
  23. package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts +413 -0
  24. package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts +868 -0
  25. package/extensions/agent-browser/lib/orchestration/browser-run/types.ts +482 -0
  26. package/extensions/agent-browser/lib/orchestration/browser-run.ts +1 -0
  27. package/extensions/agent-browser/lib/orchestration/input-plan.ts +338 -0
  28. package/extensions/agent-browser/lib/playbook.ts +22 -20
  29. package/extensions/agent-browser/lib/process.ts +106 -4
  30. package/extensions/agent-browser/lib/results/action-recommendations.ts +269 -0
  31. package/extensions/agent-browser/lib/results/artifact-manifest.ts +114 -0
  32. package/extensions/agent-browser/lib/results/artifact-state.ts +13 -0
  33. package/extensions/agent-browser/lib/results/categories.ts +106 -0
  34. package/extensions/agent-browser/lib/results/contracts.ts +220 -0
  35. package/extensions/agent-browser/lib/results/editable-ref-evidence.ts +72 -0
  36. package/extensions/agent-browser/lib/results/envelope.ts +2 -1
  37. package/extensions/agent-browser/lib/results/network.ts +64 -0
  38. package/extensions/agent-browser/lib/results/next-actions.ts +117 -0
  39. package/extensions/agent-browser/lib/results/presentation/artifacts.ts +506 -0
  40. package/extensions/agent-browser/lib/results/presentation/batch.ts +355 -0
  41. package/extensions/agent-browser/lib/results/presentation/common.ts +53 -0
  42. package/extensions/agent-browser/lib/results/presentation/content.ts +36 -0
  43. package/extensions/agent-browser/lib/results/presentation/diagnostics.ts +730 -0
  44. package/extensions/agent-browser/lib/results/presentation/errors.ts +125 -0
  45. package/extensions/agent-browser/lib/results/presentation/large-output.ts +182 -0
  46. package/extensions/agent-browser/lib/results/presentation/navigation.ts +216 -0
  47. package/extensions/agent-browser/lib/results/presentation/registry.ts +182 -0
  48. package/extensions/agent-browser/lib/results/presentation/semantic-action.ts +133 -0
  49. package/extensions/agent-browser/lib/results/presentation/skills.ts +143 -0
  50. package/extensions/agent-browser/lib/results/presentation.ts +96 -2403
  51. package/extensions/agent-browser/lib/results/recovery-actions.ts +139 -0
  52. package/extensions/agent-browser/lib/results/recovery-next-actions.ts +71 -0
  53. package/extensions/agent-browser/lib/results/selector-recovery.ts +312 -0
  54. package/extensions/agent-browser/lib/results/shared.ts +17 -789
  55. package/extensions/agent-browser/lib/results/snapshot-high-value-controls.ts +262 -0
  56. package/extensions/agent-browser/lib/results/snapshot-refs.ts +100 -0
  57. package/extensions/agent-browser/lib/results/snapshot-segments.ts +366 -0
  58. package/extensions/agent-browser/lib/results/snapshot-spill.ts +63 -0
  59. package/extensions/agent-browser/lib/results/snapshot.ts +37 -489
  60. package/extensions/agent-browser/lib/results/text.ts +40 -0
  61. package/extensions/agent-browser/lib/results.ts +16 -5
  62. package/extensions/agent-browser/lib/session-page-state.ts +486 -0
  63. package/package.json +2 -1
@@ -0,0 +1,762 @@
1
+ import { constants as fsConstants } from "node:fs";
2
+ import { access, stat } from "node:fs/promises";
3
+ import { delimiter, isAbsolute, join, resolve } from "node:path";
4
+
5
+ import type { ElectronLaunchRecord } from "../../electron/launch.js";
6
+ import type { AgentBrowserSourceLookupAnalysis, CompiledAgentBrowserJob, CompiledAgentBrowserSemanticAction } from "../../input-modes.js";
7
+ import { isHttpOrHttpsUrl } from "../../input-modes/job.js";
8
+ import type { AgentBrowserNextAction } from "../../results.js";
9
+ import { formatSessionArtifactRetentionSummary } from "../../results/artifact-manifest.js";
10
+ import { buildNextToolAction, withOptionalSessionArgs } from "../../results/next-actions.js";
11
+ import { buildVisibleRefFallbackDiagnosticFromSnapshot, getVisibleRefFallbackTarget, type VisibleRefFallbackDiagnostic } from "../../results/selector-recovery.js";
12
+ import { extractRefSnapshotFromData, normalizeComparableUrl, type SessionTabTarget } from "../../session-page-state.js";
13
+ import { redactInvocationArgs, redactSensitiveText, type CommandInfo } from "../../runtime.js";
14
+ import { isRecord } from "../../parsing.js";
15
+ import {
16
+ extractBatchResultCommand,
17
+ extractNavigationSummaryFromData,
18
+ extractStringResultField,
19
+ findElectronLaunchRecordForSession,
20
+ getGuardedRefUsage,
21
+ runSessionCommandData,
22
+ } from "./session-state.js";
23
+ import { getScreenshotPathTokenIndex } from "./prepare.js";
24
+ import type {
25
+ ArtifactCleanupGuidance,
26
+ ComboboxFocusDiagnostic,
27
+ ElectronBroadGetTextScopeDiagnostic,
28
+ ElectronHandoffSummary,
29
+ FillVerificationDiagnostic,
30
+ NavigationSummary,
31
+ OverlayBlockerCandidate,
32
+ OverlayBlockerDiagnostic,
33
+ QaAttachedPreconditionFailure,
34
+ QaAttachedTarget,
35
+ RecordingDependencyWarning,
36
+ ScrollNoopDiagnostic,
37
+ ScrollPositionSnapshot,
38
+ SelectorTextVisibilityDiagnostic,
39
+ TimeoutArtifactEvidence,
40
+ TimeoutPartialProgress,
41
+ } from "./types.js";
42
+ import type { SessionArtifactManifest } from "../../results/contracts.js";
43
+
44
+ const ELECTRON_FILL_VERIFICATION_TIMEOUT_MS = 2_000;
45
+
46
+ export function sleepMs(ms: number): Promise<void> {
47
+ return new Promise((resolve) => setTimeout(resolve, ms));
48
+ }
49
+
50
+ function boundElectronProbeString(value: string | undefined, maxLength = 240): string | undefined {
51
+ const trimmed = value?.trim();
52
+ if (!trimmed) return undefined;
53
+ return trimmed.length > maxLength ? `${trimmed.slice(0, Math.max(0, maxLength - 3))}...` : trimmed;
54
+ }
55
+
56
+ export async function collectNavigationSummary(options: {
57
+ cwd: string;
58
+ sessionName?: string;
59
+ signal?: AbortSignal;
60
+ }): Promise<NavigationSummary | undefined> {
61
+ return extractNavigationSummaryFromData(await runSessionCommandData({
62
+ args: ["eval", "--stdin"],
63
+ cwd: options.cwd,
64
+ sessionName: options.sessionName,
65
+ signal: options.signal,
66
+ stdin: `({ title: document.title, url: location.href })`,
67
+ }));
68
+ }
69
+
70
+ function extractScrollPositionSnapshot(data: unknown): ScrollPositionSnapshot | undefined {
71
+ const result = isRecord(data) && isRecord(data.result) ? data.result : data;
72
+ if (!isRecord(result)) return undefined;
73
+ const scrollX = typeof result.scrollX === "number" ? result.scrollX : undefined;
74
+ const scrollY = typeof result.scrollY === "number" ? result.scrollY : undefined;
75
+ const innerHeight = typeof result.innerHeight === "number" ? result.innerHeight : undefined;
76
+ const innerWidth = typeof result.innerWidth === "number" ? result.innerWidth : undefined;
77
+ const scrollHeight = typeof result.scrollHeight === "number" ? result.scrollHeight : undefined;
78
+ const scrollWidth = typeof result.scrollWidth === "number" ? result.scrollWidth : undefined;
79
+ if (scrollX === undefined || scrollY === undefined || innerHeight === undefined || innerWidth === undefined || scrollHeight === undefined || scrollWidth === undefined) return undefined;
80
+ const containers = Array.isArray(result.containers)
81
+ ? result.containers.flatMap((entry, index): ScrollPositionSnapshot["containers"] => {
82
+ if (!isRecord(entry)) return [];
83
+ const rawId = typeof entry.id === "string" ? entry.id : undefined;
84
+ const id = rawId && /^\d+:[a-z][a-z0-9-]*(?:\[role=[a-z-]+\])?$/i.test(rawId) ? rawId : `sample-${index}`;
85
+ const scrollTop = typeof entry.scrollTop === "number" ? entry.scrollTop : undefined;
86
+ const scrollLeft = typeof entry.scrollLeft === "number" ? entry.scrollLeft : undefined;
87
+ return scrollTop !== undefined && scrollLeft !== undefined ? [{ id, scrollLeft, scrollTop }] : [];
88
+ })
89
+ : [];
90
+ return { containerCount: typeof result.containerCount === "number" ? result.containerCount : containers.length, containers, innerHeight, innerWidth, scrollHeight, scrollWidth, scrollX, scrollY };
91
+ }
92
+
93
+ const SCROLL_POSITION_EVAL = `(() => {
94
+ const viewport = {
95
+ scrollX: window.scrollX,
96
+ scrollY: window.scrollY,
97
+ innerHeight: window.innerHeight,
98
+ innerWidth: window.innerWidth,
99
+ scrollHeight: Math.max(document.documentElement?.scrollHeight || 0, document.body?.scrollHeight || 0),
100
+ scrollWidth: Math.max(document.documentElement?.scrollWidth || 0, document.body?.scrollWidth || 0),
101
+ };
102
+ const describe = (element, index) => {
103
+ const role = element.getAttribute("role") || "";
104
+ const id = element.tagName.toLowerCase();
105
+ return { id: String(index) + ":" + id + (role ? "[role=" + role + "]" : ""), scrollTop: element.scrollTop, scrollLeft: element.scrollLeft, area: element.clientWidth * element.clientHeight };
106
+ };
107
+ const containers = Array.from(document.querySelectorAll("body *"))
108
+ .filter((element) => element instanceof HTMLElement && (element.scrollHeight > element.clientHeight + 1 || element.scrollWidth > element.clientWidth + 1))
109
+ .map(describe)
110
+ .sort((left, right) => right.area - left.area)
111
+ .slice(0, 10)
112
+ .map(({ area, ...entry }) => entry);
113
+ return { ...viewport, containerCount: containers.length, containers };
114
+ })()`;
115
+
116
+ export async function collectScrollPositionSnapshot(options: { cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<ScrollPositionSnapshot | undefined> {
117
+ return extractScrollPositionSnapshot(await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: SCROLL_POSITION_EVAL }));
118
+ }
119
+
120
+ function sameScrollPositionSnapshot(left: ScrollPositionSnapshot, right: ScrollPositionSnapshot): boolean {
121
+ return left.scrollX === right.scrollX && left.scrollY === right.scrollY && left.scrollHeight === right.scrollHeight && left.scrollWidth === right.scrollWidth && left.containers.length === right.containers.length && left.containers.every((container, index) => {
122
+ const other = right.containers[index];
123
+ return other?.id === container.id && other.scrollTop === container.scrollTop && other.scrollLeft === container.scrollLeft;
124
+ });
125
+ }
126
+
127
+ export function buildScrollNoopDiagnostic(before: ScrollPositionSnapshot | undefined, after: ScrollPositionSnapshot | undefined): ScrollNoopDiagnostic | undefined {
128
+ if (!before || !after || !sameScrollPositionSnapshot(before, after)) return undefined;
129
+ return {
130
+ after,
131
+ before,
132
+ message: "Scroll reported success, but the viewport and sampled scrollable containers did not change position.",
133
+ reason: "no-observed-scroll-position-change",
134
+ recommendations: [
135
+ "Run snapshot -i or screenshot to confirm what is visible before choosing the next action.",
136
+ "On dashboards and panes with nested scrolling, use scrollintoview <@ref> for a visible target or target the actual scrollable region instead of repeating page scrolls.",
137
+ ],
138
+ };
139
+ }
140
+
141
+ export function buildScrollNoopNextActions(sessionName: string | undefined): AgentBrowserNextAction[] {
142
+ return [
143
+ { id: "inspect-after-noop-scroll", params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) }, reason: "Refresh interactive refs and inspect whether the intended target is inside a nested scroll container.", safety: "Do not assume repeated page scrolls will move dashboard panels or nested panes.", tool: "agent_browser" },
144
+ { id: "verify-noop-scroll-visually", params: { args: withOptionalSessionArgs(sessionName, ["screenshot"]) }, reason: "Capture the current viewport to verify whether the scroll actually changed visible content.", safety: "Use screenshot evidence before concluding a dense dashboard did or did not move.", tool: "agent_browser" },
145
+ ];
146
+ }
147
+
148
+ export function formatScrollNoopDiagnosticText(diagnostic: ScrollNoopDiagnostic | undefined): string | undefined {
149
+ if (!diagnostic) return undefined;
150
+ return ["Scroll diagnostic: no observed scroll movement.", `Reason: ${diagnostic.message}`, `Sampled scrollable containers: ${diagnostic.after.containers.length}/${diagnostic.after.containerCount}.`, ...diagnostic.recommendations.map((recommendation) => `- ${recommendation}`)].join("\n");
151
+ }
152
+
153
+ const COMBOBOX_FOCUS_EVAL = `(() => {
154
+ const isVisible = (element) => {
155
+ if (!(element instanceof HTMLElement)) return false;
156
+ const style = window.getComputedStyle(element);
157
+ if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return false;
158
+ return element.getClientRects().length > 0;
159
+ };
160
+ const active = document.activeElement instanceof HTMLElement ? document.activeElement : null;
161
+ const role = active?.getAttribute("role") || undefined;
162
+ const hasPopup = active?.getAttribute("aria-haspopup") || undefined;
163
+ const expanded = active?.getAttribute("aria-expanded") || undefined;
164
+ const tagName = active?.tagName.toLowerCase();
165
+ const name = (active?.getAttribute("aria-label") || active?.getAttribute("placeholder") || active?.getAttribute("title") || active?.textContent || "").trim().slice(0, 80) || undefined;
166
+ const visibleListboxCount = Array.from(document.querySelectorAll('[role="listbox"], [role="menu"]')).filter(isVisible).length;
167
+ const visibleOptionCount = Array.from(document.querySelectorAll('[role="option"], option, [role="menuitem"]')).filter(isVisible).length;
168
+ const comboboxLike = role === "combobox" || hasPopup === "listbox" || hasPopup === "menu" || tagName === "select" || active?.getAttribute("aria-autocomplete") !== null;
169
+ return { activeElement: active ? { expanded, hasPopup, name, role, tagName } : undefined, comboboxLike, visibleListboxCount, visibleOptionCount };
170
+ })()`;
171
+
172
+ function extractComboboxFocusDiagnostic(data: unknown): ComboboxFocusDiagnostic | undefined {
173
+ const result = isRecord(data) && isRecord(data.result) ? data.result : data;
174
+ if (!isRecord(result) || result.comboboxLike !== true || !isRecord(result.activeElement)) return undefined;
175
+ const visibleListboxCount = typeof result.visibleListboxCount === "number" ? result.visibleListboxCount : 0;
176
+ const visibleOptionCount = typeof result.visibleOptionCount === "number" ? result.visibleOptionCount : 0;
177
+ const expanded = typeof result.activeElement.expanded === "string" ? result.activeElement.expanded : undefined;
178
+ if ((expanded !== "false" && expanded !== "true") || visibleListboxCount > 0 || visibleOptionCount > 0) return undefined;
179
+ return {
180
+ activeElement: {
181
+ expanded,
182
+ hasPopup: typeof result.activeElement.hasPopup === "string" ? result.activeElement.hasPopup : undefined,
183
+ name: typeof result.activeElement.name === "string" ? redactSensitiveText(result.activeElement.name) : undefined,
184
+ role: typeof result.activeElement.role === "string" ? result.activeElement.role : undefined,
185
+ tagName: typeof result.activeElement.tagName === "string" ? result.activeElement.tagName : undefined,
186
+ },
187
+ message: "A combobox-like control is focused, but no listbox or option elements are visibly open.",
188
+ reason: "focused-combobox-without-visible-options",
189
+ recommendations: ["Run snapshot -i to inspect whether options appeared under a different role or portal.", "Try ArrowDown or Enter to open the option list before selecting, or use select/visible option refs when available."],
190
+ visibleListboxCount,
191
+ visibleOptionCount,
192
+ };
193
+ }
194
+
195
+ function isComboboxFocusDiagnosticCommand(command: string | undefined, commandTokens: string[]): boolean {
196
+ const explicitlyTargetsCombobox = commandTokens.some((token) => /^(?:combobox|listbox)$/i.test(token));
197
+ if (!explicitlyTargetsCombobox) return false;
198
+ if (command === "click" || command === "fill") return true;
199
+ return command === "find" && commandTokens.some((token) => ["click", "fill"].includes(token));
200
+ }
201
+
202
+ function getCompiledSemanticActionRoleValue(compiled: CompiledAgentBrowserSemanticAction): string | undefined {
203
+ if (compiled.locator !== "role") return undefined;
204
+ const findIndex = compiled.args.indexOf("find");
205
+ if (findIndex < 0 || compiled.args[findIndex + 1] !== "role") return undefined;
206
+ return compiled.args[findIndex + 2];
207
+ }
208
+
209
+ function isComboboxFocusDiagnosticSemanticAction(compiled: CompiledAgentBrowserSemanticAction | undefined): boolean {
210
+ if (!compiled || !["click", "fill"].includes(compiled.action)) return false;
211
+ return /^(?:combobox|listbox)$/i.test(getCompiledSemanticActionRoleValue(compiled) ?? "");
212
+ }
213
+
214
+ export async function collectComboboxFocusDiagnostic(options: { command?: string; commandTokens: string[]; cwd: string; semanticAction?: CompiledAgentBrowserSemanticAction; sessionName?: string; signal?: AbortSignal }): Promise<ComboboxFocusDiagnostic | undefined> {
215
+ if (!isComboboxFocusDiagnosticCommand(options.command, options.commandTokens) && !isComboboxFocusDiagnosticSemanticAction(options.semanticAction)) return undefined;
216
+ return extractComboboxFocusDiagnostic(await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: COMBOBOX_FOCUS_EVAL }));
217
+ }
218
+
219
+ export function buildComboboxFocusNextActions(sessionName: string | undefined): AgentBrowserNextAction[] {
220
+ return [
221
+ { id: "inspect-focused-combobox", params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) }, reason: "Inspect the focused combobox and any portal/listbox refs before choosing an option.", safety: "Prefer visible option refs or select when a native/selectable option list is exposed.", tool: "agent_browser" },
222
+ { id: "try-open-combobox-with-arrow", params: { args: withOptionalSessionArgs(sessionName, ["press", "ArrowDown"]) }, reason: "Many searchable comboboxes open their option list with ArrowDown after focus.", safety: "Use only when the focused combobox is still the intended control, then re-snapshot before selecting.", tool: "agent_browser" },
223
+ { id: "try-open-combobox-with-enter", params: { args: withOptionalSessionArgs(sessionName, ["press", "Enter"]) }, reason: "Some comboboxes open or confirm their option list with Enter after focus.", safety: "Enter may select a highlighted/default option; prefer ArrowDown first unless Enter is the app's expected opener.", tool: "agent_browser" },
224
+ ];
225
+ }
226
+
227
+ export function formatComboboxFocusDiagnosticText(diagnostic: ComboboxFocusDiagnostic | undefined): string | undefined {
228
+ if (!diagnostic) return undefined;
229
+ const label = diagnostic.activeElement.name ? ` (${diagnostic.activeElement.name})` : "";
230
+ return [`Combobox diagnostic: focused combobox did not expose visible options${label}.`, `Reason: ${diagnostic.message}`, ...diagnostic.recommendations.map((recommendation) => `- ${recommendation}`)].join("\n");
231
+ }
232
+
233
+ function getRecordStartLikeCommand(command: string | undefined, commandTokens: string[]): RecordingDependencyWarning["command"] | undefined {
234
+ if (command !== "record") return undefined;
235
+ const subcommand = commandTokens[1]?.toLowerCase();
236
+ if (subcommand === "start") return "record start";
237
+ if (subcommand === "restart") return "record restart";
238
+ return undefined;
239
+ }
240
+
241
+ async function executableExistsOnPath(command: string): Promise<boolean> {
242
+ const pathValue = process.env.PATH ?? "";
243
+ const extensions = process.platform === "win32" ? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean) : [""];
244
+ for (const directory of pathValue.split(delimiter).filter(Boolean)) {
245
+ for (const extension of extensions) {
246
+ try {
247
+ const candidate = join(directory, `${command}${extension}`);
248
+ await access(candidate, fsConstants.X_OK);
249
+ if ((await stat(candidate)).isFile()) return true;
250
+ } catch {
251
+ // Try the next candidate.
252
+ }
253
+ }
254
+ }
255
+ return false;
256
+ }
257
+
258
+ export async function collectRecordingDependencyWarning(options: { command: string | undefined; commandTokens: string[]; succeeded: boolean }): Promise<RecordingDependencyWarning | undefined> {
259
+ if (!options.succeeded) return undefined;
260
+ const recordCommand = getRecordStartLikeCommand(options.command, options.commandTokens);
261
+ if (!recordCommand) return undefined;
262
+ if (await executableExistsOnPath("ffmpeg")) return undefined;
263
+ return { command: recordCommand, dependency: "ffmpeg", message: `${recordCommand} can begin recording, but record stop needs ffmpeg on PATH to encode the WebM output.`, reason: "ffmpeg-missing-for-recording", recommendations: ["Install ffmpeg before relying on this recording workflow; on macOS with Homebrew, brew install ffmpeg or brew install ffmpeg-full.", "If ffmpeg was just installed, restart pi or ensure the PATH visible to pi includes the ffmpeg binary before running record stop."] };
264
+ }
265
+
266
+ export function formatRecordingDependencyWarningText(warning: RecordingDependencyWarning | undefined): string | undefined {
267
+ if (!warning) return undefined;
268
+ return ["Recording dependency warning: ffmpeg not found on PATH.", `Reason: ${warning.message}`, ...warning.recommendations.map((recommendation) => `- ${recommendation}`)].join("\n");
269
+ }
270
+
271
+ function getSnapshotRefRecord(data: unknown): Record<string, unknown> | undefined {
272
+ return isRecord(data) && isRecord(data.refs) ? data.refs : undefined;
273
+ }
274
+
275
+ const OVERLAY_CLOSE_NAME_PATTERN = /(?:\b(?:close|dismiss|no thanks|not now|maybe later|hide|skip|continue without|x)\b|^\s*×\s*$)/i;
276
+ const OVERLAY_CONTEXT_ROLES = new Set(["alertdialog", "dialog"]);
277
+ const OVERLAY_ACTION_ROLES = new Set(["button", "link", "menuitem"]);
278
+ const OVERLAY_BLOCKER_CANDIDATE_LIMIT = 3;
279
+
280
+ function getOverlayBlockerCandidates(snapshotData: unknown): OverlayBlockerCandidate[] {
281
+ const refs = getSnapshotRefRecord(snapshotData);
282
+ if (!refs) return [];
283
+ const hasOverlayContext = Object.values(refs).some((entry) => isRecord(entry) && OVERLAY_CONTEXT_ROLES.has((typeof entry.role === "string" ? entry.role : "").toLowerCase()));
284
+ if (!hasOverlayContext) return [];
285
+ const candidates: OverlayBlockerCandidate[] = [];
286
+ for (const [ref, entry] of Object.entries(refs)) {
287
+ if (!/^e\d+$/.test(ref) || !isRecord(entry)) continue;
288
+ const role = typeof entry.role === "string" ? entry.role : undefined;
289
+ const name = typeof entry.name === "string" ? entry.name : undefined;
290
+ if (!role || !OVERLAY_ACTION_ROLES.has(role.toLowerCase()) || !name || !OVERLAY_CLOSE_NAME_PATTERN.test(name)) continue;
291
+ candidates.push({ args: ["click", `@${ref}`], name, reason: `Visible ${role} ${JSON.stringify(name)} appears in a snapshot that also contains overlay/banner/dialog context.`, ref: `@${ref}`, role });
292
+ if (candidates.length >= OVERLAY_BLOCKER_CANDIDATE_LIMIT) break;
293
+ }
294
+ return candidates;
295
+ }
296
+
297
+ export function formatOverlayBlockerText(diagnostic: OverlayBlockerDiagnostic): string {
298
+ return ["Possible overlay blockers:", ...diagnostic.candidates.map((candidate) => `- ${candidate.ref}${candidate.role ? ` ${candidate.role}` : ""}${candidate.name ? ` ${JSON.stringify(candidate.name)}` : ""}: ${candidate.reason}`)].join("\n");
299
+ }
300
+
301
+ export function buildOverlayBlockerNextActions(options: { diagnostic: OverlayBlockerDiagnostic; sessionName?: string }): AgentBrowserNextAction[] {
302
+ return [{ id: "inspect-overlay-state", params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) }, reason: "Refresh interactive refs and inspect whether an overlay, banner, modal, or dialog is blocking the intended click.", safety: "Read-only inspection; use current refs from this snapshot before interacting.", tool: "agent_browser" }, ...options.diagnostic.candidates.map((candidate, index) => ({ id: `try-overlay-blocker-candidate-${index + 1}`, params: { args: withOptionalSessionArgs(options.sessionName, candidate.args) }, reason: candidate.reason, safety: "Only click this if the candidate is clearly a close/dismiss control for an overlay that blocks the intended workflow.", tool: "agent_browser" as const }))];
303
+ }
304
+
305
+ export async function collectOverlayBlockerDiagnostic(options: { command?: string; cwd: string; data: unknown; navigationSummary?: NavigationSummary; priorTarget?: SessionTabTarget; sessionName?: string; signal?: AbortSignal }): Promise<OverlayBlockerDiagnostic | undefined> {
306
+ if (options.command !== "click" || !isRecord(options.data) || typeof options.data.clicked !== "string") return undefined;
307
+ const priorUrl = normalizeComparableUrl(options.priorTarget?.url);
308
+ const currentUrl = normalizeComparableUrl(options.navigationSummary?.url);
309
+ if (!priorUrl || !currentUrl || priorUrl !== currentUrl) return undefined;
310
+ const snapshotData = await runSessionCommandData({ args: ["snapshot", "-i"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
311
+ const candidates = getOverlayBlockerCandidates(snapshotData);
312
+ const snapshot = extractRefSnapshotFromData(snapshotData);
313
+ if (candidates.length === 0 || !snapshot) return undefined;
314
+ return { candidates, snapshot, summary: `Click completed but the page stayed on ${currentUrl}; a fresh snapshot contains likely overlay close/dismiss controls.` };
315
+ }
316
+
317
+ function buildVisibleTextProbeScript(selector: string): string {
318
+ return `(() => {\n const selector = ${JSON.stringify(selector)};\n const isVisible = (element) => {\n const style = window.getComputedStyle(element);\n if (!style || style.display === 'none' || style.visibility === 'hidden' || style.visibility === 'collapse' || Number(style.opacity) === 0) return false;\n return Array.from(element.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0);\n };\n let matches = [];\n try {\n matches = Array.from(document.querySelectorAll(selector));\n } catch (error) {\n return JSON.stringify({ selector, error: error instanceof Error ? error.message : String(error) });\n }\n const visible = matches.filter(isVisible);\n const trim = (value) => typeof value === 'string' ? value.trim().replace(/\\s+/g, ' ').slice(0, 200) : undefined;\n return JSON.stringify({ selector, matchCount: matches.length, visibleCount: visible.length, firstMatchVisible: matches[0] ? isVisible(matches[0]) : undefined, firstTextPreview: trim(matches[0]?.textContent), firstVisibleTextPreview: trim(visible[0]?.textContent) });\n})()`;
319
+ }
320
+
321
+ function parseSelectorTextVisibilityProbe(data: unknown, selector: string): Omit<SelectorTextVisibilityDiagnostic, "summary"> | undefined {
322
+ const result = extractStringResultField(data, "result");
323
+ if (!result) return undefined;
324
+ let parsed: unknown;
325
+ try { parsed = JSON.parse(result); } catch { return undefined; }
326
+ if (!isRecord(parsed) || typeof parsed.error === "string") return undefined;
327
+ const matchCount = typeof parsed.matchCount === "number" ? parsed.matchCount : undefined;
328
+ const visibleCount = typeof parsed.visibleCount === "number" ? parsed.visibleCount : undefined;
329
+ if (matchCount === undefined || visibleCount === undefined) return undefined;
330
+ return { firstMatchVisible: typeof parsed.firstMatchVisible === "boolean" ? parsed.firstMatchVisible : undefined, firstVisibleTextPreview: typeof parsed.firstVisibleTextPreview === "string" && parsed.firstVisibleTextPreview.length > 0 ? redactSensitiveText(parsed.firstVisibleTextPreview) : undefined, matchCount, selector, visibleCount };
331
+ }
332
+
333
+ function selectorMayExposeSensitiveLiteral(selector: string): boolean {
334
+ return redactSensitiveText(selector) !== selector || /\[[^\]]*[~|^$*]?=\s*(?:"[^"]*"|'[^']*'|[^\]\s]+)\s*(?:[is]\s*)?\]/.test(selector);
335
+ }
336
+
337
+ async function collectSelectorTextVisibilityDiagnosticForSelector(options: { cwd: string; selector: string | undefined; sessionName?: string; signal?: AbortSignal }): Promise<SelectorTextVisibilityDiagnostic | undefined> {
338
+ const { selector } = options;
339
+ if (!selector || /^@e\d+$/.test(selector) || selectorMayExposeSensitiveLiteral(selector)) return undefined;
340
+ const probe = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: buildVisibleTextProbeScript(selector) });
341
+ const parsed = parseSelectorTextVisibilityProbe(probe, selector);
342
+ if (!parsed || parsed.matchCount <= 1 && parsed.firstMatchVisible !== false) return undefined;
343
+ if (parsed.visibleCount === 0) return undefined;
344
+ const visibleMatchNoun = `visible match${parsed.visibleCount === 1 ? "" : "es"}`;
345
+ const visibleMatchVerb = parsed.visibleCount === 1 ? "exists" : "exist";
346
+ const summary = parsed.firstMatchVisible === false
347
+ ? `Selector ${JSON.stringify(selector)} matched ${parsed.matchCount} elements; the first match is hidden while ${parsed.visibleCount} ${visibleMatchNoun} ${visibleMatchVerb}.`
348
+ : `Selector ${JSON.stringify(selector)} matched ${parsed.matchCount} elements; get text reads the first upstream match, which may not be the intended visible tab/panel.`;
349
+ return { ...parsed, summary };
350
+ }
351
+
352
+ function getBatchGetTextSelectors(data: unknown): string[] {
353
+ if (!Array.isArray(data)) return [];
354
+ return data.flatMap((item) => {
355
+ if (!isRecord(item) || item.success === false) return [];
356
+ const [command, subcommand, selector] = extractBatchResultCommand(item);
357
+ return command === "get" && subcommand === "text" && selector ? [selector] : [];
358
+ });
359
+ }
360
+
361
+ function getSuccessfulGetTextSelectors(options: { commandInfo: CommandInfo; commandTokens: string[]; data: unknown }): string[] {
362
+ return options.commandInfo.command === "get" && options.commandInfo.subcommand === "text"
363
+ ? [options.commandTokens[2]].filter((selector): selector is string => typeof selector === "string" && selector.length > 0)
364
+ : options.commandInfo.command === "batch" ? getBatchGetTextSelectors(options.data) : [];
365
+ }
366
+
367
+ export async function collectSelectorTextVisibilityDiagnostics(options: { commandInfo: CommandInfo; commandTokens: string[]; cwd: string; data: unknown; sessionName?: string; signal?: AbortSignal }): Promise<SelectorTextVisibilityDiagnostic[]> {
368
+ const selectors = getSuccessfulGetTextSelectors(options);
369
+ const diagnostics: SelectorTextVisibilityDiagnostic[] = [];
370
+ for (const selector of selectors) {
371
+ const diagnostic = await collectSelectorTextVisibilityDiagnosticForSelector({ cwd: options.cwd, selector, sessionName: options.sessionName, signal: options.signal });
372
+ if (diagnostic) diagnostics.push(diagnostic);
373
+ }
374
+ return diagnostics.sort((left, right) => Number(right.firstMatchVisible === false) - Number(left.firstMatchVisible === false));
375
+ }
376
+
377
+ export function formatSelectorTextVisibilityText(diagnostics: SelectorTextVisibilityDiagnostic[]): string | undefined {
378
+ if (diagnostics.length === 0) return undefined;
379
+ return diagnostics.flatMap((diagnostic, index) => {
380
+ const actionId = index === 0 ? "inspect-visible-text-candidates" : `inspect-visible-text-candidates-${index + 1}`;
381
+ const lines = [`Selector text visibility warning: ${diagnostic.summary}`];
382
+ if (diagnostic.firstVisibleTextPreview) lines.push(`First visible text preview: ${JSON.stringify(diagnostic.firstVisibleTextPreview)}`);
383
+ lines.push(`Next action: use details.nextActions ${actionId} before trusting this selector text.`);
384
+ return lines;
385
+ }).join("\n");
386
+ }
387
+
388
+ export function buildSelectorTextVisibilityNextActions(options: { diagnostics: SelectorTextVisibilityDiagnostic[]; sessionName?: string }): AgentBrowserNextAction[] {
389
+ return options.diagnostics.map((diagnostic, index) => ({ id: index === 0 ? "inspect-visible-text-candidates" : `inspect-visible-text-candidates-${index + 1}`, params: { args: withOptionalSessionArgs(options.sessionName, ["eval", "--stdin"]), stdin: buildVisibleTextProbeScript(diagnostic.selector) }, reason: "Inspect selector match count and visible text before trusting get text on tabbed or hidden DOM content.", safety: "Read-only DOM inspection; use a more specific visible selector or current @ref before acting on hidden-tab text.", tool: "agent_browser" as const }));
390
+ }
391
+
392
+ function isElectronLikeRendererUrl(url: string | undefined): boolean {
393
+ return !!url && /^(?:app|file|vscode-file|vscode|chrome-extension):/i.test(url);
394
+ }
395
+
396
+ function normalizeSelectorForScopeHeuristic(selector: string): string {
397
+ return selector.trim().replace(/\s+/g, " ").toLowerCase();
398
+ }
399
+
400
+ function isBroadGetTextSelector(selector: string | undefined): selector is string {
401
+ if (!selector || /^@e\d+$/.test(selector) || selectorMayExposeSensitiveLiteral(selector)) return false;
402
+ const normalized = normalizeSelectorForScopeHeuristic(selector);
403
+ return normalized === "body" || normalized === "html" || normalized === ":root" || normalized === "*" || normalized === "main" || normalized === "div" || normalized === "section" || normalized === "article" || /^\[role=(?:"application"|'application'|application)\]$/i.test(normalized);
404
+ }
405
+
406
+ function getElectronTextScopeContext(options: { currentTarget?: SessionTabTarget; electronLaunchRecords: Map<string, ElectronLaunchRecord>; priorTarget?: SessionTabTarget; sessionName?: string }): ElectronBroadGetTextScopeDiagnostic["electronContext"] | undefined {
407
+ const record = findElectronLaunchRecordForSession(options.sessionName, options.electronLaunchRecords);
408
+ const url = options.currentTarget?.url ?? options.priorTarget?.url;
409
+ if (record) return { launchId: record.launchId, sessionName: record.sessionName ?? options.sessionName, url };
410
+ if (isElectronLikeRendererUrl(url)) return { sessionName: options.sessionName, url };
411
+ return undefined;
412
+ }
413
+
414
+ export function getSourceLookupElectronContext(options: { currentTarget?: SessionTabTarget; electronLaunchRecords: Map<string, ElectronLaunchRecord>; priorTarget?: SessionTabTarget; sessionName?: string }): AgentBrowserSourceLookupAnalysis["electronContext"] | undefined {
415
+ const record = findElectronLaunchRecordForSession(options.sessionName, options.electronLaunchRecords);
416
+ if (!record) return undefined;
417
+ const url = options.currentTarget?.url ?? options.priorTarget?.url;
418
+ return { appName: record.appName, appPath: record.appPath, executablePath: record.executablePath, launchId: record.launchId, sessionName: record.sessionName ?? options.sessionName, url };
419
+ }
420
+
421
+ export function buildSourceLookupElectronNextActions(sourceLookup: AgentBrowserSourceLookupAnalysis | undefined): AgentBrowserNextAction[] {
422
+ if (sourceLookup?.status !== "no-candidates" || !sourceLookup.electronContext) return [];
423
+ const actions: AgentBrowserNextAction[] = [];
424
+ const { launchId, sessionName } = sourceLookup.electronContext;
425
+ if (sessionName) actions.push({ id: "snapshot-electron-session", params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) }, reason: "Refresh interactive refs in the attached Electron session before retrying source lookup with a narrower target.", safety: "Read-only snapshot; no app mutation.", tool: "agent_browser" });
426
+ if (launchId) actions.push({ id: "probe-electron-launch", params: { electron: { action: "probe", launchId } }, reason: "Collect bounded wrapper/session context for the packaged Electron launch after sourceLookup found no candidates.", safety: "Read-only probe of title, URL, focus, tabs, and compact snapshot metadata.", tool: "agent_browser" });
427
+ if (sessionName) actions.push({ id: "list-electron-tabs", params: { args: withOptionalSessionArgs(sessionName, ["tab", "list"]) }, reason: "Check current Electron tabs/targets before choosing a narrower selector or @ref.", safety: "Read-only tab listing.", tool: "agent_browser" });
428
+ return actions;
429
+ }
430
+
431
+ export function collectElectronBroadGetTextScopeDiagnostics(options: { commandInfo: CommandInfo; commandTokens: string[]; currentTarget?: SessionTabTarget; data: unknown; electronLaunchRecords: Map<string, ElectronLaunchRecord>; priorTarget?: SessionTabTarget; sessionName?: string }): ElectronBroadGetTextScopeDiagnostic[] {
432
+ const electronContext = getElectronTextScopeContext(options);
433
+ if (!electronContext) return [];
434
+ return getSuccessfulGetTextSelectors(options).filter(isBroadGetTextSelector).map((selector) => ({ electronContext, selector, summary: `Broad Electron get text selector warning: selector ${JSON.stringify(selector)} may read the entire app shell; prefer snapshot -i and a current @ref or a narrower panel selector.` }));
435
+ }
436
+
437
+ export function formatElectronBroadGetTextScopeText(diagnostics: ElectronBroadGetTextScopeDiagnostic[]): string | undefined {
438
+ return diagnostics.length > 0 ? diagnostics.map((diagnostic) => diagnostic.summary).join("\n") : undefined;
439
+ }
440
+
441
+ export function buildElectronBroadGetTextScopeNextActions(options: { diagnostics: ElectronBroadGetTextScopeDiagnostic[]; sessionName?: string }): AgentBrowserNextAction[] {
442
+ return options.diagnostics.map((diagnostic, index) => ({ id: index === 0 ? "snapshot-for-electron-text-scope" : `snapshot-for-electron-text-scope-${index + 1}`, params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) }, reason: `Refresh Electron refs before trusting broad get text selector ${JSON.stringify(diagnostic.selector)}.`, safety: "Read-only snapshot; prefer a current @ref or narrower selector before extracting app-shell text.", tool: "agent_browser" as const }));
443
+ }
444
+
445
+ function looksLikeFunctionEvalStdin(stdin: string | undefined): boolean {
446
+ const trimmed = stdin?.trim();
447
+ if (!trimmed) return false;
448
+ return /^(?:async\s+)?function\b/.test(trimmed) || /^(?:async\s*)?\([^)]*\)\s*=>/.test(trimmed) || /^(?:async\s+)?[A-Za-z_$][\w$]*\s*=>/.test(trimmed);
449
+ }
450
+
451
+ function isPlainEmptyObject(value: unknown): boolean {
452
+ if (!isRecord(value) || Array.isArray(value)) return false;
453
+ const prototype = Object.getPrototypeOf(value);
454
+ return (prototype === Object.prototype || prototype === null) && Object.keys(value).length === 0;
455
+ }
456
+
457
+ export function getEvalStdinHint(options: { command?: string; data: unknown; stdin?: string }) {
458
+ if (options.command !== "eval" || !looksLikeFunctionEvalStdin(options.stdin) || !isRecord(options.data)) return undefined;
459
+ const result = options.data.result;
460
+ if (!isPlainEmptyObject(result)) return undefined;
461
+ return { reason: "eval --stdin received a function-shaped snippet and the upstream JSON result was an empty object, which often means the function itself was returned or serialized instead of invoked.", suggestion: "Pass a plain expression such as `({ title: document.title })`, or invoke the function explicitly, for example `(() => ({ title: document.title }))()`." };
462
+ }
463
+
464
+ export function formatEvalStdinHintText(hint: ReturnType<typeof getEvalStdinHint>): string | undefined {
465
+ return hint ? `Eval stdin hint: ${hint.reason} ${hint.suggestion}` : undefined;
466
+ }
467
+
468
+ export async function getArtifactCleanupGuidance(options: { command?: string; cwd: string; manifest?: SessionArtifactManifest; succeeded: boolean }): Promise<ArtifactCleanupGuidance | undefined> {
469
+ if (!options.succeeded || options.command !== "close" || !options.manifest || options.manifest.entries.length === 0) return undefined;
470
+ const explicitEntries = options.manifest.entries.filter((entry) => entry.storageScope === "explicit-path");
471
+ const explicitArtifactPaths: string[] = [];
472
+ const seenPaths = new Set<string>();
473
+ for (const entry of explicitEntries) {
474
+ if (explicitArtifactPaths.length >= 10) break;
475
+ const displayPath = entry.path;
476
+ if (seenPaths.has(displayPath)) continue;
477
+ const absolutePath = entry.absolutePath ?? (isAbsolute(entry.path) ? entry.path : resolve(options.cwd, entry.path));
478
+ try { await stat(absolutePath); } catch { continue; }
479
+ seenPaths.add(displayPath);
480
+ explicitArtifactPaths.push(displayPath);
481
+ }
482
+ return { explicitArtifactPaths, note: "Closing the browser session does not delete explicit screenshots, downloads, PDFs, traces, HAR files, or recordings; clean existing paths with host file tools when no longer needed.", owner: "host-file-tools", summary: formatSessionArtifactRetentionSummary(options.manifest) };
483
+ }
484
+
485
+ export function formatArtifactCleanupGuidanceText(guidance: ArtifactCleanupGuidance | undefined): string | undefined {
486
+ if (!guidance) return undefined;
487
+ const lines = ["Artifact lifecycle:", `- ${guidance.summary}`, `- ${guidance.note}`];
488
+ if (guidance.explicitArtifactPaths.length > 0) lines.push(`- Explicit artifact paths to review: ${guidance.explicitArtifactPaths.join(", ")}`);
489
+ return lines.join("\n");
490
+ }
491
+
492
+ async function collectManagedSessionCommandData(options: { args: string[]; cwd: string; sessionName: string; signal?: AbortSignal; timeoutMs?: number }): Promise<{ data?: unknown; error?: string }> {
493
+ try { return { data: await runSessionCommandData(options) }; } catch (error) { return { error: error instanceof Error ? error.message : String(error) }; }
494
+ }
495
+
496
+ async function collectElectronManagedSessionUrl(options: { cwd: string; sessionName: string; signal?: AbortSignal; timeoutMs?: number }): Promise<{ error?: string; url?: string }> {
497
+ const urlResult = await collectManagedSessionCommandData({
498
+ args: ["get", "url"],
499
+ cwd: options.cwd,
500
+ sessionName: options.sessionName,
501
+ signal: options.signal,
502
+ timeoutMs: options.timeoutMs,
503
+ });
504
+ const url = boundElectronProbeString(extractStringResultField(urlResult.data, "result") ?? extractStringResultField(urlResult.data, "url"), 300);
505
+ return urlResult.error ? { error: urlResult.error } : { url };
506
+ }
507
+
508
+ async function collectElectronManagedSessionTarget(options: { cwd: string; sessionName?: string; signal?: AbortSignal; timeoutMs?: number }): Promise<QaAttachedTarget | undefined> {
509
+ if (!options.sessionName) return undefined;
510
+ const [titleResult, urlResult] = await Promise.all([
511
+ collectManagedSessionCommandData({ args: ["get", "title"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, timeoutMs: options.timeoutMs }),
512
+ collectManagedSessionCommandData({ args: ["get", "url"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, timeoutMs: options.timeoutMs }),
513
+ ]);
514
+ const title = boundElectronProbeString(extractStringResultField(titleResult.data, "result") ?? extractStringResultField(titleResult.data, "title"), 160);
515
+ const url = boundElectronProbeString(extractStringResultField(urlResult.data, "result") ?? extractStringResultField(urlResult.data, "url"), 300);
516
+ const errors = [titleResult.error, urlResult.error].filter((value): value is string => value !== undefined);
517
+ return { sessionName: options.sessionName, title, url, ...(errors.length > 0 ? { error: errors.join("; ") } : {}) };
518
+ }
519
+
520
+ export async function collectQaAttachedTarget(options: { currentTarget?: SessionTabTarget; cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<QaAttachedTarget | undefined> {
521
+ if (!options.sessionName) return undefined;
522
+ if (options.currentTarget?.title || options.currentTarget?.url) return { sessionName: options.sessionName, title: options.currentTarget.title, url: options.currentTarget.url };
523
+ return collectElectronManagedSessionTarget({ cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
524
+ }
525
+
526
+ export function formatQaAttachedTargetText(target: QaAttachedTarget | undefined): string | undefined {
527
+ if (!target) return undefined;
528
+ return ["QA attached target:", target.sessionName, target.title, target.url].filter((part): part is string => typeof part === "string" && part.length > 0).join(" — ");
529
+ }
530
+
531
+ export function buildQaAttachedRecoveryNextActions(sessionName: string | undefined): AgentBrowserNextAction[] {
532
+ const sessionArgs = (args: string[]) => withOptionalSessionArgs(sessionName, args);
533
+ return [
534
+ buildNextToolAction({
535
+ args: sessionArgs(["tab", "list"]),
536
+ id: "list-tabs-before-qa-attached",
537
+ reason: "Inspect the connected session tabs before retrying qa.attached.",
538
+ safety: "Read-only tab listing for the attached session.",
539
+ }),
540
+ buildNextToolAction({
541
+ args: sessionArgs(["snapshot", "-i"]),
542
+ id: "snapshot-before-qa-attached",
543
+ reason: "Capture interactive refs on the active http(s) page before retrying qa.attached.",
544
+ safety: "Read-only snapshot; confirms a renderable page is selected.",
545
+ }),
546
+ ];
547
+ }
548
+
549
+ export async function validateQaAttachedPrecondition(options: {
550
+ cwd: string;
551
+ sessionName?: string;
552
+ signal?: AbortSignal;
553
+ }): Promise<QaAttachedPreconditionFailure | undefined> {
554
+ if (!options.sessionName) {
555
+ return {
556
+ error: "qa.attached requires an active attached session with a resolvable session name.",
557
+ nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
558
+ };
559
+ }
560
+ const urlProbe = await collectElectronManagedSessionUrl({ cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
561
+ if (urlProbe.error) {
562
+ return {
563
+ error: `qa.attached could not read the attached session URL: ${urlProbe.error}. Run tab list or snapshot -i before retrying qa.attached.`,
564
+ nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
565
+ };
566
+ }
567
+ const url = urlProbe.url?.trim();
568
+ if (!url) {
569
+ return {
570
+ error: "qa.attached requires an attached session with a readable http(s) page URL. Run tab list, select a stable tab, then snapshot -i before retrying.",
571
+ nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
572
+ };
573
+ }
574
+ if (!isHttpOrHttpsUrl(url)) {
575
+ return {
576
+ error: `qa.attached requires an http(s) page URL; the current attached URL is "${url}". Use tab list and snapshot -i to recover a web surface before retrying.`,
577
+ nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
578
+ };
579
+ }
580
+ return undefined;
581
+ }
582
+
583
+ function getTopLevelFillInvocation(commandTokens: string[]): { expected: string; selector: string } | undefined {
584
+ if (commandTokens[0] !== "fill" || commandTokens.length < 3) return undefined;
585
+ const selector = commandTokens[1];
586
+ const expected = commandTokens.slice(2).join(" ");
587
+ return selector && expected.length > 0 ? { expected, selector } : undefined;
588
+ }
589
+
590
+ export function buildFillVerificationNextActions(diagnostic: FillVerificationDiagnostic, sessionName: string | undefined): AgentBrowserNextAction[] {
591
+ return [
592
+ { id: "inspect-after-fill-verification", params: { args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]) }, reason: "Refresh the UI after a fill that reported success but did not appear to update the input value.", safety: "Read-only snapshot; use current refs before retrying.", tool: "agent_browser" },
593
+ { id: "verify-filled-value", params: { args: withOptionalSessionArgs(sessionName, ["get", "value", diagnostic.selector]) }, reason: "Check the target input value directly before submitting or creating files.", safety: "Read-only value check; selector may still be stale if the Electron UI rerendered.", tool: "agent_browser" },
594
+ ];
595
+ }
596
+
597
+ function extractFillVerificationValue(data: unknown): string | undefined {
598
+ if (typeof data === "string") return data;
599
+ if (!isRecord(data)) return undefined;
600
+ if (typeof data.value === "string") return data.value;
601
+ if (typeof data.result === "string") return data.result;
602
+ return undefined;
603
+ }
604
+
605
+ export async function collectFillVerificationDiagnostic(options: { commandTokens: string[]; cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<FillVerificationDiagnostic | undefined> {
606
+ const fill = getTopLevelFillInvocation(options.commandTokens);
607
+ if (!fill || !options.sessionName) return undefined;
608
+ let valueData: unknown | undefined;
609
+ try { valueData = await runSessionCommandData({ args: ["get", "value", fill.selector], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, timeoutMs: ELECTRON_FILL_VERIFICATION_TIMEOUT_MS }); } catch { return undefined; }
610
+ const actual = extractFillVerificationValue(valueData);
611
+ if (actual === undefined || actual === fill.expected) return undefined;
612
+ const diagnostic: FillVerificationDiagnostic = { actual: actual.length > 0 ? boundElectronProbeString(actual, 160) : "", expected: boundElectronProbeString(fill.expected, 160) ?? fill.expected, nextActionIds: [], selector: fill.selector, status: "mismatch", summary: `Fill verification warning: fill ${fill.selector} reported success, but get value returned ${actual.length > 0 ? `"${boundElectronProbeString(actual, 80)}"` : "an empty value"}.` };
613
+ diagnostic.nextActionIds = buildFillVerificationNextActions(diagnostic, options.sessionName).map((action) => action.id);
614
+ return diagnostic;
615
+ }
616
+
617
+ export function formatFillVerificationText(diagnostic: FillVerificationDiagnostic | undefined): string | undefined {
618
+ if (!diagnostic) return undefined;
619
+ const actual = diagnostic.actual !== undefined ? `actual "${diagnostic.actual}"` : "actual value unavailable";
620
+ return `${diagnostic.summary}\nExpected: "${diagnostic.expected}"; ${actual}.\nNext: re-run snapshot -i, then prefer click/focus plus keyboard type for custom Electron quick-input controls before submitting.`;
621
+ }
622
+
623
+ export async function collectVisibleRefFallbackDiagnostic(options: { commandTokens: string[]; compiledSemanticAction?: CompiledAgentBrowserSemanticAction; cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<VisibleRefFallbackDiagnostic | undefined> {
624
+ if (!options.sessionName) return undefined;
625
+ const target = getVisibleRefFallbackTarget({ commandTokens: options.commandTokens, compiledSemanticAction: options.compiledSemanticAction });
626
+ if (!target) return undefined;
627
+ const snapshotData = await runSessionCommandData({ args: ["snapshot", "-i"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
628
+ return buildVisibleRefFallbackDiagnosticFromSnapshot({ snapshotData, target });
629
+ }
630
+
631
+ export async function collectElectronHandoff(options: { cwd: string; handoff: "connect" | "snapshot" | "tabs"; sessionName?: string; signal?: AbortSignal }): Promise<ElectronHandoffSummary> {
632
+ if (options.handoff === "connect") return { handoff: "connect" };
633
+ const tabs = await runSessionCommandData({ args: ["tab", "list"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
634
+ if (options.handoff === "tabs") return { handoff: "tabs", tabs };
635
+ let snapshot = await runSessionCommandData({ args: ["snapshot", "-i"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
636
+ let refSnapshot = extractRefSnapshotFromData(snapshot);
637
+ let snapshotRetryCount = 0;
638
+ while ((!refSnapshot || refSnapshot.refIds.length === 0) && snapshotRetryCount < 2) {
639
+ snapshotRetryCount += 1;
640
+ await sleepMs(250);
641
+ snapshot = await runSessionCommandData({ args: ["snapshot", "-i"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
642
+ refSnapshot = extractRefSnapshotFromData(snapshot);
643
+ }
644
+ return { handoff: "snapshot", refSnapshot, snapshot, ...(snapshotRetryCount > 0 ? { snapshotRetryCount } : {}), tabs };
645
+ }
646
+
647
+ function getTimeoutProgressSteps(compiledJob: CompiledAgentBrowserJob | undefined, command: string | undefined, stdin: string | undefined): Array<{ args: string[]; index: number }> {
648
+ if (compiledJob) return compiledJob.steps.map((step, index) => ({ args: step.args, index: index + 1 }));
649
+ if (command !== "batch" || !stdin) return [];
650
+ try {
651
+ const parsed = JSON.parse(stdin) as unknown;
652
+ if (!Array.isArray(parsed)) return [];
653
+ return parsed.flatMap((step, index) => Array.isArray(step) && step.every((token) => typeof token === "string") ? [{ args: step as string[], index: index + 1 }] : []);
654
+ } catch { return []; }
655
+ }
656
+
657
+ function getLastPositionalToken(args: string[], startIndex = 1): string | undefined {
658
+ for (let index = args.length - 1; index >= startIndex; index -= 1) {
659
+ const token = args[index];
660
+ if (token && !token.startsWith("-")) return token;
661
+ }
662
+ return undefined;
663
+ }
664
+
665
+ function getTimeoutStepArtifactPath(args: string[]): string | undefined {
666
+ const [command] = args;
667
+ if (command === "screenshot") {
668
+ const index = getScreenshotPathTokenIndex(args);
669
+ return index === undefined ? undefined : args[index];
670
+ }
671
+ if (command === "pdf") return getLastPositionalToken(args);
672
+ if (command === "download") return getLastPositionalToken(args, 2);
673
+ if (command === "wait") {
674
+ const inlineDownload = args.find((token) => token.startsWith("--download="));
675
+ if (inlineDownload) return inlineDownload.slice("--download=".length) || undefined;
676
+ const downloadIndex = args.indexOf("--download");
677
+ const downloadPath = downloadIndex >= 0 ? args[downloadIndex + 1] : undefined;
678
+ if (downloadPath && !downloadPath.startsWith("-")) return downloadPath;
679
+ }
680
+ return undefined;
681
+ }
682
+
683
+ async function statTimeoutArtifactPath(absolutePath: string): Promise<{ exists: false } | { exists: true; sizeBytes: number }> {
684
+ for (let attempt = 0; attempt < 3; attempt += 1) {
685
+ try {
686
+ const stats = await stat(absolutePath);
687
+ return { exists: true, sizeBytes: stats.size };
688
+ } catch {
689
+ if (attempt < 2) await sleepMs(25);
690
+ }
691
+ }
692
+ return { exists: false };
693
+ }
694
+
695
+ async function collectTimeoutArtifactEvidence(cwd: string, steps: Array<{ args: string[]; index: number }>): Promise<TimeoutArtifactEvidence[]> {
696
+ const evidence: TimeoutArtifactEvidence[] = [];
697
+ for (const step of steps) {
698
+ const path = getTimeoutStepArtifactPath(step.args);
699
+ if (!path) continue;
700
+ const absolutePath = isAbsolute(path) ? path : resolve(cwd, path);
701
+ const artifact = await statTimeoutArtifactPath(absolutePath);
702
+ evidence.push(artifact.exists
703
+ ? { absolutePath, exists: true, path, sizeBytes: artifact.sizeBytes, stepIndex: step.index }
704
+ : { absolutePath, exists: false, path, stepIndex: step.index });
705
+ }
706
+ return evidence;
707
+ }
708
+
709
+ function getPlannedCurrentPageUrl(steps: Array<{ args: string[]; index: number }>): string | undefined {
710
+ for (let index = steps.length - 1; index >= 0; index -= 1) {
711
+ const args = steps[index]?.args ?? [];
712
+ if (args[0] === "open" || args[0] === "navigate" || args[0] === "pushstate") return getLastPositionalToken(args);
713
+ }
714
+ return undefined;
715
+ }
716
+
717
+ export async function collectTimeoutPartialProgress(options: { command?: string; compiledJob?: CompiledAgentBrowserJob; cwd: string; sessionName?: string; stdin?: string }): Promise<TimeoutPartialProgress | undefined> {
718
+ const steps = getTimeoutProgressSteps(options.compiledJob, options.command, options.stdin);
719
+ const artifacts = await collectTimeoutArtifactEvidence(options.cwd, steps);
720
+ const [urlData, titleData] = await Promise.all([runSessionCommandData({ args: ["get", "url"], cwd: options.cwd, sessionName: options.sessionName }), runSessionCommandData({ args: ["get", "title"], cwd: options.cwd, sessionName: options.sessionName })]);
721
+ const recoveredUrl = extractStringResultField(urlData, "result") ?? extractStringResultField(urlData, "url");
722
+ const title = extractStringResultField(titleData, "result") ?? extractStringResultField(titleData, "title");
723
+ const plannedUrl = recoveredUrl ? undefined : getPlannedCurrentPageUrl(steps);
724
+ const url = recoveredUrl ?? plannedUrl;
725
+ if (steps.length === 0 && artifacts.length === 0 && !url && !title) return undefined;
726
+ const foundArtifacts = artifacts.filter((artifact) => artifact.exists).length;
727
+ const pageStateSummary = recoveredUrl || title ? " and current page state" : plannedUrl ? " and planned page URL" : "";
728
+ return { artifacts, currentPage: url || title ? { title, url } : undefined, steps: steps.length > 0 ? steps : undefined, summary: `Timed out before upstream returned final results; recovered ${foundArtifacts}/${artifacts.length} declared artifact path${artifacts.length === 1 ? "" : "s"}${pageStateSummary}.` };
729
+ }
730
+
731
+ function redactSensitivePathSegmentsForDiagnostic(path: string): string {
732
+ return path.split(/([/\\]+)/).map((segment) => segment === "/" || segment === "\\" || /^[/\\]+$/.test(segment) ? segment : redactSensitiveText(segment) !== segment || /(?:secret|token|password|passwd|credential|auth|api[-_]?key|bearer)/i.test(segment) ? "[REDACTED]" : segment).join("");
733
+ }
734
+
735
+ function sanitizeCurrentPageUrlForTimeoutDiagnostic(url: string): string {
736
+ try {
737
+ const parsedUrl = new URL(url);
738
+ parsedUrl.pathname = parsedUrl.pathname.split("/").map((segment) => redactSensitivePathSegmentsForDiagnostic(segment)).join("/");
739
+ for (const [key, value] of parsedUrl.searchParams.entries()) {
740
+ if (redactSensitiveText(key) !== key || redactSensitiveText(value) !== value || /(?:secret|token|password|passwd|credential|auth|api[-_]?key|bearer)/i.test(`${key} ${value}`)) parsedUrl.searchParams.set(key, "[REDACTED]");
741
+ }
742
+ if (parsedUrl.hash) parsedUrl.hash = redactSensitivePathSegmentsForDiagnostic(redactSensitiveText(parsedUrl.hash));
743
+ return redactSensitiveText(parsedUrl.toString());
744
+ } catch {
745
+ return redactSensitivePathSegmentsForDiagnostic(redactSensitiveText(url));
746
+ }
747
+ }
748
+
749
+ export function formatTimeoutPartialProgressText(progress: TimeoutPartialProgress): string {
750
+ const lines = [`Timeout partial progress: ${progress.summary}`];
751
+ const currentPageTitle = progress.currentPage?.title ? redactSensitivePathSegmentsForDiagnostic(redactSensitiveText(progress.currentPage.title)) : undefined;
752
+ const currentPageUrl = progress.currentPage?.url ? sanitizeCurrentPageUrlForTimeoutDiagnostic(progress.currentPage.url) : undefined;
753
+ if (currentPageTitle || currentPageUrl) lines.push(`Current page: ${[currentPageTitle, currentPageUrl].filter(Boolean).join(" — ")}`);
754
+ if (progress.steps && progress.steps.length > 0) {
755
+ const shownSteps = progress.steps.slice(0, 6);
756
+ lines.push("Planned steps:");
757
+ for (const step of shownSteps) lines.push(`- Step ${step.index}: ${redactSensitivePathSegmentsForDiagnostic(redactInvocationArgs(step.args).join(" "))}`);
758
+ if (progress.steps.length > shownSteps.length) lines.push(`- ... ${progress.steps.length - shownSteps.length} more step${progress.steps.length - shownSteps.length === 1 ? "" : "s"} omitted`);
759
+ }
760
+ for (const artifact of progress.artifacts) lines.push(`Artifact from step ${artifact.stepIndex}: ${redactSensitivePathSegmentsForDiagnostic(artifact.path)} (${artifact.exists ? `exists${typeof artifact.sizeBytes === "number" ? `, ${artifact.sizeBytes} bytes` : ""}` : "missing"})`);
761
+ return lines.join("\n");
762
+ }