pi-agent-browser-native 0.6.17 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/README.md +55 -167
  3. package/dist/extensions/agent-browser/index.js +312 -223
  4. package/dist/extensions/agent-browser/lib/browser-transcript.js +33 -0
  5. package/dist/extensions/agent-browser/lib/execution-cwd.js +4 -2
  6. package/dist/extensions/agent-browser/lib/input-modes/job.js +2 -213
  7. package/dist/extensions/agent-browser/lib/input-modes/params.js +146 -142
  8. package/dist/extensions/agent-browser/lib/input-modes/script.js +64 -92
  9. package/dist/extensions/agent-browser/lib/managed-session-policy-lock.js +170 -24
  10. package/dist/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.js +13 -1
  11. package/dist/extensions/agent-browser/lib/orchestration/browser-run/final-result.js +26 -13
  12. package/dist/extensions/agent-browser/lib/orchestration/browser-run/index.js +21 -1
  13. package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/snapshot-filter.js +4 -2
  14. package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare.js +1 -3
  15. package/dist/extensions/agent-browser/lib/orchestration/browser-run/process-output.js +35 -20
  16. package/dist/extensions/agent-browser/lib/orchestration/browser-run/recording-recovery.js +3 -1
  17. package/dist/extensions/agent-browser/lib/orchestration/browser-run/screenshot-observation.js +79 -0
  18. package/dist/extensions/agent-browser/lib/orchestration/electron-host/index.js +3 -4
  19. package/dist/extensions/agent-browser/lib/orchestration/input-plan.js +19 -52
  20. package/dist/extensions/agent-browser/lib/orchestration/native-session-defaults.js +3 -3
  21. package/dist/extensions/agent-browser/lib/orchestration/output-file.js +3 -3
  22. package/dist/extensions/agent-browser/lib/orchestration/script-mode.js +91 -278
  23. package/dist/extensions/agent-browser/lib/pi-tool-rendering.js +9 -9
  24. package/dist/extensions/agent-browser/lib/playbook.js +46 -25
  25. package/dist/extensions/agent-browser/lib/process-identity.js +22 -6
  26. package/dist/extensions/agent-browser/lib/process.js +16 -13
  27. package/dist/extensions/agent-browser/lib/results/action-recommendations.js +2 -2
  28. package/dist/extensions/agent-browser/lib/results/next-actions.js +1 -2
  29. package/dist/extensions/agent-browser/lib/results/presentation/artifacts.js +25 -4
  30. package/dist/extensions/agent-browser/lib/results/presentation/batch.js +6 -2
  31. package/dist/extensions/agent-browser/lib/results/presentation/common.js +2 -0
  32. package/dist/extensions/agent-browser/lib/results/presentation/content.js +39 -0
  33. package/dist/extensions/agent-browser/lib/results/presentation/diagnostics.js +3 -3
  34. package/dist/extensions/agent-browser/lib/results/presentation/large-output.js +41 -1
  35. package/dist/extensions/agent-browser/lib/results/presentation.js +16 -11
  36. package/dist/extensions/agent-browser/lib/runtime.js +18 -6
  37. package/dist/extensions/agent-browser/lib/session-page-state.js +3 -4
  38. package/dist/extensions/agent-browser/lib/tool-surface.js +138 -0
  39. package/dist/extensions/agent-browser/script-worker.js +4 -1
  40. package/docs/ARCHITECTURE.md +46 -62
  41. package/docs/COMMAND_REFERENCE.md +77 -136
  42. package/docs/ELECTRON.md +51 -49
  43. package/docs/RELEASE.md +35 -21
  44. package/docs/REQUIREMENTS.md +18 -14
  45. package/docs/SUPPORT_MATRIX.md +33 -15
  46. package/docs/TOOL_CONTRACT.md +216 -405
  47. package/docs/platform-smoke.md +2 -2
  48. package/package.json +1 -1
  49. package/scripts/doctor.mjs +2 -2
@@ -4,6 +4,7 @@ import { redactNetworkSourceLookupSurface } from "../../input-modes/lookups.js";
4
4
  import { buildAgentBrowserNextActions } from "../../results/action-recommendations.js";
5
5
  import { buildAgentBrowserResultCategoryDetails } from "../../results/categories.js";
6
6
  import { extractAgentBrowserLifecycle } from "../../results/presentation/common.js";
7
+ import { projectAgentBrowserObservation } from "../../results/presentation/content.js";
7
8
  import { formatSessionArtifactRetentionSummary } from "../../results/artifact-manifest.js";
8
9
  import { alignPageChangeSummaryNextActionIds, appendUniqueAgentBrowserNextActions, applyNamespaceToNextActions, isStandaloneSnapshotNextAction, withOptionalSessionArgs, } from "../../results/next-actions.js";
9
10
  import { buildConnectedSessionNextActions, buildNoActivePageNextActions, buildPendingWebMcpNextActions, buildSessionAwareStaleRefNextActions, buildSessionTabRecoveryNextActions, } from "../../results/recovery-next-actions.js";
@@ -86,9 +87,7 @@ export function redactRecoveryHint(recoveryHint) {
86
87
  }
87
88
  export function buildJsonVisibleContent(options) {
88
89
  const { error, presentation, succeeded, warnings } = options;
89
- const payload = redactSensitiveValue({ artifacts: presentation.artifacts, data: presentation.data, error, recordingRecovery: presentation.recordingRecovery, readConfirmation: presentation.readConfirmation, success: succeeded, warnings: warnings && warnings.length > 0 ? warnings : undefined });
90
- if (isRecord(payload) && isRecord(payload.data) && isRecord(presentation.data) && typeof presentation.data.wsUrl === "string")
91
- payload.data.wsUrl = presentation.data.wsUrl;
90
+ const payload = projectAgentBrowserObservation({ ...presentation, ...options.details, error, warnings: warnings && warnings.length > 0 ? warnings : undefined }, succeeded);
92
91
  const images = presentation.content.filter((item) => item.type === "image");
93
92
  return [{ type: "text", text: JSON.stringify(payload, null, 2) }, ...images];
94
93
  }
@@ -175,8 +174,9 @@ export function buildRedactedPresentationContent(options) {
175
174
  return contentWithSessionWarnings.map((item) => {
176
175
  if (item.type !== "text")
177
176
  return item;
178
- const exactRedactedText = redactExactSensitiveText(item.text, exactSensitiveValues);
179
- return userRequestedJson && !plainTextInspection ? { ...item, text: exactRedactedText } : { ...item, text: redactSensitiveText(exactRedactedText) };
177
+ if (userRequestedJson && !plainTextInspection)
178
+ return { ...item, text: JSON.stringify(redactExactSensitiveValue(JSON.parse(item.text), exactSensitiveValues), null, 2) };
179
+ return { ...item, text: redactSensitiveText(redactExactSensitiveText(item.text, exactSensitiveValues)) };
180
180
  });
181
181
  }
182
182
  export async function prepareFinalResultRecoveryState(options) {
@@ -339,6 +339,10 @@ function buildResultNextActions(options) {
339
339
  append(buildComboboxFocusNextActions(options.executionPlan.sessionName));
340
340
  if (options.managedSessionOutcome)
341
341
  appendUnique(buildManagedSessionFreshFailureNextActions(options.managedSessionOutcome));
342
+ if (options.currentSessionTabTargetUnknown && ["aborted", "parse-failure"].includes(options.categoryDetails.failureCategory ?? "")) {
343
+ nextActions = nextActions.filter(action => !isStandaloneSnapshotNextAction(action));
344
+ appendUnique([{ id: "verify-page-target-after-interruption", tool: "agent_browser", params: { args: withOptionalSessionArgs(options.executionPlan.sessionName, ["batch", "--bail"]), stdin: JSON.stringify([["get", "url"], ["snapshot", "-i"]]) }, reason: "Verify the current URL and inspect the page after an interrupted operation.", safety: "Read-only recovery. The mutation may already have happened; inspect before deciding whether to retry." }]);
345
+ }
342
346
  if (options.categoryDetails.failureCategory === "timeout" && options.processResult.timedOut) {
343
347
  if (options.currentSessionTabTargetUnknown)
344
348
  nextActions = nextActions.filter((action) => !isStandaloneSnapshotNextAction(action));
@@ -354,14 +358,11 @@ function buildResultNextActions(options) {
354
358
  export function formatAgentBrowserNextActionsText(nextActions) {
355
359
  if (!nextActions || nextActions.length === 0)
356
360
  return undefined;
357
- const lines = nextActions.slice(0, 6).map((action) => {
358
- const params = action.params
359
- ? { ...action.params, ...(action.params.stdin !== undefined && action.params.stdin.length > 500 ? { stdin: "[omitted; use details.nextActions]" } : {}) }
360
- : undefined;
361
- const payload = action.artifactPath ? { artifactPath: action.artifactPath } : params;
361
+ const lines = nextActions.map((action) => {
362
+ const payload = action.artifactPath ? { artifactPath: action.artifactPath } : action.params;
362
363
  return `- ${action.id}${payload ? ` ${redactSensitiveText(JSON.stringify(payload))}` : ""}: ${redactSensitiveText(action.reason)}`;
363
364
  });
364
- return ["Next actions:", ...lines, "The same redacted payloads are available in details.nextActions."].join("\n");
365
+ return ["Next actions:", ...lines].join("\n");
365
366
  }
366
367
  function formatFailureNextActionsText(options, nextActions) {
367
368
  return options.categoryDetails.resultCategory === "failure" ? formatAgentBrowserNextActionsText(nextActions) : undefined;
@@ -438,6 +439,7 @@ function buildAgentBrowserResultDetails(options, nextActions) {
438
439
  managedSessionOutcome: options.managedSessionOutcome,
439
440
  imagePath: options.presentation.imagePath,
440
441
  imagePaths: options.presentation.imagePaths,
442
+ imageObservations: options.presentation.imageObservations,
441
443
  nextActions,
442
444
  pageChangeSummary,
443
445
  clickDispatch: options.clickDispatchDiagnostic,
@@ -485,6 +487,10 @@ export function buildFinalAgentBrowserToolResult(options) {
485
487
  const lifecycle = extractAgentBrowserLifecycle(options.presentationEnvelope?.data);
486
488
  const browserWindow = buildBrowserWindowStatus(options, lifecycle);
487
489
  const details = buildAgentBrowserResultDetails(options, nextActions);
490
+ if (options.modelVisible === false) {
491
+ const result = { content: [], details: redactToolDetails(details, options.exactSensitiveValues), isError: !options.succeeded };
492
+ return options.compiledNetworkSourceLookup ? redactNetworkSourceLookupSurface(result) : result;
493
+ }
488
494
  const visibleRefFallbackText = formatVisibleRefFallbackText(options.visibleRefFallbackDiagnostic);
489
495
  const richInputRecoveryText = formatRichInputRecoveryText(options.richInputRecoveryDiagnostic);
490
496
  const semanticActionCandidateText = nextActions ? formatSemanticActionCandidateText(nextActions) : undefined;
@@ -509,10 +515,17 @@ export function buildFinalAgentBrowserToolResult(options) {
509
515
  const appendedDiagnosticText = redactSensitiveText(redactExactSensitiveText(rawAppendedDiagnosticText, options.exactSensitiveValues));
510
516
  const shouldAppendDiagnosticText = appendedDiagnosticText.length > 0 && (!options.userRequestedJson || options.plainTextInspection);
511
517
  let content = shouldAppendDiagnosticText && options.redactedContent[0]?.type === "text" ? [{ ...options.redactedContent[0], text: `${options.redactedContent[0].text}\n\n${appendedDiagnosticText}` }, ...options.redactedContent.slice(1)] : options.redactedContent;
512
- if (options.electronLaunchRecord && options.succeeded && content[0]?.type === "text") {
518
+ if (options.electronLaunchRecord && options.succeeded && !options.userRequestedJson && content[0]?.type === "text") {
513
519
  content = [{ ...content[0], text: redactSensitiveText(formatElectronLaunchText({ handoff: options.electronHandoff, record: options.electronLaunchRecord, targets: options.electronLaunch?.targets ?? [], upstreamText: content[0].text })) }, ...content.slice(1)];
514
520
  }
515
- const result = { content, details: redactToolDetails(details, options.exactSensitiveValues), isError: !options.succeeded };
521
+ const redactedDetails = redactToolDetails(details, options.exactSensitiveValues);
522
+ if (options.userRequestedJson && !options.plainTextInspection) {
523
+ const prior = options.redactedContent[0]?.type === "text" ? JSON.parse(options.redactedContent[0].text) : undefined;
524
+ if (Array.isArray(prior?.warnings))
525
+ redactedDetails.warnings = prior.warnings;
526
+ content = buildJsonVisibleContent({ error: redactedDetails.error, details: redactedDetails, presentation: options.presentation, succeeded: options.succeeded, warnings: prior?.warnings });
527
+ }
528
+ const result = { content, details: redactedDetails, isError: !options.succeeded };
516
529
  return options.compiledNetworkSourceLookup ? redactNetworkSourceLookupSurface(result) : result;
517
530
  }
518
531
  export function isMissingAgentBrowserBinary(processResult) {
@@ -6,6 +6,7 @@ import { formatWebMcpCatalogUpdate } from "../../results/presentation/common.js"
6
6
  import { withOwnedManagedSessionContext } from "../../managed-session-restore.js";
7
7
  import { cleanupClickDispatchProbe } from "./click-dispatch.js";
8
8
  import { applyBrowserRunStatePatch, getSessionContextKey } from "./session-state.js";
9
+ import { buildScreenshotGeometry, collectScreenshotSample } from "./screenshot-observation.js";
9
10
  import { buildJsonVisibleContent, buildMissingBinaryFailureResult } from "./final-result.js";
10
11
  import { prepareBrowserRun } from "./prepare.js";
11
12
  import { processBrowserOutput } from "./process-output.js";
@@ -27,6 +28,12 @@ export async function runAgentBrowserTool(options) {
27
28
  result.content.push({ type: "text", text: notice });
28
29
  }
29
30
  }
31
+ if (result.isError && details?.error === undefined) {
32
+ details = { ...details, error: details?.validationError ?? details?.summary ?? result.content.filter(part => part.type === "text").map(part => part.text).join("\n") };
33
+ result.details = details;
34
+ }
35
+ if (options.modelVisible === false)
36
+ result.content = [];
30
37
  const sessionKey = getSessionContextKey(typeof details?.sessionName === "string" ? details.sessionName : undefined, typeof details?.namespace === "string" ? details.namespace : undefined);
31
38
  const page = options.state.sessionPageState.get(sessionKey);
32
39
  return page.tabReopenPending === undefined ? result : {
@@ -39,11 +46,12 @@ async function runAgentBrowserToolInContext(options) {
39
46
  applyBrowserRunStatePatch(options.state, preparedResult.kind === "ready" ? preparedResult.prepared.statePatch : preparedResult.statePatch);
40
47
  if (preparedResult.kind === "early-result") {
41
48
  const result = preparedResult.result;
42
- if (options.input.toolArgs.includes("--json") && !isPlainTextInspectionArgs(options.input.toolArgs)) {
49
+ if (options.modelVisible !== false && options.input.toolArgs.includes("--json") && !isPlainTextInspectionArgs(options.input.toolArgs)) {
43
50
  const details = isRecord(result.details) ? result.details : {};
44
51
  const summary = result.content.filter(item => item.type === "text").map(item => item.text).join("\n");
45
52
  result.content = buildJsonVisibleContent({
46
53
  error: result.isError ? details.validationError ?? summary : null,
54
+ details,
47
55
  presentation: { content: result.content, data: details.data, summary },
48
56
  succeeded: result.isError !== true,
49
57
  });
@@ -54,6 +62,11 @@ async function runAgentBrowserToolInContext(options) {
54
62
  const ownedManagedSession = prepared.ownedManagedSessionContext;
55
63
  return await withOwnedManagedSessionContext(ownedManagedSession, async () => {
56
64
  try {
65
+ const geometryOptions = { command: prepared.commandTokens, cwd: options.cwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, signal: options.signal,
66
+ env: ownedManagedSession ? { AGENT_BROWSER_IDLE_TIMEOUT_MS: options.implicitSessionIdleTimeoutMs } : undefined };
67
+ // Do not start a cold browser through a probe or consume launch flags ahead of the requested capture.
68
+ const probeGeometry = prepared.commandTokens[0] === "screenshot" && !!prepared.priorSessionTabTarget && !prepared.priorSessionTabTargetUnknown && !prepared.executionPlan.startupScopedFlags?.length;
69
+ const before = probeGeometry ? await withChromeStartupArgs(prepared.chromeStartupArgs, () => collectScreenshotSample(geometryOptions)) : undefined;
57
70
  const artifactRunStartedAtMs = Date.now();
58
71
  const processResult = await withChromeStartupArgs(prepared.chromeStartupArgs, () => runAgentBrowserProcess({
59
72
  args: prepared.processArgs,
@@ -70,6 +83,8 @@ async function runAgentBrowserToolInContext(options) {
70
83
  stdin: prepared.processStdin,
71
84
  timeoutMs: prepared.processTimeoutMs,
72
85
  }));
86
+ const after = probeGeometry && !processResult.aborted && !processResult.timedOut && processResult.exitCode === 0
87
+ ? await withChromeStartupArgs(prepared.chromeStartupArgs, () => collectScreenshotSample(geometryOptions)) : undefined;
73
88
  const missingBinaryResult = await buildMissingBinaryFailureResult({
74
89
  compatibilityWorkaround: prepared.compatibilityWorkaround,
75
90
  electronLaunch: prepared.electronLaunch,
@@ -88,6 +103,11 @@ async function runAgentBrowserToolInContext(options) {
88
103
  return missingBinaryResult;
89
104
  const output = await processBrowserOutput({ ...options, artifactRunStartedAtMs, prepared, processResult });
90
105
  applyBrowserRunStatePatch(options.state, output.statePatch);
106
+ if (isRecord(output.result.details) && Array.isArray(output.result.details.imageObservations) && prepared.commandTokens[0] === "screenshot") {
107
+ output.result.details.imageObservations = redactSensitiveValue(output.result.details.imageObservations.map(image => ({
108
+ ...image, geometry: buildScreenshotGeometry({ capture: image.capture, pixels: image.pixels, before, after }),
109
+ })));
110
+ }
91
111
  return output.result;
92
112
  }
93
113
  finally {
@@ -2,7 +2,7 @@ import { isRecord } from "../../../parsing.js";
2
2
  import { buildAgentBrowserResultCategoryDetails } from "../../../results/categories.js";
3
3
  import { buildSnapshotPresentation } from "../../../results/snapshot.js";
4
4
  import { extractRefSnapshotFromData } from "../../../session-page-state.js";
5
- import { redactSensitiveText } from "../../../runtime.js";
5
+ import { redactSensitiveText, redactSensitiveValue } from "../../../runtime.js";
6
6
  import { collectScrollPositionSnapshot } from "../diagnostics.js";
7
7
  import { buildSessionDetailFields, runSessionCommandData } from "../session-state.js";
8
8
  function parseSnapshotFilterRequest(commandTokens) {
@@ -237,7 +237,9 @@ export async function trySnapshotFilter(options) {
237
237
  const diff = request.diff ? buildSnapshotDiff(options.previousRefSnapshot, fullSnapshot) : undefined;
238
238
  if (fullSnapshot)
239
239
  options.sessionPageState.applyRefSnapshot({ sessionName: options.sessionStateKey ?? options.sessionName, snapshot: fullSnapshot, update: options.sessionPageStateUpdate });
240
- const presentation = await buildSnapshotPresentation(filtered.data, options.persistentArtifactStore, options.artifactManifest);
240
+ const presentation = options.modelVisible === false
241
+ ? { content: [], data: redactSensitiveValue(filtered.data), summary: "Snapshot" }
242
+ : await buildSnapshotPresentation(filtered.data, options.persistentArtifactStore, options.artifactManifest);
241
243
  const summary = request.role || request.search
242
244
  ? `Snapshot filter: ${filtered.matchedRefs}/${filtered.totalRefs} direct refs matched${request.role ? ` role=${request.role}` : ""}${request.search ? ` search ${JSON.stringify(request.search)}` : ""}; ${filtered.visibleLines} surrounding snapshot line${filtered.visibleLines === 1 ? "" : "s"} shown.`
243
245
  : request.diff
@@ -36,8 +36,6 @@ export function normalizeRunInput(input) {
36
36
  switch (input.kind) {
37
37
  case "electron":
38
38
  return { ...base, compiledElectron: input.compiledElectron, redactedCompiledElectron: input.redactedCompiledElectron };
39
- case "job":
40
- return { ...base, compiledJob: input.compiledJob, redactedCompiledJob: input.redactedCompiledJob };
41
39
  case "networkSourceLookup":
42
40
  return { ...base, compiledNetworkSourceLookup: input.compiledNetworkSourceLookup, redactedCompiledNetworkSourceLookup: input.redactedCompiledNetworkSourceLookup };
43
41
  case "qa":
@@ -46,7 +44,6 @@ export function normalizeRunInput(input) {
46
44
  return { ...base, compiledSemanticAction: input.compiledSemanticAction, redactedCompiledSemanticAction: input.redactedCompiledSemanticAction };
47
45
  case "sourceLookup":
48
46
  return { ...base, compiledSourceLookup: input.compiledSourceLookup, redactedCompiledSourceLookup: input.redactedCompiledSourceLookup };
49
- case "script":
50
47
  case "args":
51
48
  return base;
52
49
  }
@@ -937,6 +934,7 @@ export async function prepareBrowserRun(options) {
937
934
  }
938
935
  const persistentArtifactStore = getPersistentSessionArtifactStore(options.ctx);
939
936
  const snapshotFilter = await trySnapshotFilter({
937
+ modelVisible: options.modelVisible,
940
938
  artifactManifest: state.artifactManifest,
941
939
  commandTokens,
942
940
  compatibilityWorkaround,
@@ -4,7 +4,7 @@ import { parseArgvDescriptor } from "../../argv-descriptor.js";
4
4
  import { isBrowserIndependentRead, needsManagedSession } from "../../command-policy.js";
5
5
  import { deleteIdentityKeysInNamespace, getAgentBrowserSessionIdentityKey, isAgentBrowserSessionIdentityKeyInNamespace } from "../../argv-grammar.js";
6
6
  import { batchHasSuccessfulCloseAll, getSuccessfulBatchCloseLifecycle } from "../../batch-lifecycle.js";
7
- import { isCloseAllCommand, isCloseCommand, isOpenNavigationCommand, isRecordPageTransitionCommand, isUnverifiedPageTransitionCommand, isWindowOrDiffPageTransitionCommand } from "../../command-taxonomy.js";
7
+ import { isCloseAllCommand, isCloseCommand, isOpenNavigationCommand, isPageMutationCommand, isRecordPageTransitionCommand, isUnverifiedPageTransitionCommand, isWindowOrDiffPageTransitionCommand } from "../../command-taxonomy.js";
8
8
  import { OPEN_RESULT_TAB_CORRECTION_FLAGS } from "../../launch-scoped-flags.js";
9
9
  import { cleanupElectronLaunchResources, inspectElectronLaunchStatus } from "../../electron/cleanup.js";
10
10
  import { getResultingPageTargetState, commandRequiresLivePageVerification } from "../../page-target-validation.js";
@@ -168,6 +168,7 @@ export async function processBrowserOutput(input) {
168
168
  observeNativeWebMcp(parsed.envelope?.data);
169
169
  let parseError = parsed.parseError;
170
170
  const recordingStopRecovery = await recoverRecordingStop({
171
+ modelVisible: input.modelVisible,
171
172
  artifactManifest, artifactRunStartedAtMs: input.artifactRunStartedAtMs, commandTokens: prepared.commandTokens, cwd,
172
173
  envelope: parsed.envelope, namespace: prepared.executionPlan.namespace, parseError, processResult,
173
174
  reservation: prepared.executionPlan.sessionName ? state.activeRecordingReservations?.get(getAgentBrowserSessionIdentityKey(prepared.executionPlan.sessionName, prepared.executionPlan.namespace)) : undefined,
@@ -241,6 +242,12 @@ export async function processBrowserOutput(input) {
241
242
  const envelopeSuccess = plainTextInspection ? true : presentationEnvelope?.success !== false;
242
243
  let succeeded = (processSucceeded && parseSucceeded && envelopeSuccess) || recordingStopRecovery?.recovery.healed === true;
243
244
  const inspectionText = plainTextInspection ? processResult.stdout.trim() : undefined;
245
+ const unobservedMutation = processResult.agentBrowserStarted && !plainTextInspection
246
+ && (processResult.aborted || processResult.timedOut || !parseSucceeded || presentationEnvelope === undefined)
247
+ && (prepared.executionPlan.commandInfo.command === "batch" ? batchCommandSteps : [prepared.commandTokens]).some(step => {
248
+ const action = step[0] === "find" ? step[step[1] === "nth" ? 4 : 3] ?? "click" : step[0];
249
+ return isPageMutationCommand(action, step[1]);
250
+ });
244
251
  const sessionStateKey = getSessionContextKey(prepared.executionPlan.sessionName, prepared.executionPlan.namespace);
245
252
  const closeAllApplied = nestedBatchClosesAll || (directCloseAllRequested && succeeded);
246
253
  if (sessionStateKey && processResult.agentBrowserStarted && sessionPageState.get(sessionStateKey).tabReopenPending === true) {
@@ -335,14 +342,14 @@ export async function processBrowserOutput(input) {
335
342
  const failedWebMcpSettlement = prepared.priorSessionTabTargetUnknown === true && ((!succeeded && isWebMcpSettlementCommand(prepared.executionPlan.commandInfo.command, prepared.executionPlan.commandInfo.subcommand))
336
343
  || (prepared.executionPlan.commandInfo.command === "batch" && batchHasFailedWebMcpSettlement(presentationEnvelope?.data)));
337
344
  const unsettledWebMcpMutation = pendingWebMcpMutation || failedWebMcpSettlement;
338
- const observedSessionTabTarget = unsettledWebMcpMutation
345
+ const observedSessionTabTarget = unsettledWebMcpMutation || (unobservedMutation && !failedTransitionReverification)
339
346
  ? undefined
340
347
  : normalizeSessionTabTarget(navigationSummary)
341
348
  ?? (trustsReportedPageTarget ? extractSessionTabTargetFromBatchResults(presentationEnvelope?.data) : undefined)
342
349
  ?? (succeeded && trustsReportedPageTarget ? extractSessionTabTargetFromCommandData(prepared.commandTokens, presentationEnvelope?.data) : undefined);
343
350
  const safeObservedSessionTabTarget = observedSessionTabTarget;
344
351
  let currentSessionTabTarget = safeObservedSessionTabTarget;
345
- if (!currentSessionTabTarget && nestedBatchClose === undefined) {
352
+ if (!currentSessionTabTarget && nestedBatchClose === undefined && !unobservedMutation) {
346
353
  // Window/diff responses do not report the final URL; URL2 is intent, not redirect evidence.
347
354
  currentSessionTabTarget = resultingPageState.pageTargetMayHaveChanged
348
355
  ? succeeded && !destinationTransition ? normalizeSessionTabTarget({ url: resultingPageState.currentPageUrl }) : undefined
@@ -397,7 +404,7 @@ export async function processBrowserOutput(input) {
397
404
  let selectorTextVisibilityDiagnostics = [];
398
405
  let electronBroadGetTextScopeDiagnostics = [];
399
406
  const timeoutPartialProgress = processResult.timedOut && !recordingStopRecovery && !prepared.readConfirmation ? await collectTimeoutPartialProgress({ commandTokens: prepared.commandTokens, compiledJob: prepared.compiledJob, cwd, operationCwd: input.operationCwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, stdin: prepared.runtimeToolStdin }) : undefined;
400
- if (!currentSessionTabTarget && timeoutPartialProgress?.currentPage?.source === "live") {
407
+ if (!currentSessionTabTarget && timeoutPartialProgress?.liveUrlRecovered === true) {
401
408
  currentSessionTabTarget = normalizeSessionTabTarget(timeoutPartialProgress.currentPage);
402
409
  }
403
410
  if (succeeded) {
@@ -445,24 +452,26 @@ export async function processBrowserOutput(input) {
445
452
  const plannedBatchTransitionInvalidation = !Array.isArray(presentationEnvelope?.data)
446
453
  ? batchCommandSteps.map(getCommandRefSnapshotInvalidation).find((invalidation) => invalidation !== undefined)
447
454
  : undefined;
448
- const pageTransitionInvalidation = unsettledWebMcpMutation
449
- ? buildPageTransitionRefSnapshotInvalidation("A detached WebMCP invocation is still pending or failed to settle and can mutate, rerender, or navigate the page, so prior snapshot refs remain invalid after URL verification. Run webmcp result or cancel, then take a fresh snapshot before using page-scoped refs.")
450
- : processResult.agentBrowserStarted && (directTransitionInvalidation || plannedBatchTransitionInvalidation)
451
- ? directTransitionInvalidation ?? plannedBatchTransitionInvalidation
452
- : failedTransitionReverification
453
- ? buildPageTransitionRefSnapshotInvalidation("A failed eval/back/forward/reload/connect/state-load/tab command may still have changed the page, so the prior snapshot refs were invalidated. Run snapshot -i before using page-scoped refs.")
454
- : batchRefSnapshotState?.invalidation?.reason === "page-transition"
455
- ? batchRefSnapshotState.invalidation
456
- : undefined;
455
+ const pageTransitionInvalidation = unobservedMutation && !failedTransitionReverification
456
+ ? buildPageTransitionRefSnapshotInvalidation("A dispatched mutation was interrupted without a final outcome. Verify the current URL and take a fresh snapshot before deciding whether to retry.")
457
+ : unsettledWebMcpMutation
458
+ ? buildPageTransitionRefSnapshotInvalidation("A detached WebMCP invocation is still pending or failed to settle and can mutate, rerender, or navigate the page, so prior snapshot refs remain invalid after URL verification. Run webmcp result or cancel, then take a fresh snapshot before using page-scoped refs.")
459
+ : processResult.agentBrowserStarted && (directTransitionInvalidation || plannedBatchTransitionInvalidation)
460
+ ? directTransitionInvalidation ?? plannedBatchTransitionInvalidation
461
+ : failedTransitionReverification
462
+ ? buildPageTransitionRefSnapshotInvalidation("A failed eval/back/forward/reload/connect/state-load/tab command may still have changed the page, so the prior snapshot refs were invalidated. Run snapshot -i before using page-scoped refs.")
463
+ : batchRefSnapshotState?.invalidation?.reason === "page-transition"
464
+ ? batchRefSnapshotState.invalidation
465
+ : undefined;
457
466
  if (currentSessionTabTarget) {
458
467
  const tabUpdate = sessionPageState.applyTabTarget({ sessionName: sessionStateKey, target: currentSessionTabTarget, update: sessionPageStateUpdate });
459
468
  if (!tabUpdate.applied && succeeded)
460
469
  sessionPageState.markPinning(sessionStateKey, "drift");
461
470
  }
462
- else if (processResult.agentBrowserStarted && (resultingPageState.pageUrlUnknown || resultingPageState.pageTargetMayHaveChanged) && !(prepared.commandTokens[0] === "session" && prepared.commandTokens[1] === "info")) {
471
+ else if (processResult.agentBrowserStarted && (unobservedMutation || resultingPageState.pageUrlUnknown || resultingPageState.pageTargetMayHaveChanged) && !(prepared.commandTokens[0] === "session" && prepared.commandTokens[1] === "info")) {
463
472
  sessionPageState.markTabTargetUnknown({ sessionName: sessionStateKey, update: sessionPageStateUpdate });
464
473
  }
465
- let refSnapshot = unsettledWebMcpMutation
474
+ let refSnapshot = unsettledWebMcpMutation || unobservedMutation
466
475
  ? undefined
467
476
  : prepared.executionPlan.commandInfo.command === "batch"
468
477
  ? batchRefSnapshotState?.snapshot
@@ -473,7 +482,7 @@ export async function processBrowserOutput(input) {
473
482
  ? prepared.commandTokens.filter((token) => token !== "--delta" && token !== "--full") : undefined);
474
483
  // Native owns delta baselines. Read complete refs without advancing that baseline
475
484
  // rather than maintaining a second revision/tree cache in the wrapper.
476
- if (refreshArgs && !unsettledWebMcpMutation) {
485
+ if (refreshArgs && !unsettledWebMcpMutation && !unobservedMutation) {
477
486
  refSnapshot = extractRefSnapshotFromData(await runSessionCommandData({ args: refreshArgs, cwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, signal }));
478
487
  if (!refSnapshot)
479
488
  sessionPageState.applyRefSnapshotInvalidation({ invalidation: buildPageTransitionRefSnapshotInvalidation("The native snapshot delta did not include complete refs and the full ref read failed. Run snapshot -i before using refs."), sessionName: sessionStateKey, update: sessionPageStateUpdate });
@@ -683,7 +692,7 @@ export async function processBrowserOutput(input) {
683
692
  }
684
693
  if (plainTextUpgrade && errorText)
685
694
  presentationEnvelope = { ...presentationEnvelope, success: false, error: errorText };
686
- let presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text", text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : recordingStopRecovery && !recordingStopRecovery.batch ? recordingStopRecovery.presentation : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactMaxUpdatedAtMs: Date.now(), artifactMinUpdatedAtMs: input.artifactRunStartedAtMs, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd: operationCwd, envelope: presentationEnvelope, errorText, namespace: prepared.executionPlan.namespace, networkRouteDiagnostics, networkRoutes: activeNetworkRoutes, persistentArtifactStore, previousRecordingContactSheetPath: sessionStateKey ? state.activeRecordingReservations?.get(sessionStateKey)?.contactSheetPath : undefined, piCleanupOwnership: sessionStateKey && (state.ownedManagedSessions.has(sessionStateKey) || prepared.executionPlan.managedSessionName !== undefined) ? "wrapper-managed" : "caller-owned", sessionName: prepared.executionPlan.sessionName });
695
+ let presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text", text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : recordingStopRecovery && !recordingStopRecovery.batch ? recordingStopRecovery.presentation : await buildToolPresentation({ modelVisible: input.modelVisible, args: prepared.redactedProcessArgs, artifactManifest, artifactMaxUpdatedAtMs: Date.now(), artifactMinUpdatedAtMs: input.artifactRunStartedAtMs, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd: operationCwd, envelope: presentationEnvelope, errorText, namespace: prepared.executionPlan.namespace, networkRouteDiagnostics, networkRoutes: activeNetworkRoutes, persistentArtifactStore, previousRecordingContactSheetPath: sessionStateKey ? state.activeRecordingReservations?.get(sessionStateKey)?.contactSheetPath : undefined, piCleanupOwnership: sessionStateKey && (state.ownedManagedSessions.has(sessionStateKey) || prepared.executionPlan.managedSessionName !== undefined) ? "wrapper-managed" : "caller-owned", sessionName: prepared.executionPlan.sessionName });
687
696
  if (recordingStopRecovery)
688
697
  presentation = mergeRecordingRecoveryPresentation(presentation, recordingStopRecovery);
689
698
  const confirmation = readConfirmationEvent ?? prepared.readConfirmation;
@@ -703,7 +712,8 @@ export async function processBrowserOutput(input) {
703
712
  const errorContent = presentation.content[0];
704
713
  if (errorContent?.type === "text" && presentationEnvelope.data)
705
714
  errorContent.text += `\n\n${presentationEnvelope.data}`;
706
- presentation = await compactLargePresentationOutput({ artifactManifest, commandInfo: prepared.executionPlan.commandInfo, data: presentation.data, persistentArtifactStore, presentation });
715
+ if (input.modelVisible !== false)
716
+ presentation = await compactLargePresentationOutput({ artifactManifest, commandInfo: prepared.executionPlan.commandInfo, data: presentation.data, persistentArtifactStore, presentation });
707
717
  }
708
718
  if (electronHandoff?.error && electronHandoff.failureCategory)
709
719
  presentation.failureCategory = electronHandoff.failureCategory;
@@ -743,6 +753,11 @@ export async function processBrowserOutput(input) {
743
753
  const qaPreset = prepared.compiledQaPreset
744
754
  ? (processResult.timedOut ? analyzeQaPresetTimeout(prepared.compiledQaPreset) ?? analyzeQaPresetResults(presentationEnvelope?.data, prepared.compiledQaPreset) : analyzeQaPresetResults(presentationEnvelope?.data, prepared.compiledQaPreset))
745
755
  : undefined;
756
+ if (qaPreset?.passed && presentation.resultCategory === "failure") {
757
+ qaPreset.passed = false;
758
+ qaPreset.summary = presentation.summary;
759
+ qaPreset.failedChecks.push(presentation.summary);
760
+ }
746
761
  let qaAttachedTarget = prepared.compiledQaPreset?.checks.attached
747
762
  ? await collectQaAttachedTarget({ currentTarget: currentSessionTabTarget ?? prepared.priorSessionTabTarget, cwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, signal })
748
763
  : undefined;
@@ -819,7 +834,7 @@ export async function processBrowserOutput(input) {
819
834
  : undefined;
820
835
  const sessionWarning = electronPostCommandHealth ? formatElectronPostCommandHealthText(electronPostCommandHealth) : electronSessionMismatch ? formatElectronSessionMismatchText(electronSessionMismatch) : aboutBlankSessionMismatch ? buildAboutBlankWarning(aboutBlankSessionMismatch) : undefined;
821
836
  const warningText = [sessionWarning, recordingPageWarning].filter(Boolean).join("\n\n") || undefined;
822
- const redactedContent = buildRedactedPresentationContent({ exactSensitiveValues: prepared.exactSensitiveValues, plainTextInspection, presentation, presentationEnvelope, succeeded, userRequestedJson: prepared.userRequestedJson, warningText });
837
+ const redactedContent = input.modelVisible === false ? [] : buildRedactedPresentationContent({ exactSensitiveValues: prepared.exactSensitiveValues, plainTextInspection, presentation, presentationEnvelope, succeeded, userRequestedJson: prepared.userRequestedJson, warningText });
823
838
  const finalRecoveryState = await prepareFinalResultRecoveryState({ aboutBlankSessionMismatch, batchRefSnapshotState, commandTokens: prepared.commandTokens, compiledSemanticAction: prepared.compiledSemanticAction, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, cwd, electronPostCommandHealth, errorText, executionPlan: prepared.executionPlan, parseError, plainTextInspection, presentation, processResult, redactedProcessArgs: prepared.redactedProcessArgs, runtimeToolArgs: prepared.runtimeToolArgs, sessionPageState, sessionPageStateUpdate, sessionTabCorrection, signal, succeeded });
824
839
  currentRefSnapshot = finalRecoveryState.currentRefSnapshot;
825
840
  currentRefSnapshotInvalidation = finalRecoveryState.currentRefSnapshotInvalidation;
@@ -836,7 +851,7 @@ export async function processBrowserOutput(input) {
836
851
  const resultHeadedManagedAutosaveInterval = resultRetainsPreparedManagedSession && !prepared.ownedManagedSessionContext?.reuseOnly && !(commandClosesSession && succeeded)
837
852
  ? prepared.ownedManagedSessionContext?.headedManagedAutosaveInterval
838
853
  : undefined;
839
- const result = buildFinalAgentBrowserToolResult({ aboutBlankSessionMismatch, artifactCleanup, categoryDetails: finalRecoveryState.categoryDetails, clickDispatchDiagnostic, commandTokens: prepared.commandTokens, comboboxFocusDiagnostic, compiledNetworkSourceLookup: prepared.compiledNetworkSourceLookup, compiledSemanticAction: prepared.compiledSemanticAction, compatibilityWorkaround: prepared.compatibilityWorkaround, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, currentSessionTabTargetUnknown, electronBroadGetTextScopeDiagnostics, electronFailedConnectCleanup, electronHandoff, electronLaunch: prepared.electronLaunch, electronLaunchRecord, electronLaunchRecords, electronPostCommandHealth, electronProfileIsolationDetails: input.electronProfileIsolationDetails, electronRefFreshnessDiagnostic, electronSessionMismatch, errorText, evalResultWarning, evalStdinHint, exactSensitiveValues: prepared.exactSensitiveValues, executionPlan: prepared.executionPlan, fillVerificationDiagnostic, headedLaunch: prepared.headedLaunch, inspectionText, preserveAttachedBrowserSession: input.preserveAttachedBrowserSession === true, providerLaunch: prepared.providerLaunch, managedSessionHeadedAutosaveDisabled: resultHeadedManagedAutosaveDisabled || undefined, managedSessionHeadedAutosaveInterval: resultHeadedManagedAutosaveInterval, managedSessionOutcome, managedSessionRestoreDisabled: state.managedSessionRestoreState.isDisabled(prepared.executionPlan.sessionName, prepared.executionPlan.namespace), navigationSummary, networkSourceLookup, noActivePageSnapshotFailure: finalRecoveryState.noActivePageSnapshotFailure, openResultTabCorrection, overlayBlockerDiagnostic, parseError, parseFailureOutput, parseSucceeded, plainTextInspection, presentation, presentationEnvelope, priorSessionTabTarget: prepared.priorSessionTabTarget, processResult, qaAttachedTarget, qaPreset, recordingDependencyWarning, redactedArgs: prepared.redactedArgs, redactedCompiledElectron: prepared.redactedCompiledElectron, redactedCompiledJob: prepared.redactedCompiledJob, redactedCompiledNetworkSourceLookup: prepared.redactedCompiledNetworkSourceLookup, redactedCompiledQaPreset: prepared.redactedCompiledQaPreset, redactedCompiledSemanticAction: prepared.redactedCompiledSemanticAction, redactedCompiledSourceLookup: prepared.redactedCompiledSourceLookup, redactedContent, redactedProcessArgs: prepared.redactedProcessArgs, redactedRecoveryHint: prepared.redactedRecoveryHint, resultArtifactManifest, richInputRecoveryDiagnostic: finalRecoveryState.richInputRecoveryDiagnostic, scrollNoopDiagnostic, selectorTextVisibilityDiagnostics, sessionMode: prepared.sessionMode, sessionTabCorrection, sourceLookup, succeeded, timeoutPartialProgress, unsettledWebMcpMutation, userRequestedJson: prepared.userRequestedJson, visibleRefFallbackDiagnostic: finalRecoveryState.visibleRefFallbackDiagnostic, visibleRefFallbackSessionName: finalRecoveryState.visibleRefFallbackSessionName });
854
+ const result = buildFinalAgentBrowserToolResult({ modelVisible: input.modelVisible, aboutBlankSessionMismatch, artifactCleanup, categoryDetails: finalRecoveryState.categoryDetails, clickDispatchDiagnostic, commandTokens: prepared.commandTokens, comboboxFocusDiagnostic, compiledNetworkSourceLookup: prepared.compiledNetworkSourceLookup, compiledSemanticAction: prepared.compiledSemanticAction, compatibilityWorkaround: prepared.compatibilityWorkaround, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, currentSessionTabTargetUnknown, electronBroadGetTextScopeDiagnostics, electronFailedConnectCleanup, electronHandoff, electronLaunch: prepared.electronLaunch, electronLaunchRecord, electronLaunchRecords, electronPostCommandHealth, electronProfileIsolationDetails: input.electronProfileIsolationDetails, electronRefFreshnessDiagnostic, electronSessionMismatch, errorText, evalResultWarning, evalStdinHint, exactSensitiveValues: prepared.exactSensitiveValues, executionPlan: prepared.executionPlan, fillVerificationDiagnostic, headedLaunch: prepared.headedLaunch, inspectionText, preserveAttachedBrowserSession: input.preserveAttachedBrowserSession === true, providerLaunch: prepared.providerLaunch, managedSessionHeadedAutosaveDisabled: resultHeadedManagedAutosaveDisabled || undefined, managedSessionHeadedAutosaveInterval: resultHeadedManagedAutosaveInterval, managedSessionOutcome, managedSessionRestoreDisabled: state.managedSessionRestoreState.isDisabled(prepared.executionPlan.sessionName, prepared.executionPlan.namespace), navigationSummary, networkSourceLookup, noActivePageSnapshotFailure: finalRecoveryState.noActivePageSnapshotFailure, openResultTabCorrection, overlayBlockerDiagnostic, parseError, parseFailureOutput, parseSucceeded, plainTextInspection, presentation, presentationEnvelope, priorSessionTabTarget: prepared.priorSessionTabTarget, processResult, qaAttachedTarget, qaPreset, recordingDependencyWarning, redactedArgs: prepared.redactedArgs, redactedCompiledElectron: prepared.redactedCompiledElectron, redactedCompiledJob: prepared.redactedCompiledJob, redactedCompiledNetworkSourceLookup: prepared.redactedCompiledNetworkSourceLookup, redactedCompiledQaPreset: prepared.redactedCompiledQaPreset, redactedCompiledSemanticAction: prepared.redactedCompiledSemanticAction, redactedCompiledSourceLookup: prepared.redactedCompiledSourceLookup, redactedContent, redactedProcessArgs: prepared.redactedProcessArgs, redactedRecoveryHint: prepared.redactedRecoveryHint, resultArtifactManifest, richInputRecoveryDiagnostic: finalRecoveryState.richInputRecoveryDiagnostic, scrollNoopDiagnostic, selectorTextVisibilityDiagnostics, sessionMode: prepared.sessionMode, sessionTabCorrection, sourceLookup, succeeded, timeoutPartialProgress, unsettledWebMcpMutation, userRequestedJson: prepared.userRequestedJson, visibleRefFallbackDiagnostic: finalRecoveryState.visibleRefFallbackDiagnostic, visibleRefFallbackSessionName: finalRecoveryState.visibleRefFallbackSessionName });
840
855
  const resultWithCloseAll = closeAllApplied
841
856
  ? { ...result, details: { ...(isRecord(result.details) ? result.details : {}), closeAllApplied: true } }
842
857
  : result;
@@ -95,6 +95,7 @@ export async function recoverRecordingStop(options) {
95
95
  && !currentUsesSamePath;
96
96
  const data = matchedData ?? (expected ? { path: expected.absolutePath, recordingId: expected.recordingId, success: null } : undefined);
97
97
  const presentation = await buildToolPresentation({
98
+ modelVisible: options.modelVisible,
98
99
  artifactManifest: options.artifactManifest, artifactMinUpdatedAtMs: expected?.startedAtMs ?? options.artifactRunStartedAtMs, artifactMaxUpdatedAtMs: Date.now(),
99
100
  artifactRequest: expected ? { path: expected.path, absolutePath: expected.absolutePath, status: terminalMeasurements ? undefined : "unverified" } : undefined,
100
101
  commandInfo: { command: "record", subcommand: "stop" }, cwd: expected?.cwd ?? options.cwd,
@@ -139,7 +140,8 @@ export async function recoverRecordingStop(options) {
139
140
  presentation.nextActions = followups;
140
141
  presentation.recordingRecovery = recovery;
141
142
  const text = `Recording receipt recovery (${recovery.status}): ${recovery.reason}\nOriginal attempt: ${attemptError}`;
142
- presentation.content.unshift({ type: "text", text });
143
+ if (options.modelVisible !== false)
144
+ presentation.content.unshift({ type: "text", text });
143
145
  return { batch, envelope, partialBatch: batch && !rows, stopIndex, presentation, recovery };
144
146
  }
145
147
  export function mergeRecordingRecoveryPresentation(base, result) {
@@ -0,0 +1,79 @@
1
+ import { isRecord } from "../../parsing.js";
2
+ import { getScreenshotPathTokenIndex, getScreenshotPositionalIndices } from "./artifact-paths.js";
3
+ import { runSessionCommandData } from "./session-state.js";
4
+ export function getScreenshotCapture(command) {
5
+ if (command[0] !== "screenshot")
6
+ return { kind: "unknown" };
7
+ if (command.includes("--full") || command.includes("-f"))
8
+ return { kind: "full-page" };
9
+ const positional = getScreenshotPositionalIndices(command);
10
+ const selectorIndex = positional.find(index => index !== getScreenshotPathTokenIndex(command));
11
+ return selectorIndex === undefined ? { kind: "viewport" } : { kind: "element", selector: command[selectorIndex] };
12
+ }
13
+ function finiteRecord(value, keys) {
14
+ return isRecord(value) && keys.every(key => typeof value[key] === "number" && Number.isFinite(value[key]));
15
+ }
16
+ export async function collectScreenshotSample(options) {
17
+ if (options.command[0] !== "screenshot" || options.signal?.aborted)
18
+ return undefined;
19
+ const { selector } = getScreenshotCapture(options.command);
20
+ const script = `(() => {
21
+ const root = document.documentElement, body = document.body, v = window.visualViewport;
22
+ let element;
23
+ try {
24
+ const selector = ${JSON.stringify(selector ?? null)};
25
+ const matches = selector ? document.querySelectorAll(selector) : [];
26
+ if (matches.length === 1) {
27
+ const rect = matches[0].getBoundingClientRect();
28
+ element = { x: rect.x, y: rect.y, width: rect.width, height: rect.height };
29
+ }
30
+ } catch {}
31
+ return { url: location.href, frame: window === window.top ? "main" : "child", childFrameCount: window.frames.length,
32
+ viewport: { width: innerWidth, height: innerHeight },
33
+ document: { width: Math.max(root.scrollWidth, body?.scrollWidth || 0, innerWidth), height: Math.max(root.scrollHeight, body?.scrollHeight || 0, innerHeight) },
34
+ scroll: { x: scrollX, y: scrollY }, dpr: devicePixelRatio,
35
+ visualViewport: { x: v?.offsetLeft ?? 0, y: v?.offsetTop ?? 0, scale: v?.scale ?? 1 }, element };
36
+ })()`;
37
+ try {
38
+ const data = await runSessionCommandData({ ...options, args: ["eval", "--stdin"], stdin: script, timeoutMs: 2_000 });
39
+ const sample = isRecord(data) ? data.result : undefined;
40
+ if (!isRecord(sample) || typeof sample.url !== "string" || !["main", "child"].includes(String(sample.frame))
41
+ || !finiteRecord(sample.viewport, ["width", "height"]) || !finiteRecord(sample.document, ["width", "height"])
42
+ || !finiteRecord(sample.scroll, ["x", "y"]) || !finiteRecord(sample.visualViewport, ["x", "y", "scale"])
43
+ || typeof sample.childFrameCount !== "number" || !Number.isInteger(sample.childFrameCount) || sample.childFrameCount < 0
44
+ || typeof sample.dpr !== "number" || !Number.isFinite(sample.dpr) || sample.dpr <= 0
45
+ || (sample.element !== undefined && !finiteRecord(sample.element, ["x", "y", "width", "height"])))
46
+ return undefined;
47
+ return sample;
48
+ }
49
+ catch {
50
+ return undefined;
51
+ }
52
+ }
53
+ export function buildScreenshotGeometry(options) {
54
+ const { before, after, capture, pixels } = options;
55
+ const unknown = (reason) => ({ status: "unknown", reason, before, after });
56
+ if (!before || !after)
57
+ return unknown("Capture was not bracketed by browser geometry samples; coordinates are unknown.");
58
+ if (JSON.stringify(before) !== JSON.stringify(after))
59
+ return unknown("Browser geometry changed across capture; no coordinate mapping is asserted.");
60
+ if (before.frame !== "main")
61
+ return unknown("Probe observed a child frame; its relation to the captured page is unknown.");
62
+ if (before.visualViewport.scale !== 1 || before.visualViewport.x !== 0 || before.visualViewport.y !== 0)
63
+ return unknown("Visual viewport is zoomed or offset; capture origin is unknown.");
64
+ let crop;
65
+ if (capture === "viewport")
66
+ crop = { ...before.scroll, ...before.viewport };
67
+ else if (capture === "full-page")
68
+ crop = { x: 0, y: 0, ...before.document };
69
+ else if (capture === "element" && before.element && before.childFrameCount === 0 && before.scroll.x === 0 && before.scroll.y === 0)
70
+ crop = before.element;
71
+ else
72
+ return unknown("Element crop/frame provenance is unverified (scrolled page, child frames, non-CSS selector, or ambiguous target).");
73
+ if (!pixels || crop.width <= 0 || crop.height <= 0 || crop.x < 0 || crop.y < 0
74
+ || !Object.values(crop).every(Number.isInteger)
75
+ || pixels.width !== crop.width * before.dpr || pixels.height !== crop.height * before.dpr)
76
+ return unknown("Image dimensions do not establish the sampled CSS crop at this DPR.");
77
+ return { status: "measured", before, after, crop, pixelsPerCssPixel: { x: pixels.width / crop.width, y: pixels.height / crop.height },
78
+ reason: "Matching pre/post samples, not an atomic capture guarantee. Pixels map to CSS document coordinates within crop; native mouse uses current viewport CSS coordinates. Recheck scroll/frame before input." };
79
+ }
@@ -4,6 +4,7 @@ import { boundElectronProbeString } from "../../electron/cdp.js";
4
4
  import { buildOwnedManagedSessionRestoreContext, withOwnedManagedSessionContext, } from "../../managed-session-restore.js";
5
5
  import { getPageTargetValidationError } from "../../page-target-validation.js";
6
6
  import { isRecord } from "../../parsing.js";
7
+ import { getBrowserResultMessage } from "../../browser-transcript.js";
7
8
  import { withAttachedBrowserSessionContext } from "../../process.js";
8
9
  import { buildAgentBrowserNextActions } from "../../results/action-recommendations.js";
9
10
  import { buildAgentBrowserResultCategoryDetails } from "../../results/categories.js";
@@ -125,10 +126,8 @@ function isElectronLaunchRecord(value) {
125
126
  export function restoreElectronLaunchRecordsFromBranch(branch) {
126
127
  const records = new Map();
127
128
  for (const entry of branch) {
128
- if (!isRecord(entry) || entry.type !== "message")
129
- continue;
130
- const message = isRecord(entry.message) ? entry.message : undefined;
131
- if (!message || message.toolName !== "agent_browser")
129
+ const message = getBrowserResultMessage(entry);
130
+ if (!message)
132
131
  continue;
133
132
  const details = isRecord(message.details) ? message.details : undefined;
134
133
  const electron = isRecord(details?.electron) ? details.electron : undefined;