pi-agent-browser-native 0.6.17 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +55 -167
- package/dist/extensions/agent-browser/index.js +317 -223
- package/dist/extensions/agent-browser/lib/browser-transcript.js +33 -0
- package/dist/extensions/agent-browser/lib/execution-cwd.js +4 -2
- package/dist/extensions/agent-browser/lib/input-modes/job.js +2 -213
- package/dist/extensions/agent-browser/lib/input-modes/params.js +146 -142
- package/dist/extensions/agent-browser/lib/input-modes/script.js +64 -92
- package/dist/extensions/agent-browser/lib/managed-session-policy-lock.js +170 -24
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.js +13 -1
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/final-result.js +26 -13
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/index.js +21 -1
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/snapshot-filter.js +4 -2
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare.js +1 -3
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/process-output.js +35 -20
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/recording-recovery.js +3 -1
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/screenshot-observation.js +79 -0
- package/dist/extensions/agent-browser/lib/orchestration/electron-host/index.js +3 -4
- package/dist/extensions/agent-browser/lib/orchestration/input-plan.js +19 -52
- package/dist/extensions/agent-browser/lib/orchestration/native-session-defaults.js +3 -3
- package/dist/extensions/agent-browser/lib/orchestration/output-file.js +3 -3
- package/dist/extensions/agent-browser/lib/orchestration/script-mode.js +91 -278
- package/dist/extensions/agent-browser/lib/pi-tool-rendering.js +9 -9
- package/dist/extensions/agent-browser/lib/playbook.js +46 -25
- package/dist/extensions/agent-browser/lib/process-identity.js +22 -6
- package/dist/extensions/agent-browser/lib/process.js +16 -13
- package/dist/extensions/agent-browser/lib/results/action-recommendations.js +2 -2
- package/dist/extensions/agent-browser/lib/results/next-actions.js +1 -2
- package/dist/extensions/agent-browser/lib/results/presentation/artifacts.js +25 -4
- package/dist/extensions/agent-browser/lib/results/presentation/batch.js +6 -2
- package/dist/extensions/agent-browser/lib/results/presentation/common.js +2 -0
- package/dist/extensions/agent-browser/lib/results/presentation/content.js +39 -0
- package/dist/extensions/agent-browser/lib/results/presentation/diagnostics.js +3 -3
- package/dist/extensions/agent-browser/lib/results/presentation/large-output.js +41 -1
- package/dist/extensions/agent-browser/lib/results/presentation.js +16 -11
- package/dist/extensions/agent-browser/lib/runtime.js +18 -6
- package/dist/extensions/agent-browser/lib/session-page-state.js +3 -4
- package/dist/extensions/agent-browser/lib/tool-surface.js +131 -0
- package/dist/extensions/agent-browser/script-worker.js +4 -1
- package/docs/ARCHITECTURE.md +46 -62
- package/docs/COMMAND_REFERENCE.md +77 -136
- package/docs/ELECTRON.md +51 -49
- package/docs/RELEASE.md +31 -15
- package/docs/REQUIREMENTS.md +18 -14
- package/docs/SUPPORT_MATRIX.md +33 -15
- package/docs/TOOL_CONTRACT.md +216 -405
- package/package.json +1 -1
- package/scripts/doctor.mjs +2 -2
|
@@ -4,6 +4,7 @@ import { redactNetworkSourceLookupSurface } from "../../input-modes/lookups.js";
|
|
|
4
4
|
import { buildAgentBrowserNextActions } from "../../results/action-recommendations.js";
|
|
5
5
|
import { buildAgentBrowserResultCategoryDetails } from "../../results/categories.js";
|
|
6
6
|
import { extractAgentBrowserLifecycle } from "../../results/presentation/common.js";
|
|
7
|
+
import { projectAgentBrowserObservation } from "../../results/presentation/content.js";
|
|
7
8
|
import { formatSessionArtifactRetentionSummary } from "../../results/artifact-manifest.js";
|
|
8
9
|
import { alignPageChangeSummaryNextActionIds, appendUniqueAgentBrowserNextActions, applyNamespaceToNextActions, isStandaloneSnapshotNextAction, withOptionalSessionArgs, } from "../../results/next-actions.js";
|
|
9
10
|
import { buildConnectedSessionNextActions, buildNoActivePageNextActions, buildPendingWebMcpNextActions, buildSessionAwareStaleRefNextActions, buildSessionTabRecoveryNextActions, } from "../../results/recovery-next-actions.js";
|
|
@@ -86,9 +87,7 @@ export function redactRecoveryHint(recoveryHint) {
|
|
|
86
87
|
}
|
|
87
88
|
export function buildJsonVisibleContent(options) {
|
|
88
89
|
const { error, presentation, succeeded, warnings } = options;
|
|
89
|
-
const payload =
|
|
90
|
-
if (isRecord(payload) && isRecord(payload.data) && isRecord(presentation.data) && typeof presentation.data.wsUrl === "string")
|
|
91
|
-
payload.data.wsUrl = presentation.data.wsUrl;
|
|
90
|
+
const payload = projectAgentBrowserObservation({ ...presentation, ...options.details, error, warnings: warnings && warnings.length > 0 ? warnings : undefined }, succeeded);
|
|
92
91
|
const images = presentation.content.filter((item) => item.type === "image");
|
|
93
92
|
return [{ type: "text", text: JSON.stringify(payload, null, 2) }, ...images];
|
|
94
93
|
}
|
|
@@ -175,8 +174,9 @@ export function buildRedactedPresentationContent(options) {
|
|
|
175
174
|
return contentWithSessionWarnings.map((item) => {
|
|
176
175
|
if (item.type !== "text")
|
|
177
176
|
return item;
|
|
178
|
-
|
|
179
|
-
|
|
177
|
+
if (userRequestedJson && !plainTextInspection)
|
|
178
|
+
return { ...item, text: JSON.stringify(redactExactSensitiveValue(JSON.parse(item.text), exactSensitiveValues), null, 2) };
|
|
179
|
+
return { ...item, text: redactSensitiveText(redactExactSensitiveText(item.text, exactSensitiveValues)) };
|
|
180
180
|
});
|
|
181
181
|
}
|
|
182
182
|
export async function prepareFinalResultRecoveryState(options) {
|
|
@@ -339,6 +339,10 @@ function buildResultNextActions(options) {
|
|
|
339
339
|
append(buildComboboxFocusNextActions(options.executionPlan.sessionName));
|
|
340
340
|
if (options.managedSessionOutcome)
|
|
341
341
|
appendUnique(buildManagedSessionFreshFailureNextActions(options.managedSessionOutcome));
|
|
342
|
+
if (options.currentSessionTabTargetUnknown && ["aborted", "parse-failure"].includes(options.categoryDetails.failureCategory ?? "")) {
|
|
343
|
+
nextActions = nextActions.filter(action => !isStandaloneSnapshotNextAction(action));
|
|
344
|
+
appendUnique([{ id: "verify-page-target-after-interruption", tool: "agent_browser", params: { args: withOptionalSessionArgs(options.executionPlan.sessionName, ["batch", "--bail"]), stdin: JSON.stringify([["get", "url"], ["snapshot", "-i"]]) }, reason: "Verify the current URL and inspect the page after an interrupted operation.", safety: "Read-only recovery. The mutation may already have happened; inspect before deciding whether to retry." }]);
|
|
345
|
+
}
|
|
342
346
|
if (options.categoryDetails.failureCategory === "timeout" && options.processResult.timedOut) {
|
|
343
347
|
if (options.currentSessionTabTargetUnknown)
|
|
344
348
|
nextActions = nextActions.filter((action) => !isStandaloneSnapshotNextAction(action));
|
|
@@ -354,14 +358,11 @@ function buildResultNextActions(options) {
|
|
|
354
358
|
export function formatAgentBrowserNextActionsText(nextActions) {
|
|
355
359
|
if (!nextActions || nextActions.length === 0)
|
|
356
360
|
return undefined;
|
|
357
|
-
const lines = nextActions.
|
|
358
|
-
const
|
|
359
|
-
? { ...action.params, ...(action.params.stdin !== undefined && action.params.stdin.length > 500 ? { stdin: "[omitted; use details.nextActions]" } : {}) }
|
|
360
|
-
: undefined;
|
|
361
|
-
const payload = action.artifactPath ? { artifactPath: action.artifactPath } : params;
|
|
361
|
+
const lines = nextActions.map((action) => {
|
|
362
|
+
const payload = action.artifactPath ? { artifactPath: action.artifactPath } : action.params;
|
|
362
363
|
return `- ${action.id}${payload ? ` ${redactSensitiveText(JSON.stringify(payload))}` : ""}: ${redactSensitiveText(action.reason)}`;
|
|
363
364
|
});
|
|
364
|
-
return ["Next actions:", ...lines
|
|
365
|
+
return ["Next actions:", ...lines].join("\n");
|
|
365
366
|
}
|
|
366
367
|
function formatFailureNextActionsText(options, nextActions) {
|
|
367
368
|
return options.categoryDetails.resultCategory === "failure" ? formatAgentBrowserNextActionsText(nextActions) : undefined;
|
|
@@ -438,6 +439,7 @@ function buildAgentBrowserResultDetails(options, nextActions) {
|
|
|
438
439
|
managedSessionOutcome: options.managedSessionOutcome,
|
|
439
440
|
imagePath: options.presentation.imagePath,
|
|
440
441
|
imagePaths: options.presentation.imagePaths,
|
|
442
|
+
imageObservations: options.presentation.imageObservations,
|
|
441
443
|
nextActions,
|
|
442
444
|
pageChangeSummary,
|
|
443
445
|
clickDispatch: options.clickDispatchDiagnostic,
|
|
@@ -485,6 +487,10 @@ export function buildFinalAgentBrowserToolResult(options) {
|
|
|
485
487
|
const lifecycle = extractAgentBrowserLifecycle(options.presentationEnvelope?.data);
|
|
486
488
|
const browserWindow = buildBrowserWindowStatus(options, lifecycle);
|
|
487
489
|
const details = buildAgentBrowserResultDetails(options, nextActions);
|
|
490
|
+
if (options.modelVisible === false) {
|
|
491
|
+
const result = { content: [], details: redactToolDetails(details, options.exactSensitiveValues), isError: !options.succeeded };
|
|
492
|
+
return options.compiledNetworkSourceLookup ? redactNetworkSourceLookupSurface(result) : result;
|
|
493
|
+
}
|
|
488
494
|
const visibleRefFallbackText = formatVisibleRefFallbackText(options.visibleRefFallbackDiagnostic);
|
|
489
495
|
const richInputRecoveryText = formatRichInputRecoveryText(options.richInputRecoveryDiagnostic);
|
|
490
496
|
const semanticActionCandidateText = nextActions ? formatSemanticActionCandidateText(nextActions) : undefined;
|
|
@@ -509,10 +515,17 @@ export function buildFinalAgentBrowserToolResult(options) {
|
|
|
509
515
|
const appendedDiagnosticText = redactSensitiveText(redactExactSensitiveText(rawAppendedDiagnosticText, options.exactSensitiveValues));
|
|
510
516
|
const shouldAppendDiagnosticText = appendedDiagnosticText.length > 0 && (!options.userRequestedJson || options.plainTextInspection);
|
|
511
517
|
let content = shouldAppendDiagnosticText && options.redactedContent[0]?.type === "text" ? [{ ...options.redactedContent[0], text: `${options.redactedContent[0].text}\n\n${appendedDiagnosticText}` }, ...options.redactedContent.slice(1)] : options.redactedContent;
|
|
512
|
-
if (options.electronLaunchRecord && options.succeeded && content[0]?.type === "text") {
|
|
518
|
+
if (options.electronLaunchRecord && options.succeeded && !options.userRequestedJson && content[0]?.type === "text") {
|
|
513
519
|
content = [{ ...content[0], text: redactSensitiveText(formatElectronLaunchText({ handoff: options.electronHandoff, record: options.electronLaunchRecord, targets: options.electronLaunch?.targets ?? [], upstreamText: content[0].text })) }, ...content.slice(1)];
|
|
514
520
|
}
|
|
515
|
-
const
|
|
521
|
+
const redactedDetails = redactToolDetails(details, options.exactSensitiveValues);
|
|
522
|
+
if (options.userRequestedJson && !options.plainTextInspection) {
|
|
523
|
+
const prior = options.redactedContent[0]?.type === "text" ? JSON.parse(options.redactedContent[0].text) : undefined;
|
|
524
|
+
if (Array.isArray(prior?.warnings))
|
|
525
|
+
redactedDetails.warnings = prior.warnings;
|
|
526
|
+
content = buildJsonVisibleContent({ error: redactedDetails.error, details: redactedDetails, presentation: options.presentation, succeeded: options.succeeded, warnings: prior?.warnings });
|
|
527
|
+
}
|
|
528
|
+
const result = { content, details: redactedDetails, isError: !options.succeeded };
|
|
516
529
|
return options.compiledNetworkSourceLookup ? redactNetworkSourceLookupSurface(result) : result;
|
|
517
530
|
}
|
|
518
531
|
export function isMissingAgentBrowserBinary(processResult) {
|
|
@@ -6,6 +6,7 @@ import { formatWebMcpCatalogUpdate } from "../../results/presentation/common.js"
|
|
|
6
6
|
import { withOwnedManagedSessionContext } from "../../managed-session-restore.js";
|
|
7
7
|
import { cleanupClickDispatchProbe } from "./click-dispatch.js";
|
|
8
8
|
import { applyBrowserRunStatePatch, getSessionContextKey } from "./session-state.js";
|
|
9
|
+
import { buildScreenshotGeometry, collectScreenshotSample } from "./screenshot-observation.js";
|
|
9
10
|
import { buildJsonVisibleContent, buildMissingBinaryFailureResult } from "./final-result.js";
|
|
10
11
|
import { prepareBrowserRun } from "./prepare.js";
|
|
11
12
|
import { processBrowserOutput } from "./process-output.js";
|
|
@@ -27,6 +28,12 @@ export async function runAgentBrowserTool(options) {
|
|
|
27
28
|
result.content.push({ type: "text", text: notice });
|
|
28
29
|
}
|
|
29
30
|
}
|
|
31
|
+
if (result.isError && details?.error === undefined) {
|
|
32
|
+
details = { ...details, error: details?.validationError ?? details?.summary ?? result.content.filter(part => part.type === "text").map(part => part.text).join("\n") };
|
|
33
|
+
result.details = details;
|
|
34
|
+
}
|
|
35
|
+
if (options.modelVisible === false)
|
|
36
|
+
result.content = [];
|
|
30
37
|
const sessionKey = getSessionContextKey(typeof details?.sessionName === "string" ? details.sessionName : undefined, typeof details?.namespace === "string" ? details.namespace : undefined);
|
|
31
38
|
const page = options.state.sessionPageState.get(sessionKey);
|
|
32
39
|
return page.tabReopenPending === undefined ? result : {
|
|
@@ -39,11 +46,12 @@ async function runAgentBrowserToolInContext(options) {
|
|
|
39
46
|
applyBrowserRunStatePatch(options.state, preparedResult.kind === "ready" ? preparedResult.prepared.statePatch : preparedResult.statePatch);
|
|
40
47
|
if (preparedResult.kind === "early-result") {
|
|
41
48
|
const result = preparedResult.result;
|
|
42
|
-
if (options.input.toolArgs.includes("--json") && !isPlainTextInspectionArgs(options.input.toolArgs)) {
|
|
49
|
+
if (options.modelVisible !== false && options.input.toolArgs.includes("--json") && !isPlainTextInspectionArgs(options.input.toolArgs)) {
|
|
43
50
|
const details = isRecord(result.details) ? result.details : {};
|
|
44
51
|
const summary = result.content.filter(item => item.type === "text").map(item => item.text).join("\n");
|
|
45
52
|
result.content = buildJsonVisibleContent({
|
|
46
53
|
error: result.isError ? details.validationError ?? summary : null,
|
|
54
|
+
details,
|
|
47
55
|
presentation: { content: result.content, data: details.data, summary },
|
|
48
56
|
succeeded: result.isError !== true,
|
|
49
57
|
});
|
|
@@ -54,6 +62,11 @@ async function runAgentBrowserToolInContext(options) {
|
|
|
54
62
|
const ownedManagedSession = prepared.ownedManagedSessionContext;
|
|
55
63
|
return await withOwnedManagedSessionContext(ownedManagedSession, async () => {
|
|
56
64
|
try {
|
|
65
|
+
const geometryOptions = { command: prepared.commandTokens, cwd: options.cwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, signal: options.signal,
|
|
66
|
+
env: ownedManagedSession ? { AGENT_BROWSER_IDLE_TIMEOUT_MS: options.implicitSessionIdleTimeoutMs } : undefined };
|
|
67
|
+
// Do not start a cold browser through a probe or consume launch flags ahead of the requested capture.
|
|
68
|
+
const probeGeometry = prepared.commandTokens[0] === "screenshot" && !!prepared.priorSessionTabTarget && !prepared.priorSessionTabTargetUnknown && !prepared.executionPlan.startupScopedFlags?.length;
|
|
69
|
+
const before = probeGeometry ? await withChromeStartupArgs(prepared.chromeStartupArgs, () => collectScreenshotSample(geometryOptions)) : undefined;
|
|
57
70
|
const artifactRunStartedAtMs = Date.now();
|
|
58
71
|
const processResult = await withChromeStartupArgs(prepared.chromeStartupArgs, () => runAgentBrowserProcess({
|
|
59
72
|
args: prepared.processArgs,
|
|
@@ -70,6 +83,8 @@ async function runAgentBrowserToolInContext(options) {
|
|
|
70
83
|
stdin: prepared.processStdin,
|
|
71
84
|
timeoutMs: prepared.processTimeoutMs,
|
|
72
85
|
}));
|
|
86
|
+
const after = probeGeometry && !processResult.aborted && !processResult.timedOut && processResult.exitCode === 0
|
|
87
|
+
? await withChromeStartupArgs(prepared.chromeStartupArgs, () => collectScreenshotSample(geometryOptions)) : undefined;
|
|
73
88
|
const missingBinaryResult = await buildMissingBinaryFailureResult({
|
|
74
89
|
compatibilityWorkaround: prepared.compatibilityWorkaround,
|
|
75
90
|
electronLaunch: prepared.electronLaunch,
|
|
@@ -88,6 +103,11 @@ async function runAgentBrowserToolInContext(options) {
|
|
|
88
103
|
return missingBinaryResult;
|
|
89
104
|
const output = await processBrowserOutput({ ...options, artifactRunStartedAtMs, prepared, processResult });
|
|
90
105
|
applyBrowserRunStatePatch(options.state, output.statePatch);
|
|
106
|
+
if (isRecord(output.result.details) && Array.isArray(output.result.details.imageObservations) && prepared.commandTokens[0] === "screenshot") {
|
|
107
|
+
output.result.details.imageObservations = redactSensitiveValue(output.result.details.imageObservations.map(image => ({
|
|
108
|
+
...image, geometry: buildScreenshotGeometry({ capture: image.capture, pixels: image.pixels, before, after }),
|
|
109
|
+
})));
|
|
110
|
+
}
|
|
91
111
|
return output.result;
|
|
92
112
|
}
|
|
93
113
|
finally {
|
package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/snapshot-filter.js
CHANGED
|
@@ -2,7 +2,7 @@ import { isRecord } from "../../../parsing.js";
|
|
|
2
2
|
import { buildAgentBrowserResultCategoryDetails } from "../../../results/categories.js";
|
|
3
3
|
import { buildSnapshotPresentation } from "../../../results/snapshot.js";
|
|
4
4
|
import { extractRefSnapshotFromData } from "../../../session-page-state.js";
|
|
5
|
-
import { redactSensitiveText } from "../../../runtime.js";
|
|
5
|
+
import { redactSensitiveText, redactSensitiveValue } from "../../../runtime.js";
|
|
6
6
|
import { collectScrollPositionSnapshot } from "../diagnostics.js";
|
|
7
7
|
import { buildSessionDetailFields, runSessionCommandData } from "../session-state.js";
|
|
8
8
|
function parseSnapshotFilterRequest(commandTokens) {
|
|
@@ -237,7 +237,9 @@ export async function trySnapshotFilter(options) {
|
|
|
237
237
|
const diff = request.diff ? buildSnapshotDiff(options.previousRefSnapshot, fullSnapshot) : undefined;
|
|
238
238
|
if (fullSnapshot)
|
|
239
239
|
options.sessionPageState.applyRefSnapshot({ sessionName: options.sessionStateKey ?? options.sessionName, snapshot: fullSnapshot, update: options.sessionPageStateUpdate });
|
|
240
|
-
const presentation =
|
|
240
|
+
const presentation = options.modelVisible === false
|
|
241
|
+
? { content: [], data: redactSensitiveValue(filtered.data), summary: "Snapshot" }
|
|
242
|
+
: await buildSnapshotPresentation(filtered.data, options.persistentArtifactStore, options.artifactManifest);
|
|
241
243
|
const summary = request.role || request.search
|
|
242
244
|
? `Snapshot filter: ${filtered.matchedRefs}/${filtered.totalRefs} direct refs matched${request.role ? ` role=${request.role}` : ""}${request.search ? ` search ${JSON.stringify(request.search)}` : ""}; ${filtered.visibleLines} surrounding snapshot line${filtered.visibleLines === 1 ? "" : "s"} shown.`
|
|
243
245
|
: request.diff
|
|
@@ -36,8 +36,6 @@ export function normalizeRunInput(input) {
|
|
|
36
36
|
switch (input.kind) {
|
|
37
37
|
case "electron":
|
|
38
38
|
return { ...base, compiledElectron: input.compiledElectron, redactedCompiledElectron: input.redactedCompiledElectron };
|
|
39
|
-
case "job":
|
|
40
|
-
return { ...base, compiledJob: input.compiledJob, redactedCompiledJob: input.redactedCompiledJob };
|
|
41
39
|
case "networkSourceLookup":
|
|
42
40
|
return { ...base, compiledNetworkSourceLookup: input.compiledNetworkSourceLookup, redactedCompiledNetworkSourceLookup: input.redactedCompiledNetworkSourceLookup };
|
|
43
41
|
case "qa":
|
|
@@ -46,7 +44,6 @@ export function normalizeRunInput(input) {
|
|
|
46
44
|
return { ...base, compiledSemanticAction: input.compiledSemanticAction, redactedCompiledSemanticAction: input.redactedCompiledSemanticAction };
|
|
47
45
|
case "sourceLookup":
|
|
48
46
|
return { ...base, compiledSourceLookup: input.compiledSourceLookup, redactedCompiledSourceLookup: input.redactedCompiledSourceLookup };
|
|
49
|
-
case "script":
|
|
50
47
|
case "args":
|
|
51
48
|
return base;
|
|
52
49
|
}
|
|
@@ -937,6 +934,7 @@ export async function prepareBrowserRun(options) {
|
|
|
937
934
|
}
|
|
938
935
|
const persistentArtifactStore = getPersistentSessionArtifactStore(options.ctx);
|
|
939
936
|
const snapshotFilter = await trySnapshotFilter({
|
|
937
|
+
modelVisible: options.modelVisible,
|
|
940
938
|
artifactManifest: state.artifactManifest,
|
|
941
939
|
commandTokens,
|
|
942
940
|
compatibilityWorkaround,
|
|
@@ -4,7 +4,7 @@ import { parseArgvDescriptor } from "../../argv-descriptor.js";
|
|
|
4
4
|
import { isBrowserIndependentRead, needsManagedSession } from "../../command-policy.js";
|
|
5
5
|
import { deleteIdentityKeysInNamespace, getAgentBrowserSessionIdentityKey, isAgentBrowserSessionIdentityKeyInNamespace } from "../../argv-grammar.js";
|
|
6
6
|
import { batchHasSuccessfulCloseAll, getSuccessfulBatchCloseLifecycle } from "../../batch-lifecycle.js";
|
|
7
|
-
import { isCloseAllCommand, isCloseCommand, isOpenNavigationCommand, isRecordPageTransitionCommand, isUnverifiedPageTransitionCommand, isWindowOrDiffPageTransitionCommand } from "../../command-taxonomy.js";
|
|
7
|
+
import { isCloseAllCommand, isCloseCommand, isOpenNavigationCommand, isPageMutationCommand, isRecordPageTransitionCommand, isUnverifiedPageTransitionCommand, isWindowOrDiffPageTransitionCommand } from "../../command-taxonomy.js";
|
|
8
8
|
import { OPEN_RESULT_TAB_CORRECTION_FLAGS } from "../../launch-scoped-flags.js";
|
|
9
9
|
import { cleanupElectronLaunchResources, inspectElectronLaunchStatus } from "../../electron/cleanup.js";
|
|
10
10
|
import { getResultingPageTargetState, commandRequiresLivePageVerification } from "../../page-target-validation.js";
|
|
@@ -168,6 +168,7 @@ export async function processBrowserOutput(input) {
|
|
|
168
168
|
observeNativeWebMcp(parsed.envelope?.data);
|
|
169
169
|
let parseError = parsed.parseError;
|
|
170
170
|
const recordingStopRecovery = await recoverRecordingStop({
|
|
171
|
+
modelVisible: input.modelVisible,
|
|
171
172
|
artifactManifest, artifactRunStartedAtMs: input.artifactRunStartedAtMs, commandTokens: prepared.commandTokens, cwd,
|
|
172
173
|
envelope: parsed.envelope, namespace: prepared.executionPlan.namespace, parseError, processResult,
|
|
173
174
|
reservation: prepared.executionPlan.sessionName ? state.activeRecordingReservations?.get(getAgentBrowserSessionIdentityKey(prepared.executionPlan.sessionName, prepared.executionPlan.namespace)) : undefined,
|
|
@@ -241,6 +242,12 @@ export async function processBrowserOutput(input) {
|
|
|
241
242
|
const envelopeSuccess = plainTextInspection ? true : presentationEnvelope?.success !== false;
|
|
242
243
|
let succeeded = (processSucceeded && parseSucceeded && envelopeSuccess) || recordingStopRecovery?.recovery.healed === true;
|
|
243
244
|
const inspectionText = plainTextInspection ? processResult.stdout.trim() : undefined;
|
|
245
|
+
const unobservedMutation = processResult.agentBrowserStarted && !plainTextInspection
|
|
246
|
+
&& (processResult.aborted || processResult.timedOut || !parseSucceeded || presentationEnvelope === undefined)
|
|
247
|
+
&& (prepared.executionPlan.commandInfo.command === "batch" ? batchCommandSteps : [prepared.commandTokens]).some(step => {
|
|
248
|
+
const action = step[0] === "find" ? step[step[1] === "nth" ? 4 : 3] ?? "click" : step[0];
|
|
249
|
+
return isPageMutationCommand(action, step[1]);
|
|
250
|
+
});
|
|
244
251
|
const sessionStateKey = getSessionContextKey(prepared.executionPlan.sessionName, prepared.executionPlan.namespace);
|
|
245
252
|
const closeAllApplied = nestedBatchClosesAll || (directCloseAllRequested && succeeded);
|
|
246
253
|
if (sessionStateKey && processResult.agentBrowserStarted && sessionPageState.get(sessionStateKey).tabReopenPending === true) {
|
|
@@ -335,14 +342,14 @@ export async function processBrowserOutput(input) {
|
|
|
335
342
|
const failedWebMcpSettlement = prepared.priorSessionTabTargetUnknown === true && ((!succeeded && isWebMcpSettlementCommand(prepared.executionPlan.commandInfo.command, prepared.executionPlan.commandInfo.subcommand))
|
|
336
343
|
|| (prepared.executionPlan.commandInfo.command === "batch" && batchHasFailedWebMcpSettlement(presentationEnvelope?.data)));
|
|
337
344
|
const unsettledWebMcpMutation = pendingWebMcpMutation || failedWebMcpSettlement;
|
|
338
|
-
const observedSessionTabTarget = unsettledWebMcpMutation
|
|
345
|
+
const observedSessionTabTarget = unsettledWebMcpMutation || (unobservedMutation && !failedTransitionReverification)
|
|
339
346
|
? undefined
|
|
340
347
|
: normalizeSessionTabTarget(navigationSummary)
|
|
341
348
|
?? (trustsReportedPageTarget ? extractSessionTabTargetFromBatchResults(presentationEnvelope?.data) : undefined)
|
|
342
349
|
?? (succeeded && trustsReportedPageTarget ? extractSessionTabTargetFromCommandData(prepared.commandTokens, presentationEnvelope?.data) : undefined);
|
|
343
350
|
const safeObservedSessionTabTarget = observedSessionTabTarget;
|
|
344
351
|
let currentSessionTabTarget = safeObservedSessionTabTarget;
|
|
345
|
-
if (!currentSessionTabTarget && nestedBatchClose === undefined) {
|
|
352
|
+
if (!currentSessionTabTarget && nestedBatchClose === undefined && !unobservedMutation) {
|
|
346
353
|
// Window/diff responses do not report the final URL; URL2 is intent, not redirect evidence.
|
|
347
354
|
currentSessionTabTarget = resultingPageState.pageTargetMayHaveChanged
|
|
348
355
|
? succeeded && !destinationTransition ? normalizeSessionTabTarget({ url: resultingPageState.currentPageUrl }) : undefined
|
|
@@ -397,7 +404,7 @@ export async function processBrowserOutput(input) {
|
|
|
397
404
|
let selectorTextVisibilityDiagnostics = [];
|
|
398
405
|
let electronBroadGetTextScopeDiagnostics = [];
|
|
399
406
|
const timeoutPartialProgress = processResult.timedOut && !recordingStopRecovery && !prepared.readConfirmation ? await collectTimeoutPartialProgress({ commandTokens: prepared.commandTokens, compiledJob: prepared.compiledJob, cwd, operationCwd: input.operationCwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, stdin: prepared.runtimeToolStdin }) : undefined;
|
|
400
|
-
if (!currentSessionTabTarget && timeoutPartialProgress?.
|
|
407
|
+
if (!currentSessionTabTarget && timeoutPartialProgress?.liveUrlRecovered === true) {
|
|
401
408
|
currentSessionTabTarget = normalizeSessionTabTarget(timeoutPartialProgress.currentPage);
|
|
402
409
|
}
|
|
403
410
|
if (succeeded) {
|
|
@@ -445,24 +452,26 @@ export async function processBrowserOutput(input) {
|
|
|
445
452
|
const plannedBatchTransitionInvalidation = !Array.isArray(presentationEnvelope?.data)
|
|
446
453
|
? batchCommandSteps.map(getCommandRefSnapshotInvalidation).find((invalidation) => invalidation !== undefined)
|
|
447
454
|
: undefined;
|
|
448
|
-
const pageTransitionInvalidation =
|
|
449
|
-
? buildPageTransitionRefSnapshotInvalidation("A
|
|
450
|
-
:
|
|
451
|
-
?
|
|
452
|
-
:
|
|
453
|
-
?
|
|
454
|
-
:
|
|
455
|
-
?
|
|
456
|
-
:
|
|
455
|
+
const pageTransitionInvalidation = unobservedMutation && !failedTransitionReverification
|
|
456
|
+
? buildPageTransitionRefSnapshotInvalidation("A dispatched mutation was interrupted without a final outcome. Verify the current URL and take a fresh snapshot before deciding whether to retry.")
|
|
457
|
+
: unsettledWebMcpMutation
|
|
458
|
+
? buildPageTransitionRefSnapshotInvalidation("A detached WebMCP invocation is still pending or failed to settle and can mutate, rerender, or navigate the page, so prior snapshot refs remain invalid after URL verification. Run webmcp result or cancel, then take a fresh snapshot before using page-scoped refs.")
|
|
459
|
+
: processResult.agentBrowserStarted && (directTransitionInvalidation || plannedBatchTransitionInvalidation)
|
|
460
|
+
? directTransitionInvalidation ?? plannedBatchTransitionInvalidation
|
|
461
|
+
: failedTransitionReverification
|
|
462
|
+
? buildPageTransitionRefSnapshotInvalidation("A failed eval/back/forward/reload/connect/state-load/tab command may still have changed the page, so the prior snapshot refs were invalidated. Run snapshot -i before using page-scoped refs.")
|
|
463
|
+
: batchRefSnapshotState?.invalidation?.reason === "page-transition"
|
|
464
|
+
? batchRefSnapshotState.invalidation
|
|
465
|
+
: undefined;
|
|
457
466
|
if (currentSessionTabTarget) {
|
|
458
467
|
const tabUpdate = sessionPageState.applyTabTarget({ sessionName: sessionStateKey, target: currentSessionTabTarget, update: sessionPageStateUpdate });
|
|
459
468
|
if (!tabUpdate.applied && succeeded)
|
|
460
469
|
sessionPageState.markPinning(sessionStateKey, "drift");
|
|
461
470
|
}
|
|
462
|
-
else if (processResult.agentBrowserStarted && (resultingPageState.pageUrlUnknown || resultingPageState.pageTargetMayHaveChanged) && !(prepared.commandTokens[0] === "session" && prepared.commandTokens[1] === "info")) {
|
|
471
|
+
else if (processResult.agentBrowserStarted && (unobservedMutation || resultingPageState.pageUrlUnknown || resultingPageState.pageTargetMayHaveChanged) && !(prepared.commandTokens[0] === "session" && prepared.commandTokens[1] === "info")) {
|
|
463
472
|
sessionPageState.markTabTargetUnknown({ sessionName: sessionStateKey, update: sessionPageStateUpdate });
|
|
464
473
|
}
|
|
465
|
-
let refSnapshot = unsettledWebMcpMutation
|
|
474
|
+
let refSnapshot = unsettledWebMcpMutation || unobservedMutation
|
|
466
475
|
? undefined
|
|
467
476
|
: prepared.executionPlan.commandInfo.command === "batch"
|
|
468
477
|
? batchRefSnapshotState?.snapshot
|
|
@@ -473,7 +482,7 @@ export async function processBrowserOutput(input) {
|
|
|
473
482
|
? prepared.commandTokens.filter((token) => token !== "--delta" && token !== "--full") : undefined);
|
|
474
483
|
// Native owns delta baselines. Read complete refs without advancing that baseline
|
|
475
484
|
// rather than maintaining a second revision/tree cache in the wrapper.
|
|
476
|
-
if (refreshArgs && !unsettledWebMcpMutation) {
|
|
485
|
+
if (refreshArgs && !unsettledWebMcpMutation && !unobservedMutation) {
|
|
477
486
|
refSnapshot = extractRefSnapshotFromData(await runSessionCommandData({ args: refreshArgs, cwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, signal }));
|
|
478
487
|
if (!refSnapshot)
|
|
479
488
|
sessionPageState.applyRefSnapshotInvalidation({ invalidation: buildPageTransitionRefSnapshotInvalidation("The native snapshot delta did not include complete refs and the full ref read failed. Run snapshot -i before using refs."), sessionName: sessionStateKey, update: sessionPageStateUpdate });
|
|
@@ -683,7 +692,7 @@ export async function processBrowserOutput(input) {
|
|
|
683
692
|
}
|
|
684
693
|
if (plainTextUpgrade && errorText)
|
|
685
694
|
presentationEnvelope = { ...presentationEnvelope, success: false, error: errorText };
|
|
686
|
-
let presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text", text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : recordingStopRecovery && !recordingStopRecovery.batch ? recordingStopRecovery.presentation : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactMaxUpdatedAtMs: Date.now(), artifactMinUpdatedAtMs: input.artifactRunStartedAtMs, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd: operationCwd, envelope: presentationEnvelope, errorText, namespace: prepared.executionPlan.namespace, networkRouteDiagnostics, networkRoutes: activeNetworkRoutes, persistentArtifactStore, previousRecordingContactSheetPath: sessionStateKey ? state.activeRecordingReservations?.get(sessionStateKey)?.contactSheetPath : undefined, piCleanupOwnership: sessionStateKey && (state.ownedManagedSessions.has(sessionStateKey) || prepared.executionPlan.managedSessionName !== undefined) ? "wrapper-managed" : "caller-owned", sessionName: prepared.executionPlan.sessionName });
|
|
695
|
+
let presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text", text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : recordingStopRecovery && !recordingStopRecovery.batch ? recordingStopRecovery.presentation : await buildToolPresentation({ modelVisible: input.modelVisible, args: prepared.redactedProcessArgs, artifactManifest, artifactMaxUpdatedAtMs: Date.now(), artifactMinUpdatedAtMs: input.artifactRunStartedAtMs, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd: operationCwd, envelope: presentationEnvelope, errorText, namespace: prepared.executionPlan.namespace, networkRouteDiagnostics, networkRoutes: activeNetworkRoutes, persistentArtifactStore, previousRecordingContactSheetPath: sessionStateKey ? state.activeRecordingReservations?.get(sessionStateKey)?.contactSheetPath : undefined, piCleanupOwnership: sessionStateKey && (state.ownedManagedSessions.has(sessionStateKey) || prepared.executionPlan.managedSessionName !== undefined) ? "wrapper-managed" : "caller-owned", sessionName: prepared.executionPlan.sessionName });
|
|
687
696
|
if (recordingStopRecovery)
|
|
688
697
|
presentation = mergeRecordingRecoveryPresentation(presentation, recordingStopRecovery);
|
|
689
698
|
const confirmation = readConfirmationEvent ?? prepared.readConfirmation;
|
|
@@ -703,7 +712,8 @@ export async function processBrowserOutput(input) {
|
|
|
703
712
|
const errorContent = presentation.content[0];
|
|
704
713
|
if (errorContent?.type === "text" && presentationEnvelope.data)
|
|
705
714
|
errorContent.text += `\n\n${presentationEnvelope.data}`;
|
|
706
|
-
|
|
715
|
+
if (input.modelVisible !== false)
|
|
716
|
+
presentation = await compactLargePresentationOutput({ artifactManifest, commandInfo: prepared.executionPlan.commandInfo, data: presentation.data, persistentArtifactStore, presentation });
|
|
707
717
|
}
|
|
708
718
|
if (electronHandoff?.error && electronHandoff.failureCategory)
|
|
709
719
|
presentation.failureCategory = electronHandoff.failureCategory;
|
|
@@ -743,6 +753,11 @@ export async function processBrowserOutput(input) {
|
|
|
743
753
|
const qaPreset = prepared.compiledQaPreset
|
|
744
754
|
? (processResult.timedOut ? analyzeQaPresetTimeout(prepared.compiledQaPreset) ?? analyzeQaPresetResults(presentationEnvelope?.data, prepared.compiledQaPreset) : analyzeQaPresetResults(presentationEnvelope?.data, prepared.compiledQaPreset))
|
|
745
755
|
: undefined;
|
|
756
|
+
if (qaPreset?.passed && presentation.resultCategory === "failure") {
|
|
757
|
+
qaPreset.passed = false;
|
|
758
|
+
qaPreset.summary = presentation.summary;
|
|
759
|
+
qaPreset.failedChecks.push(presentation.summary);
|
|
760
|
+
}
|
|
746
761
|
let qaAttachedTarget = prepared.compiledQaPreset?.checks.attached
|
|
747
762
|
? await collectQaAttachedTarget({ currentTarget: currentSessionTabTarget ?? prepared.priorSessionTabTarget, cwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, signal })
|
|
748
763
|
: undefined;
|
|
@@ -819,7 +834,7 @@ export async function processBrowserOutput(input) {
|
|
|
819
834
|
: undefined;
|
|
820
835
|
const sessionWarning = electronPostCommandHealth ? formatElectronPostCommandHealthText(electronPostCommandHealth) : electronSessionMismatch ? formatElectronSessionMismatchText(electronSessionMismatch) : aboutBlankSessionMismatch ? buildAboutBlankWarning(aboutBlankSessionMismatch) : undefined;
|
|
821
836
|
const warningText = [sessionWarning, recordingPageWarning].filter(Boolean).join("\n\n") || undefined;
|
|
822
|
-
const redactedContent = buildRedactedPresentationContent({ exactSensitiveValues: prepared.exactSensitiveValues, plainTextInspection, presentation, presentationEnvelope, succeeded, userRequestedJson: prepared.userRequestedJson, warningText });
|
|
837
|
+
const redactedContent = input.modelVisible === false ? [] : buildRedactedPresentationContent({ exactSensitiveValues: prepared.exactSensitiveValues, plainTextInspection, presentation, presentationEnvelope, succeeded, userRequestedJson: prepared.userRequestedJson, warningText });
|
|
823
838
|
const finalRecoveryState = await prepareFinalResultRecoveryState({ aboutBlankSessionMismatch, batchRefSnapshotState, commandTokens: prepared.commandTokens, compiledSemanticAction: prepared.compiledSemanticAction, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, cwd, electronPostCommandHealth, errorText, executionPlan: prepared.executionPlan, parseError, plainTextInspection, presentation, processResult, redactedProcessArgs: prepared.redactedProcessArgs, runtimeToolArgs: prepared.runtimeToolArgs, sessionPageState, sessionPageStateUpdate, sessionTabCorrection, signal, succeeded });
|
|
824
839
|
currentRefSnapshot = finalRecoveryState.currentRefSnapshot;
|
|
825
840
|
currentRefSnapshotInvalidation = finalRecoveryState.currentRefSnapshotInvalidation;
|
|
@@ -836,7 +851,7 @@ export async function processBrowserOutput(input) {
|
|
|
836
851
|
const resultHeadedManagedAutosaveInterval = resultRetainsPreparedManagedSession && !prepared.ownedManagedSessionContext?.reuseOnly && !(commandClosesSession && succeeded)
|
|
837
852
|
? prepared.ownedManagedSessionContext?.headedManagedAutosaveInterval
|
|
838
853
|
: undefined;
|
|
839
|
-
const result = buildFinalAgentBrowserToolResult({ aboutBlankSessionMismatch, artifactCleanup, categoryDetails: finalRecoveryState.categoryDetails, clickDispatchDiagnostic, commandTokens: prepared.commandTokens, comboboxFocusDiagnostic, compiledNetworkSourceLookup: prepared.compiledNetworkSourceLookup, compiledSemanticAction: prepared.compiledSemanticAction, compatibilityWorkaround: prepared.compatibilityWorkaround, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, currentSessionTabTargetUnknown, electronBroadGetTextScopeDiagnostics, electronFailedConnectCleanup, electronHandoff, electronLaunch: prepared.electronLaunch, electronLaunchRecord, electronLaunchRecords, electronPostCommandHealth, electronProfileIsolationDetails: input.electronProfileIsolationDetails, electronRefFreshnessDiagnostic, electronSessionMismatch, errorText, evalResultWarning, evalStdinHint, exactSensitiveValues: prepared.exactSensitiveValues, executionPlan: prepared.executionPlan, fillVerificationDiagnostic, headedLaunch: prepared.headedLaunch, inspectionText, preserveAttachedBrowserSession: input.preserveAttachedBrowserSession === true, providerLaunch: prepared.providerLaunch, managedSessionHeadedAutosaveDisabled: resultHeadedManagedAutosaveDisabled || undefined, managedSessionHeadedAutosaveInterval: resultHeadedManagedAutosaveInterval, managedSessionOutcome, managedSessionRestoreDisabled: state.managedSessionRestoreState.isDisabled(prepared.executionPlan.sessionName, prepared.executionPlan.namespace), navigationSummary, networkSourceLookup, noActivePageSnapshotFailure: finalRecoveryState.noActivePageSnapshotFailure, openResultTabCorrection, overlayBlockerDiagnostic, parseError, parseFailureOutput, parseSucceeded, plainTextInspection, presentation, presentationEnvelope, priorSessionTabTarget: prepared.priorSessionTabTarget, processResult, qaAttachedTarget, qaPreset, recordingDependencyWarning, redactedArgs: prepared.redactedArgs, redactedCompiledElectron: prepared.redactedCompiledElectron, redactedCompiledJob: prepared.redactedCompiledJob, redactedCompiledNetworkSourceLookup: prepared.redactedCompiledNetworkSourceLookup, redactedCompiledQaPreset: prepared.redactedCompiledQaPreset, redactedCompiledSemanticAction: prepared.redactedCompiledSemanticAction, redactedCompiledSourceLookup: prepared.redactedCompiledSourceLookup, redactedContent, redactedProcessArgs: prepared.redactedProcessArgs, redactedRecoveryHint: prepared.redactedRecoveryHint, resultArtifactManifest, richInputRecoveryDiagnostic: finalRecoveryState.richInputRecoveryDiagnostic, scrollNoopDiagnostic, selectorTextVisibilityDiagnostics, sessionMode: prepared.sessionMode, sessionTabCorrection, sourceLookup, succeeded, timeoutPartialProgress, unsettledWebMcpMutation, userRequestedJson: prepared.userRequestedJson, visibleRefFallbackDiagnostic: finalRecoveryState.visibleRefFallbackDiagnostic, visibleRefFallbackSessionName: finalRecoveryState.visibleRefFallbackSessionName });
|
|
854
|
+
const result = buildFinalAgentBrowserToolResult({ modelVisible: input.modelVisible, aboutBlankSessionMismatch, artifactCleanup, categoryDetails: finalRecoveryState.categoryDetails, clickDispatchDiagnostic, commandTokens: prepared.commandTokens, comboboxFocusDiagnostic, compiledNetworkSourceLookup: prepared.compiledNetworkSourceLookup, compiledSemanticAction: prepared.compiledSemanticAction, compatibilityWorkaround: prepared.compatibilityWorkaround, currentRefSnapshot, currentRefSnapshotInvalidation, currentSessionTabTarget, currentSessionTabTargetUnknown, electronBroadGetTextScopeDiagnostics, electronFailedConnectCleanup, electronHandoff, electronLaunch: prepared.electronLaunch, electronLaunchRecord, electronLaunchRecords, electronPostCommandHealth, electronProfileIsolationDetails: input.electronProfileIsolationDetails, electronRefFreshnessDiagnostic, electronSessionMismatch, errorText, evalResultWarning, evalStdinHint, exactSensitiveValues: prepared.exactSensitiveValues, executionPlan: prepared.executionPlan, fillVerificationDiagnostic, headedLaunch: prepared.headedLaunch, inspectionText, preserveAttachedBrowserSession: input.preserveAttachedBrowserSession === true, providerLaunch: prepared.providerLaunch, managedSessionHeadedAutosaveDisabled: resultHeadedManagedAutosaveDisabled || undefined, managedSessionHeadedAutosaveInterval: resultHeadedManagedAutosaveInterval, managedSessionOutcome, managedSessionRestoreDisabled: state.managedSessionRestoreState.isDisabled(prepared.executionPlan.sessionName, prepared.executionPlan.namespace), navigationSummary, networkSourceLookup, noActivePageSnapshotFailure: finalRecoveryState.noActivePageSnapshotFailure, openResultTabCorrection, overlayBlockerDiagnostic, parseError, parseFailureOutput, parseSucceeded, plainTextInspection, presentation, presentationEnvelope, priorSessionTabTarget: prepared.priorSessionTabTarget, processResult, qaAttachedTarget, qaPreset, recordingDependencyWarning, redactedArgs: prepared.redactedArgs, redactedCompiledElectron: prepared.redactedCompiledElectron, redactedCompiledJob: prepared.redactedCompiledJob, redactedCompiledNetworkSourceLookup: prepared.redactedCompiledNetworkSourceLookup, redactedCompiledQaPreset: prepared.redactedCompiledQaPreset, redactedCompiledSemanticAction: prepared.redactedCompiledSemanticAction, redactedCompiledSourceLookup: prepared.redactedCompiledSourceLookup, redactedContent, redactedProcessArgs: prepared.redactedProcessArgs, redactedRecoveryHint: prepared.redactedRecoveryHint, resultArtifactManifest, richInputRecoveryDiagnostic: finalRecoveryState.richInputRecoveryDiagnostic, scrollNoopDiagnostic, selectorTextVisibilityDiagnostics, sessionMode: prepared.sessionMode, sessionTabCorrection, sourceLookup, succeeded, timeoutPartialProgress, unsettledWebMcpMutation, userRequestedJson: prepared.userRequestedJson, visibleRefFallbackDiagnostic: finalRecoveryState.visibleRefFallbackDiagnostic, visibleRefFallbackSessionName: finalRecoveryState.visibleRefFallbackSessionName });
|
|
840
855
|
const resultWithCloseAll = closeAllApplied
|
|
841
856
|
? { ...result, details: { ...(isRecord(result.details) ? result.details : {}), closeAllApplied: true } }
|
|
842
857
|
: result;
|
|
@@ -95,6 +95,7 @@ export async function recoverRecordingStop(options) {
|
|
|
95
95
|
&& !currentUsesSamePath;
|
|
96
96
|
const data = matchedData ?? (expected ? { path: expected.absolutePath, recordingId: expected.recordingId, success: null } : undefined);
|
|
97
97
|
const presentation = await buildToolPresentation({
|
|
98
|
+
modelVisible: options.modelVisible,
|
|
98
99
|
artifactManifest: options.artifactManifest, artifactMinUpdatedAtMs: expected?.startedAtMs ?? options.artifactRunStartedAtMs, artifactMaxUpdatedAtMs: Date.now(),
|
|
99
100
|
artifactRequest: expected ? { path: expected.path, absolutePath: expected.absolutePath, status: terminalMeasurements ? undefined : "unverified" } : undefined,
|
|
100
101
|
commandInfo: { command: "record", subcommand: "stop" }, cwd: expected?.cwd ?? options.cwd,
|
|
@@ -139,7 +140,8 @@ export async function recoverRecordingStop(options) {
|
|
|
139
140
|
presentation.nextActions = followups;
|
|
140
141
|
presentation.recordingRecovery = recovery;
|
|
141
142
|
const text = `Recording receipt recovery (${recovery.status}): ${recovery.reason}\nOriginal attempt: ${attemptError}`;
|
|
142
|
-
|
|
143
|
+
if (options.modelVisible !== false)
|
|
144
|
+
presentation.content.unshift({ type: "text", text });
|
|
143
145
|
return { batch, envelope, partialBatch: batch && !rows, stopIndex, presentation, recovery };
|
|
144
146
|
}
|
|
145
147
|
export function mergeRecordingRecoveryPresentation(base, result) {
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { isRecord } from "../../parsing.js";
|
|
2
|
+
import { getScreenshotPathTokenIndex, getScreenshotPositionalIndices } from "./artifact-paths.js";
|
|
3
|
+
import { runSessionCommandData } from "./session-state.js";
|
|
4
|
+
export function getScreenshotCapture(command) {
|
|
5
|
+
if (command[0] !== "screenshot")
|
|
6
|
+
return { kind: "unknown" };
|
|
7
|
+
if (command.includes("--full") || command.includes("-f"))
|
|
8
|
+
return { kind: "full-page" };
|
|
9
|
+
const positional = getScreenshotPositionalIndices(command);
|
|
10
|
+
const selectorIndex = positional.find(index => index !== getScreenshotPathTokenIndex(command));
|
|
11
|
+
return selectorIndex === undefined ? { kind: "viewport" } : { kind: "element", selector: command[selectorIndex] };
|
|
12
|
+
}
|
|
13
|
+
function finiteRecord(value, keys) {
|
|
14
|
+
return isRecord(value) && keys.every(key => typeof value[key] === "number" && Number.isFinite(value[key]));
|
|
15
|
+
}
|
|
16
|
+
export async function collectScreenshotSample(options) {
|
|
17
|
+
if (options.command[0] !== "screenshot" || options.signal?.aborted)
|
|
18
|
+
return undefined;
|
|
19
|
+
const { selector } = getScreenshotCapture(options.command);
|
|
20
|
+
const script = `(() => {
|
|
21
|
+
const root = document.documentElement, body = document.body, v = window.visualViewport;
|
|
22
|
+
let element;
|
|
23
|
+
try {
|
|
24
|
+
const selector = ${JSON.stringify(selector ?? null)};
|
|
25
|
+
const matches = selector ? document.querySelectorAll(selector) : [];
|
|
26
|
+
if (matches.length === 1) {
|
|
27
|
+
const rect = matches[0].getBoundingClientRect();
|
|
28
|
+
element = { x: rect.x, y: rect.y, width: rect.width, height: rect.height };
|
|
29
|
+
}
|
|
30
|
+
} catch {}
|
|
31
|
+
return { url: location.href, frame: window === window.top ? "main" : "child", childFrameCount: window.frames.length,
|
|
32
|
+
viewport: { width: innerWidth, height: innerHeight },
|
|
33
|
+
document: { width: Math.max(root.scrollWidth, body?.scrollWidth || 0, innerWidth), height: Math.max(root.scrollHeight, body?.scrollHeight || 0, innerHeight) },
|
|
34
|
+
scroll: { x: scrollX, y: scrollY }, dpr: devicePixelRatio,
|
|
35
|
+
visualViewport: { x: v?.offsetLeft ?? 0, y: v?.offsetTop ?? 0, scale: v?.scale ?? 1 }, element };
|
|
36
|
+
})()`;
|
|
37
|
+
try {
|
|
38
|
+
const data = await runSessionCommandData({ ...options, args: ["eval", "--stdin"], stdin: script, timeoutMs: 2_000 });
|
|
39
|
+
const sample = isRecord(data) ? data.result : undefined;
|
|
40
|
+
if (!isRecord(sample) || typeof sample.url !== "string" || !["main", "child"].includes(String(sample.frame))
|
|
41
|
+
|| !finiteRecord(sample.viewport, ["width", "height"]) || !finiteRecord(sample.document, ["width", "height"])
|
|
42
|
+
|| !finiteRecord(sample.scroll, ["x", "y"]) || !finiteRecord(sample.visualViewport, ["x", "y", "scale"])
|
|
43
|
+
|| typeof sample.childFrameCount !== "number" || !Number.isInteger(sample.childFrameCount) || sample.childFrameCount < 0
|
|
44
|
+
|| typeof sample.dpr !== "number" || !Number.isFinite(sample.dpr) || sample.dpr <= 0
|
|
45
|
+
|| (sample.element !== undefined && !finiteRecord(sample.element, ["x", "y", "width", "height"])))
|
|
46
|
+
return undefined;
|
|
47
|
+
return sample;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
export function buildScreenshotGeometry(options) {
|
|
54
|
+
const { before, after, capture, pixels } = options;
|
|
55
|
+
const unknown = (reason) => ({ status: "unknown", reason, before, after });
|
|
56
|
+
if (!before || !after)
|
|
57
|
+
return unknown("Capture was not bracketed by browser geometry samples; coordinates are unknown.");
|
|
58
|
+
if (JSON.stringify(before) !== JSON.stringify(after))
|
|
59
|
+
return unknown("Browser geometry changed across capture; no coordinate mapping is asserted.");
|
|
60
|
+
if (before.frame !== "main")
|
|
61
|
+
return unknown("Probe observed a child frame; its relation to the captured page is unknown.");
|
|
62
|
+
if (before.visualViewport.scale !== 1 || before.visualViewport.x !== 0 || before.visualViewport.y !== 0)
|
|
63
|
+
return unknown("Visual viewport is zoomed or offset; capture origin is unknown.");
|
|
64
|
+
let crop;
|
|
65
|
+
if (capture === "viewport")
|
|
66
|
+
crop = { ...before.scroll, ...before.viewport };
|
|
67
|
+
else if (capture === "full-page")
|
|
68
|
+
crop = { x: 0, y: 0, ...before.document };
|
|
69
|
+
else if (capture === "element" && before.element && before.childFrameCount === 0 && before.scroll.x === 0 && before.scroll.y === 0)
|
|
70
|
+
crop = before.element;
|
|
71
|
+
else
|
|
72
|
+
return unknown("Element crop/frame provenance is unverified (scrolled page, child frames, non-CSS selector, or ambiguous target).");
|
|
73
|
+
if (!pixels || crop.width <= 0 || crop.height <= 0 || crop.x < 0 || crop.y < 0
|
|
74
|
+
|| !Object.values(crop).every(Number.isInteger)
|
|
75
|
+
|| pixels.width !== crop.width * before.dpr || pixels.height !== crop.height * before.dpr)
|
|
76
|
+
return unknown("Image dimensions do not establish the sampled CSS crop at this DPR.");
|
|
77
|
+
return { status: "measured", before, after, crop, pixelsPerCssPixel: { x: pixels.width / crop.width, y: pixels.height / crop.height },
|
|
78
|
+
reason: "Matching pre/post samples, not an atomic capture guarantee. Pixels map to CSS document coordinates within crop; native mouse uses current viewport CSS coordinates. Recheck scroll/frame before input." };
|
|
79
|
+
}
|
|
@@ -4,6 +4,7 @@ import { boundElectronProbeString } from "../../electron/cdp.js";
|
|
|
4
4
|
import { buildOwnedManagedSessionRestoreContext, withOwnedManagedSessionContext, } from "../../managed-session-restore.js";
|
|
5
5
|
import { getPageTargetValidationError } from "../../page-target-validation.js";
|
|
6
6
|
import { isRecord } from "../../parsing.js";
|
|
7
|
+
import { getBrowserResultMessage } from "../../browser-transcript.js";
|
|
7
8
|
import { withAttachedBrowserSessionContext } from "../../process.js";
|
|
8
9
|
import { buildAgentBrowserNextActions } from "../../results/action-recommendations.js";
|
|
9
10
|
import { buildAgentBrowserResultCategoryDetails } from "../../results/categories.js";
|
|
@@ -125,10 +126,8 @@ function isElectronLaunchRecord(value) {
|
|
|
125
126
|
export function restoreElectronLaunchRecordsFromBranch(branch) {
|
|
126
127
|
const records = new Map();
|
|
127
128
|
for (const entry of branch) {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
const message = isRecord(entry.message) ? entry.message : undefined;
|
|
131
|
-
if (!message || message.toolName !== "agent_browser")
|
|
129
|
+
const message = getBrowserResultMessage(entry);
|
|
130
|
+
if (!message)
|
|
132
131
|
continue;
|
|
133
132
|
const details = isRecord(message.details) ? message.details : undefined;
|
|
134
133
|
const electron = isRecord(details?.electron) ? details.electron : undefined;
|