pi-agent-browser-native 0.2.33 → 0.2.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +47 -17
- package/docs/ARCHITECTURE.md +25 -13
- package/docs/COMMAND_REFERENCE.md +285 -47
- package/docs/ELECTRON.md +3 -3
- package/docs/RELEASE.md +22 -14
- package/docs/REQUIREMENTS.md +5 -5
- package/docs/SUPPORT_MATRIX.md +26 -22
- package/docs/TOOL_CONTRACT.md +97 -32
- package/extensions/agent-browser/index.ts +519 -2402
- package/extensions/agent-browser/lib/argv-descriptor.ts +90 -0
- package/extensions/agent-browser/lib/argv-grammar.ts +128 -0
- package/extensions/agent-browser/lib/command-policy.ts +71 -0
- package/extensions/agent-browser/lib/command-taxonomy.ts +336 -0
- package/extensions/agent-browser/lib/electron/cleanup.ts +1 -0
- package/extensions/agent-browser/lib/executable-path.ts +19 -0
- package/extensions/agent-browser/lib/input-modes/job.ts +62 -0
- package/extensions/agent-browser/lib/input-modes/params.ts +8 -8
- package/extensions/agent-browser/lib/input-modes.ts +3 -0
- package/extensions/agent-browser/lib/orchestration/batch-stdin.ts +65 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/browser-action-model.ts +154 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.ts +149 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts +77 -29
- package/extensions/agent-browser/lib/orchestration/browser-run/final-result.ts +6 -2
- package/extensions/agent-browser/lib/orchestration/browser-run/index.ts +33 -27
- package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts +74 -23
- package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts +67 -17
- package/extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.ts +93 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts +19 -123
- package/extensions/agent-browser/lib/orchestration/browser-run/types.ts +32 -1
- package/extensions/agent-browser/lib/orchestration/electron-host/index.ts +860 -0
- package/extensions/agent-browser/lib/playbook.ts +24 -23
- package/extensions/agent-browser/lib/prompt-policy.ts +122 -0
- package/extensions/agent-browser/lib/results/action-recommendations.ts +3 -23
- package/extensions/agent-browser/lib/results/categories.ts +1 -1
- package/extensions/agent-browser/lib/results/presentation/navigation.ts +2 -34
- package/extensions/agent-browser/lib/results/presentation/registry.ts +34 -6
- package/extensions/agent-browser/lib/results/presentation/semantic-action.ts +133 -0
- package/extensions/agent-browser/lib/results/presentation.ts +11 -6
- package/extensions/agent-browser/lib/runtime.ts +93 -227
- package/extensions/agent-browser/lib/session-page-state.ts +31 -14
- package/extensions/agent-browser/lib/temp.ts +148 -23
- package/package.json +4 -4
- package/scripts/agent-browser-capability-baseline.mjs +198 -1
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { delimiter, isAbsolute, join, resolve } from "node:path";
|
|
1
|
+
import { stat } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, resolve } from "node:path";
|
|
4
3
|
|
|
4
|
+
import { isCloseCommand, isOpenNavigationCommand } from "../../command-taxonomy.js";
|
|
5
5
|
import type { ElectronLaunchRecord } from "../../electron/launch.js";
|
|
6
|
+
import { executableExistsOnPath } from "../../executable-path.js";
|
|
6
7
|
import type { AgentBrowserSourceLookupAnalysis, CompiledAgentBrowserJob, CompiledAgentBrowserSemanticAction } from "../../input-modes.js";
|
|
8
|
+
import { isHttpOrHttpsUrl } from "../../input-modes/job.js";
|
|
7
9
|
import type { AgentBrowserNextAction } from "../../results.js";
|
|
8
10
|
import { formatSessionArtifactRetentionSummary } from "../../results/artifact-manifest.js";
|
|
9
|
-
import { withOptionalSessionArgs } from "../../results/next-actions.js";
|
|
11
|
+
import { buildNextToolAction, withOptionalSessionArgs } from "../../results/next-actions.js";
|
|
10
12
|
import { buildVisibleRefFallbackDiagnosticFromSnapshot, getVisibleRefFallbackTarget, type VisibleRefFallbackDiagnostic } from "../../results/selector-recovery.js";
|
|
11
13
|
import { extractRefSnapshotFromData, normalizeComparableUrl, type SessionTabTarget } from "../../session-page-state.js";
|
|
12
14
|
import { redactInvocationArgs, redactSensitiveText, type CommandInfo } from "../../runtime.js";
|
|
@@ -19,16 +21,19 @@ import {
|
|
|
19
21
|
getGuardedRefUsage,
|
|
20
22
|
runSessionCommandData,
|
|
21
23
|
} from "./session-state.js";
|
|
24
|
+
import { parseValidBatchStepEntries } from "../batch-stdin.js";
|
|
22
25
|
import { getScreenshotPathTokenIndex } from "./prepare.js";
|
|
23
26
|
import type {
|
|
24
27
|
ArtifactCleanupGuidance,
|
|
25
28
|
ComboboxFocusDiagnostic,
|
|
26
29
|
ElectronBroadGetTextScopeDiagnostic,
|
|
27
30
|
ElectronHandoffSummary,
|
|
31
|
+
ElectronManagedSessionTarget,
|
|
28
32
|
FillVerificationDiagnostic,
|
|
29
33
|
NavigationSummary,
|
|
30
34
|
OverlayBlockerCandidate,
|
|
31
35
|
OverlayBlockerDiagnostic,
|
|
36
|
+
QaAttachedPreconditionFailure,
|
|
32
37
|
QaAttachedTarget,
|
|
33
38
|
RecordingDependencyWarning,
|
|
34
39
|
ScrollNoopDiagnostic,
|
|
@@ -236,23 +241,6 @@ function getRecordStartLikeCommand(command: string | undefined, commandTokens: s
|
|
|
236
241
|
return undefined;
|
|
237
242
|
}
|
|
238
243
|
|
|
239
|
-
async function executableExistsOnPath(command: string): Promise<boolean> {
|
|
240
|
-
const pathValue = process.env.PATH ?? "";
|
|
241
|
-
const extensions = process.platform === "win32" ? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean) : [""];
|
|
242
|
-
for (const directory of pathValue.split(delimiter).filter(Boolean)) {
|
|
243
|
-
for (const extension of extensions) {
|
|
244
|
-
try {
|
|
245
|
-
const candidate = join(directory, `${command}${extension}`);
|
|
246
|
-
await access(candidate, fsConstants.X_OK);
|
|
247
|
-
if ((await stat(candidate)).isFile()) return true;
|
|
248
|
-
} catch {
|
|
249
|
-
// Try the next candidate.
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
return false;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
244
|
export async function collectRecordingDependencyWarning(options: { command: string | undefined; commandTokens: string[]; succeeded: boolean }): Promise<RecordingDependencyWarning | undefined> {
|
|
257
245
|
if (!options.succeeded) return undefined;
|
|
258
246
|
const recordCommand = getRecordStartLikeCommand(options.command, options.commandTokens);
|
|
@@ -464,7 +452,7 @@ export function formatEvalStdinHintText(hint: ReturnType<typeof getEvalStdinHint
|
|
|
464
452
|
}
|
|
465
453
|
|
|
466
454
|
export async function getArtifactCleanupGuidance(options: { command?: string; cwd: string; manifest?: SessionArtifactManifest; succeeded: boolean }): Promise<ArtifactCleanupGuidance | undefined> {
|
|
467
|
-
if (!options.succeeded || options.command
|
|
455
|
+
if (!options.succeeded || !isCloseCommand(options.command) || !options.manifest || options.manifest.entries.length === 0) return undefined;
|
|
468
456
|
const explicitEntries = options.manifest.entries.filter((entry) => entry.storageScope === "explicit-path");
|
|
469
457
|
const explicitArtifactPaths: string[] = [];
|
|
470
458
|
const seenPaths = new Set<string>();
|
|
@@ -491,7 +479,19 @@ async function collectManagedSessionCommandData(options: { args: string[]; cwd:
|
|
|
491
479
|
try { return { data: await runSessionCommandData(options) }; } catch (error) { return { error: error instanceof Error ? error.message : String(error) }; }
|
|
492
480
|
}
|
|
493
481
|
|
|
494
|
-
async function
|
|
482
|
+
async function collectElectronManagedSessionUrl(options: { cwd: string; sessionName: string; signal?: AbortSignal; timeoutMs?: number }): Promise<{ error?: string; url?: string }> {
|
|
483
|
+
const urlResult = await collectManagedSessionCommandData({
|
|
484
|
+
args: ["get", "url"],
|
|
485
|
+
cwd: options.cwd,
|
|
486
|
+
sessionName: options.sessionName,
|
|
487
|
+
signal: options.signal,
|
|
488
|
+
timeoutMs: options.timeoutMs,
|
|
489
|
+
});
|
|
490
|
+
const url = boundElectronProbeString(extractStringResultField(urlResult.data, "result") ?? extractStringResultField(urlResult.data, "url"), 300);
|
|
491
|
+
return urlResult.error ? { error: urlResult.error } : { url };
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
export async function collectElectronManagedSessionTarget(options: { cwd: string; sessionName?: string; signal?: AbortSignal; timeoutMs?: number }): Promise<ElectronManagedSessionTarget | undefined> {
|
|
495
495
|
if (!options.sessionName) return undefined;
|
|
496
496
|
const [titleResult, urlResult] = await Promise.all([
|
|
497
497
|
collectManagedSessionCommandData({ args: ["get", "title"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, timeoutMs: options.timeoutMs }),
|
|
@@ -514,6 +514,58 @@ export function formatQaAttachedTargetText(target: QaAttachedTarget | undefined)
|
|
|
514
514
|
return ["QA attached target:", target.sessionName, target.title, target.url].filter((part): part is string => typeof part === "string" && part.length > 0).join(" — ");
|
|
515
515
|
}
|
|
516
516
|
|
|
517
|
+
export function buildQaAttachedRecoveryNextActions(sessionName: string | undefined): AgentBrowserNextAction[] {
|
|
518
|
+
const sessionArgs = (args: string[]) => withOptionalSessionArgs(sessionName, args);
|
|
519
|
+
return [
|
|
520
|
+
buildNextToolAction({
|
|
521
|
+
args: sessionArgs(["tab", "list"]),
|
|
522
|
+
id: "list-tabs-before-qa-attached",
|
|
523
|
+
reason: "Inspect the connected session tabs before retrying qa.attached.",
|
|
524
|
+
safety: "Read-only tab listing for the attached session.",
|
|
525
|
+
}),
|
|
526
|
+
buildNextToolAction({
|
|
527
|
+
args: sessionArgs(["snapshot", "-i"]),
|
|
528
|
+
id: "snapshot-before-qa-attached",
|
|
529
|
+
reason: "Capture interactive refs on the active http(s) page before retrying qa.attached.",
|
|
530
|
+
safety: "Read-only snapshot; confirms a renderable page is selected.",
|
|
531
|
+
}),
|
|
532
|
+
];
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
export async function validateQaAttachedPrecondition(options: {
|
|
536
|
+
cwd: string;
|
|
537
|
+
sessionName?: string;
|
|
538
|
+
signal?: AbortSignal;
|
|
539
|
+
}): Promise<QaAttachedPreconditionFailure | undefined> {
|
|
540
|
+
if (!options.sessionName) {
|
|
541
|
+
return {
|
|
542
|
+
error: "qa.attached requires an active attached session with a resolvable session name.",
|
|
543
|
+
nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
const urlProbe = await collectElectronManagedSessionUrl({ cwd: options.cwd, sessionName: options.sessionName, signal: options.signal });
|
|
547
|
+
if (urlProbe.error) {
|
|
548
|
+
return {
|
|
549
|
+
error: `qa.attached could not read the attached session URL: ${urlProbe.error}. Run tab list or snapshot -i before retrying qa.attached.`,
|
|
550
|
+
nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
const url = urlProbe.url?.trim();
|
|
554
|
+
if (!url) {
|
|
555
|
+
return {
|
|
556
|
+
error: "qa.attached requires an attached session with a readable http(s) page URL. Run tab list, select a stable tab, then snapshot -i before retrying.",
|
|
557
|
+
nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
if (!isHttpOrHttpsUrl(url)) {
|
|
561
|
+
return {
|
|
562
|
+
error: `qa.attached requires an http(s) page URL; the current attached URL is "${url}". Use tab list and snapshot -i to recover a web surface before retrying.`,
|
|
563
|
+
nextActions: buildQaAttachedRecoveryNextActions(options.sessionName),
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
return undefined;
|
|
567
|
+
}
|
|
568
|
+
|
|
517
569
|
function getTopLevelFillInvocation(commandTokens: string[]): { expected: string; selector: string } | undefined {
|
|
518
570
|
if (commandTokens[0] !== "fill" || commandTokens.length < 3) return undefined;
|
|
519
571
|
const selector = commandTokens[1];
|
|
@@ -581,11 +633,7 @@ export async function collectElectronHandoff(options: { cwd: string; handoff: "c
|
|
|
581
633
|
function getTimeoutProgressSteps(compiledJob: CompiledAgentBrowserJob | undefined, command: string | undefined, stdin: string | undefined): Array<{ args: string[]; index: number }> {
|
|
582
634
|
if (compiledJob) return compiledJob.steps.map((step, index) => ({ args: step.args, index: index + 1 }));
|
|
583
635
|
if (command !== "batch" || !stdin) return [];
|
|
584
|
-
|
|
585
|
-
const parsed = JSON.parse(stdin) as unknown;
|
|
586
|
-
if (!Array.isArray(parsed)) return [];
|
|
587
|
-
return parsed.flatMap((step, index) => Array.isArray(step) && step.every((token) => typeof token === "string") ? [{ args: step as string[], index: index + 1 }] : []);
|
|
588
|
-
} catch { return []; }
|
|
636
|
+
return parseValidBatchStepEntries(stdin).map(({ index, step }) => ({ args: step, index: index + 1 }));
|
|
589
637
|
}
|
|
590
638
|
|
|
591
639
|
function getLastPositionalToken(args: string[], startIndex = 1): string | undefined {
|
|
@@ -643,7 +691,7 @@ async function collectTimeoutArtifactEvidence(cwd: string, steps: Array<{ args:
|
|
|
643
691
|
function getPlannedCurrentPageUrl(steps: Array<{ args: string[]; index: number }>): string | undefined {
|
|
644
692
|
for (let index = steps.length - 1; index >= 0; index -= 1) {
|
|
645
693
|
const args = steps[index]?.args ?? [];
|
|
646
|
-
if (args[0]
|
|
694
|
+
if (isOpenNavigationCommand(args[0]) || args[0] === "pushstate") return getLastPositionalToken(args);
|
|
647
695
|
}
|
|
648
696
|
return undefined;
|
|
649
697
|
}
|
|
@@ -43,6 +43,7 @@ import {
|
|
|
43
43
|
} from "../../session-page-state.js";
|
|
44
44
|
import { extractExplicitSessionName, redactInvocationArgs, redactSensitiveText, redactSensitiveValue, type OpenResultTabCorrection } from "../../runtime.js";
|
|
45
45
|
import { isRecord } from "../../parsing.js";
|
|
46
|
+
import { buildClickDispatchNextActions, formatClickDispatchDiagnosticText } from "./click-dispatch.js";
|
|
46
47
|
import {
|
|
47
48
|
buildComboboxFocusNextActions,
|
|
48
49
|
buildElectronBroadGetTextScopeNextActions,
|
|
@@ -212,7 +213,7 @@ export function buildElectronHostFailureResult(options: {
|
|
|
212
213
|
return { content: [{ type: "text", text: redactSensitiveText(text) }], details: redactToolDetails(details, []), isError: true };
|
|
213
214
|
}
|
|
214
215
|
|
|
215
|
-
function formatElectronTargetLines(targets: ElectronCdpTarget[], limit = 8): string[] {
|
|
216
|
+
export function formatElectronTargetLines(targets: ElectronCdpTarget[], limit = 8): string[] {
|
|
216
217
|
const shownTargets = targets.slice(0, limit);
|
|
217
218
|
const lines = shownTargets.map((target) => {
|
|
218
219
|
const label = [target.type, target.title].filter(Boolean).join(" ") || target.id || "target";
|
|
@@ -319,6 +320,7 @@ function buildResultNextActions(options: FinalResultInput): AgentBrowserNextActi
|
|
|
319
320
|
if (options.selectorTextVisibilityDiagnostics.length > 0) nextActionCollector.append(buildSelectorTextVisibilityNextActions({ diagnostics: options.selectorTextVisibilityDiagnostics, sessionName: options.executionPlan.sessionName }));
|
|
320
321
|
if (options.electronBroadGetTextScopeDiagnostics.length > 0) nextActionCollector.append(buildElectronBroadGetTextScopeNextActions({ diagnostics: options.electronBroadGetTextScopeDiagnostics, sessionName: options.executionPlan.sessionName }));
|
|
321
322
|
if (options.sourceLookup?.electronContext) nextActionCollector.appendUnique(buildSourceLookupElectronNextActions(options.sourceLookup));
|
|
323
|
+
if (options.clickDispatchDiagnostic) nextActionCollector.append(buildClickDispatchNextActions({ commandTokens: options.commandTokens, sessionName: options.executionPlan.sessionName }));
|
|
322
324
|
if (options.scrollNoopDiagnostic) nextActionCollector.append(buildScrollNoopNextActions(options.executionPlan.sessionName));
|
|
323
325
|
if (options.comboboxFocusDiagnostic) nextActionCollector.append(buildComboboxFocusNextActions(options.executionPlan.sessionName));
|
|
324
326
|
if (options.categoryDetails.failureCategory === "stale-ref" && options.redactedCompiledSemanticAction && isCompiledSemanticActionFindCommand(options.compiledSemanticAction)) nextActionCollector.append([{ id: "retry-semantic-action-after-stale-ref", params: { args: options.redactedCompiledSemanticAction.args }, reason: "Retry the same semantic target via its compiled find command after the upstream stale-ref failure proves the prior action did not execute.", safety: "Use only for the same intended target; direct stale @refs still require a fresh snapshot or stable locator before retrying.", tool: "agent_browser" as const }]);
|
|
@@ -369,6 +371,7 @@ function buildAgentBrowserResultDetails(options: FinalResultInput, nextActions:
|
|
|
369
371
|
imagePaths: options.presentation.imagePaths,
|
|
370
372
|
nextActions,
|
|
371
373
|
pageChangeSummary,
|
|
374
|
+
clickDispatch: options.clickDispatchDiagnostic,
|
|
372
375
|
overlayBlockers: options.overlayBlockerDiagnostic,
|
|
373
376
|
fillVerification: options.fillVerificationDiagnostic,
|
|
374
377
|
visibleRefFallback: publicVisibleRefFallbackDiagnostic,
|
|
@@ -411,6 +414,7 @@ export function buildFinalAgentBrowserToolResult(options: FinalResultInput): Age
|
|
|
411
414
|
const visibleRefFallbackText = formatVisibleRefFallbackText(options.visibleRefFallbackDiagnostic);
|
|
412
415
|
const richInputRecoveryText = formatRichInputRecoveryText(options.richInputRecoveryDiagnostic);
|
|
413
416
|
const semanticActionCandidateText = nextActions ? formatSemanticActionCandidateText(nextActions) : undefined;
|
|
417
|
+
const clickDispatchText = options.clickDispatchDiagnostic ? formatClickDispatchDiagnosticText(options.clickDispatchDiagnostic) : undefined;
|
|
414
418
|
const overlayBlockerText = options.overlayBlockerDiagnostic ? formatOverlayBlockerText(options.overlayBlockerDiagnostic) : undefined;
|
|
415
419
|
const fillVerificationText = formatFillVerificationText(options.fillVerificationDiagnostic);
|
|
416
420
|
const electronRefFreshnessText = formatElectronRefFreshnessText(options.electronRefFreshnessDiagnostic);
|
|
@@ -423,7 +427,7 @@ export function buildFinalAgentBrowserToolResult(options: FinalResultInput): Age
|
|
|
423
427
|
const artifactCleanupText = formatArtifactCleanupGuidanceText(options.artifactCleanup);
|
|
424
428
|
const timeoutPartialProgressText = options.timeoutPartialProgress ? formatTimeoutPartialProgressText(options.timeoutPartialProgress) : undefined;
|
|
425
429
|
const managedSessionOutcomeText = formatManagedSessionOutcomeText(options.managedSessionOutcome);
|
|
426
|
-
const rawAppendedDiagnosticText = [visibleRefFallbackText, richInputRecoveryText, semanticActionCandidateText, overlayBlockerText, fillVerificationText, electronRefFreshnessText, selectorTextVisibilityText, electronBroadGetTextScopeText, scrollNoopDiagnosticText, comboboxFocusDiagnosticText, recordingDependencyWarningText, evalStdinHintText, artifactCleanupText, timeoutPartialProgressText, managedSessionOutcomeText].filter((item): item is string => item !== undefined).join("\n\n");
|
|
430
|
+
const rawAppendedDiagnosticText = [visibleRefFallbackText, richInputRecoveryText, semanticActionCandidateText, clickDispatchText, overlayBlockerText, fillVerificationText, electronRefFreshnessText, selectorTextVisibilityText, electronBroadGetTextScopeText, scrollNoopDiagnosticText, comboboxFocusDiagnosticText, recordingDependencyWarningText, evalStdinHintText, artifactCleanupText, timeoutPartialProgressText, managedSessionOutcomeText].filter((item): item is string => item !== undefined).join("\n\n");
|
|
427
431
|
const appendedDiagnosticText = redactSensitiveText(redactExactSensitiveText(rawAppendedDiagnosticText, options.exactSensitiveValues));
|
|
428
432
|
const shouldAppendDiagnosticText = appendedDiagnosticText.length > 0 && (!options.userRequestedJson || options.plainTextInspection);
|
|
429
433
|
let content = shouldAppendDiagnosticText && options.redactedContent[0]?.type === "text" ? [{ ...options.redactedContent[0], text: `${options.redactedContent[0].text}\n\n${appendedDiagnosticText}` }, ...options.redactedContent.slice(1)] : options.redactedContent;
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { runAgentBrowserProcess } from "../../process.js";
|
|
2
|
+
import { cleanupClickDispatchProbe } from "./click-dispatch.js";
|
|
2
3
|
import { applyBrowserRunStatePatch } from "./session-state.js";
|
|
3
4
|
import { buildMissingBinaryFailureResult } from "./final-result.js";
|
|
4
5
|
import { prepareBrowserRun } from "./prepare.js";
|
|
5
6
|
import { processBrowserOutput } from "./process-output.js";
|
|
6
7
|
import type { AgentBrowserToolResult, BrowserRunOptions } from "./types.js";
|
|
7
8
|
|
|
8
|
-
export
|
|
9
|
+
export { closeManagedSession } from "./session-state.js";
|
|
10
|
+
export type { AgentBrowserToolResult, BrowserRunOptions, BrowserRunState, TraceOwner } from "./types.js";
|
|
9
11
|
|
|
10
12
|
export async function runAgentBrowserTool(options: BrowserRunOptions): Promise<AgentBrowserToolResult> {
|
|
11
13
|
const preparedResult = await prepareBrowserRun(options);
|
|
@@ -15,32 +17,36 @@ export async function runAgentBrowserTool(options: BrowserRunOptions): Promise<A
|
|
|
15
17
|
}
|
|
16
18
|
|
|
17
19
|
const { prepared } = preparedResult;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
try {
|
|
21
|
+
const processResult = await runAgentBrowserProcess({
|
|
22
|
+
args: prepared.processArgs,
|
|
23
|
+
cwd: options.cwd,
|
|
24
|
+
env: prepared.executionPlan.managedSessionName ? { AGENT_BROWSER_IDLE_TIMEOUT_MS: options.implicitSessionIdleTimeoutMs } : undefined,
|
|
25
|
+
signal: options.signal,
|
|
26
|
+
stdin: prepared.processStdin,
|
|
27
|
+
});
|
|
25
28
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
29
|
+
const missingBinaryResult = await buildMissingBinaryFailureResult({
|
|
30
|
+
compatibilityWorkaround: prepared.compatibilityWorkaround,
|
|
31
|
+
electronLaunch: prepared.electronLaunch,
|
|
32
|
+
executionPlan: prepared.executionPlan,
|
|
33
|
+
implicitSessionCloseTimeoutMs: options.implicitSessionCloseTimeoutMs,
|
|
34
|
+
managedSessionActive: options.state.managedSessionActive,
|
|
35
|
+
managedSessionName: options.state.managedSessionName,
|
|
36
|
+
processResult,
|
|
37
|
+
redactedArgs: prepared.redactedArgs,
|
|
38
|
+
redactedProcessArgs: prepared.redactedProcessArgs,
|
|
39
|
+
sessionMode: prepared.sessionMode,
|
|
40
|
+
sessionTabCorrection: prepared.sessionTabCorrection,
|
|
41
|
+
});
|
|
42
|
+
if (missingBinaryResult) {
|
|
43
|
+
return missingBinaryResult;
|
|
44
|
+
}
|
|
42
45
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
+
const output = await processBrowserOutput({ ...options, prepared, processResult });
|
|
47
|
+
applyBrowserRunStatePatch(options.state, output.statePatch);
|
|
48
|
+
return output.result;
|
|
49
|
+
} finally {
|
|
50
|
+
await cleanupClickDispatchProbe({ cwd: options.cwd, probe: prepared.clickDispatchProbe, sessionName: prepared.executionPlan.sessionName });
|
|
51
|
+
}
|
|
46
52
|
}
|
|
@@ -26,8 +26,11 @@ import {
|
|
|
26
26
|
runSessionCommandData,
|
|
27
27
|
shouldPinSessionTabForCommand,
|
|
28
28
|
} from "./session-state.js";
|
|
29
|
+
import { parseBatchStdinJsonArray, parseValidBatchStepEntries } from "../batch-stdin.js";
|
|
29
30
|
import { buildElectronHostFailureResult, getElectronLaunchFailureCategory, redactRecoveryHint } from "./final-result.js";
|
|
30
|
-
import {
|
|
31
|
+
import { prepareClickDispatchProbe } from "./click-dispatch.js";
|
|
32
|
+
import { collectScrollPositionSnapshot, validateQaAttachedPrecondition } from "./diagnostics.js";
|
|
33
|
+
import { findRequestedArtifactCloseViolation, findStopBoundaryViolation } from "./prompt-guards.js";
|
|
31
34
|
import type {
|
|
32
35
|
BrowserRunInputFields,
|
|
33
36
|
BrowserRunOptions,
|
|
@@ -141,19 +144,14 @@ async function prepareBatchScreenshotPaths(args: string[], stdin: string | undef
|
|
|
141
144
|
if (commandTokens[0] !== "batch" || stdin === undefined) {
|
|
142
145
|
return undefined;
|
|
143
146
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
steps = JSON.parse(stdin);
|
|
147
|
-
} catch {
|
|
148
|
-
return undefined;
|
|
149
|
-
}
|
|
150
|
-
if (!Array.isArray(steps)) {
|
|
147
|
+
const parsed = parseBatchStdinJsonArray(stdin);
|
|
148
|
+
if (parsed.error || parsed.steps === undefined) {
|
|
151
149
|
return undefined;
|
|
152
150
|
}
|
|
153
151
|
|
|
154
152
|
let changed = false;
|
|
155
153
|
const batchScreenshotPathRequests: Array<ScreenshotPathRequest | undefined> = [];
|
|
156
|
-
const preparedSteps = await Promise.all(steps.map(async (step, index) => {
|
|
154
|
+
const preparedSteps = await Promise.all(parsed.steps.map(async (step, index) => {
|
|
157
155
|
if (!Array.isArray(step) || !step.every((item) => typeof item === "string") || step[0] !== "screenshot") {
|
|
158
156
|
return step;
|
|
159
157
|
}
|
|
@@ -282,20 +280,7 @@ export function validateWaitIpcTimeoutContract(commandTokens: string[], stdin: s
|
|
|
282
280
|
if (commandTokens[0] !== "batch" || stdin === undefined) {
|
|
283
281
|
return undefined;
|
|
284
282
|
}
|
|
285
|
-
|
|
286
|
-
try {
|
|
287
|
-
steps = JSON.parse(stdin);
|
|
288
|
-
} catch {
|
|
289
|
-
return undefined;
|
|
290
|
-
}
|
|
291
|
-
if (!Array.isArray(steps)) {
|
|
292
|
-
return undefined;
|
|
293
|
-
}
|
|
294
|
-
for (let index = 0; index < steps.length; index += 1) {
|
|
295
|
-
const step = steps[index];
|
|
296
|
-
if (!Array.isArray(step) || !step.every((item) => typeof item === "string")) {
|
|
297
|
-
continue;
|
|
298
|
-
}
|
|
283
|
+
for (const { index, step } of parseValidBatchStepEntries(stdin)) {
|
|
299
284
|
const waitTimeout = findWaitTimeoutMs(step);
|
|
300
285
|
if (waitTimeout && waitTimeout.timeoutMs > SAFE_AGENT_BROWSER_OPERATION_TIMEOUT_MS) {
|
|
301
286
|
return buildIpcUnsafeWaitError(waitTimeout.source, waitTimeout.timeoutMs, index);
|
|
@@ -528,6 +513,43 @@ export async function prepareBrowserRun(options: BrowserRunOptions): Promise<Pre
|
|
|
528
513
|
const resolvedSemanticActionRefSnapshot: SessionRefSnapshot | undefined = semanticActionVisibleRefResolution?.snapshot
|
|
529
514
|
? { ...semanticActionVisibleRefResolution.snapshot, target: semanticActionVisibleRefResolution.snapshot.target ?? priorSessionTabTarget }
|
|
530
515
|
: undefined;
|
|
516
|
+
const promptRefSnapshot = resolvedSemanticActionRefSnapshot ?? priorRefSnapshotState;
|
|
517
|
+
const stopBoundaryViolation = findStopBoundaryViolation({ commandTokens, promptPolicy: options.promptPolicy, refSnapshot: promptRefSnapshot, stdin: runtimeToolStdin });
|
|
518
|
+
if (stopBoundaryViolation) {
|
|
519
|
+
return { kind: "early-result", statePatch, result: {
|
|
520
|
+
content: [{ type: "text", text: stopBoundaryViolation.message }],
|
|
521
|
+
details: {
|
|
522
|
+
args: redactedArgs,
|
|
523
|
+
command: executionPlan.commandInfo.command,
|
|
524
|
+
compatibilityWorkaround,
|
|
525
|
+
effectiveArgs: redactedEffectiveArgs,
|
|
526
|
+
promptGuard: stopBoundaryViolation,
|
|
527
|
+
sessionMode,
|
|
528
|
+
...buildAgentBrowserResultCategoryDetails({ args: redactedEffectiveArgs, command: executionPlan.commandInfo.command, errorText: stopBoundaryViolation.message, failureCategory: "policy-blocked", succeeded: false, validationError: stopBoundaryViolation.message }),
|
|
529
|
+
validationError: stopBoundaryViolation.message,
|
|
530
|
+
...buildSessionDetailFields(executionPlan.sessionName, executionPlan.usedImplicitSession),
|
|
531
|
+
},
|
|
532
|
+
isError: true,
|
|
533
|
+
} };
|
|
534
|
+
}
|
|
535
|
+
const requestedArtifactCloseViolation = await findRequestedArtifactCloseViolation({ artifactManifest: state.artifactManifest, command: executionPlan.commandInfo.command, cwd, promptPolicy: options.promptPolicy });
|
|
536
|
+
if (requestedArtifactCloseViolation) {
|
|
537
|
+
return { kind: "early-result", statePatch, result: {
|
|
538
|
+
content: [{ type: "text", text: requestedArtifactCloseViolation.message }],
|
|
539
|
+
details: {
|
|
540
|
+
args: redactedArgs,
|
|
541
|
+
command: executionPlan.commandInfo.command,
|
|
542
|
+
compatibilityWorkaround,
|
|
543
|
+
effectiveArgs: redactedEffectiveArgs,
|
|
544
|
+
promptGuard: requestedArtifactCloseViolation,
|
|
545
|
+
sessionMode,
|
|
546
|
+
...buildAgentBrowserResultCategoryDetails({ args: redactedEffectiveArgs, command: executionPlan.commandInfo.command, errorText: requestedArtifactCloseViolation.message, failureCategory: "policy-blocked", succeeded: false, validationError: requestedArtifactCloseViolation.message }),
|
|
547
|
+
validationError: requestedArtifactCloseViolation.message,
|
|
548
|
+
...buildSessionDetailFields(executionPlan.sessionName, executionPlan.usedImplicitSession),
|
|
549
|
+
},
|
|
550
|
+
isError: true,
|
|
551
|
+
} };
|
|
552
|
+
}
|
|
531
553
|
const staleRefPreflight = buildStaleRefPreflight({
|
|
532
554
|
commandTokens,
|
|
533
555
|
currentTarget: priorSessionTabTarget,
|
|
@@ -555,6 +577,31 @@ export async function prepareBrowserRun(options: BrowserRunOptions): Promise<Pre
|
|
|
555
577
|
} };
|
|
556
578
|
}
|
|
557
579
|
|
|
580
|
+
if (compiledQaPreset?.checks.attached) {
|
|
581
|
+
const qaAttachedPrecondition = await validateQaAttachedPrecondition({
|
|
582
|
+
cwd,
|
|
583
|
+
sessionName: executionPlan.sessionName,
|
|
584
|
+
signal,
|
|
585
|
+
});
|
|
586
|
+
if (qaAttachedPrecondition) {
|
|
587
|
+
return { kind: "early-result", statePatch, result: {
|
|
588
|
+
content: [{ type: "text", text: qaAttachedPrecondition.error }],
|
|
589
|
+
details: {
|
|
590
|
+
args: redactedArgs,
|
|
591
|
+
compiledQaPreset: redactedCompiledQaPreset,
|
|
592
|
+
compatibilityWorkaround,
|
|
593
|
+
effectiveArgs: redactedEffectiveArgs,
|
|
594
|
+
nextActions: qaAttachedPrecondition.nextActions,
|
|
595
|
+
sessionMode,
|
|
596
|
+
...buildAgentBrowserResultCategoryDetails({ args: redactedEffectiveArgs, command: executionPlan.commandInfo.command, errorText: qaAttachedPrecondition.error, succeeded: false, validationError: qaAttachedPrecondition.error }),
|
|
597
|
+
validationError: qaAttachedPrecondition.error,
|
|
598
|
+
...buildSessionDetailFields(executionPlan.sessionName, executionPlan.usedImplicitSession),
|
|
599
|
+
},
|
|
600
|
+
isError: true,
|
|
601
|
+
} };
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
558
605
|
let pinnedBatchUnwrapMode: PreparedBrowserRun["pinnedBatchUnwrapMode"];
|
|
559
606
|
let includePinnedNavigationSummary = false;
|
|
560
607
|
let sessionTabCorrection: PreparedBrowserRun["sessionTabCorrection"];
|
|
@@ -651,6 +698,9 @@ export async function prepareBrowserRun(options: BrowserRunOptions): Promise<Pre
|
|
|
651
698
|
}
|
|
652
699
|
}
|
|
653
700
|
}
|
|
701
|
+
const clickDispatchProbe = pinnedBatchUnwrapMode === undefined && compiledElectron === undefined
|
|
702
|
+
? await prepareClickDispatchProbe({ commandTokens, cwd, sessionName: executionPlan.sessionName, signal })
|
|
703
|
+
: undefined;
|
|
654
704
|
const redactedProcessArgs = redactInvocationArgs(processArgs);
|
|
655
705
|
const shouldProbeScrollNoop = executionPlan.commandInfo.command === "scroll" && executionPlan.startupScopedFlags.length === 0;
|
|
656
706
|
const scrollPositionBefore = shouldProbeScrollNoop
|
|
@@ -677,6 +727,7 @@ export async function prepareBrowserRun(options: BrowserRunOptions): Promise<Pre
|
|
|
677
727
|
compiledSemanticAction,
|
|
678
728
|
compiledSourceLookup,
|
|
679
729
|
compatibilityWorkaround,
|
|
730
|
+
clickDispatchProbe,
|
|
680
731
|
electronLaunch,
|
|
681
732
|
exactSensitiveValues,
|
|
682
733
|
executionPlan,
|