pi-agent-browser-native 0.2.23 → 0.2.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +123 -9
- package/docs/ARCHITECTURE.md +48 -7
- package/docs/COMMAND_REFERENCE.md +567 -40
- package/docs/RELEASE.md +57 -7
- package/docs/REQUIREMENTS.md +13 -1
- package/docs/SUPPORT_MATRIX.md +65 -0
- package/docs/TOOL_CONTRACT.md +268 -19
- package/extensions/agent-browser/index.ts +1154 -25
- package/extensions/agent-browser/lib/playbook.ts +20 -10
- package/extensions/agent-browser/lib/results/presentation.ts +624 -33
- package/extensions/agent-browser/lib/results/shared.ts +365 -0
- package/extensions/agent-browser/lib/results.ts +12 -0
- package/extensions/agent-browser/lib/runtime.ts +75 -8
- package/package.json +4 -2
- package/scripts/agent-browser-capability-baseline.mjs +499 -110
- package/scripts/doctor.mjs +1 -1
|
@@ -19,6 +19,55 @@ export interface AgentBrowserBatchResult {
|
|
|
19
19
|
success?: boolean;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
export type AgentBrowserResultCategory = "failure" | "success";
|
|
23
|
+
|
|
24
|
+
export type AgentBrowserSuccessCategory = "artifact-saved" | "artifact-unverified" | "completed" | "inspection";
|
|
25
|
+
|
|
26
|
+
export type AgentBrowserFailureCategory =
|
|
27
|
+
| "aborted"
|
|
28
|
+
| "confirmation-required"
|
|
29
|
+
| "download-not-verified"
|
|
30
|
+
| "missing-binary"
|
|
31
|
+
| "parse-failure"
|
|
32
|
+
| "qa-failure"
|
|
33
|
+
| "selector-not-found"
|
|
34
|
+
| "selector-unsupported"
|
|
35
|
+
| "stale-ref"
|
|
36
|
+
| "tab-drift"
|
|
37
|
+
| "timeout"
|
|
38
|
+
| "upstream-error"
|
|
39
|
+
| "validation-error";
|
|
40
|
+
|
|
41
|
+
export interface AgentBrowserResultCategoryDetails {
|
|
42
|
+
failureCategory?: AgentBrowserFailureCategory;
|
|
43
|
+
resultCategory: AgentBrowserResultCategory;
|
|
44
|
+
successCategory?: AgentBrowserSuccessCategory;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface AgentBrowserPageChangeSummary {
|
|
48
|
+
artifactCount?: number;
|
|
49
|
+
changeType: "artifact" | "confirmation" | "mutation" | "navigation";
|
|
50
|
+
command?: string;
|
|
51
|
+
nextActionIds?: string[];
|
|
52
|
+
savedFilePath?: string;
|
|
53
|
+
summary: string;
|
|
54
|
+
title?: string;
|
|
55
|
+
url?: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface AgentBrowserNextAction {
|
|
59
|
+
artifactPath?: string;
|
|
60
|
+
id: string;
|
|
61
|
+
params?: {
|
|
62
|
+
args: string[];
|
|
63
|
+
sessionMode?: "auto" | "fresh";
|
|
64
|
+
stdin?: string;
|
|
65
|
+
};
|
|
66
|
+
reason: string;
|
|
67
|
+
safety?: string;
|
|
68
|
+
tool: "agent_browser";
|
|
69
|
+
}
|
|
70
|
+
|
|
22
71
|
export type FileArtifactKind = "download" | "file" | "har" | "image" | "pdf" | "profile" | "trace" | "video";
|
|
23
72
|
|
|
24
73
|
export type FileArtifactStatus = "missing" | "repaired-from-temp" | "saved" | "upstream-temp-only";
|
|
@@ -41,6 +90,32 @@ export interface FileArtifactMetadata {
|
|
|
41
90
|
tempPath?: string;
|
|
42
91
|
}
|
|
43
92
|
|
|
93
|
+
export type ArtifactVerificationState = "missing" | "pending" | "unverified" | "verified";
|
|
94
|
+
|
|
95
|
+
export interface ArtifactVerificationEntry {
|
|
96
|
+
absolutePath?: string;
|
|
97
|
+
exists?: boolean;
|
|
98
|
+
kind: FileArtifactKind | "spill";
|
|
99
|
+
limitation?: string;
|
|
100
|
+
mediaType?: string;
|
|
101
|
+
path: string;
|
|
102
|
+
requestedPath?: string;
|
|
103
|
+
retentionState?: ArtifactRetentionState;
|
|
104
|
+
sizeBytes?: number;
|
|
105
|
+
state: ArtifactVerificationState;
|
|
106
|
+
status?: FileArtifactStatus;
|
|
107
|
+
storageScope?: ArtifactStorageScope;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export interface ArtifactVerificationSummary {
|
|
111
|
+
artifacts: ArtifactVerificationEntry[];
|
|
112
|
+
missingCount: number;
|
|
113
|
+
pendingCount: number;
|
|
114
|
+
unverifiedCount: number;
|
|
115
|
+
verified: boolean;
|
|
116
|
+
verifiedCount: number;
|
|
117
|
+
}
|
|
118
|
+
|
|
44
119
|
export interface SavedFilePresentationDetails {
|
|
45
120
|
command: "download" | "pdf" | "wait";
|
|
46
121
|
kind: "download" | "pdf";
|
|
@@ -187,18 +262,24 @@ export function mergeSessionArtifactManifest(options: {
|
|
|
187
262
|
}
|
|
188
263
|
|
|
189
264
|
export interface BatchStepPresentationDetails {
|
|
265
|
+
artifactVerification?: ArtifactVerificationSummary;
|
|
190
266
|
artifacts?: FileArtifactMetadata[];
|
|
191
267
|
command?: string[];
|
|
192
268
|
commandText: string;
|
|
193
269
|
data?: unknown;
|
|
270
|
+
failureCategory?: AgentBrowserFailureCategory;
|
|
271
|
+
nextActions?: AgentBrowserNextAction[];
|
|
272
|
+
pageChangeSummary?: AgentBrowserPageChangeSummary;
|
|
194
273
|
fullOutputPath?: string;
|
|
195
274
|
fullOutputPaths?: string[];
|
|
196
275
|
imagePath?: string;
|
|
197
276
|
imagePaths?: string[];
|
|
198
277
|
index: number;
|
|
278
|
+
resultCategory: AgentBrowserResultCategory;
|
|
199
279
|
savedFile?: SavedFilePresentationDetails;
|
|
200
280
|
savedFilePath?: string;
|
|
201
281
|
success: boolean;
|
|
282
|
+
successCategory?: AgentBrowserSuccessCategory;
|
|
202
283
|
summary: string;
|
|
203
284
|
text: string;
|
|
204
285
|
}
|
|
@@ -213,20 +294,304 @@ export interface BatchFailurePresentationDetails {
|
|
|
213
294
|
export interface ToolPresentation {
|
|
214
295
|
artifactManifest?: SessionArtifactManifest;
|
|
215
296
|
artifactRetentionSummary?: string;
|
|
297
|
+
artifactVerification?: ArtifactVerificationSummary;
|
|
216
298
|
artifacts?: FileArtifactMetadata[];
|
|
217
299
|
batchFailure?: BatchFailurePresentationDetails;
|
|
218
300
|
batchSteps?: BatchStepPresentationDetails[];
|
|
219
301
|
content: Array<{ text: string; type: "text" } | { data: string; mimeType: string; type: "image" }>;
|
|
220
302
|
data?: unknown;
|
|
303
|
+
failureCategory?: AgentBrowserFailureCategory;
|
|
221
304
|
fullOutputPath?: string;
|
|
222
305
|
fullOutputPaths?: string[];
|
|
223
306
|
imagePath?: string;
|
|
224
307
|
imagePaths?: string[];
|
|
308
|
+
nextActions?: AgentBrowserNextAction[];
|
|
309
|
+
pageChangeSummary?: AgentBrowserPageChangeSummary;
|
|
310
|
+
resultCategory?: AgentBrowserResultCategory;
|
|
225
311
|
savedFile?: SavedFilePresentationDetails;
|
|
226
312
|
savedFilePath?: string;
|
|
313
|
+
successCategory?: AgentBrowserSuccessCategory;
|
|
227
314
|
summary: string;
|
|
228
315
|
}
|
|
229
316
|
|
|
317
|
+
function isPendingFileArtifact(artifact: FileArtifactMetadata): boolean {
|
|
318
|
+
return artifact.command === "record" && artifact.subcommand === "start" && artifact.kind === "video";
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function hasUnverifiedFileArtifact(artifacts: FileArtifactMetadata[] | undefined): boolean {
|
|
322
|
+
return (artifacts ?? []).some((artifact) => !isPendingFileArtifact(artifact) && artifact.exists !== true);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
export function classifyAgentBrowserSuccessCategory(options: {
|
|
326
|
+
artifacts?: FileArtifactMetadata[];
|
|
327
|
+
inspection?: boolean;
|
|
328
|
+
savedFile?: SavedFilePresentationDetails;
|
|
329
|
+
}): AgentBrowserSuccessCategory {
|
|
330
|
+
if (options.inspection) return "inspection";
|
|
331
|
+
if ((options.artifacts ?? []).length > 0) return hasUnverifiedFileArtifact(options.artifacts) ? "artifact-unverified" : "artifact-saved";
|
|
332
|
+
if (options.savedFile) return "artifact-saved";
|
|
333
|
+
return "completed";
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
export function classifyAgentBrowserFailureCategory(options: {
|
|
337
|
+
args?: string[];
|
|
338
|
+
command?: string;
|
|
339
|
+
confirmationRequired?: boolean;
|
|
340
|
+
errorText?: string;
|
|
341
|
+
parseError?: string;
|
|
342
|
+
spawnError?: string;
|
|
343
|
+
stderr?: string;
|
|
344
|
+
tabDrift?: boolean;
|
|
345
|
+
timedOut?: boolean;
|
|
346
|
+
validationError?: string;
|
|
347
|
+
}): AgentBrowserFailureCategory {
|
|
348
|
+
const text = [options.errorText, options.validationError, options.parseError, options.spawnError, options.stderr].filter(Boolean).join("\n");
|
|
349
|
+
const command = options.command ?? "";
|
|
350
|
+
const usedRef = options.args?.some((arg) => /^@e\d+\b/.test(arg)) ?? false;
|
|
351
|
+
if (options.confirmationRequired || /confirmation required|pending confirmation|requires confirmation/i.test(text)) return "confirmation-required";
|
|
352
|
+
if (options.timedOut || /timeout|timed out|watchdog|IPC read timeout|must stay under its 30s IPC read timeout/i.test(text)) return "timeout";
|
|
353
|
+
if (/ENOENT|not found on PATH|could not find.*agent-browser|agent-browser is required but was not found/i.test(text)) return "missing-binary";
|
|
354
|
+
if (options.parseError || /invalid JSON|missing boolean success|success field must be boolean|returned no JSON output/i.test(text)) return "parse-failure";
|
|
355
|
+
if (/aborted/i.test(text)) return "aborted";
|
|
356
|
+
if (options.tabDrift || /could not re-select the intended tab|about:blank|selected tab looks wrong|tab drift|tab.*wrong/i.test(text)) return "tab-drift";
|
|
357
|
+
if (/\bUnknown ref\b|\bstale ref\b|@ref may be stale|\bref\b.*\b(?:not found|missing|expired)\b/i.test(text)) return "stale-ref";
|
|
358
|
+
if (usedRef && /could not locate element|element not found|no element/i.test(text)) return "stale-ref";
|
|
359
|
+
const mentionsPlaywrightSelectorDialect = /(?:\btext=|:has-text\(|\bgetByRole\b|\bgetByText\b)/i.test(text);
|
|
360
|
+
const reportsSelectorMatchFailure =
|
|
361
|
+
/\b(?:no elements? found|failed to find|could not find|unable to find)\b.*\b(?:selector|locator)\b/i.test(text) ||
|
|
362
|
+
/\b(?:selector|locator)\b.*\b(?:no elements? found|not found|missing|failed to find|could not find|unable to find)\b/i.test(text);
|
|
363
|
+
if (
|
|
364
|
+
/\b(?:unsupported|unknown|invalid)\s+(?:selector|locator)\b/i.test(text) ||
|
|
365
|
+
/\bfailed to parse selector\b/i.test(text) ||
|
|
366
|
+
/\bselector\b.*\b(?:parse|syntax|unsupported|invalid)\b/i.test(text) ||
|
|
367
|
+
(mentionsPlaywrightSelectorDialect && reportsSelectorMatchFailure)
|
|
368
|
+
) {
|
|
369
|
+
return "selector-unsupported";
|
|
370
|
+
}
|
|
371
|
+
if (reportsSelectorMatchFailure) return "selector-not-found";
|
|
372
|
+
if ((command === "download" || text.includes("wait --download") || /\bdownload\b/i.test(text)) && /missing|not verified|not found|failed|timeout|timed out/i.test(text)) {
|
|
373
|
+
return "download-not-verified";
|
|
374
|
+
}
|
|
375
|
+
if (options.validationError) return "validation-error";
|
|
376
|
+
return "upstream-error";
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
function buildNextToolAction(options: {
|
|
380
|
+
args: string[];
|
|
381
|
+
id: string;
|
|
382
|
+
reason: string;
|
|
383
|
+
safety?: string;
|
|
384
|
+
sessionMode?: "auto" | "fresh";
|
|
385
|
+
stdin?: string;
|
|
386
|
+
}): AgentBrowserNextAction {
|
|
387
|
+
return {
|
|
388
|
+
id: options.id,
|
|
389
|
+
params: {
|
|
390
|
+
args: options.args,
|
|
391
|
+
...(options.sessionMode ? { sessionMode: options.sessionMode } : {}),
|
|
392
|
+
...(options.stdin ? { stdin: options.stdin } : {}),
|
|
393
|
+
},
|
|
394
|
+
reason: options.reason,
|
|
395
|
+
...(options.safety ? { safety: options.safety } : {}),
|
|
396
|
+
tool: "agent_browser",
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function buildArtifactAction(path: string): AgentBrowserNextAction {
|
|
401
|
+
return {
|
|
402
|
+
artifactPath: path,
|
|
403
|
+
id: "use-saved-artifact",
|
|
404
|
+
reason: "Use the saved artifact path from the structured result instead of scraping it from text.",
|
|
405
|
+
safety: "Verify artifact metadata such as exists/status before treating the file as durable.",
|
|
406
|
+
tool: "agent_browser",
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function buildArtifactVerificationAction(artifact: FileArtifactMetadata): AgentBrowserNextAction {
|
|
411
|
+
return {
|
|
412
|
+
artifactPath: artifact.path,
|
|
413
|
+
id: "verify-artifact-path",
|
|
414
|
+
reason: "The wrapper has artifact metadata but did not verify this file as present on disk.",
|
|
415
|
+
safety: "Check details.artifactVerification and the filesystem before treating the artifact as durable.",
|
|
416
|
+
tool: "agent_browser",
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const MUTATING_COMMANDS = new Set([
|
|
421
|
+
"back",
|
|
422
|
+
"check",
|
|
423
|
+
"click",
|
|
424
|
+
"dblclick",
|
|
425
|
+
"dialog",
|
|
426
|
+
"fill",
|
|
427
|
+
"forward",
|
|
428
|
+
"hover",
|
|
429
|
+
"press",
|
|
430
|
+
"pushstate",
|
|
431
|
+
"reload",
|
|
432
|
+
"scroll",
|
|
433
|
+
"scrollintoview",
|
|
434
|
+
"select",
|
|
435
|
+
"swipe",
|
|
436
|
+
"tap",
|
|
437
|
+
"type",
|
|
438
|
+
"uncheck",
|
|
439
|
+
]);
|
|
440
|
+
|
|
441
|
+
function getDownloadRetryPath(args: string[] | undefined, fallback: string | undefined): string | undefined {
|
|
442
|
+
if (fallback) return fallback;
|
|
443
|
+
if (!args || args.length === 0) return undefined;
|
|
444
|
+
const downloadFlagIndex = args.indexOf("--download");
|
|
445
|
+
if (downloadFlagIndex >= 0) {
|
|
446
|
+
const candidate = args[downloadFlagIndex + 1];
|
|
447
|
+
return candidate && !candidate.startsWith("-") ? candidate : undefined;
|
|
448
|
+
}
|
|
449
|
+
const downloadCommandIndex = args.indexOf("download");
|
|
450
|
+
if (downloadCommandIndex >= 0 && args.length > downloadCommandIndex + 2) {
|
|
451
|
+
return args[args.length - 1];
|
|
452
|
+
}
|
|
453
|
+
return undefined;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
export function buildAgentBrowserNextActions(options: {
|
|
457
|
+
artifacts?: FileArtifactMetadata[];
|
|
458
|
+
args?: string[];
|
|
459
|
+
command?: string;
|
|
460
|
+
confirmationId?: string;
|
|
461
|
+
failureCategory?: AgentBrowserFailureCategory;
|
|
462
|
+
resultCategory: AgentBrowserResultCategory;
|
|
463
|
+
savedFilePath?: string;
|
|
464
|
+
successCategory?: AgentBrowserSuccessCategory;
|
|
465
|
+
}): AgentBrowserNextAction[] | undefined {
|
|
466
|
+
const actions: AgentBrowserNextAction[] = [];
|
|
467
|
+
if (options.resultCategory === "success") {
|
|
468
|
+
if (options.command === "open") {
|
|
469
|
+
actions.push(buildNextToolAction({
|
|
470
|
+
args: ["snapshot", "-i"],
|
|
471
|
+
id: "inspect-opened-page",
|
|
472
|
+
reason: "Inspect the opened page before choosing interactive refs.",
|
|
473
|
+
}));
|
|
474
|
+
} else if (options.command && MUTATING_COMMANDS.has(options.command)) {
|
|
475
|
+
actions.push(buildNextToolAction({
|
|
476
|
+
args: ["snapshot", "-i"],
|
|
477
|
+
id: "inspect-after-mutation",
|
|
478
|
+
reason: "Refresh interactive refs after a browser mutation, navigation, scroll, or rerender.",
|
|
479
|
+
safety: "Do not reuse prior @refs until a fresh snapshot confirms they still exist.",
|
|
480
|
+
}));
|
|
481
|
+
}
|
|
482
|
+
const artifacts = options.artifacts ?? [];
|
|
483
|
+
const savedFileArtifact = options.savedFilePath ? artifacts.find((artifact) => artifact.path === options.savedFilePath) : undefined;
|
|
484
|
+
if (options.savedFilePath && savedFileArtifact?.exists !== false) {
|
|
485
|
+
actions.push(buildArtifactAction(options.savedFilePath));
|
|
486
|
+
}
|
|
487
|
+
for (const artifact of artifacts) {
|
|
488
|
+
if (isPendingFileArtifact(artifact)) {
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
if (artifact.exists === false) {
|
|
492
|
+
if (artifact.kind === "download") {
|
|
493
|
+
actions.push(buildNextToolAction({
|
|
494
|
+
args: ["wait", "--download", artifact.path],
|
|
495
|
+
id: "wait-for-download",
|
|
496
|
+
reason: "Upstream reported a download path, but the wrapper did not verify the file on disk.",
|
|
497
|
+
safety: "Use a bounded wait timeout that stays below the native wrapper IPC budget.",
|
|
498
|
+
}));
|
|
499
|
+
} else {
|
|
500
|
+
actions.push(buildArtifactVerificationAction(artifact));
|
|
501
|
+
}
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
504
|
+
if (artifact.path !== options.savedFilePath) {
|
|
505
|
+
actions.push(buildArtifactAction(artifact.path));
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
} else {
|
|
509
|
+
switch (options.failureCategory) {
|
|
510
|
+
case "confirmation-required":
|
|
511
|
+
if (options.confirmationId) {
|
|
512
|
+
actions.push(
|
|
513
|
+
buildNextToolAction({
|
|
514
|
+
args: ["confirm", options.confirmationId],
|
|
515
|
+
id: "approve-confirmation",
|
|
516
|
+
reason: "Approve the pending upstream confirmation when the requested action is safe.",
|
|
517
|
+
safety: "Only confirm after reviewing the guarded action shown in the result.",
|
|
518
|
+
}),
|
|
519
|
+
buildNextToolAction({
|
|
520
|
+
args: ["deny", options.confirmationId],
|
|
521
|
+
id: "deny-confirmation",
|
|
522
|
+
reason: "Deny the pending upstream confirmation when the guarded action is unsafe or unintended.",
|
|
523
|
+
}),
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
break;
|
|
527
|
+
case "stale-ref":
|
|
528
|
+
case "selector-not-found":
|
|
529
|
+
case "selector-unsupported":
|
|
530
|
+
actions.push(buildNextToolAction({
|
|
531
|
+
args: ["snapshot", "-i"],
|
|
532
|
+
id: "refresh-interactive-refs",
|
|
533
|
+
reason: "Get current interactive refs before retrying the element action.",
|
|
534
|
+
safety: "Prefer a current @ref or a stable find locator; do not retry stale refs blindly.",
|
|
535
|
+
}));
|
|
536
|
+
break;
|
|
537
|
+
case "download-not-verified":
|
|
538
|
+
{
|
|
539
|
+
const retryPath = getDownloadRetryPath(options.args, options.savedFilePath);
|
|
540
|
+
actions.push(buildNextToolAction({
|
|
541
|
+
args: retryPath ? ["wait", "--download", retryPath] : ["wait", "--download"],
|
|
542
|
+
id: "wait-for-download",
|
|
543
|
+
reason: "Wait for the browser download and let the wrapper verify saved-file metadata.",
|
|
544
|
+
safety: "Use a bounded wait timeout that stays below the native wrapper IPC budget.",
|
|
545
|
+
}));
|
|
546
|
+
}
|
|
547
|
+
break;
|
|
548
|
+
case "tab-drift":
|
|
549
|
+
actions.push(
|
|
550
|
+
buildNextToolAction({
|
|
551
|
+
args: ["tab", "list"],
|
|
552
|
+
id: "list-tabs-for-recovery",
|
|
553
|
+
reason: "Inspect available tabs before selecting the intended target.",
|
|
554
|
+
}),
|
|
555
|
+
buildNextToolAction({
|
|
556
|
+
args: ["snapshot", "-i"],
|
|
557
|
+
id: "inspect-current-tab",
|
|
558
|
+
reason: "Inspect the currently selected tab after tab recovery.",
|
|
559
|
+
}),
|
|
560
|
+
);
|
|
561
|
+
break;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
return actions.length > 0 ? actions : undefined;
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
export function buildAgentBrowserResultCategoryDetails(options: {
|
|
568
|
+
artifacts?: FileArtifactMetadata[];
|
|
569
|
+
args?: string[];
|
|
570
|
+
command?: string;
|
|
571
|
+
confirmationRequired?: boolean;
|
|
572
|
+
errorText?: string;
|
|
573
|
+
failureCategory?: AgentBrowserFailureCategory;
|
|
574
|
+
inspection?: boolean;
|
|
575
|
+
parseError?: string;
|
|
576
|
+
savedFile?: SavedFilePresentationDetails;
|
|
577
|
+
spawnError?: string;
|
|
578
|
+
succeeded: boolean;
|
|
579
|
+
tabDrift?: boolean;
|
|
580
|
+
timedOut?: boolean;
|
|
581
|
+
validationError?: string;
|
|
582
|
+
}): AgentBrowserResultCategoryDetails {
|
|
583
|
+
if (options.succeeded) {
|
|
584
|
+
return {
|
|
585
|
+
resultCategory: "success",
|
|
586
|
+
successCategory: classifyAgentBrowserSuccessCategory(options),
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
return {
|
|
590
|
+
failureCategory: options.failureCategory ?? classifyAgentBrowserFailureCategory(options),
|
|
591
|
+
resultCategory: "failure",
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
|
|
230
595
|
export function stringifyUnknown(value: unknown): string {
|
|
231
596
|
if (typeof value === "string") return value;
|
|
232
597
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
@@ -8,9 +8,21 @@
|
|
|
8
8
|
|
|
9
9
|
export { getAgentBrowserErrorText, parseAgentBrowserEnvelope } from "./results/envelope.js";
|
|
10
10
|
export { buildToolPresentation } from "./results/presentation.js";
|
|
11
|
+
export {
|
|
12
|
+
buildAgentBrowserNextActions,
|
|
13
|
+
buildAgentBrowserResultCategoryDetails,
|
|
14
|
+
classifyAgentBrowserFailureCategory,
|
|
15
|
+
classifyAgentBrowserSuccessCategory,
|
|
16
|
+
} from "./results/shared.js";
|
|
11
17
|
export type {
|
|
12
18
|
AgentBrowserBatchResult,
|
|
13
19
|
AgentBrowserEnvelope,
|
|
20
|
+
AgentBrowserFailureCategory,
|
|
21
|
+
AgentBrowserResultCategory,
|
|
22
|
+
AgentBrowserNextAction,
|
|
23
|
+
AgentBrowserPageChangeSummary,
|
|
24
|
+
AgentBrowserResultCategoryDetails,
|
|
25
|
+
AgentBrowserSuccessCategory,
|
|
14
26
|
FileArtifactKind,
|
|
15
27
|
FileArtifactMetadata,
|
|
16
28
|
ToolPresentation,
|
|
@@ -47,10 +47,22 @@ const LAUNCH_SCOPED_FLAG_DEFINITIONS = [
|
|
|
47
47
|
flag: "--init-script",
|
|
48
48
|
reason: "registers page init scripts before the upstream browser session is launched",
|
|
49
49
|
},
|
|
50
|
+
{
|
|
51
|
+
flag: "--device",
|
|
52
|
+
reason: "selects the provider device for the upstream launch",
|
|
53
|
+
},
|
|
50
54
|
{
|
|
51
55
|
flag: "--profile",
|
|
52
56
|
reason: "selects Chrome profile state for the upstream launch",
|
|
53
57
|
},
|
|
58
|
+
{
|
|
59
|
+
flag: "--provider",
|
|
60
|
+
reason: "selects the upstream browser provider for the launch",
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
flag: "-p",
|
|
64
|
+
reason: "selects the upstream browser provider for the launch",
|
|
65
|
+
},
|
|
54
66
|
{
|
|
55
67
|
flag: "--session-name",
|
|
56
68
|
reason: "selects upstream saved auth/session state for the launch",
|
|
@@ -92,7 +104,7 @@ const BROWSER_PROMPT_PATTERNS = [
|
|
|
92
104
|
/\b(?:browse|click|fill|login|navigate|open|visit)\b.*\b(?:https?:\/\/\S+|page|site|tab|url|web(?:site| page)?)\b/i,
|
|
93
105
|
];
|
|
94
106
|
const INSPECTION_FLAGS = new Set(["--help", "-h", "--version", "-V"]);
|
|
95
|
-
const SENSITIVE_VALUE_FLAGS = new Set(["--headers", "--password", "--proxy"]);
|
|
107
|
+
const SENSITIVE_VALUE_FLAGS = new Set(["--body", "--headers", "--password", "--proxy"]);
|
|
96
108
|
const GLOBAL_VALUE_FLAGS_ALLOWING_DASH_VALUE = new Set(["--args"]);
|
|
97
109
|
const GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES = new Set([
|
|
98
110
|
"--allow-file-access",
|
|
@@ -115,7 +127,7 @@ const SENSITIVE_QUERY_PARAM_PATTERN =
|
|
|
115
127
|
const SENSITIVE_FIELD_NAME_PATTERN =
|
|
116
128
|
/^(?:access(?:_|-)?token|api(?:_|-)?key|auth(?:orization)?|bearer|client(?:_|-)?secret|cookie|id(?:_|-)?token|pass(?:word)?|proxy(?:_|-)?authorization|refresh(?:_|-)?token|secret|session(?:_|-)?id|set(?:_|-)?cookie|sig(?:nature)?|token|x(?:_|-)?api(?:_|-)?key)$/i;
|
|
117
129
|
|
|
118
|
-
const
|
|
130
|
+
const VALUE_FLAGS = new Set([
|
|
119
131
|
"--session",
|
|
120
132
|
"--cdp",
|
|
121
133
|
"--config",
|
|
@@ -146,6 +158,29 @@ const GLOBAL_FLAGS_WITH_VALUES = new Set([
|
|
|
146
158
|
"--confirm-actions",
|
|
147
159
|
"--max-output",
|
|
148
160
|
"--model",
|
|
161
|
+
"--baseline",
|
|
162
|
+
"--body",
|
|
163
|
+
"--categories",
|
|
164
|
+
"--curl",
|
|
165
|
+
"--depth",
|
|
166
|
+
"-d",
|
|
167
|
+
"--domain",
|
|
168
|
+
"--expires",
|
|
169
|
+
"--filter",
|
|
170
|
+
"--fn",
|
|
171
|
+
"--label",
|
|
172
|
+
"--load",
|
|
173
|
+
"--name",
|
|
174
|
+
"--path",
|
|
175
|
+
"--resource-type",
|
|
176
|
+
"--sameSite",
|
|
177
|
+
"--selector",
|
|
178
|
+
"-s",
|
|
179
|
+
"--text",
|
|
180
|
+
"--timeout",
|
|
181
|
+
"--url",
|
|
182
|
+
"--username",
|
|
183
|
+
"--password",
|
|
149
184
|
]);
|
|
150
185
|
const DEFAULT_HEADLESS_COMPAT_USER_AGENT_BY_PLATFORM: Partial<Record<NodeJS.Platform, string>> = {
|
|
151
186
|
darwin: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
|
@@ -440,7 +475,7 @@ export function redactInvocationArgs(args: string[]): string[] {
|
|
|
440
475
|
continue;
|
|
441
476
|
}
|
|
442
477
|
|
|
443
|
-
redacted.push(redactUrlToken(token));
|
|
478
|
+
redacted.push(redactSensitiveText(redactUrlToken(token)));
|
|
444
479
|
}
|
|
445
480
|
|
|
446
481
|
const commandStartIndex = findCommandStartIndex(args);
|
|
@@ -452,6 +487,20 @@ export function redactInvocationArgs(args: string[]): string[] {
|
|
|
452
487
|
}
|
|
453
488
|
}
|
|
454
489
|
|
|
490
|
+
if (commandStartIndex !== undefined && args[commandStartIndex] === "cookies" && args[commandStartIndex + 1] === "set" && redacted[commandStartIndex + 3] !== undefined) {
|
|
491
|
+
redacted[commandStartIndex + 3] = "[REDACTED]";
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
if (
|
|
495
|
+
commandStartIndex !== undefined
|
|
496
|
+
&& args[commandStartIndex] === "storage"
|
|
497
|
+
&& ["local", "session"].includes(args[commandStartIndex + 1] ?? "")
|
|
498
|
+
&& args[commandStartIndex + 2] === "set"
|
|
499
|
+
&& redacted[commandStartIndex + 4] !== undefined
|
|
500
|
+
) {
|
|
501
|
+
redacted[commandStartIndex + 4] = "[REDACTED]";
|
|
502
|
+
}
|
|
503
|
+
|
|
455
504
|
return redacted;
|
|
456
505
|
}
|
|
457
506
|
|
|
@@ -467,6 +516,10 @@ export function isPlainTextInspectionArgs(args: string[]): boolean {
|
|
|
467
516
|
return args.some((token) => INSPECTION_FLAGS.has(token));
|
|
468
517
|
}
|
|
469
518
|
|
|
519
|
+
function isStatelessInspectionCommand(commandInfo: CommandInfo): boolean {
|
|
520
|
+
return commandInfo.command === "skills" && ["list", "get", "path"].includes(commandInfo.subcommand ?? "");
|
|
521
|
+
}
|
|
522
|
+
|
|
470
523
|
export function hasUsableBraveApiKey(apiKey: string | null | undefined = process.env[BRAVE_API_KEY_ENV]): boolean {
|
|
471
524
|
return typeof apiKey === "string" && apiKey.trim().length > 0;
|
|
472
525
|
}
|
|
@@ -693,7 +746,7 @@ function getInvalidValueFlagDetails(args: string[]): InvalidValueFlagDetails | u
|
|
|
693
746
|
continue;
|
|
694
747
|
}
|
|
695
748
|
const normalizedToken = token.split("=", 1)[0] ?? token;
|
|
696
|
-
if (!
|
|
749
|
+
if (!VALUE_FLAGS.has(normalizedToken)) {
|
|
697
750
|
continue;
|
|
698
751
|
}
|
|
699
752
|
if (token.includes("=")) {
|
|
@@ -861,10 +914,23 @@ export function extractExplicitSessionName(args: string[]): string | undefined {
|
|
|
861
914
|
return undefined;
|
|
862
915
|
}
|
|
863
916
|
|
|
917
|
+
function hasLaunchScopedFlagToken(args: string[], flag: string): boolean {
|
|
918
|
+
const commandStartIndex = findCommandStartIndex(args);
|
|
919
|
+
const command = commandStartIndex === undefined ? undefined : args[commandStartIndex];
|
|
920
|
+
return args.some((token, index) => {
|
|
921
|
+
if (token !== flag && !token.startsWith(`${flag}=`)) return false;
|
|
922
|
+
if (flag === "--auto-connect") return isBooleanFlagEnabled(args, flag);
|
|
923
|
+
if (flag === "--state" && command === "wait" && commandStartIndex !== undefined && index > commandStartIndex) {
|
|
924
|
+
return false;
|
|
925
|
+
}
|
|
926
|
+
return true;
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
|
|
864
930
|
export function getStartupScopedFlags(args: string[]): string[] {
|
|
865
931
|
return LAUNCH_SCOPED_FLAG_DEFINITIONS
|
|
866
932
|
.map((definition) => definition.flag)
|
|
867
|
-
.filter((flag) =>
|
|
933
|
+
.filter((flag) => hasLaunchScopedFlagToken(args, flag));
|
|
868
934
|
}
|
|
869
935
|
|
|
870
936
|
export function hasLaunchScopedTabCorrectionFlag(args: string[]): boolean {
|
|
@@ -923,6 +989,7 @@ export function buildExecutionPlan(
|
|
|
923
989
|
const startupScopedFlags = getStartupScopedFlags(args);
|
|
924
990
|
const plainTextInspection = isPlainTextInspectionArgs(args);
|
|
925
991
|
const commandInfo = parseCommandInfo(args);
|
|
992
|
+
const statelessInspection = plainTextInspection || isStatelessInspectionCommand(commandInfo);
|
|
926
993
|
const effectiveArgs = plainTextInspection ? [...args] : args.includes("--json") ? [] : ["--json"];
|
|
927
994
|
if (invalidValueFlag) {
|
|
928
995
|
return {
|
|
@@ -956,7 +1023,7 @@ export function buildExecutionPlan(
|
|
|
956
1023
|
let usedImplicitSession = false;
|
|
957
1024
|
let validationError: string | undefined;
|
|
958
1025
|
|
|
959
|
-
if (!explicitSessionName && options.sessionMode === "auto") {
|
|
1026
|
+
if (!explicitSessionName && options.sessionMode === "auto" && !statelessInspection) {
|
|
960
1027
|
if (options.managedSessionActive && startupScopedFlags.length > 0) {
|
|
961
1028
|
recoveryHint = {
|
|
962
1029
|
exampleArgs: args,
|
|
@@ -975,7 +1042,7 @@ export function buildExecutionPlan(
|
|
|
975
1042
|
sessionName = options.managedSessionName;
|
|
976
1043
|
usedImplicitSession = true;
|
|
977
1044
|
}
|
|
978
|
-
} else if (shouldCreateFreshManagedSession) {
|
|
1045
|
+
} else if (shouldCreateFreshManagedSession && !statelessInspection) {
|
|
979
1046
|
effectiveArgs.push("--session", options.freshSessionName);
|
|
980
1047
|
managedSessionName = options.freshSessionName;
|
|
981
1048
|
sessionName = options.freshSessionName;
|
|
@@ -1080,7 +1147,7 @@ function findCommandStartIndex(args: string[]): number | undefined {
|
|
|
1080
1147
|
}
|
|
1081
1148
|
if (token.startsWith("-")) {
|
|
1082
1149
|
const normalizedToken = token.split("=", 1)[0] ?? token;
|
|
1083
|
-
if (
|
|
1150
|
+
if (VALUE_FLAGS.has(normalizedToken) && !token.includes("=")) {
|
|
1084
1151
|
index += 1;
|
|
1085
1152
|
} else if (
|
|
1086
1153
|
GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES.has(normalizedToken) &&
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-agent-browser-native",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.25",
|
|
4
4
|
"description": "pi extension that exposes agent-browser as a native tool for browser automation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "Mitch Fultz (https://github.com/fitchmultz)",
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
"docs/COMMAND_REFERENCE.md",
|
|
41
41
|
"docs/RELEASE.md",
|
|
42
42
|
"docs/REQUIREMENTS.md",
|
|
43
|
+
"docs/SUPPORT_MATRIX.md",
|
|
43
44
|
"docs/TOOL_CONTRACT.md"
|
|
44
45
|
],
|
|
45
46
|
"pi": {
|
|
@@ -66,9 +67,10 @@
|
|
|
66
67
|
"scripts": {
|
|
67
68
|
"docs": "node ./scripts/project.mjs docs",
|
|
68
69
|
"doctor": "node ./scripts/doctor.mjs",
|
|
70
|
+
"benchmark:agent-browser": "node ./scripts/agent-browser-efficiency-benchmark.mjs",
|
|
69
71
|
"test": "tsx --test test/**/*.test.ts",
|
|
70
72
|
"verify": "node ./scripts/project.mjs verify",
|
|
71
|
-
"prepublishOnly": "npm run verify && npm pack --dry-run"
|
|
73
|
+
"prepublishOnly": "npm run verify -- release && npm pack --dry-run"
|
|
72
74
|
},
|
|
73
75
|
"packageManager": "npm@11.14.0"
|
|
74
76
|
}
|