pi-agent-browser-native 0.2.34 → 0.2.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CHANGELOG.md +44 -0
  2. package/README.md +25 -15
  3. package/docs/ARCHITECTURE.md +19 -13
  4. package/docs/COMMAND_REFERENCE.md +274 -44
  5. package/docs/ELECTRON.md +3 -3
  6. package/docs/RELEASE.md +11 -11
  7. package/docs/REQUIREMENTS.md +5 -5
  8. package/docs/SUPPORT_MATRIX.md +43 -24
  9. package/docs/TOOL_CONTRACT.md +50 -30
  10. package/extensions/agent-browser/index.ts +518 -2402
  11. package/extensions/agent-browser/lib/argv-descriptor.ts +90 -0
  12. package/extensions/agent-browser/lib/argv-grammar.ts +128 -0
  13. package/extensions/agent-browser/lib/command-policy.ts +71 -0
  14. package/extensions/agent-browser/lib/command-taxonomy.ts +336 -0
  15. package/extensions/agent-browser/lib/electron/cleanup.ts +1 -0
  16. package/extensions/agent-browser/lib/executable-path.ts +19 -0
  17. package/extensions/agent-browser/lib/input-modes/params.ts +6 -6
  18. package/extensions/agent-browser/lib/orchestration/batch-stdin.ts +65 -0
  19. package/extensions/agent-browser/lib/orchestration/browser-run/browser-action-model.ts +154 -0
  20. package/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.ts +149 -0
  21. package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts +56 -30
  22. package/extensions/agent-browser/lib/orchestration/browser-run/final-result.ts +13 -3
  23. package/extensions/agent-browser/lib/orchestration/browser-run/index.ts +33 -27
  24. package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts +48 -22
  25. package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts +39 -10
  26. package/extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.ts +93 -0
  27. package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts +98 -124
  28. package/extensions/agent-browser/lib/orchestration/browser-run/types.ts +40 -1
  29. package/extensions/agent-browser/lib/orchestration/electron-host/index.ts +860 -0
  30. package/extensions/agent-browser/lib/playbook.ts +10 -10
  31. package/extensions/agent-browser/lib/prompt-policy.ts +122 -0
  32. package/extensions/agent-browser/lib/results/action-recommendations.ts +3 -23
  33. package/extensions/agent-browser/lib/results/presentation/navigation.ts +2 -34
  34. package/extensions/agent-browser/lib/runtime.ts +93 -227
  35. package/extensions/agent-browser/lib/session-page-state.ts +31 -14
  36. package/extensions/agent-browser/lib/temp.ts +148 -23
  37. package/package.json +4 -4
  38. package/scripts/agent-browser-capability-baseline.mjs +198 -1
@@ -12,6 +12,22 @@ export const COMMAND_REFERENCE_DOC_PATH = "docs/COMMAND_REFERENCE.md";
12
12
  export const CAPABILITY_BASELINE_BLOCK_MARKER_PREFIX = "agent-browser-capability-baseline";
13
13
  export const COMMAND_REFERENCE_BASELINE_BLOCK_IDS = Object.freeze(["upstream-baseline", "capability-token-baseline"]);
14
14
 
15
+ const sourceEvidence = Object.freeze({
16
+ repository: "vercel-labs/agent-browser",
17
+ upstreamHead: "4ad284890cb59564af603e6de403dd75dd19e832",
18
+ upstreamPackageVersion: "0.27.0",
19
+ inspectedSources: Object.freeze([
20
+ "agent-browser --version",
21
+ "agent-browser --help",
22
+ "selected agent-browser <command> --help output",
23
+ "README.md",
24
+ "CHANGELOG.md",
25
+ "agent-browser.schema.json",
26
+ "cli/src/commands.rs",
27
+ "cli/src/flags.rs",
28
+ ]),
29
+ });
30
+
15
31
  const helpCommand = (label, args) => Object.freeze({ label, args: Object.freeze(args) });
16
32
  const expectation = (help, token) => Object.freeze({ help, token });
17
33
  const section = (id, title, docTokens, upstreamExpectations = []) =>
@@ -28,15 +44,30 @@ const helpCommands = Object.freeze([
28
44
  helpCommand("skills help", ["skills", "--help"]),
29
45
  helpCommand("skills list", ["skills", "list"]),
30
46
  helpCommand("core skill full", ["skills", "get", "core", "--full"]),
47
+ helpCommand("open help", ["open", "--help"]),
48
+ helpCommand("click help", ["click", "--help"]),
49
+ helpCommand("key help", ["key", "--help"]),
50
+ helpCommand("scroll help", ["scroll", "--help"]),
51
+ helpCommand("scrollinto help", ["scrollinto", "--help"]),
52
+ helpCommand("keydown help", ["keydown", "--help"]),
53
+ helpCommand("keyup help", ["keyup", "--help"]),
54
+ helpCommand("get help", ["get", "--help"]),
55
+ helpCommand("is help", ["is", "--help"]),
56
+ helpCommand("mouse help", ["mouse", "--help"]),
57
+ helpCommand("set help", ["set", "--help"]),
31
58
  helpCommand("tab help", ["tab", "--help"]),
32
59
  helpCommand("snapshot help", ["snapshot", "--help"]),
60
+ helpCommand("eval help", ["eval", "--help"]),
33
61
  helpCommand("wait help", ["wait", "--help"]),
34
62
  helpCommand("screenshot help", ["screenshot", "--help"]),
63
+ helpCommand("pdf help", ["pdf", "--help"]),
64
+ helpCommand("close help", ["close", "--help"]),
35
65
  helpCommand("find help", ["find", "--help"]),
36
66
  helpCommand("network help", ["network", "--help"]),
37
67
  helpCommand("cookies help", ["cookies", "--help"]),
38
68
  helpCommand("storage help", ["storage", "--help"]),
39
69
  helpCommand("state help", ["state", "--help"]),
70
+ helpCommand("session help", ["session", "--help"]),
40
71
  helpCommand("frame help", ["frame", "--help"]),
41
72
  helpCommand("dialog help", ["dialog", "--help"]),
42
73
  helpCommand("window help", ["window", "--help"]),
@@ -51,6 +82,15 @@ const helpCommands = Object.freeze([
51
82
  helpCommand("trace help", ["trace", "--help"]),
52
83
  helpCommand("profiler help", ["profiler", "--help"]),
53
84
  helpCommand("record help", ["record", "--help"]),
85
+ helpCommand("console help", ["console", "--help"]),
86
+ helpCommand("errors help", ["errors", "--help"]),
87
+ helpCommand("clipboard help", ["clipboard", "--help"]),
88
+ helpCommand("tap help", ["tap", "--help"]),
89
+ helpCommand("swipe help", ["swipe", "--help"]),
90
+ helpCommand("device help", ["device", "--help"]),
91
+ helpCommand("install help", ["install", "--help"]),
92
+ helpCommand("upgrade help", ["upgrade", "--help"]),
93
+ helpCommand("profiles help", ["profiles", "--help"]),
54
94
  ]);
55
95
 
56
96
  const inventorySections = Object.freeze([
@@ -62,16 +102,21 @@ const inventorySections = Object.freeze([
62
102
  "skills get core",
63
103
  "skills get core --full",
64
104
  "skills get <name>",
105
+ "skills get <name> --full",
106
+ "skills get --all",
65
107
  "skills get electron",
66
108
  "skills get slack",
67
109
  "skills get dogfood",
68
110
  "skills get vercel-sandbox",
69
111
  "skills get agentcore",
70
112
  "skills path [name]",
113
+ "AGENT_BROWSER_SKILLS_DIR",
71
114
  ],
72
115
  [
73
116
  root("skills get core --full"),
74
117
  ["skills help", "get <name> --full"],
118
+ ["skills help", "get --all"],
119
+ ["skills help", "AGENT_BROWSER_SKILLS_DIR"],
75
120
  ["skills list", "core"],
76
121
  ["skills list", "electron"],
77
122
  ["skills list", "slack"],
@@ -87,12 +132,19 @@ const inventorySections = Object.freeze([
87
132
  "core-commands",
88
133
  "Core page, element, navigation, and extraction commands",
89
134
  [
135
+ "open [url]",
90
136
  "open <url>",
137
+ "goto <url>",
138
+ "navigate <url>",
91
139
  "click <sel>",
140
+ "click <sel> --new-tab",
92
141
  "dblclick <sel>",
93
142
  "type <sel> <text>",
94
143
  "fill <sel> <text>",
95
144
  "press <key>",
145
+ "key <key>",
146
+ "keydown <key>",
147
+ "keyup <key>",
96
148
  "keyboard type <text>",
97
149
  "keyboard inserttext <text>",
98
150
  "keydown Shift",
@@ -106,33 +158,68 @@ const inventorySections = Object.freeze([
106
158
  "upload <sel> <files...>",
107
159
  "download <sel> <path>",
108
160
  "scroll <dir> [px]",
161
+ "scroll <dir> [px] --selector <sel>",
109
162
  "scrollintoview <sel>",
163
+ "scrollinto <sel>",
110
164
  "wait <sel|ms>",
165
+ "wait --url <pattern>",
166
+ "wait --load <state>",
167
+ "wait --fn <expression>",
168
+ "wait --text <text>",
169
+ "wait --download [path]",
170
+ "screenshot [selector] [path]",
111
171
  "screenshot [path]",
112
172
  "screenshot --full",
113
173
  "screenshot --annotate",
114
174
  "pdf <path>",
115
175
  "snapshot",
176
+ "snapshot --cursor",
177
+ "snapshot --interactive",
178
+ "snapshot --urls",
179
+ "snapshot --compact",
180
+ "snapshot --depth <n>",
181
+ "snapshot --selector <sel>",
116
182
  "eval <js>",
183
+ "eval --stdin",
184
+ "eval -b <base64>",
117
185
  "connect <port|url>",
118
186
  "close [--all]",
187
+ "quit",
188
+ "exit",
119
189
  "back",
120
190
  "forward",
121
191
  "reload",
122
192
  "pushstate <url>",
123
193
  "get <what> [selector]",
194
+ "get cdp-url",
195
+ "get box <selector>",
196
+ "get styles <selector>",
124
197
  "is <what> <selector>",
125
198
  "find <locator> <value> <action>",
199
+ "find first <sel>",
200
+ "find last <sel>",
201
+ "find nth <n> <sel>",
202
+ "find role <role> --name <name>",
203
+ "find ... --exact",
126
204
  "mouse <action> [args]",
127
205
  "set <setting> [value]",
206
+ "set media <features>",
207
+ "tap <selector>",
208
+ "swipe <direction> [distance]",
128
209
  ],
129
210
  [
211
+ ["open help", "open [url]"],
212
+ ["open help", "aliases still require a URL."],
130
213
  root("open <url>"),
131
214
  root("click <sel>"),
215
+ ["click help", "--new-tab"],
132
216
  root("dblclick <sel>"),
133
217
  root("type <sel> <text>"),
134
218
  root("fill <sel> <text>"),
135
219
  root("press <key>"),
220
+ ["key help", "Aliases: key"],
221
+ ["keydown help", "keydown <key>"],
222
+ ["keyup help", "keyup <key>"],
136
223
  root("keyboard type <text>"),
137
224
  root("keyboard inserttext <text>"),
138
225
  root("hover <sel>"),
@@ -144,29 +231,57 @@ const inventorySections = Object.freeze([
144
231
  root("upload <sel> <files...>"),
145
232
  root("download <sel> <path>"),
146
233
  root("scroll <dir> [px]"),
234
+ ["scroll help", "--selector <sel>"],
147
235
  root("scrollintoview <sel>"),
236
+ ["scrollinto help", "Aliases: scrollinto"],
148
237
  root("wait <sel|ms>"),
238
+ ["wait help", "--url <pattern>"],
239
+ ["wait help", "--load <state>"],
240
+ ["wait help", "--fn <expression>"],
241
+ ["wait help", "--text <text>"],
242
+ ["wait help", "--download [path]"],
149
243
  root("screenshot [path]"),
244
+ ["screenshot help", "screenshot [selector] [path]"],
150
245
  root("pdf <path>"),
246
+ ["pdf help", "Save page as PDF"],
151
247
  root("snapshot"),
248
+ ["snapshot help", "--interactive"],
249
+ ["snapshot help", "--urls"],
250
+ ["snapshot help", "--compact"],
251
+ ["snapshot help", "--depth <n>"],
252
+ ["snapshot help", "--selector <sel>"],
152
253
  root("eval <js>"),
254
+ ["eval help", "--stdin"],
255
+ ["eval help", "-b, --base64"],
153
256
  root("connect <port|url>"),
154
257
  root("close [--all]"),
258
+ ["close help", "Aliases: quit, exit"],
155
259
  root("back"),
156
260
  root("forward"),
157
261
  root("reload"),
158
262
  root("pushstate <url>"),
159
263
  root("Get Info: agent-browser get <what> [selector]"),
264
+ ["get help", "box <selector>"],
265
+ ["get help", "styles <selector>"],
266
+ ["get help", "cdp-url"],
160
267
  root("Check State: agent-browser is <what> <selector>"),
161
268
  root("Find Elements: agent-browser find <locator> <value> <action> [text]"),
269
+ ["find help", "first <selector>"],
270
+ ["find help", "last <selector>"],
271
+ ["find help", "nth <index> <selector>"],
272
+ ["find help", "--name <name>"],
273
+ ["find help", "--exact"],
162
274
  root("Mouse: agent-browser mouse <action> [args]"),
163
275
  root("Browser Settings: agent-browser set <setting> [value]"),
276
+ ["set help", "media [dark|light]"],
164
277
  ["keyboard help", "type <text>"],
165
278
  ["keyboard help", "inserttext <text>"],
166
279
  ["screenshot help", "--full, -f"],
167
280
  ["screenshot help", "--annotate"],
168
281
  ["find help", "role <role>"],
169
282
  ["find help", "testid <id>"],
283
+ ["tap help", "tap <selector>"],
284
+ ["swipe help", "swipe <direction> [distance]"],
170
285
  ],
171
286
  ),
172
287
  section(
@@ -177,8 +292,16 @@ const inventorySections = Object.freeze([
177
292
  "session list",
178
293
  "state save <path>",
179
294
  "state load <path>",
295
+ "state list",
296
+ "state show <filename>",
297
+ "state rename <old-name> <new-name>",
298
+ "state clear [session-name] [--all]",
299
+ "state clear -a",
300
+ "state clean --older-than <days>",
180
301
  "tab list",
302
+ "tab new [url]",
181
303
  "tab new --label <name> [url]",
304
+ "tab close [target]",
182
305
  "tab <t<N>|label>",
183
306
  "frame <selector|main>",
184
307
  "dialog accept [text]",
@@ -190,7 +313,15 @@ const inventorySections = Object.freeze([
190
313
  root("session list"),
191
314
  ["state help", "save <path>"],
192
315
  ["state help", "load <path>"],
316
+ ["state help", "list"],
317
+ ["state help", "show <filename>"],
318
+ ["state help", "rename <old-name> <new-name>"],
319
+ ["state help", "clear [session-name] [--all]"],
320
+ ["state help", "agent-browser state clear --all"],
321
+ ["state help", "clean --older-than <days>"],
322
+ ["tab help", "new [url]"],
193
323
  ["tab help", "new --label <name> [url]"],
324
+ ["tab help", "close [t<N>|label]"],
194
325
  ["tab help", "Stable tab ids"],
195
326
  ["frame help", "frame <selector|main>"],
196
327
  ["dialog help", "dialog <accept|dismiss|status> [text]"],
@@ -203,13 +334,21 @@ const inventorySections = Object.freeze([
203
334
  [
204
335
  "network <action>",
205
336
  "network route <url> [--abort|--body <json>] [--resource-type <csv>]",
337
+ "network unroute [url]",
338
+ "network requests [--clear] [--filter <pattern>] [--type <csv>] [--method <method>] [--status <code|range>]",
206
339
  "network request <requestId>",
340
+ "network har start",
341
+ "network har stop [path]",
207
342
  "cookies [get|set|clear]",
343
+ "cookies set <name> <value> --url <url> --domain <domain> --path <path> --httpOnly --secure --sameSite <Strict|Lax|None> --expires <timestamp>",
208
344
  "cookies set --curl <file>",
209
345
  "storage <local|session>",
210
346
  "diff snapshot",
347
+ "diff snapshot --baseline <file> --selector <sel> --compact --depth <n>",
211
348
  "diff screenshot --baseline",
349
+ "diff screenshot --baseline <file> --output <file> --threshold <0-1> --selector <sel> --full",
212
350
  "diff url <u1> <u2>",
351
+ "diff url <u1> <u2> --screenshot --wait-until <strategy> --selector <sel> --compact --depth <n>",
213
352
  "trace start|stop [path]",
214
353
  "profiler start|stop [path]",
215
354
  "record start <path> [url]",
@@ -220,6 +359,10 @@ const inventorySections = Object.freeze([
220
359
  "highlight <sel>",
221
360
  "inspect",
222
361
  "clipboard <op> [text]",
362
+ "clipboard read",
363
+ "clipboard write <text>",
364
+ "clipboard copy",
365
+ "clipboard paste",
223
366
  "stream enable [--port <n>]",
224
367
  "stream disable",
225
368
  "stream status",
@@ -229,11 +372,15 @@ const inventorySections = Object.freeze([
229
372
  "react renders stop [--json]",
230
373
  "react suspense [--only-dynamic] [--json]",
231
374
  "vitals [url] [--json]",
375
+ "web-vitals [url] [--json]",
232
376
  "removeinitscript <id>",
233
377
  ],
234
378
  [
235
379
  root("network <action>"),
236
380
  root("--resource-type <csv>"),
381
+ ["network help", "unroute [url]"],
382
+ ["network help", "network har start"],
383
+ ["network help", "network har stop ./capture.har"],
237
384
  root("cookies [get|set|clear]"),
238
385
  root("cookies set --curl <file>"),
239
386
  root("storage <local|session>"),
@@ -248,6 +395,10 @@ const inventorySections = Object.freeze([
248
395
  root("highlight <sel>"),
249
396
  root("inspect"),
250
397
  root("clipboard <op> [text]"),
398
+ ["clipboard help", "read"],
399
+ ["clipboard help", "write <text>"],
400
+ ["clipboard help", "copy"],
401
+ ["clipboard help", "paste"],
251
402
  root("stream enable [--port <n>]"),
252
403
  root("stream disable"),
253
404
  root("stream status"),
@@ -258,31 +409,47 @@ const inventorySections = Object.freeze([
258
409
  root("react suspense [--only-dynamic] [--json]"),
259
410
  root("vitals [url] [--json]"),
260
411
  root("removeinitscript <id>"),
412
+ ["network help", "requests [options]"],
413
+ ["network help", "--type <types>"],
414
+ ["network help", "--method <method>"],
415
+ ["network help", "--status <code>"],
261
416
  ["network help", "request <requestId>"],
262
417
  ["network help", "har <start|stop>"],
263
418
  ["storage help", "set <key> <value>"],
419
+ ["diff help", "diff snapshot [options]"],
420
+ ["diff help", "--baseline <f>"],
421
+ ["diff help", "--output <file>"],
422
+ ["diff help", "--threshold <0-1>"],
423
+ ["diff help", "--wait-until <strategy>"],
264
424
  ["diff help", "diff screenshot --baseline <f>"],
265
425
  ["trace help", "trace <operation> [path]"],
266
426
  ["profiler help", "--categories <list>"],
267
427
  ["record help", "record restart <path.webm> [url]"],
428
+ ["console help", "--clear"],
429
+ ["errors help", "--clear"],
268
430
  ],
269
431
  ),
270
432
  section(
271
433
  "batch-auth-setup-ai",
272
- "Batch, auth, confirmations, setup, dashboard, and AI commands",
434
+ "Batch, auth, confirmations, setup, dashboard, devices, and AI commands",
273
435
  [
274
436
  "batch [--bail]",
275
437
  "auth save <name>",
438
+ "auth save <name> --url <url> --username <user> --password <pass>",
439
+ "auth save <name> --username-selector <s> --password-selector <s> --submit-selector <s>",
276
440
  "auth save <name> --password-stdin",
277
441
  "auth login <name>",
278
442
  "auth list",
279
443
  "auth show <name>",
280
444
  "auth delete <name>",
445
+ "auth remove <name>",
281
446
  "confirm <id>",
282
447
  "deny <id>",
283
448
  "chat <message>",
449
+ "dashboard [start]",
284
450
  "dashboard start --port <n>",
285
451
  "dashboard stop",
452
+ "device list",
286
453
  "install",
287
454
  "install --with-deps",
288
455
  "upgrade",
@@ -299,12 +466,19 @@ const inventorySections = Object.freeze([
299
466
  root("deny <id>"),
300
467
  root("chat <message>"),
301
468
  root("dashboard start --port <n>"),
469
+ ["device help", "device list"],
302
470
  root("install --with-deps"),
303
471
  root("upgrade"),
304
472
  root("doctor [--fix]"),
305
473
  root("profiles"),
306
474
  ["batch help", "--bail"],
475
+ ["auth help", "--url <url>"],
476
+ ["auth help", "--username <user>"],
477
+ ["auth help", "--password <pass>"],
307
478
  ["auth help", "--password-stdin"],
479
+ ["auth help", "--username-selector <s>"],
480
+ ["auth help", "--password-selector <s>"],
481
+ ["auth help", "--submit-selector <s>"],
308
482
  ["dashboard help", "dashboard [start|stop] [options]"],
309
483
  ["chat help", "chat <message>"],
310
484
  ["doctor help", "--offline"],
@@ -403,6 +577,7 @@ const inventorySections = Object.freeze([
403
577
  "AGENT_BROWSER_DEBUG",
404
578
  "AGENT_BROWSER_CONFIG",
405
579
  "AGENT_BROWSER_DEFAULT_TIMEOUT",
580
+ "--idle-timeout <ms>",
406
581
  "AGENT_BROWSER_STREAM_PORT",
407
582
  "AGENT_BROWSER_IDLE_TIMEOUT_MS",
408
583
  "AGENT_BROWSER_ENCRYPTION_KEY",
@@ -410,6 +585,27 @@ const inventorySections = Object.freeze([
410
585
  "AGENT_BROWSER_IOS_UDID",
411
586
  "AI_GATEWAY_URL",
412
587
  "AI_GATEWAY_API_KEY",
588
+ "BROWSERBASE_API_KEY",
589
+ "BROWSERBASE_PROJECT_ID",
590
+ "BROWSERLESS_API_KEY",
591
+ "BROWSERLESS_API_URL",
592
+ "BROWSERLESS_BROWSER_TYPE",
593
+ "BROWSERLESS_STEALTH",
594
+ "BROWSERLESS_TTL",
595
+ "BROWSER_USE_API_KEY",
596
+ "KERNEL_API_KEY",
597
+ "KERNEL_HEADLESS",
598
+ "KERNEL_STEALTH",
599
+ "KERNEL_TIMEOUT_SECONDS",
600
+ "KERNEL_PROFILE_NAME",
601
+ "AGENTCORE_API_KEY",
602
+ "AGENTCORE_REGION",
603
+ "AGENTCORE_BROWSER_ID",
604
+ "AGENTCORE_PROFILE_ID",
605
+ "AGENTCORE_SESSION_TIMEOUT",
606
+ "AWS_PROFILE",
607
+ "AWS_ACCESS_KEY_ID",
608
+ "AWS_SECRET_ACCESS_KEY",
413
609
  ],
414
610
  [
415
611
  root("--profile <name|path>"),
@@ -508,6 +704,7 @@ const inventorySections = Object.freeze([
508
704
 
509
705
  export const CAPABILITY_BASELINE = Object.freeze({
510
706
  targetVersion: "0.27.0",
707
+ sourceEvidence,
511
708
  helpCommands,
512
709
  inventorySections,
513
710
  docRequiredTokens: Object.freeze(inventorySections.flatMap((entry) => entry.docTokens)),