pi-agent-browser-native 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +127 -0
  2. package/README.md +63 -20
  3. package/dist/extensions/agent-browser/index.js +787 -105
  4. package/dist/extensions/agent-browser/lib/argv-descriptor.js +35 -3
  5. package/dist/extensions/agent-browser/lib/argv-grammar.js +44 -2
  6. package/dist/extensions/agent-browser/lib/batch-lifecycle.js +71 -0
  7. package/dist/extensions/agent-browser/lib/command-policy.js +1 -1
  8. package/dist/extensions/agent-browser/lib/command-taxonomy.js +35 -2
  9. package/dist/extensions/agent-browser/lib/input-modes/job.js +61 -4
  10. package/dist/extensions/agent-browser/lib/input-modes/lookups.js +2 -2
  11. package/dist/extensions/agent-browser/lib/input-modes/params.js +22 -23
  12. package/dist/extensions/agent-browser/lib/input-modes/script.js +462 -0
  13. package/dist/extensions/agent-browser/lib/input-modes/semantic-action.js +51 -12
  14. package/dist/extensions/agent-browser/lib/launch-scoped-flags.js +8 -0
  15. package/dist/extensions/agent-browser/lib/managed-session-policy-lock.js +3 -1
  16. package/dist/extensions/agent-browser/lib/managed-session-restore.js +26 -36
  17. package/dist/extensions/agent-browser/lib/managed-session-snapshots.js +2 -4
  18. package/dist/extensions/agent-browser/lib/managed-session-state-policy.js +47 -29
  19. package/dist/extensions/agent-browser/lib/managed-session-storage.js +50 -24
  20. package/dist/extensions/agent-browser/lib/orchestration/batch-stdin.js +26 -5
  21. package/dist/extensions/agent-browser/lib/orchestration/browser-run/artifact-paths.js +110 -30
  22. package/dist/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.js +2 -1
  23. package/dist/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.js +26 -25
  24. package/dist/extensions/agent-browser/lib/orchestration/browser-run/final-result.js +17 -3
  25. package/dist/extensions/agent-browser/lib/orchestration/browser-run/index.js +2 -1
  26. package/dist/extensions/agent-browser/lib/orchestration/browser-run/managed-session-daemon-policy.js +6 -4
  27. package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare.js +129 -32
  28. package/dist/extensions/agent-browser/lib/orchestration/browser-run/process-output.js +191 -55
  29. package/dist/extensions/agent-browser/lib/orchestration/browser-run/session-state.js +57 -29
  30. package/dist/extensions/agent-browser/lib/orchestration/electron-host/index.js +15 -11
  31. package/dist/extensions/agent-browser/lib/orchestration/input-plan.js +36 -18
  32. package/dist/extensions/agent-browser/lib/orchestration/script-mode.js +299 -0
  33. package/dist/extensions/agent-browser/lib/pi-tool-rendering.js +32 -10
  34. package/dist/extensions/agent-browser/lib/playbook.js +18 -15
  35. package/dist/extensions/agent-browser/lib/process-environment.js +14 -0
  36. package/dist/extensions/agent-browser/lib/process-identity.js +4 -4
  37. package/dist/extensions/agent-browser/lib/process.js +131 -41
  38. package/dist/extensions/agent-browser/lib/recording-reservations.js +183 -0
  39. package/dist/extensions/agent-browser/lib/results/action-recommendations.js +62 -5
  40. package/dist/extensions/agent-browser/lib/results/artifact-manifest.js +62 -4
  41. package/dist/extensions/agent-browser/lib/results/categories.js +6 -1
  42. package/dist/extensions/agent-browser/lib/results/next-actions.js +19 -5
  43. package/dist/extensions/agent-browser/lib/results/presentation/artifacts.js +85 -38
  44. package/dist/extensions/agent-browser/lib/results/presentation/batch.js +66 -11
  45. package/dist/extensions/agent-browser/lib/results/presentation/common.js +18 -0
  46. package/dist/extensions/agent-browser/lib/results/presentation/diagnostics.js +7 -1
  47. package/dist/extensions/agent-browser/lib/results/presentation/errors.js +2 -1
  48. package/dist/extensions/agent-browser/lib/results/presentation/navigation.js +26 -11
  49. package/dist/extensions/agent-browser/lib/results/presentation/registry.js +58 -13
  50. package/dist/extensions/agent-browser/lib/results/presentation/semantic-action.js +1 -10
  51. package/dist/extensions/agent-browser/lib/results/presentation.js +6 -3
  52. package/dist/extensions/agent-browser/lib/results/recovery-actions.js +2 -0
  53. package/dist/extensions/agent-browser/lib/results/selector-recovery.js +51 -8
  54. package/dist/extensions/agent-browser/lib/results/snapshot-high-value-controls.js +13 -7
  55. package/dist/extensions/agent-browser/lib/runtime.js +116 -39
  56. package/dist/extensions/agent-browser/lib/session-page-state.js +62 -10
  57. package/dist/extensions/agent-browser/lib/upstream-version.js +14 -0
  58. package/dist/extensions/agent-browser/script-worker.js +169 -0
  59. package/dist/scripts/agent-browser-target.mjs +3 -0
  60. package/docs/ARCHITECTURE.md +40 -21
  61. package/docs/COMMAND_REFERENCE.md +90 -35
  62. package/docs/RELEASE.md +3 -3
  63. package/docs/REQUIREMENTS.md +4 -2
  64. package/docs/SUPPORT_MATRIX.md +26 -19
  65. package/docs/TOOL_CONTRACT.md +93 -52
  66. package/package.json +3 -1
  67. package/platform-smoke.config.mjs +2 -2
  68. package/scripts/agent-browser-capability-baseline.mjs +24 -6
  69. package/scripts/agent-browser-target.mjs +3 -0
  70. package/scripts/build.mjs +41 -0
  71. package/scripts/doctor.mjs +7 -6
  72. package/scripts/platform-smoke/browser-dogfood-windows.ps1 +9 -3
  73. package/scripts/platform-smoke/targets.mjs +12 -6
@@ -1,11 +1,13 @@
1
+ import { isCloseCommand } from "../../command-taxonomy.js";
1
2
  import { isRecord } from "../../parsing.js";
2
- import { extractCommandTokens, parseCommandInfo, redactInvocationArgs, redactSensitiveText, redactSensitiveValue } from "../../runtime.js";
3
+ import { getAgentBrowserSessionIdentityKey } from "../../argv-grammar.js";
4
+ import { extractUpstreamCommandTokens, parseCommandInfo, redactInvocationArgs, redactSensitiveText, redactSensitiveValue } from "../../runtime.js";
3
5
  import { buildAgentBrowserNextActions } from "../action-recommendations.js";
4
- import { formatSessionArtifactRetentionSummary } from "../artifact-manifest.js";
6
+ import { formatSessionArtifactRetentionSummary, isPendingRecordingArtifact } from "../artifact-manifest.js";
5
7
  import { classifyAgentBrowserFailureCategory } from "../categories.js";
6
8
  import { detectConfirmationRequired } from "../confirmation.js";
7
9
  import { applyNetworkRouteRecords, buildNetworkRouteDiagnostics } from "../network-routes.js";
8
- import { applyNamespaceToNextActions, withOptionalSessionArgs } from "../next-actions.js";
10
+ import { appendUniqueAgentBrowserNextActions, applyNamespaceToNextActions, withOptionalSessionArgs } from "../next-actions.js";
9
11
  import { stringifyModelFacing } from "./common.js";
10
12
  import { buildArtifactVerificationSummary, classifyPresentationSuccessCategory, manifestHasNewNoticeWorthyEntries } from "./artifacts.js";
11
13
  import { formatBatchStepCommand, getPresentationImages, getPresentationPaths, getPresentationText, isStringArray } from "./content.js";
@@ -62,7 +64,7 @@ function hasModelFacingArgRedaction(args) {
62
64
  function getStatefulCommandSensitiveValues(command) {
63
65
  if (!command)
64
66
  return [];
65
- const tokens = extractCommandTokens(command);
67
+ const tokens = extractUpstreamCommandTokens(command);
66
68
  const values = [];
67
69
  if (tokens[0] === "cookies" && tokens[1] === "set" && tokens[3])
68
70
  values.push(tokens[3]);
@@ -181,10 +183,11 @@ function formatBatchStepsText(steps) {
181
183
  return lines.join("\n\n");
182
184
  }
183
185
  async function buildBatchStepPresentation(options) {
184
- const { artifactManifest, artifactRequest, buildNestedToolPresentation, cwd, index, item, namespace, networkRoutes, persistentArtifactStore, sessionName } = options;
186
+ const { artifactManifest, artifactMaxUpdatedAtMs, artifactMinUpdatedAtMs, artifactRequest, buildNestedToolPresentation, cwd, index, item, namespace, networkRoutes, persistentArtifactStore, sessionName } = options;
185
187
  const command = isStringArray(item.command) ? item.command : undefined;
186
188
  const redactedCommand = command ? redactInvocationArgs(command) : undefined;
187
189
  const commandText = formatBatchStepCommand(hasModelFacingArgRedaction(redactedCommand) ? redactedCommand : command, index);
190
+ const lifecycle = extractBatchStepLifecycle(item.result);
188
191
  if (item.success === false) {
189
192
  const redactedErrorData = command?.[0] === "clipboard"
190
193
  ? redactSensitiveValue(redactClipboardPermissionErrorValue({ command: "clipboard", subcommand: command[1] }, item.error, getClipboardWritePayloadCandidates(command)))
@@ -196,13 +199,14 @@ async function buildBatchStepPresentation(options) {
196
199
  errorText,
197
200
  });
198
201
  const confirmationRequired = detectConfirmationRequired(item.error);
199
- const nextActions = applyNamespaceToNextActions(mergePresentationNextActions(buildAgentBrowserNextActions({
202
+ const nextActions = applyNamespaceToNextActions(mergePresentationNextActions(isWaitTextAssertionCommand(command) ? [buildWaitTextAssertionFailureNextAction(sessionName)] : undefined, buildAgentBrowserNextActions({
200
203
  args: command,
201
204
  command: command?.[0],
202
205
  confirmationId: confirmationRequired?.id,
203
206
  failureCategory,
204
207
  resultCategory: "failure",
205
- }), isWaitTextAssertionCommand(command) ? [buildWaitTextAssertionFailureNextAction(sessionName)] : undefined), namespace);
208
+ sessionName,
209
+ })), namespace);
206
210
  const presentation = {
207
211
  content: [{ type: "text", text: errorText }],
208
212
  failureCategory,
@@ -219,6 +223,7 @@ async function buildBatchStepPresentation(options) {
219
223
  data: redactedErrorData,
220
224
  failureCategory,
221
225
  index,
226
+ lifecycle,
222
227
  nextActions,
223
228
  resultCategory: "failure",
224
229
  success: false,
@@ -235,12 +240,15 @@ async function buildBatchStepPresentation(options) {
235
240
  : undefined;
236
241
  const presentation = await buildNestedToolPresentation({
237
242
  artifactManifest,
243
+ artifactMaxUpdatedAtMs,
244
+ artifactMinUpdatedAtMs,
238
245
  artifactRequest,
239
246
  commandInfo: commandInfoWithTokens,
240
247
  cwd,
241
248
  args: command,
242
249
  envelope: { data: item.result, success: true },
243
250
  networkRouteDiagnostics,
251
+ namespace,
244
252
  persistentArtifactStore,
245
253
  sessionName,
246
254
  });
@@ -261,6 +269,7 @@ async function buildBatchStepPresentation(options) {
261
269
  failureCategory: presentation.failureCategory,
262
270
  resultCategory: stepSucceeded ? "success" : "failure",
263
271
  savedFilePath: presentation.savedFilePath,
272
+ sessionName,
264
273
  successCategory: presentation.successCategory,
265
274
  }), namespace);
266
275
  const pageChangeSummary = buildPageChangeSummary({
@@ -284,6 +293,7 @@ async function buildBatchStepPresentation(options) {
284
293
  imagePath: imagePaths[0],
285
294
  imagePaths: imagePaths.length > 0 ? imagePaths : undefined,
286
295
  index,
296
+ lifecycle,
287
297
  networkRouteDiagnostics: presentation.networkRouteDiagnostics,
288
298
  nextActions,
289
299
  pageChangeSummary,
@@ -298,6 +308,48 @@ async function buildBatchStepPresentation(options) {
298
308
  presentation,
299
309
  };
300
310
  }
311
+ function extractBatchStepLifecycle(result) {
312
+ if (!isRecord(result) || !isRecord(result.lifecycle) || !isRecord(result.lifecycle.effectiveLaunch))
313
+ return undefined;
314
+ const browserLaunched = result.lifecycle.effectiveLaunch.browserLaunched;
315
+ return typeof browserLaunched === "boolean" ? { effectiveLaunch: { browserLaunched } } : undefined;
316
+ }
317
+ function abandonedRecordingArtifact(artifact) {
318
+ const { recordingState: _recordingState, willExistOnStop: _willExistOnStop, ...terminal } = artifact;
319
+ return { ...terminal, exists: false, status: "missing", subcommand: "close-abandoned" };
320
+ }
321
+ function coalesceTerminalBatchRecordingArtifacts(steps, sessionName, namespace) {
322
+ const artifacts = [];
323
+ const pendingIndexesBySession = new Map();
324
+ const removedPendingIndexes = new Set();
325
+ for (const step of steps) {
326
+ for (const artifact of step.presentation.artifacts ?? []) {
327
+ const index = artifacts.push(artifact) - 1;
328
+ if (artifact.command !== "record" || artifact.kind !== "video")
329
+ continue;
330
+ const session = artifact.session ? getAgentBrowserSessionIdentityKey(artifact.session, artifact.namespace) : "";
331
+ if (isPendingRecordingArtifact(artifact)) {
332
+ const pendingIndexes = pendingIndexesBySession.get(session) ?? [];
333
+ pendingIndexes.push(index);
334
+ pendingIndexesBySession.set(session, pendingIndexes);
335
+ continue;
336
+ }
337
+ const pendingIndex = pendingIndexesBySession.get(session)?.pop();
338
+ if (pendingIndex !== undefined)
339
+ removedPendingIndexes.add(pendingIndex);
340
+ }
341
+ const command = step.details.command;
342
+ if (step.details.success !== true || !command || !isCloseCommand(extractUpstreamCommandTokens(command)[0]))
343
+ continue;
344
+ const session = sessionName ? getAgentBrowserSessionIdentityKey(sessionName, namespace) : "";
345
+ for (const pendingIndex of pendingIndexesBySession.get(session) ?? []) {
346
+ const pending = artifacts[pendingIndex];
347
+ if (pending && !removedPendingIndexes.has(pendingIndex))
348
+ artifacts[pendingIndex] = abandonedRecordingArtifact(pending);
349
+ }
350
+ }
351
+ return artifacts.filter((_, index) => !removedPendingIndexes.has(index));
352
+ }
301
353
  export async function buildBatchPresentation(options) {
302
354
  const { artifactRequests, buildNestedToolPresentation, cwd, data, namespace, networkRoutes, persistentArtifactStore, sessionName, summary } = options;
303
355
  const steps = [];
@@ -307,6 +359,8 @@ export async function buildBatchPresentation(options) {
307
359
  for (const [index, item] of data.entries()) {
308
360
  const step = await buildBatchStepPresentation({
309
361
  artifactManifest: currentArtifactManifest,
362
+ artifactMaxUpdatedAtMs: options.artifactMaxUpdatedAtMs,
363
+ artifactMinUpdatedAtMs: options.artifactMinUpdatedAtMs,
310
364
  artifactRequest: artifactRequests?.[index],
311
365
  buildNestedToolPresentation,
312
366
  cwd,
@@ -319,7 +373,7 @@ export async function buildBatchPresentation(options) {
319
373
  });
320
374
  steps.push(step);
321
375
  currentArtifactManifest = step.presentation.artifactManifest ?? currentArtifactManifest;
322
- currentNetworkRoutes = applyNetworkRouteRecords(currentNetworkRoutes, isStringArray(item.command) ? extractCommandTokens(item.command) : undefined, item.success !== false && step.details.success);
376
+ currentNetworkRoutes = applyNetworkRouteRecords(currentNetworkRoutes, isStringArray(item.command) ? extractUpstreamCommandTokens(item.command) : undefined, item.success !== false && step.details.success);
323
377
  protectedPersistentPaths.push(...getPresentationPaths({
324
378
  primaryPath: step.presentation.fullOutputPath,
325
379
  secondaryPaths: step.presentation.fullOutputPaths,
@@ -327,7 +381,7 @@ export async function buildBatchPresentation(options) {
327
381
  }
328
382
  const batchFailure = getBatchFailureDetails(steps);
329
383
  const images = steps.flatMap((step) => getPresentationImages(step.presentation));
330
- const artifacts = steps.flatMap((step) => step.presentation.artifacts ?? []);
384
+ const artifacts = coalesceTerminalBatchRecordingArtifacts(steps, sessionName, namespace);
331
385
  const artifactVerification = buildArtifactVerificationSummary(artifacts);
332
386
  const fullOutputPaths = steps.flatMap((step) => getPresentationPaths({
333
387
  primaryPath: step.presentation.fullOutputPath,
@@ -356,9 +410,10 @@ export async function buildBatchPresentation(options) {
356
410
  const contentText = artifactRetentionSummary && manifestHasNewNoticeWorthyEntries(options.artifactManifest, currentArtifactManifest)
357
411
  ? `${text}\n\n${artifactRetentionSummary}`
358
412
  : text;
413
+ const artifactLifecycleActions = applyNamespaceToNextActions(buildAgentBrowserNextActions({ artifacts, command: "batch", resultCategory: batchFailure ? "failure" : "success", sessionName }), namespace);
359
414
  const nextActions = batchFailure
360
- ? batchFailure.failedStep.nextActions
361
- : buildAgentBrowserNextActions({ artifacts, command: "batch", resultCategory: "success" });
415
+ ? appendUniqueAgentBrowserNextActions([...(batchFailure.failedStep.nextActions ?? [])], artifactLifecycleActions)
416
+ : artifactLifecycleActions;
362
417
  const changedSteps = steps.map((step) => step.details).filter((details) => details.pageChangeSummary !== undefined);
363
418
  const pageChangeSummary = artifacts.length > 0
364
419
  ? buildPageChangeSummary({
@@ -1,6 +1,7 @@
1
1
  import { containsManagedSessionRestoreKey } from "../../managed-session-capabilities.js";
2
2
  import { redactSensitiveText, redactSensitiveValue } from "../../runtime.js";
3
3
  import { stringifyUnknown, truncateText } from "../text.js";
4
+ const UNTITLED_PAGE_SUMMARY = "(untitled page)";
4
5
  export function stringifyModelFacing(value) {
5
6
  return stringifyUnknown(redactSensitiveValue(value));
6
7
  }
@@ -34,6 +35,23 @@ export function getStringField(data, key) {
34
35
  const value = data[key];
35
36
  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
36
37
  }
38
+ // `lifecycle` is upstream launch/reuse bookkeeping, never page content, so it must not be the
39
+ // answer an agent reads when a command has no dedicated presenter.
40
+ export function omitUpstreamLifecycle(data) {
41
+ const { lifecycle: _lifecycle, ...rest } = data;
42
+ return rest;
43
+ }
44
+ export function getPageSummary(data) {
45
+ const title = typeof data.title === "string" ? data.title : undefined;
46
+ const url = typeof data.url === "string" ? data.url : undefined;
47
+ if (title === undefined && url === undefined)
48
+ return undefined;
49
+ if (title && url)
50
+ return `${title}\n${url}`;
51
+ if (url)
52
+ return url;
53
+ return title || UNTITLED_PAGE_SUMMARY;
54
+ }
37
55
  export function formatCount(count, singular, plural = `${singular}s`) {
38
56
  return `${count} ${count === 1 ? singular : plural}`;
39
57
  }
@@ -61,7 +61,9 @@ export function getTabSummary(data) {
61
61
  ? String(tab.index)
62
62
  : String(index);
63
63
  const labelText = label && label !== tabSelector ? ` label=${redactModelFacingText(label)}` : "";
64
- return `${marker} [${tabSelector}]${labelText} ${title} ${url}`;
64
+ const targetId = typeof tab.targetId === "string" && tab.targetId.trim().length > 0 ? tab.targetId.trim() : undefined;
65
+ const targetText = targetId && targetId !== tabSelector ? ` target=${redactModelFacingText(targetId)}` : "";
66
+ return `${marker} [${tabSelector}]${labelText}${targetText} ${title} — ${url}`;
65
67
  });
66
68
  return lines.join("\n");
67
69
  }
@@ -348,6 +350,8 @@ function formatNetworkRequestLine(item, index) {
348
350
  }
349
351
  function formatNetworkRequestsText(data, commandInfo) {
350
352
  const requests = getArrayField(data, "requests");
353
+ if (isClearDiagnosticCommand(commandInfo) && data.cleared === true && !requests)
354
+ return "Network request buffer cleared.";
351
355
  if (!requests)
352
356
  return undefined;
353
357
  if (isClearDiagnosticCommand(commandInfo)) {
@@ -593,6 +597,8 @@ export function buildStreamNextActions(commandInfo, data, sessionName) {
593
597
  }
594
598
  function formatConsoleText(data, commandInfo) {
595
599
  const messages = getArrayField(data, "messages");
600
+ if (isClearDiagnosticCommand(commandInfo) && data.cleared === true && !messages)
601
+ return "Console buffer cleared.";
596
602
  if (!messages)
597
603
  return undefined;
598
604
  if (isClearDiagnosticCommand(commandInfo)) {
@@ -189,11 +189,12 @@ export function buildErrorPresentation(options) {
189
189
  const nextActions = [
190
190
  ...(buildUnknownCommandSuggestionActions(unknownCommandSuggestions, sessionName) ?? []),
191
191
  ...(browserProfileConfigRecovery?.actions ?? []),
192
- ...(buildAgentBrowserNextActions({
192
+ ...(browserProfileConfigRecovery ? [] : buildAgentBrowserNextActions({
193
193
  args,
194
194
  command: commandInfo.command,
195
195
  failureCategory: categoryDetails.failureCategory,
196
196
  resultCategory: "failure",
197
+ sessionName,
197
198
  }) ?? []),
198
199
  ];
199
200
  return {
@@ -1,16 +1,31 @@
1
1
  import { isNavigationObservableCommandName, isPageChangeSummaryCommand } from "../../command-taxonomy.js";
2
2
  import { isRecord } from "../../parsing.js";
3
3
  import { detectConfirmationRequired } from "../confirmation.js";
4
- import { redactModelFacingText, stringifyModelFacing } from "./common.js";
4
+ import { omitUpstreamLifecycle, redactModelFacingText, stringifyModelFacing } from "./common.js";
5
5
  const NAVIGATION_SUMMARY_FIELD = "navigationSummary";
6
- function getScalarExtractionResult(data) {
7
- const { result } = data;
8
- if (typeof result === "string") {
9
- return result.trim().length > 0 ? result : undefined;
10
- }
11
- if (typeof result === "number" || typeof result === "boolean") {
6
+ const GET_RESULT_FIELDS = {
7
+ attr: "value",
8
+ count: "count",
9
+ html: "html",
10
+ text: "text",
11
+ title: "title",
12
+ url: "url",
13
+ value: "value",
14
+ };
15
+ function getScalarExtractionResult(commandInfo, data) {
16
+ const fallbackField = commandInfo.command === "get" && commandInfo.subcommand ? GET_RESULT_FIELDS[commandInfo.subcommand] ?? "" : "";
17
+ const resultField = Object.hasOwn(data, "result") ? "result" : fallbackField.length > 0 && Object.hasOwn(data, fallbackField) ? fallbackField : undefined;
18
+ if (resultField === undefined)
19
+ return undefined;
20
+ const result = data[resultField];
21
+ if (typeof result === "string")
22
+ return result.trim().length > 0 ? result : "(empty string)";
23
+ if (typeof result === "number" || typeof result === "boolean")
12
24
  return String(result);
13
- }
25
+ if (result === null || result === undefined)
26
+ return "null";
27
+ if (typeof result === "object")
28
+ return JSON.stringify(result);
14
29
  return undefined;
15
30
  }
16
31
  function getExtractionOrigin(data) {
@@ -32,7 +47,7 @@ function formatGetSummaryLabel(subcommand) {
32
47
  return `${subcommand.slice(0, 1).toUpperCase()}${subcommand.slice(1)}`;
33
48
  }
34
49
  export function formatExtractionSummary(commandInfo, data) {
35
- const scalarResult = getScalarExtractionResult(data);
50
+ const scalarResult = getScalarExtractionResult(commandInfo, data);
36
51
  if (!scalarResult) {
37
52
  return undefined;
38
53
  }
@@ -50,7 +65,7 @@ export function formatExtractionText(commandInfo, data) {
50
65
  if (commandInfo.command !== "get" && commandInfo.command !== "eval") {
51
66
  return undefined;
52
67
  }
53
- const scalarResult = getScalarExtractionResult(data);
68
+ const scalarResult = getScalarExtractionResult(commandInfo, data);
54
69
  if (!scalarResult) {
55
70
  return undefined;
56
71
  }
@@ -132,7 +147,7 @@ function stripNavigationSummary(data) {
132
147
  return rest;
133
148
  }
134
149
  export function formatNavigationActionResult(data) {
135
- const actionData = stripNavigationSummary(data);
150
+ const actionData = omitUpstreamLifecycle(stripNavigationSummary(data));
136
151
  const lines = [];
137
152
  if (typeof actionData.clicked === "string" || typeof actionData.clicked === "boolean") {
138
153
  lines.push(`Clicked: ${String(actionData.clicked)}`);
@@ -1,24 +1,54 @@
1
1
  import { isRecord } from "../../parsing.js";
2
2
  import { detectConfirmationRequired } from "../confirmation.js";
3
3
  import { formatRawSnapshotText, formatSnapshotSummary } from "../snapshot.js";
4
- import { redactModelFacingText, stringifyModelFacing } from "./common.js";
4
+ import { getPageSummary, omitUpstreamLifecycle, redactModelFacingText, stringifyModelFacing } from "./common.js";
5
5
  import { formatDiagnosticSummary, formatDiagnosticText, formatProfilesText, getStreamSummary, getTabSummary } from "./diagnostics.js";
6
6
  import { getScreenshotSummary } from "./artifacts.js";
7
7
  import { formatSkillsText } from "./skills.js";
8
8
  import { formatExtractionSummary, formatExtractionText, formatNavigationActionResult, formatNavigationSummary, getNavigationSummary, isNavigationObservableCommand, } from "./navigation.js";
9
9
  import { formatSemanticActionPresentationSummary, formatSemanticActionPresentationText, resolvePresentationCommandInfo, } from "./semantic-action.js";
10
- function getPageSummary(data) {
11
- const title = typeof data.title === "string" ? data.title : undefined;
12
- const url = typeof data.url === "string" ? data.url : undefined;
13
- if (!title && !url)
14
- return undefined;
15
- if (title && url)
16
- return `${title}\n${url}`;
17
- return title ?? url;
18
- }
19
10
  function formatConfirmationRequiredSummary(confirmation) {
20
11
  return `Confirmation required: ${confirmation.id}`;
21
12
  }
13
+ const SIMPLE_ACTION_RESULTS = {
14
+ check: { field: "checked", label: "Checked" },
15
+ click: { field: "clicked", label: "Clicked" },
16
+ fill: { field: "filled", label: "Filled" },
17
+ focus: { field: "focused", label: "Focused" },
18
+ hover: { field: "hovered", label: "Hovered" },
19
+ press: { field: "pressed", label: "Pressed" },
20
+ select: { field: "selected", label: "Selected" },
21
+ type: { field: "typed", label: "Typed" },
22
+ uncheck: { field: "unchecked", label: "Unchecked" },
23
+ };
24
+ function formatSimpleActionResult(command, data) {
25
+ if (!isRecord(data))
26
+ return undefined;
27
+ if (command === "click" && getNavigationSummary(data))
28
+ return undefined;
29
+ const definition = SIMPLE_ACTION_RESULTS[command];
30
+ if (!definition)
31
+ return undefined;
32
+ const value = data[definition.field];
33
+ if (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean")
34
+ return undefined;
35
+ return `${definition.label}: ${redactModelFacingText(String(value))}`;
36
+ }
37
+ function formatWaitResult(data) {
38
+ if (!isRecord(data))
39
+ return undefined;
40
+ if (data.waited === "timeout")
41
+ return "Fixed wait elapsed; no page condition was verified.";
42
+ for (const field of ["selector", "text", "url", "state", "waited"]) {
43
+ const value = data[field];
44
+ if (typeof value === "string" && value.length > 0)
45
+ return `Wait completed: ${redactModelFacingText(value)}`;
46
+ }
47
+ return undefined;
48
+ }
49
+ function formatCloseResult(data) {
50
+ return isRecord(data) && data.closed === true ? "Browser session closed." : undefined;
51
+ }
22
52
  const VITALS_METRICS = ["lcp", "fcp", "ttfb", "inp", "cls"];
23
53
  function coerceVitalsMetricValue(value) {
24
54
  if (typeof value === "number" && Number.isFinite(value))
@@ -80,6 +110,14 @@ function formatConfirmationRequiredText(confirmation) {
80
110
  return lines.join("\n");
81
111
  }
82
112
  const COMMAND_PRESENTERS = {
113
+ ...Object.fromEntries(Object.keys(SIMPLE_ACTION_RESULTS).map((command) => [command, {
114
+ summary: (_commandInfo, data) => formatSimpleActionResult(command, data),
115
+ text: (_commandInfo, data) => formatSimpleActionResult(command, data),
116
+ }])),
117
+ close: { summary: (_commandInfo, data) => formatCloseResult(data), text: (_commandInfo, data) => formatCloseResult(data) },
118
+ exit: { summary: (_commandInfo, data) => formatCloseResult(data), text: (_commandInfo, data) => formatCloseResult(data) },
119
+ quit: { summary: (_commandInfo, data) => formatCloseResult(data), text: (_commandInfo, data) => formatCloseResult(data) },
120
+ wait: { summary: (_commandInfo, data) => formatWaitResult(data), text: (_commandInfo, data) => formatWaitResult(data) },
83
121
  profiles: {
84
122
  summary: (_commandInfo, data) => Array.isArray(data) ? `Chrome profiles: ${data.length}` : undefined,
85
123
  text: (_commandInfo, data) => Array.isArray(data) ? formatProfilesText(data, "Chrome profiles") : undefined,
@@ -118,8 +156,12 @@ const COMMAND_PRESENTERS = {
118
156
  text: (commandInfo, data) => isRecord(data) && commandInfo.subcommand === "status" ? getStreamSummary(data) : undefined,
119
157
  },
120
158
  tab: {
121
- summary: (_commandInfo, data) => isRecord(data) && Array.isArray(data.tabs) ? `Tabs: ${data.tabs.length}` : undefined,
122
- text: (_commandInfo, data) => isRecord(data) ? getTabSummary(data) : undefined,
159
+ summary: (_commandInfo, data) => isRecord(data) && Array.isArray(data.tabs)
160
+ ? `Tabs: ${data.tabs.length}`
161
+ : isRecord(data) && data.closed === true ? "Tab closed" : undefined,
162
+ text: (_commandInfo, data) => isRecord(data) && data.closed === true
163
+ ? `Tab closed${typeof data.tabId === "string" ? `: ${redactModelFacingText(data.tabId)}` : "."}`
164
+ : isRecord(data) ? getTabSummary(data) : undefined,
123
165
  },
124
166
  vitals: {
125
167
  summary: (_commandInfo, data) => isRecord(data) ? formatVitalsSummary(data) : undefined,
@@ -216,5 +258,8 @@ export function formatPresentationContentText(commandInfo, data, compiledSemanti
216
258
  const pageSummary = getPageSummary(data);
217
259
  if (pageSummary)
218
260
  return redactModelFacingText(pageSummary);
219
- return stringifyModelFacing(data);
261
+ const { navigationSummary: _navigationSummary, ...pageData } = omitUpstreamLifecycle(data);
262
+ if (Object.keys(pageData).length > 0)
263
+ return stringifyModelFacing(pageData);
264
+ return `${presentationCommandInfo.command ?? commandInfo.command ?? "agent-browser"} completed`;
220
265
  }
@@ -1,18 +1,9 @@
1
1
  import { getCompiledSemanticActionCommandIndex, isCompiledSemanticActionFindCommand, } from "../../input-modes/semantic-action.js";
2
2
  import { isRecord } from "../../parsing.js";
3
3
  import { formatNavigationSummary, getNavigationSummary, isNavigationObservableCommand, } from "./navigation.js";
4
- import { redactModelFacingText } from "./common.js";
4
+ import { getPageSummary, redactModelFacingText } from "./common.js";
5
5
  const SEMANTIC_NAVIGATION_PROBE_ACTIONS = new Set(["check", "click"]);
6
6
  const SEMANTIC_PRESENTATION_ACTIONS = new Set(["check", "click", "fill", "select"]);
7
- function getPageSummary(data) {
8
- const title = typeof data.title === "string" ? data.title : undefined;
9
- const url = typeof data.url === "string" ? data.url : undefined;
10
- if (!title && !url)
11
- return undefined;
12
- if (title && url)
13
- return `${title}\n${url}`;
14
- return title ?? url;
15
- }
16
7
  function formatSemanticActionTarget(compiled) {
17
8
  if (compiled.action === "select") {
18
9
  const selector = compiled.selector ?? "selector";
@@ -1,5 +1,5 @@
1
1
  import { isRecord } from "../parsing.js";
2
- import { extractCommandTokens } from "../runtime.js";
2
+ import { extractUpstreamCommandTokens } from "../runtime.js";
3
3
  import { buildAgentBrowserNextActions } from "./action-recommendations.js";
4
4
  import { buildAgentBrowserResultCategoryDetails } from "./categories.js";
5
5
  import { detectConfirmationRequired } from "./confirmation.js";
@@ -33,14 +33,14 @@ function shouldAddAnnotatedScreenshotGuidance(commandInfo, args) {
33
33
  }
34
34
  export async function buildToolPresentation(options) {
35
35
  const { args, artifactManifest, artifactRequest, commandInfo, compiledSemanticAction, cwd, envelope, errorText, namespace, networkRouteDiagnostics, networkRoutes, persistentArtifactStore, sessionName, } = options;
36
- const commandInfoWithTokens = commandInfo.commandTokens || !args ? commandInfo : { ...commandInfo, commandTokens: extractCommandTokens(args) };
36
+ const commandInfoWithTokens = commandInfo.commandTokens || !args ? commandInfo : { ...commandInfo, commandTokens: extractUpstreamCommandTokens(args) };
37
37
  const presentationCommandInfo = resolvePresentationCommandInfo(commandInfoWithTokens, compiledSemanticAction);
38
38
  if (errorText) {
39
39
  return buildErrorPresentation({ args, commandInfo, errorText, sessionName });
40
40
  }
41
41
  const data = enrichStreamStatusData(commandInfoWithTokens, envelope?.data);
42
42
  const presentationData = redactPresentationData(commandInfoWithTokens, data);
43
- const artifacts = await extractFileArtifacts({ artifactManifest, artifactRequest, commandInfo: presentationCommandInfo, cwd, data, sessionName });
43
+ const artifacts = await extractFileArtifacts({ artifactManifest, artifactMaxUpdatedAtMs: options.artifactMaxUpdatedAtMs, artifactMinUpdatedAtMs: options.artifactMinUpdatedAtMs, artifactRequest, commandInfo: presentationCommandInfo, cwd, data, namespace, sessionName });
44
44
  const artifactVerification = buildArtifactVerificationSummary(artifacts);
45
45
  const artifactSummary = formatArtifactSummary(artifacts);
46
46
  const summary = artifactSummary ?? formatPresentationSummary(commandInfoWithTokens, data, compiledSemanticAction);
@@ -49,6 +49,8 @@ export async function buildToolPresentation(options) {
49
49
  if (commandInfo.command === "batch" && isAgentBrowserBatchResultArray(data)) {
50
50
  presentation = await buildBatchPresentation({
51
51
  artifactManifest,
52
+ artifactMaxUpdatedAtMs: options.artifactMaxUpdatedAtMs,
53
+ artifactMinUpdatedAtMs: options.artifactMinUpdatedAtMs,
52
54
  artifactRequests: options.batchArtifactRequests,
53
55
  buildNestedToolPresentation: buildToolPresentation,
54
56
  cwd,
@@ -156,6 +158,7 @@ export async function buildToolPresentation(options) {
156
158
  failureCategory: presentationWithManifest.failureCategory,
157
159
  resultCategory: presentationWithManifest.resultCategory ?? "success",
158
160
  savedFilePath: presentationWithManifest.savedFilePath,
161
+ sessionName,
159
162
  successCategory: presentationWithManifest.successCategory,
160
163
  });
161
164
  const networkNextActions = commandInfoWithTokens.command === "network" && commandInfoWithTokens.subcommand === "requests" && presentationWithManifest.resultCategory === "success"
@@ -8,6 +8,8 @@ export const AGENT_BROWSER_RECOVERY_NEXT_ACTION_IDS = {
8
8
  selectIntendedTabAfterDrift: "select-intended-tab-after-drift",
9
9
  snapshotAfterTabRecovery: "snapshot-after-tab-recovery",
10
10
  tabDriftListTabs: "list-tabs-for-tab-drift-recovery",
11
+ tabGoneListTabs: "list-tabs-after-tab-gone",
12
+ tabGoneNewTab: "open-tab-after-tab-gone",
11
13
  };
12
14
  export const AGENT_BROWSER_RICH_INPUT_RECOVERY_NEXT_ACTION_IDS = {
13
15
  click: "click-current-editable-ref",
@@ -16,7 +16,17 @@ function isSelectorRecoveryActionName(action) {
16
16
  function getFindNameFlagValue(args, startIndex) {
17
17
  const nameFlagIndex = args.indexOf("--name", startIndex);
18
18
  const name = nameFlagIndex >= 0 ? args[nameFlagIndex + 1] : undefined;
19
- return name && !name.startsWith("-") ? name : undefined;
19
+ return typeof name === "string" && name.length > 0 ? name : undefined;
20
+ }
21
+ function collectFindTrailingValues(args, startIndex) {
22
+ const values = [];
23
+ for (let index = startIndex; index < args.length; index += 1) {
24
+ const token = args[index];
25
+ if (!token || token.startsWith("-"))
26
+ break;
27
+ values.push(token);
28
+ }
29
+ return values;
20
30
  }
21
31
  function getFindVisibleRefFallbackTarget(args, options = {}) {
22
32
  const findIndex = args[0] === "--session" ? 2 : 0;
@@ -25,8 +35,21 @@ function getFindVisibleRefFallbackTarget(args, options = {}) {
25
35
  const locator = args[findIndex + 1];
26
36
  const value = args[findIndex + 2];
27
37
  const action = args[findIndex + 3];
28
- if (!locator || !value || !isSelectorRecoveryActionName(action) || action === "select")
38
+ if (!locator || !value || !isSelectorRecoveryActionName(action))
29
39
  return undefined;
40
+ if (action === "select") {
41
+ const optionValues = collectFindTrailingValues(args, findIndex + 4);
42
+ if (locator === "role") {
43
+ if (!/^(?:combobox|listbox)$/i.test(value))
44
+ return undefined;
45
+ const targetName = getFindNameFlagValue(args, findIndex + 4);
46
+ return targetName ? { action, optionValues, roles: [value.toLowerCase()], targetName } : undefined;
47
+ }
48
+ if (locator === "label") {
49
+ return { action, optionValues, roles: ["combobox", "listbox"], targetName: value };
50
+ }
51
+ return undefined;
52
+ }
30
53
  const text = action === "fill" ? args[findIndex + 4] : undefined;
31
54
  if (action === "fill" && (!text || (!options.allowLeadingDashFillText && text.startsWith("-"))))
32
55
  return undefined;
@@ -69,7 +92,13 @@ function getVisibleRefFallbackCandidates(target, snapshotData) {
69
92
  return [];
70
93
  if (target.action === "fill" && editableEvidence === false && EDITABLE_CONTROL_ROLES.has(role.toLowerCase()))
71
94
  return [];
72
- const directRefArgs = target.action === "fill" ? undefined : [target.action, `@${ref}`];
95
+ const directRefArgs = target.action === "fill"
96
+ ? undefined
97
+ : target.action === "select" && target.optionValues && target.optionValues.length > 0
98
+ ? ["select", `@${ref}`, ...target.optionValues]
99
+ : target.action === "select"
100
+ ? undefined
101
+ : [target.action, `@${ref}`];
73
102
  return [{
74
103
  action: target.action,
75
104
  ...(directRefArgs ? { args: directRefArgs } : {}),
@@ -101,19 +130,33 @@ export function buildVisibleRefFallbackDiagnosticFromSnapshot(options) {
101
130
  }
102
131
  export function resolveVisibleRefActionFromSnapshot(options) {
103
132
  const target = getFindVisibleRefFallbackTarget(options.compiledAction.args, { allowLeadingDashFillText: true });
104
- if (!target || target.action === "select")
133
+ if (!target)
105
134
  return undefined;
106
135
  const snapshot = extractRefSnapshotFromData(options.snapshotData);
107
136
  if (!snapshot)
108
137
  return undefined;
109
- const candidates = getVisibleRefFallbackCandidates(target, options.snapshotData);
110
- if (target.action === "fill") {
111
- if (!options.allowFill || candidates.length !== 1 || target.text === undefined)
138
+ const selectOptionValues = options.compiledAction.values && options.compiledAction.values.length > 0
139
+ ? options.compiledAction.values
140
+ : target.optionValues;
141
+ const effectiveTarget = target.action === "select" && selectOptionValues
142
+ ? { ...target, optionValues: selectOptionValues }
143
+ : target;
144
+ const candidates = getVisibleRefFallbackCandidates(effectiveTarget, options.snapshotData);
145
+ if (effectiveTarget.action === "fill") {
146
+ if (!options.allowFill || candidates.length !== 1 || effectiveTarget.text === undefined)
112
147
  return undefined;
113
148
  const [candidate] = candidates;
114
149
  if (!candidate || candidate.editableEvidence === false || !EDITABLE_CONTROL_ROLES.has(candidate.role.toLowerCase()))
115
150
  return undefined;
116
- return { args: ["fill", candidate.ref, target.text], snapshot };
151
+ return { args: ["fill", candidate.ref, effectiveTarget.text], snapshot };
152
+ }
153
+ if (effectiveTarget.action === "select") {
154
+ if (candidates.length !== 1 || !selectOptionValues || selectOptionValues.length === 0)
155
+ return undefined;
156
+ const [candidate] = candidates;
157
+ if (!candidate)
158
+ return undefined;
159
+ return { args: ["select", candidate.ref, ...selectOptionValues], snapshot };
117
160
  }
118
161
  const candidate = candidates.find((item) => item.args !== undefined);
119
162
  if (!candidate?.args)
@@ -2,6 +2,8 @@ import { compareRefIds } from "./text.js";
2
2
  const SNAPSHOT_HIGH_VALUE_EDITABLE_REF_FILL_TARGET_LINES = 4;
3
3
  const SNAPSHOT_HIGH_VALUE_SURFACE_REF_FILL_TARGET_LINES = 3;
4
4
  const SNAPSHOT_HIGH_VALUE_PRIMARY_ACTION_REF_FILL_TARGET_LINES = 3;
5
+ const SNAPSHOT_HIGH_VALUE_NAMED_LINK_REF_FILL_TARGET_LINES = 6;
6
+ const SNAPSHOT_HIGH_VALUE_LINK_NAME_MAX_LENGTH = 80;
5
7
  const SNAPSHOT_HIGH_VALUE_CONTROL_ROLES = new Set([
6
8
  "button",
7
9
  "checkbox",
@@ -33,9 +35,6 @@ const SNAPSHOT_PRIMARY_ACTION_BUTTON_NAME_PATTERNS = [
33
35
  /^(?:add|apply|ask|attach|choose|confirm|connect|continue|create|deploy|done|download|go|insert|launch|log in|new|next|ok|open|publish|refresh|retry|run|save|search|select|send|sign in|sign up|start|submit|upload)$/i,
34
36
  /^(?:add|apply|ask|confirm|connect|continue|create|launch|new|open|refresh|retry|run|save|search|send|start|submit)\b/i,
35
37
  ];
36
- const SNAPSHOT_HIGH_VALUE_LINK_NAME_PATTERNS = [
37
- /^[a-z0-9_.-]+\/[a-z0-9_.-]+$/i,
38
- ];
39
38
  function getHighValueControlRole(entry) {
40
39
  return entry.isEditable === true && (entry.role === "unknown" || entry.role === "generic") ? "textbox" : entry.role;
41
40
  }
@@ -82,22 +81,29 @@ const SNAPSHOT_HIGH_VALUE_CONTROL_CATEGORY_RULES = [
82
81
  matches: (entry) => isPrimaryActionButtonRef(entry),
83
82
  priority: 2,
84
83
  },
84
+ {
85
+ bucketKey: () => "named-link",
86
+ fillTarget: SNAPSHOT_HIGH_VALUE_NAMED_LINK_REF_FILL_TARGET_LINES,
87
+ id: "named-link",
88
+ matches: (entry) => getHighValueControlRole(entry) === "link" && isNamedActionLinkRef(entry),
89
+ priority: 3,
90
+ },
85
91
  {
86
92
  bucketKey: (_entry, role) => role,
87
93
  id: "role",
88
94
  matches: () => true,
89
- priority: 3,
95
+ priority: 4,
90
96
  },
91
97
  ];
92
- function isHighValueLinkRef(entry) {
93
- return entry.name.length > 0 && SNAPSHOT_HIGH_VALUE_LINK_NAME_PATTERNS.some((pattern) => pattern.test(entry.name));
98
+ function isNamedActionLinkRef(entry) {
99
+ return entry.name.length > 0 && entry.name.length <= SNAPSHOT_HIGH_VALUE_LINK_NAME_MAX_LENGTH;
94
100
  }
95
101
  export function isHighValueControlEntry(entry) {
96
102
  const role = getHighValueControlRole(entry);
97
103
  if (!SNAPSHOT_HIGH_VALUE_CONTROL_ROLES.has(role))
98
104
  return false;
99
105
  if (role === "link")
100
- return isHighValueLinkRef(entry);
106
+ return isNamedActionLinkRef(entry);
101
107
  if (entry.isEditable === false && (role === "searchbox" || role === "textbox" || role === "combobox"))
102
108
  return false;
103
109
  return entry.name.length > 0 || isEditableControlRef(entry);