pi-agent-browser-native 0.2.15 → 0.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +5 -1
- package/docs/COMMAND_REFERENCE.md +11 -1
- package/docs/TOOL_CONTRACT.md +4 -3
- package/extensions/agent-browser/index.ts +415 -12
- package/extensions/agent-browser/lib/playbook.ts +1 -0
- package/extensions/agent-browser/lib/results/presentation.ts +93 -20
- package/extensions/agent-browser/lib/results/shared.ts +11 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.2.16 - 2026-05-02
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
- made screenshot artifact paths reliable for agent workflows by normalizing explicit screenshot output paths, including dot-directory paths such as `.dogfood/...`, to absolute paths before invoking upstream `agent-browser`
|
|
9
|
+
- repaired screenshot outputs from upstream temp files when needed and made the requested path the primary visible artifact path
|
|
10
|
+
- extended the screenshot path contract to annotated batch screenshots, so top-level `--annotate` batch calls preserve and verify per-step requested output paths
|
|
11
|
+
- blocked per-step batch `--annotate` screenshot forms that upstream parses unsafely and now point agents to the safe top-level `--annotate batch` form
|
|
12
|
+
- added wrapper-observed trace/profiler owner guards to prevent known conflicting start/stop sequences from corrupting upstream tracing state
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
- artifact-producing commands now render direct visible artifact metadata, including artifact type, requested path, absolute path, existence, size, status, cwd, session, and temp path when repaired
|
|
16
|
+
- explicit `--json` calls now render valid JSON in visible tool content; `stream status` JSON is enriched with `wsUrl` and frame format metadata
|
|
17
|
+
- documented the artifact contract, batch annotation guidance, trace/profiler caveat, and package-development bash bypass for upstream debugging
|
|
18
|
+
|
|
5
19
|
## 0.2.15 - 2026-05-01
|
|
6
20
|
|
|
7
21
|
### Changed
|
package/README.md
CHANGED
|
@@ -242,9 +242,10 @@ Validated workflow examples:
|
|
|
242
242
|
- in configured-source lifecycle mode, verify `/reload` and full restart + `/resume` keep following the same implicit managed browser session
|
|
243
243
|
- run `batch` with JSON via `stdin`
|
|
244
244
|
- run `eval --stdin`
|
|
245
|
-
- take a screenshot with inline attachment support
|
|
245
|
+
- take a screenshot with inline attachment support and visible artifact metadata: artifact type, requested path, absolute path, existence, size, cwd, session, and repair/copy status when applicable
|
|
246
246
|
- inspect upstream help/version through native tool calls like `{ "args": ["--help"] }` and `{ "args": ["--version"] }` via the tool's stateless plain-text inspection fallback
|
|
247
247
|
- use `download <selector> <path>` for direct attachment/file-save workflows instead of trying to infer downloads from generic clicks or large eval dumps
|
|
248
|
+
- for `.dogfood/...` or other dot-directory screenshot paths, rely on the wrapper's path normalization/repair contract; the visible result reports the requested path and absolute path rather than only an upstream temp path
|
|
248
249
|
- use `click` plus `wait --download <path>` for asynchronous export flows, confirm `details.savedFilePath`/`details.savedFile` are present on the wait result or batch wait step, and check `details.artifacts[].exists` before relying on requested-path persistence
|
|
249
250
|
- confirm oversized outputs show the actual spill file path directly in tool content, not just a details key name
|
|
250
251
|
- inspect `details.artifactManifest` / `details.artifactRetentionSummary` during artifact-heavy flows to recover recent saved files, spill files, and visible eviction state after reload/resume
|
|
@@ -274,6 +275,9 @@ Current cautions:
|
|
|
274
275
|
- If a known session target unexpectedly reports about:blank, agent_browser preserves the prior intended target, best-effort re-selects it when it still exists, and reports exact recovery guidance when it cannot be re-selected.
|
|
275
276
|
<!-- agent-browser-playbook:end wrapper-tab-recovery -->
|
|
276
277
|
- oversized snapshots and oversized generic outputs compact inline content and print the actual spill file path directly in the tool result when a spill file exists; recent spills and explicit saved artifacts are also summarized in `details.artifactManifest`, including `evicted` entries when retention budgets remove older persisted files
|
|
278
|
+
- artifact-producing commands render direct readable artifact metadata in visible content and `details.artifacts`: `kind`/`artifactType`, `path`, `requestedPath`, `absolutePath`, `exists`, `sizeBytes`, `status`, `cwd`, `session`, and `tempPath` when the wrapper repaired an upstream temp fallback
|
|
279
|
+
- if the caller explicitly passes `--json`, the visible text content is valid JSON; for `stream status`, the wrapper enriches data with `wsUrl` and `frameFormat`
|
|
280
|
+
- `trace` and `profiler` share upstream tracing machinery; the wrapper blocks starts/stops that conflict with owner state it observed in the current Pi session, but the message says "wrapper believes" because upstream or external CLI calls can desynchronize that local state
|
|
277
281
|
- explicit caller-provided `--session` values are treated as user-managed and are not auto-closed by the extension
|
|
278
282
|
- explicit caller-provided `--user-agent` values win over the ChatGPT/OpenAI compatibility workaround
|
|
279
283
|
- tool progress/details redact sensitive invocation values such as `--headers`, proxy credentials, and auth-bearing URL parameters before echoing them back into Pi
|
|
@@ -128,11 +128,19 @@ A successful wait-based download renders a readable summary such as `Download co
|
|
|
128
128
|
Prefer `download <selector> <path>` when the target element itself is the downloadable link/control. Use `click` plus `wait --download [path]` when a previous action starts the download indirectly.
|
|
129
129
|
|
|
130
130
|
Wrapper result rendering is metadata-first for saved files:
|
|
131
|
-
- screenshots return a saved-path summary, structured `details.artifacts` metadata, and an inline image attachment when safe
|
|
131
|
+
- screenshots return a saved-path summary, visible artifact metadata, structured `details.artifacts` metadata, and an inline image attachment when safe; the visible block includes artifact type, requested path, absolute path, existence, size, cwd, session, and repair/copy status when applicable
|
|
132
132
|
- downloads, PDFs, `wait --download` files, traces, CPU profiles, completed WebM recordings from `record stop`, and path-bearing HAR captures return concise saved-path summaries plus structured `details.artifacts` metadata without inlining large files
|
|
133
133
|
- `record start <path>` reports that recording started and that output will be written on `record stop`; the target file may not exist until recording stops
|
|
134
134
|
- `batch` keeps each step's artifacts in `details.batchSteps[].artifacts` and aggregates them in top-level `details.artifacts` in step order
|
|
135
135
|
|
|
136
|
+
For screenshot paths under dot-directories such as `.dogfood/run/foo.png`, the wrapper normalizes the requested path to an absolute path before invoking upstream `agent-browser`, verifies the requested file exists, and repairs from an upstream temp screenshot when possible. The requested path remains visible as `Requested path`, while `Absolute path` shows the actual on-disk location.
|
|
137
|
+
|
|
138
|
+
For annotated screenshots in `batch`, put `--annotate` in top-level args instead of inside the screenshot step:
|
|
139
|
+
|
|
140
|
+
```json
|
|
141
|
+
{ "args": ["--annotate", "batch"], "stdin": "[[\"screenshot\",\"/tmp/page.png\"]]" }
|
|
142
|
+
```
|
|
143
|
+
|
|
136
144
|
#### Artifact retention and dogfood-heavy QA runs
|
|
137
145
|
|
|
138
146
|
The wrapper keeps a bounded, metadata-only `details.artifactManifest` of recent artifacts so long sessions do not grow unbounded. The default recent window is 100 entries and can be raised for screenshot/video-heavy QA sessions with `PI_AGENT_BROWSER_SESSION_ARTIFACT_MANIFEST_MAX_ENTRIES=<count>`.
|
|
@@ -325,6 +333,8 @@ Stable tab ids look like `t1`, `t2`, and `t3`. Optional user labels such as `doc
|
|
|
325
333
|
|
|
326
334
|
When these diagnostic commands are invoked through the native `agent_browser` tool, structured console and page-error outputs render as compact summaries with counts and key fields. Large outputs are previewed with a `Full output path:` spill file instead of dumping the entire payload into context.
|
|
327
335
|
|
|
336
|
+
`trace` and `profiler` share upstream Chrome tracing machinery. Do not run them at the same time. The wrapper tracks owner state it observes in the current Pi session and blocks conflicting starts/stops with "wrapper believes ..." wording because direct upstream CLI use or browser restarts can desynchronize wrapper-local state.
|
|
337
|
+
|
|
328
338
|
### Batch, auth, confirmations, sessions, chat, dashboard, and setup
|
|
329
339
|
|
|
330
340
|
| Command | Purpose |
|
package/docs/TOOL_CONTRACT.md
CHANGED
|
@@ -171,7 +171,7 @@ Additional structured fields can appear when relevant:
|
|
|
171
171
|
- `batchFailure` and `batchSteps` for `batch` rendering, including mixed-success runs
|
|
172
172
|
- `navigationSummary` for navigation-style commands like `click`, `back`, `forward`, and `reload`
|
|
173
173
|
- `imagePath` / `imagePaths` for screenshots and batched image outputs
|
|
174
|
-
- `artifacts` for upstream saved files such as screenshots, PDFs, downloads, `wait --download` files, traces, CPU profiles, completed WebM recordings, path-bearing HAR captures, and future recording output paths reported by `record start`. Each artifact includes the original saved or requested `path`, resolved `absolutePath`, `kind`, optional `mediaType`, optional `extension`,
|
|
174
|
+
- `artifacts` for upstream saved files such as screenshots, PDFs, downloads, `wait --download` files, traces, CPU profiles, completed WebM recordings, path-bearing HAR captures, and future recording output paths reported by `record start`. Each artifact includes the original saved or requested `path`, resolved `absolutePath`, `kind`/`artifactType`, optional `mediaType`, optional `extension`, best-effort disk metadata such as `exists` and `sizeBytes`, plus `requestedPath`, `status`, `cwd`, `session`, and `tempPath` when applicable.
|
|
175
175
|
- `savedFilePath` / `savedFile` for direct `download`, `pdf`, and `wait --download` saved-file workflows; batch results preserve the same fields on the relevant `batchSteps` entry.
|
|
176
176
|
- `batchSteps[].artifacts` for per-step artifacts in `batch` output; top-level `artifacts` aggregates all step artifacts in order
|
|
177
177
|
- `fullOutputPath` / `fullOutputPaths` when large snapshot output or other oversized tool output is compacted and spilled to a private file; persisted sessions keep that path under a private session-scoped artifact directory with a bounded per-session budget so it survives reload/resume without unbounded growth
|
|
@@ -189,15 +189,16 @@ For oversized snapshots and other oversized tool outputs, details should switch
|
|
|
189
189
|
"Rendering" here means how results appear inside `pi`, not embedding a browser UI.
|
|
190
190
|
|
|
191
191
|
Worth doing in v1:
|
|
192
|
-
- screenshots → saved-path summary, `details.artifacts` metadata, and inline image attachment when safe
|
|
192
|
+
- screenshots → saved-path summary, visible artifact metadata, `details.artifacts` metadata, and inline image attachment when safe; screenshot paths that upstream would treat ambiguously, such as `.dogfood/run/foo.png`, are normalized to absolute paths before launch and repaired from upstream temp output when possible
|
|
193
193
|
- file artifacts such as PDFs, downloads, `wait --download` files, traces, CPU profiles, completed WebM recordings, and path-bearing HAR captures → concise saved-path summaries plus metadata in `details.artifacts` and bounded recent metadata in `details.artifactManifest`; `record start` reports recording lifecycle state and the future output path without adding a missing manifest entry; direct saved-file workflows also expose `details.savedFilePath` / `details.savedFile`; large or binary artifacts are not inlined into model context; the recent manifest cap can age out explicit-file metadata but does not remove explicit saved files from disk
|
|
194
194
|
- snapshots → origin + ref count + main-content-first compact preview, with the raw snapshot spill path printed directly in content and kept in `details.fullOutputPath` plus `details.artifactManifest` when the inline result would otherwise be too large
|
|
195
195
|
- oversized generic outputs such as large `eval --stdin` payloads → compact preview plus the actual spill file path instead of dumping the whole payload into model context
|
|
196
196
|
- extraction-style commands like `eval --stdin` and `get title` → scalar-first text with lightweight origin context when available
|
|
197
197
|
- navigation actions like `click`, `back`, `forward`, and `reload` → lightweight post-action title/url summary when available
|
|
198
198
|
- tab lists → compact summary/table
|
|
199
|
-
- stream status → enabled/connected/port summary
|
|
199
|
+
- stream status → enabled/connected/port summary plus WebSocket URL and frame format when a port is known; if the caller explicitly passed `--json`, visible text is valid JSON instead of a prose summary
|
|
200
200
|
- diagnostic/status families (`session`, `session list`, `profiles`, `doctor`, `auth list`/`show`, `network requests`, `console`, `errors`, and dashboard start/stop/status outputs) → compact readable summaries with counts and stable fields; large log/request/error outputs use previews plus `fullOutputPath` spill files; sensitive nested auth/header/token fields are not expanded in the model-facing text
|
|
201
|
+
- trace/profiler owner conflicts → when the wrapper has observed one owner active for a session, block conflicting starts/stops with "wrapper believes ..." wording because upstream or external CLI use can desynchronize wrapper-local state
|
|
201
202
|
|
|
202
203
|
## Missing binary behavior
|
|
203
204
|
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* Invariants/Assumptions: agent-browser is installed separately on PATH, the wrapper targets the current locally installed upstream version only, and no backward-compatibility shims are provided.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { readFile, rm } from "node:fs/promises";
|
|
9
|
+
import { copyFile, mkdir, readFile, rm, stat } from "node:fs/promises";
|
|
10
|
+
import { dirname, extname, isAbsolute, join, resolve } from "node:path";
|
|
10
11
|
|
|
11
12
|
import { StringEnum } from "@mariozechner/pi-ai";
|
|
12
13
|
import { isToolCallEventType, type AgentToolResult, type ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
@@ -64,6 +65,8 @@ import {
|
|
|
64
65
|
} from "./lib/results/shared.js";
|
|
65
66
|
|
|
66
67
|
const DEFAULT_SESSION_MODE = "auto" as const;
|
|
68
|
+
const DIRECT_AGENT_BROWSER_BASH_BYPASS_ENV = "PI_AGENT_BROWSER_ALLOW_DIRECT_BASH";
|
|
69
|
+
const PACKAGE_NAME = "pi-agent-browser-native";
|
|
67
70
|
|
|
68
71
|
const AGENT_BROWSER_PARAMS = Type.Object({
|
|
69
72
|
args: Type.Array(Type.String({ description: "Exact agent-browser CLI arguments, excluding the binary name." }), {
|
|
@@ -292,6 +295,23 @@ function isHarmlessAgentBrowserInspectionCommand(command: string): boolean {
|
|
|
292
295
|
return HARMLESS_AGENT_BROWSER_INSPECTION_PATTERN.test(command);
|
|
293
296
|
}
|
|
294
297
|
|
|
298
|
+
function isTruthyEnvValue(value: string | undefined): boolean {
|
|
299
|
+
return value === "1" || value?.toLowerCase() === "true" || value?.toLowerCase() === "yes";
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async function isPackageDevelopmentCwd(cwd: string): Promise<boolean> {
|
|
303
|
+
try {
|
|
304
|
+
const packageJson = JSON.parse(await readFile(join(cwd, "package.json"), "utf8")) as { name?: unknown };
|
|
305
|
+
return packageJson.name === PACKAGE_NAME;
|
|
306
|
+
} catch {
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
async function isDirectAgentBrowserBashAllowed(cwd: string): Promise<boolean> {
|
|
312
|
+
return isTruthyEnvValue(process.env[DIRECT_AGENT_BROWSER_BASH_BYPASS_ENV]) || await isPackageDevelopmentCwd(cwd);
|
|
313
|
+
}
|
|
314
|
+
|
|
295
315
|
const NAVIGATION_SUMMARY_COMMANDS = new Set(["back", "click", "dblclick", "forward", "reload"]);
|
|
296
316
|
|
|
297
317
|
interface NavigationSummary {
|
|
@@ -303,6 +323,330 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
303
323
|
return typeof value === "object" && value !== null;
|
|
304
324
|
}
|
|
305
325
|
|
|
326
|
+
const SCREENSHOT_VALUE_FLAGS = new Set(["--screenshot-dir", "--screenshot-format", "--screenshot-quality"]);
|
|
327
|
+
const SCREENSHOT_IMAGE_EXTENSIONS = new Set([".jpeg", ".jpg", ".png", ".webp"]);
|
|
328
|
+
|
|
329
|
+
interface ScreenshotPathRequest {
|
|
330
|
+
absolutePath: string;
|
|
331
|
+
path: string;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
interface PreparedAgentBrowserArgs {
|
|
335
|
+
args: string[];
|
|
336
|
+
batchScreenshotPathRequests?: Array<ScreenshotPathRequest | undefined>;
|
|
337
|
+
screenshotPathRequest?: ScreenshotPathRequest;
|
|
338
|
+
stdin?: string;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
interface ScreenshotArtifactRequest extends ScreenshotPathRequest {
|
|
342
|
+
status?: "missing" | "repaired-from-temp" | "saved" | "upstream-temp-only";
|
|
343
|
+
tempPath?: string;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
type TraceOwner = "profiler" | "trace";
|
|
347
|
+
|
|
348
|
+
function isImagePathToken(token: string): boolean {
|
|
349
|
+
const extension = extname(token).toLowerCase();
|
|
350
|
+
return SCREENSHOT_IMAGE_EXTENSIONS.has(extension);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function getScreenshotPathTokenIndex(commandTokens: string[]): number | undefined {
|
|
354
|
+
if (commandTokens[0] !== "screenshot") {
|
|
355
|
+
return undefined;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const positionalIndices: number[] = [];
|
|
359
|
+
for (let index = 1; index < commandTokens.length; index += 1) {
|
|
360
|
+
const token = commandTokens[index];
|
|
361
|
+
if (token === "--") {
|
|
362
|
+
for (let positionalIndex = index + 1; positionalIndex < commandTokens.length; positionalIndex += 1) {
|
|
363
|
+
positionalIndices.push(positionalIndex);
|
|
364
|
+
}
|
|
365
|
+
break;
|
|
366
|
+
}
|
|
367
|
+
if (token.startsWith("-")) {
|
|
368
|
+
const normalizedToken = token.split("=", 1)[0] ?? token;
|
|
369
|
+
if (SCREENSHOT_VALUE_FLAGS.has(normalizedToken) && !token.includes("=")) {
|
|
370
|
+
index += 1;
|
|
371
|
+
}
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
positionalIndices.push(index);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
if (positionalIndices.length === 0) {
|
|
378
|
+
return undefined;
|
|
379
|
+
}
|
|
380
|
+
const candidateIndex = positionalIndices[positionalIndices.length - 1];
|
|
381
|
+
const candidate = commandTokens[candidateIndex];
|
|
382
|
+
if (positionalIndices.length >= 2 || isImagePathToken(candidate) || isAbsolute(candidate) || candidate.startsWith("./") || candidate.startsWith("../")) {
|
|
383
|
+
return candidateIndex;
|
|
384
|
+
}
|
|
385
|
+
return undefined;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
async function normalizeScreenshotPathInTokens(commandTokens: string[], cwd: string): Promise<{
|
|
389
|
+
request?: ScreenshotPathRequest;
|
|
390
|
+
tokens: string[];
|
|
391
|
+
}> {
|
|
392
|
+
const screenshotPathTokenIndex = getScreenshotPathTokenIndex(commandTokens);
|
|
393
|
+
if (screenshotPathTokenIndex === undefined) {
|
|
394
|
+
return { tokens: commandTokens };
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
const requestedPath = commandTokens[screenshotPathTokenIndex];
|
|
398
|
+
const absolutePath = resolve(cwd, requestedPath);
|
|
399
|
+
await mkdir(dirname(absolutePath), { recursive: true });
|
|
400
|
+
|
|
401
|
+
const tokens = [...commandTokens];
|
|
402
|
+
tokens[screenshotPathTokenIndex] = absolutePath;
|
|
403
|
+
const terminatorIndex = tokens.indexOf("--");
|
|
404
|
+
if (terminatorIndex >= 0) {
|
|
405
|
+
tokens.splice(terminatorIndex, 1);
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
return {
|
|
409
|
+
request: {
|
|
410
|
+
absolutePath,
|
|
411
|
+
path: requestedPath,
|
|
412
|
+
},
|
|
413
|
+
tokens,
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
async function prepareBatchScreenshotPaths(args: string[], stdin: string | undefined, cwd: string): Promise<PreparedAgentBrowserArgs | undefined> {
|
|
418
|
+
const commandTokens = extractCommandTokens(args);
|
|
419
|
+
if (commandTokens[0] !== "batch" || stdin === undefined) {
|
|
420
|
+
return undefined;
|
|
421
|
+
}
|
|
422
|
+
let steps: unknown;
|
|
423
|
+
try {
|
|
424
|
+
steps = JSON.parse(stdin);
|
|
425
|
+
} catch {
|
|
426
|
+
return undefined;
|
|
427
|
+
}
|
|
428
|
+
if (!Array.isArray(steps)) {
|
|
429
|
+
return undefined;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
let changed = false;
|
|
433
|
+
const batchScreenshotPathRequests: Array<ScreenshotPathRequest | undefined> = [];
|
|
434
|
+
const preparedSteps = await Promise.all(steps.map(async (step, index) => {
|
|
435
|
+
if (!Array.isArray(step) || !step.every((item) => typeof item === "string") || step[0] !== "screenshot") {
|
|
436
|
+
return step;
|
|
437
|
+
}
|
|
438
|
+
const normalized = await normalizeScreenshotPathInTokens(step, cwd);
|
|
439
|
+
batchScreenshotPathRequests[index] = normalized.request;
|
|
440
|
+
if (normalized.request) {
|
|
441
|
+
changed = true;
|
|
442
|
+
}
|
|
443
|
+
return normalized.tokens;
|
|
444
|
+
}));
|
|
445
|
+
|
|
446
|
+
return changed
|
|
447
|
+
? {
|
|
448
|
+
args,
|
|
449
|
+
batchScreenshotPathRequests,
|
|
450
|
+
stdin: JSON.stringify(preparedSteps),
|
|
451
|
+
}
|
|
452
|
+
: undefined;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
async function prepareAgentBrowserArgs(args: string[], stdin: string | undefined, cwd: string): Promise<PreparedAgentBrowserArgs> {
|
|
456
|
+
const preparedBatch = await prepareBatchScreenshotPaths(args, stdin, cwd);
|
|
457
|
+
if (preparedBatch) {
|
|
458
|
+
return preparedBatch;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const commandTokens = extractCommandTokens(args);
|
|
462
|
+
const normalized = await normalizeScreenshotPathInTokens(commandTokens, cwd);
|
|
463
|
+
if (!normalized.request) {
|
|
464
|
+
return { args };
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
const commandStartIndex = args.length - commandTokens.length;
|
|
468
|
+
return {
|
|
469
|
+
args: [...args.slice(0, commandStartIndex), ...normalized.tokens],
|
|
470
|
+
screenshotPathRequest: normalized.request,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async function pathExists(path: string): Promise<boolean> {
|
|
475
|
+
try {
|
|
476
|
+
await stat(path);
|
|
477
|
+
return true;
|
|
478
|
+
} catch {
|
|
479
|
+
return false;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
async function repairScreenshotData(options: {
|
|
484
|
+
cwd: string;
|
|
485
|
+
data: Record<string, unknown>;
|
|
486
|
+
request: ScreenshotPathRequest;
|
|
487
|
+
}): Promise<{ data: Record<string, unknown>; request: ScreenshotArtifactRequest }> {
|
|
488
|
+
const { cwd, data, request } = options;
|
|
489
|
+
const reportedPath = typeof data.path === "string" ? data.path : undefined;
|
|
490
|
+
const reportedAbsolutePath = reportedPath ? resolve(cwd, reportedPath) : undefined;
|
|
491
|
+
let status: ScreenshotArtifactRequest["status"] = await pathExists(request.absolutePath) ? "saved" : "missing";
|
|
492
|
+
let tempPath: string | undefined;
|
|
493
|
+
|
|
494
|
+
if (reportedAbsolutePath && reportedAbsolutePath !== request.absolutePath) {
|
|
495
|
+
tempPath = reportedAbsolutePath;
|
|
496
|
+
if (status === "missing" && await pathExists(reportedAbsolutePath)) {
|
|
497
|
+
await mkdir(dirname(request.absolutePath), { recursive: true });
|
|
498
|
+
await copyFile(reportedAbsolutePath, request.absolutePath);
|
|
499
|
+
status = "repaired-from-temp";
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
return {
|
|
504
|
+
data: {
|
|
505
|
+
...data,
|
|
506
|
+
path: request.absolutePath,
|
|
507
|
+
},
|
|
508
|
+
request: {
|
|
509
|
+
...request,
|
|
510
|
+
status,
|
|
511
|
+
tempPath,
|
|
512
|
+
},
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
async function repairScreenshotArtifact(options: {
|
|
517
|
+
cwd: string;
|
|
518
|
+
envelope?: AgentBrowserEnvelope;
|
|
519
|
+
request?: ScreenshotPathRequest;
|
|
520
|
+
}): Promise<{ envelope?: AgentBrowserEnvelope; request?: ScreenshotArtifactRequest }> {
|
|
521
|
+
const { cwd, envelope, request } = options;
|
|
522
|
+
if (!request || !envelope || !isRecord(envelope.data)) {
|
|
523
|
+
return { envelope, request };
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
const repaired = await repairScreenshotData({ cwd, data: envelope.data, request });
|
|
527
|
+
return {
|
|
528
|
+
envelope: { ...envelope, data: repaired.data },
|
|
529
|
+
request: repaired.request,
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
async function repairBatchScreenshotArtifacts(options: {
|
|
534
|
+
cwd: string;
|
|
535
|
+
envelope?: AgentBrowserEnvelope;
|
|
536
|
+
requests?: Array<ScreenshotPathRequest | undefined>;
|
|
537
|
+
}): Promise<{ envelope?: AgentBrowserEnvelope; requests?: Array<ScreenshotArtifactRequest | undefined> }> {
|
|
538
|
+
const { cwd, envelope, requests } = options;
|
|
539
|
+
if (!envelope || !Array.isArray(envelope.data) || !requests?.some((request) => request !== undefined)) {
|
|
540
|
+
return { envelope, requests };
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
const repairedRequests: Array<ScreenshotArtifactRequest | undefined> = [];
|
|
544
|
+
const repairedData = await Promise.all(envelope.data.map(async (item, index) => {
|
|
545
|
+
const request = requests[index];
|
|
546
|
+
if (!request || !isRecord(item) || !isRecord(item.result)) {
|
|
547
|
+
return item;
|
|
548
|
+
}
|
|
549
|
+
const repaired = await repairScreenshotData({ cwd, data: item.result, request });
|
|
550
|
+
repairedRequests[index] = repaired.request;
|
|
551
|
+
return {
|
|
552
|
+
...item,
|
|
553
|
+
result: repaired.data,
|
|
554
|
+
};
|
|
555
|
+
}));
|
|
556
|
+
|
|
557
|
+
return {
|
|
558
|
+
envelope: { ...envelope, data: repairedData },
|
|
559
|
+
requests: repairedRequests,
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
function buildJsonVisibleContent(options: {
|
|
564
|
+
error: unknown;
|
|
565
|
+
presentation: Awaited<ReturnType<typeof buildToolPresentation>>;
|
|
566
|
+
succeeded: boolean;
|
|
567
|
+
warnings?: string[];
|
|
568
|
+
}): Array<{ text: string; type: "text" } | { data: string; mimeType: string; type: "image" }> {
|
|
569
|
+
const { error, presentation, succeeded, warnings } = options;
|
|
570
|
+
const payload = redactSensitiveValue({
|
|
571
|
+
artifacts: presentation.artifacts,
|
|
572
|
+
data: presentation.data,
|
|
573
|
+
error,
|
|
574
|
+
success: succeeded,
|
|
575
|
+
warnings: warnings && warnings.length > 0 ? warnings : undefined,
|
|
576
|
+
});
|
|
577
|
+
if (isRecord(payload) && isRecord(payload.data) && isRecord(presentation.data) && typeof presentation.data.wsUrl === "string") {
|
|
578
|
+
payload.data.wsUrl = presentation.data.wsUrl;
|
|
579
|
+
}
|
|
580
|
+
const images = presentation.content.filter((item): item is { data: string; mimeType: string; type: "image" } => item.type === "image");
|
|
581
|
+
return [{ type: "text", text: JSON.stringify(payload, null, 2) }, ...images];
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
function getBatchAnnotateValidationError(args: string[], stdin: string | undefined): string | undefined {
|
|
585
|
+
const commandTokens = extractCommandTokens(args);
|
|
586
|
+
if (commandTokens[0] !== "batch" || stdin === undefined) {
|
|
587
|
+
return undefined;
|
|
588
|
+
}
|
|
589
|
+
let steps: unknown;
|
|
590
|
+
try {
|
|
591
|
+
steps = JSON.parse(stdin);
|
|
592
|
+
} catch {
|
|
593
|
+
return undefined;
|
|
594
|
+
}
|
|
595
|
+
if (!Array.isArray(steps)) {
|
|
596
|
+
return undefined;
|
|
597
|
+
}
|
|
598
|
+
const badStepIndex = steps.findIndex((step) => Array.isArray(step) && step[0] === "screenshot" && step.includes("--annotate"));
|
|
599
|
+
if (badStepIndex < 0) {
|
|
600
|
+
return undefined;
|
|
601
|
+
}
|
|
602
|
+
return [
|
|
603
|
+
`Unsupported batch screenshot annotation in step ${badStepIndex + 1}: put --annotate in top-level args, not inside the batch step.`,
|
|
604
|
+
`Use: { "args": ["--annotate", "batch"], "stdin": "[[\\"screenshot\\",\\"/path/to/image.png\\"]]" }`,
|
|
605
|
+
].join("\n");
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
function getTraceOwner(command: string | undefined): TraceOwner | undefined {
|
|
609
|
+
return command === "trace" || command === "profiler" ? command : undefined;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
function getTraceOwnerGuardMessage(options: {
|
|
613
|
+
command: string | undefined;
|
|
614
|
+
sessionName: string | undefined;
|
|
615
|
+
subcommand: string | undefined;
|
|
616
|
+
traceOwners: Map<string, TraceOwner>;
|
|
617
|
+
}): string | undefined {
|
|
618
|
+
const owner = getTraceOwner(options.command);
|
|
619
|
+
if (!owner || !options.sessionName || (options.subcommand !== "start" && options.subcommand !== "stop")) {
|
|
620
|
+
return undefined;
|
|
621
|
+
}
|
|
622
|
+
const activeOwner = options.traceOwners.get(options.sessionName);
|
|
623
|
+
if (!activeOwner || activeOwner === owner) {
|
|
624
|
+
return undefined;
|
|
625
|
+
}
|
|
626
|
+
return options.subcommand === "start"
|
|
627
|
+
? `Wrapper believes ${activeOwner} tracing is active for session ${options.sessionName}; stop ${activeOwner} before starting ${owner}.`
|
|
628
|
+
: `Wrapper believes tracing for session ${options.sessionName} is owned by ${activeOwner}; run ${activeOwner} stop instead of ${owner} stop.`;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
function updateTraceOwnerState(options: {
|
|
632
|
+
command: string | undefined;
|
|
633
|
+
sessionName: string | undefined;
|
|
634
|
+
subcommand: string | undefined;
|
|
635
|
+
succeeded: boolean;
|
|
636
|
+
traceOwners: Map<string, TraceOwner>;
|
|
637
|
+
}): void {
|
|
638
|
+
const owner = getTraceOwner(options.command);
|
|
639
|
+
if (!owner || !options.sessionName || !options.succeeded) {
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
if (options.subcommand === "start") {
|
|
643
|
+
options.traceOwners.set(options.sessionName, owner);
|
|
644
|
+
}
|
|
645
|
+
if (options.subcommand === "stop" && options.traceOwners.get(options.sessionName) === owner) {
|
|
646
|
+
options.traceOwners.delete(options.sessionName);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
306
650
|
function shouldCaptureNavigationSummary(command: string | undefined, data: unknown): boolean {
|
|
307
651
|
return (
|
|
308
652
|
command !== undefined &&
|
|
@@ -1025,6 +1369,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1025
1369
|
let freshSessionOrdinal = 0;
|
|
1026
1370
|
let sessionTabTargets = new Map<string, OrderedSessionTabTarget>();
|
|
1027
1371
|
let sessionTabTargetUpdateOrder = 0;
|
|
1372
|
+
let traceOwners = new Map<string, TraceOwner>();
|
|
1028
1373
|
let artifactManifest: SessionArtifactManifest | undefined;
|
|
1029
1374
|
const managedSessionExecutionQueue = new AsyncExecutionQueue();
|
|
1030
1375
|
|
|
@@ -1044,6 +1389,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1044
1389
|
managedSessionActive = false;
|
|
1045
1390
|
sessionTabTargets = new Map<string, OrderedSessionTabTarget>();
|
|
1046
1391
|
sessionTabTargetUpdateOrder = 0;
|
|
1392
|
+
traceOwners = new Map<string, TraceOwner>();
|
|
1047
1393
|
artifactManifest = undefined;
|
|
1048
1394
|
await cleanupSecureTempArtifacts();
|
|
1049
1395
|
});
|
|
@@ -1063,7 +1409,8 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1063
1409
|
isToolCallEventType("bash", event) &&
|
|
1064
1410
|
!promptPolicy.allowLegacyAgentBrowserBash &&
|
|
1065
1411
|
looksLikeDirectAgentBrowserBash(event.input.command) &&
|
|
1066
|
-
!isHarmlessAgentBrowserInspectionCommand(event.input.command)
|
|
1412
|
+
!isHarmlessAgentBrowserInspectionCommand(event.input.command) &&
|
|
1413
|
+
!(await isDirectAgentBrowserBashAllowed(ctx.cwd))
|
|
1067
1414
|
) {
|
|
1068
1415
|
return {
|
|
1069
1416
|
block: true,
|
|
@@ -1083,7 +1430,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1083
1430
|
parameters: AGENT_BROWSER_PARAMS,
|
|
1084
1431
|
async execute(_toolCallId, params, signal, onUpdate, ctx) {
|
|
1085
1432
|
const redactedArgs = redactInvocationArgs(params.args);
|
|
1086
|
-
const validationError = validateToolArgs(params.args);
|
|
1433
|
+
const validationError = validateToolArgs(params.args) ?? getBatchAnnotateValidationError(params.args, params.stdin);
|
|
1087
1434
|
if (validationError) {
|
|
1088
1435
|
return {
|
|
1089
1436
|
content: [{ type: "text", text: validationError }],
|
|
@@ -1091,12 +1438,14 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1091
1438
|
isError: true,
|
|
1092
1439
|
};
|
|
1093
1440
|
}
|
|
1441
|
+
const preparedArgs = await prepareAgentBrowserArgs(params.args, params.stdin, ctx.cwd);
|
|
1442
|
+
const userRequestedJson = params.args.includes("--json");
|
|
1094
1443
|
|
|
1095
1444
|
const tabTargetUpdateOrder = ++sessionTabTargetUpdateOrder;
|
|
1096
1445
|
const runTool = async (): Promise<AgentBrowserToolResult> => {
|
|
1097
1446
|
const sessionMode = params.sessionMode ?? DEFAULT_SESSION_MODE;
|
|
1098
1447
|
const freshSessionName = createFreshSessionName(managedSessionBaseName, ephemeralSessionSeed, freshSessionOrdinal + 1);
|
|
1099
|
-
const executionPlan = buildExecutionPlan(
|
|
1448
|
+
const executionPlan = buildExecutionPlan(preparedArgs.args, {
|
|
1100
1449
|
freshSessionName,
|
|
1101
1450
|
managedSessionActive,
|
|
1102
1451
|
managedSessionName,
|
|
@@ -1124,7 +1473,28 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1124
1473
|
};
|
|
1125
1474
|
}
|
|
1126
1475
|
|
|
1127
|
-
const commandTokens = extractCommandTokens(
|
|
1476
|
+
const commandTokens = extractCommandTokens(preparedArgs.args);
|
|
1477
|
+
const traceOwnerGuardMessage = getTraceOwnerGuardMessage({
|
|
1478
|
+
command: executionPlan.commandInfo.command,
|
|
1479
|
+
sessionName: executionPlan.sessionName,
|
|
1480
|
+
subcommand: executionPlan.commandInfo.subcommand,
|
|
1481
|
+
traceOwners,
|
|
1482
|
+
});
|
|
1483
|
+
if (traceOwnerGuardMessage) {
|
|
1484
|
+
return {
|
|
1485
|
+
content: [{ type: "text", text: traceOwnerGuardMessage }],
|
|
1486
|
+
details: {
|
|
1487
|
+
args: redactedArgs,
|
|
1488
|
+
command: executionPlan.commandInfo.command,
|
|
1489
|
+
compatibilityWorkaround,
|
|
1490
|
+
effectiveArgs: redactedEffectiveArgs,
|
|
1491
|
+
sessionMode,
|
|
1492
|
+
validationError: traceOwnerGuardMessage,
|
|
1493
|
+
...buildSessionDetailFields(executionPlan.sessionName, executionPlan.usedImplicitSession),
|
|
1494
|
+
},
|
|
1495
|
+
isError: true,
|
|
1496
|
+
};
|
|
1497
|
+
}
|
|
1128
1498
|
const stdinValidationError = validateStdinCommandContract({
|
|
1129
1499
|
command: executionPlan.commandInfo.command,
|
|
1130
1500
|
commandTokens,
|
|
@@ -1152,7 +1522,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1152
1522
|
let includePinnedNavigationSummary = false;
|
|
1153
1523
|
let sessionTabCorrection: OpenResultTabCorrection | undefined;
|
|
1154
1524
|
let processArgs = executionPlan.effectiveArgs;
|
|
1155
|
-
let processStdin = params.stdin;
|
|
1525
|
+
let processStdin = preparedArgs.stdin ?? params.stdin;
|
|
1156
1526
|
if (
|
|
1157
1527
|
priorSessionTabTarget &&
|
|
1158
1528
|
shouldPinSessionTabForCommand({
|
|
@@ -1283,6 +1653,20 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1283
1653
|
presentationEnvelope = pinnedBatchResult.envelope ?? presentationEnvelope;
|
|
1284
1654
|
navigationSummary = pinnedBatchResult.navigationSummary;
|
|
1285
1655
|
}
|
|
1656
|
+
const repairedScreenshot = await repairScreenshotArtifact({
|
|
1657
|
+
cwd: ctx.cwd,
|
|
1658
|
+
envelope: presentationEnvelope,
|
|
1659
|
+
request: preparedArgs.screenshotPathRequest,
|
|
1660
|
+
});
|
|
1661
|
+
presentationEnvelope = repairedScreenshot.envelope;
|
|
1662
|
+
const repairedBatchScreenshots = await repairBatchScreenshotArtifacts({
|
|
1663
|
+
cwd: ctx.cwd,
|
|
1664
|
+
envelope: presentationEnvelope,
|
|
1665
|
+
requests: preparedArgs.batchScreenshotPathRequests,
|
|
1666
|
+
});
|
|
1667
|
+
presentationEnvelope = repairedBatchScreenshots.envelope;
|
|
1668
|
+
const screenshotArtifactRequest = repairedScreenshot.request;
|
|
1669
|
+
const batchScreenshotArtifactRequests = repairedBatchScreenshots.requests;
|
|
1286
1670
|
const parseFailureOutput = parseError
|
|
1287
1671
|
? await preserveParseFailureOutput({
|
|
1288
1672
|
artifactManifest,
|
|
@@ -1296,6 +1680,13 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1296
1680
|
const envelopeSuccess = plainTextInspection ? true : presentationEnvelope?.success !== false;
|
|
1297
1681
|
const succeeded = processSucceeded && parseSucceeded && envelopeSuccess;
|
|
1298
1682
|
const inspectionText = plainTextInspection ? processResult.stdout.trim() : undefined;
|
|
1683
|
+
updateTraceOwnerState({
|
|
1684
|
+
command: executionPlan.commandInfo.command,
|
|
1685
|
+
sessionName: executionPlan.sessionName,
|
|
1686
|
+
subcommand: executionPlan.commandInfo.subcommand,
|
|
1687
|
+
succeeded,
|
|
1688
|
+
traceOwners,
|
|
1689
|
+
});
|
|
1299
1690
|
|
|
1300
1691
|
if (succeeded && !navigationSummary && shouldCaptureNavigationSummary(executionPlan.commandInfo.command, presentationEnvelope?.data)) {
|
|
1301
1692
|
navigationSummary = await collectNavigationSummary({
|
|
@@ -1477,11 +1868,14 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1477
1868
|
}
|
|
1478
1869
|
: await buildToolPresentation({
|
|
1479
1870
|
artifactManifest,
|
|
1871
|
+
artifactRequest: screenshotArtifactRequest,
|
|
1872
|
+
batchArtifactRequests: batchScreenshotArtifactRequests,
|
|
1480
1873
|
commandInfo: executionPlan.commandInfo,
|
|
1481
1874
|
cwd: ctx.cwd,
|
|
1482
1875
|
envelope: presentationEnvelope,
|
|
1483
1876
|
errorText,
|
|
1484
1877
|
persistentArtifactStore,
|
|
1878
|
+
sessionName: executionPlan.sessionName,
|
|
1485
1879
|
});
|
|
1486
1880
|
if (parseFailureOutput.artifactManifest) {
|
|
1487
1881
|
presentation.artifactManifest = parseFailureOutput.artifactManifest;
|
|
@@ -1504,20 +1898,29 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
|
|
|
1504
1898
|
if (presentation.artifactManifest) {
|
|
1505
1899
|
artifactManifest = presentation.artifactManifest;
|
|
1506
1900
|
}
|
|
1507
|
-
const
|
|
1508
|
-
|
|
1509
|
-
|
|
1901
|
+
const warningText = aboutBlankSessionMismatch ? buildAboutBlankWarning(aboutBlankSessionMismatch) : undefined;
|
|
1902
|
+
const contentWithSessionWarnings = userRequestedJson && !plainTextInspection
|
|
1903
|
+
? buildJsonVisibleContent({
|
|
1904
|
+
error: presentationEnvelope?.error,
|
|
1905
|
+
presentation,
|
|
1906
|
+
succeeded,
|
|
1907
|
+
warnings: warningText ? [warningText] : undefined,
|
|
1908
|
+
})
|
|
1909
|
+
: warningText
|
|
1910
|
+
? [...presentation.content]
|
|
1911
|
+
: presentation.content;
|
|
1912
|
+
if (warningText && !userRequestedJson) {
|
|
1510
1913
|
if (contentWithSessionWarnings[0]?.type === "text") {
|
|
1511
1914
|
contentWithSessionWarnings[0] = {
|
|
1512
1915
|
...contentWithSessionWarnings[0],
|
|
1513
|
-
text: `${
|
|
1916
|
+
text: `${warningText}\n\n${contentWithSessionWarnings[0].text}`,
|
|
1514
1917
|
};
|
|
1515
1918
|
} else {
|
|
1516
|
-
contentWithSessionWarnings.unshift({ type: "text", text:
|
|
1919
|
+
contentWithSessionWarnings.unshift({ type: "text", text: warningText });
|
|
1517
1920
|
}
|
|
1518
1921
|
}
|
|
1519
1922
|
const redactedContent = contentWithSessionWarnings.map((item) =>
|
|
1520
|
-
item.type === "text" ? { ...item, text: redactSensitiveText(item.text) } : item,
|
|
1923
|
+
item.type === "text" && !(userRequestedJson && !plainTextInspection) ? { ...item, text: redactSensitiveText(item.text) } : item,
|
|
1521
1924
|
);
|
|
1522
1925
|
|
|
1523
1926
|
return {
|
|
@@ -18,6 +18,7 @@ export const QUICK_START_GUIDELINES = [
|
|
|
18
18
|
"Common first calls: { args: [\"open\", \"https://example.com\"] } then { args: [\"snapshot\", \"-i\"] }; after navigation, use { args: [\"click\", \"@e2\"] } then { args: [\"snapshot\", \"-i\"] }.",
|
|
19
19
|
"Common advanced calls: { args: [\"batch\"], stdin: \"[[\\\"open\\\",\\\"https://example.com\\\"],[\\\"snapshot\\\",\\\"-i\\\"]]\" }, { args: [\"eval\", \"--stdin\"], stdin: \"document.title\" }, and { args: [\"--profile\", \"Default\", \"open\", \"https://example.com/account\"], sessionMode: \"fresh\" }.",
|
|
20
20
|
"High-value command reference: download <selector> <path> saves a file triggered by a click; get title/url/text/html/value/attr/count reads page state; screenshot [path] captures an image; pdf <path> saves a PDF; tab list and tab <tab-id-or-label> inspect or recover the active tab.",
|
|
21
|
+
"For artifact-producing commands, read the visible artifact block for requested path, absolute path, existence, size, type, cwd, and session; details.artifacts contains the same machine-readable metadata. For annotated screenshots inside batch, put --annotate in top-level args (for example { args: [\"--annotate\", \"batch\"], stdin: \"[[\\\"screenshot\\\",\\\"/tmp/page.png\\\"]]\" }) rather than inside the screenshot step.",
|
|
21
22
|
] as const;
|
|
22
23
|
|
|
23
24
|
export const BRAVE_SEARCH_PROMPT_GUIDELINE =
|
|
@@ -173,10 +173,27 @@ function getStreamSummary(data: Record<string, unknown>): string | undefined {
|
|
|
173
173
|
];
|
|
174
174
|
if (typeof data.port === "number") {
|
|
175
175
|
lines.push(`Port: ${data.port}`);
|
|
176
|
+
lines.push(`WebSocket URL: ${getStreamWebSocketUrl(data.port)}`);
|
|
177
|
+
lines.push(`Frame format: JSON messages with base64 JPEG frame data`);
|
|
176
178
|
}
|
|
177
179
|
return lines.join("\n");
|
|
178
180
|
}
|
|
179
181
|
|
|
182
|
+
function getStreamWebSocketUrl(port: number): string {
|
|
183
|
+
return `ws://127.0.0.1:${port}`;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function enrichStreamStatusData(commandInfo: CommandInfo, data: unknown): unknown {
|
|
187
|
+
if (commandInfo.command !== "stream" || commandInfo.subcommand !== "status" || !isRecord(data) || typeof data.port !== "number") {
|
|
188
|
+
return data;
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
...data,
|
|
192
|
+
frameFormat: "JSON messages with base64 JPEG frame data",
|
|
193
|
+
wsUrl: getStreamWebSocketUrl(data.port),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
180
197
|
function getArrayField(data: Record<string, unknown>, key: string): unknown[] | undefined {
|
|
181
198
|
return Array.isArray(data[key]) ? data[key] : undefined;
|
|
182
199
|
}
|
|
@@ -662,18 +679,28 @@ function extractPathStrings(data: unknown): string[] {
|
|
|
662
679
|
return [...new Set(paths)];
|
|
663
680
|
}
|
|
664
681
|
|
|
682
|
+
interface ArtifactRequestContext {
|
|
683
|
+
absolutePath: string;
|
|
684
|
+
path: string;
|
|
685
|
+
status?: FileArtifactMetadata["status"];
|
|
686
|
+
tempPath?: string;
|
|
687
|
+
}
|
|
688
|
+
|
|
665
689
|
async function buildFileArtifactMetadata(options: {
|
|
690
|
+
artifactRequest?: ArtifactRequestContext;
|
|
666
691
|
commandInfo: CommandInfo;
|
|
667
692
|
cwd: string;
|
|
668
693
|
path: string;
|
|
694
|
+
sessionName?: string;
|
|
669
695
|
}): Promise<FileArtifactMetadata | undefined> {
|
|
670
696
|
const kind = getArtifactKind(options.commandInfo);
|
|
671
697
|
if (!kind) {
|
|
672
698
|
return undefined;
|
|
673
699
|
}
|
|
674
700
|
|
|
675
|
-
const absolutePath = resolve(options.cwd, options.path);
|
|
676
|
-
const
|
|
701
|
+
const absolutePath = options.artifactRequest?.absolutePath ?? resolve(options.cwd, options.path);
|
|
702
|
+
const displayPath = options.artifactRequest?.path ?? options.path;
|
|
703
|
+
const extension = extname(absolutePath || options.path).toLowerCase() || undefined;
|
|
677
704
|
let exists: boolean | undefined;
|
|
678
705
|
let sizeBytes: number | undefined;
|
|
679
706
|
try {
|
|
@@ -686,20 +713,32 @@ async function buildFileArtifactMetadata(options: {
|
|
|
686
713
|
|
|
687
714
|
return {
|
|
688
715
|
absolutePath,
|
|
716
|
+
artifactType: kind,
|
|
689
717
|
command: options.commandInfo.command,
|
|
718
|
+
cwd: options.cwd,
|
|
690
719
|
exists,
|
|
691
720
|
extension,
|
|
692
721
|
kind,
|
|
693
722
|
mediaType: extension ? ARTIFACT_EXTENSION_TO_MEDIA_TYPE[extension] : undefined,
|
|
694
|
-
path:
|
|
723
|
+
path: displayPath,
|
|
724
|
+
requestedPath: options.artifactRequest?.path,
|
|
725
|
+
session: options.sessionName,
|
|
695
726
|
sizeBytes,
|
|
727
|
+
status: options.artifactRequest?.status ?? (exists === false ? "missing" : "saved"),
|
|
696
728
|
subcommand: options.commandInfo.subcommand,
|
|
729
|
+
tempPath: options.artifactRequest?.tempPath,
|
|
697
730
|
};
|
|
698
731
|
}
|
|
699
732
|
|
|
700
|
-
async function extractFileArtifacts(
|
|
701
|
-
|
|
702
|
-
|
|
733
|
+
async function extractFileArtifacts(options: {
|
|
734
|
+
artifactRequest?: ArtifactRequestContext;
|
|
735
|
+
commandInfo: CommandInfo;
|
|
736
|
+
cwd: string;
|
|
737
|
+
data: unknown;
|
|
738
|
+
sessionName?: string;
|
|
739
|
+
}): Promise<FileArtifactMetadata[]> {
|
|
740
|
+
const candidates = extractPathStrings(options.data);
|
|
741
|
+
const artifacts = await Promise.all(candidates.map((path) => buildFileArtifactMetadata({ ...options, path })));
|
|
703
742
|
return artifacts.filter((artifact): artifact is FileArtifactMetadata => artifact !== undefined);
|
|
704
743
|
}
|
|
705
744
|
|
|
@@ -708,12 +747,15 @@ function buildManifestEntriesForFileArtifacts(artifacts: FileArtifactMetadata[],
|
|
|
708
747
|
absolutePath: artifact.absolutePath,
|
|
709
748
|
command: artifact.command,
|
|
710
749
|
createdAtMs: nowMs,
|
|
750
|
+
cwd: artifact.cwd,
|
|
711
751
|
exists: artifact.exists,
|
|
712
752
|
extension: artifact.extension,
|
|
713
753
|
kind: artifact.kind,
|
|
714
754
|
mediaType: artifact.mediaType,
|
|
715
755
|
path: artifact.path,
|
|
756
|
+
requestedPath: artifact.requestedPath,
|
|
716
757
|
retentionState: artifact.exists === false ? "missing" : "live",
|
|
758
|
+
session: artifact.session,
|
|
717
759
|
sizeBytes: artifact.sizeBytes,
|
|
718
760
|
storageScope: "explicit-path",
|
|
719
761
|
subcommand: artifact.subcommand,
|
|
@@ -761,17 +803,37 @@ function formatArtifactSummary(artifacts: FileArtifactMetadata[]): string | unde
|
|
|
761
803
|
}
|
|
762
804
|
|
|
763
805
|
function formatArtifactMetadataLines(artifacts: FileArtifactMetadata[]): string[] {
|
|
764
|
-
return artifacts.map((artifact) => {
|
|
806
|
+
return artifacts.map((artifact, index) => {
|
|
765
807
|
if (isRecordingStartArtifact(artifact)) {
|
|
766
|
-
return
|
|
808
|
+
return [
|
|
809
|
+
`${formatArtifactLabel(artifact)}: ${artifact.path}`,
|
|
810
|
+
`Artifact type: ${artifact.kind}`,
|
|
811
|
+
`Requested path: ${artifact.requestedPath ?? artifact.path}`,
|
|
812
|
+
`Absolute path: ${artifact.absolutePath}`,
|
|
813
|
+
`Exists: ${artifact.exists === true}`,
|
|
814
|
+
`Status: ${artifact.status ?? (artifact.exists === false ? "missing" : "saved")}`,
|
|
815
|
+
artifact.session ? `Session: ${artifact.session}` : undefined,
|
|
816
|
+
artifact.cwd ? `CWD: ${artifact.cwd}` : undefined,
|
|
817
|
+
`Machine data: details.artifacts[${index}]`,
|
|
818
|
+
].filter((item): item is string => item !== undefined).join("\n");
|
|
767
819
|
}
|
|
768
820
|
|
|
769
|
-
|
|
770
|
-
artifact.
|
|
771
|
-
|
|
821
|
+
return [
|
|
822
|
+
`${formatArtifactLabel(artifact)}: ${artifact.path}`,
|
|
823
|
+
`Artifact type: ${artifact.kind}`,
|
|
824
|
+
`Requested path: ${artifact.requestedPath ?? artifact.path}`,
|
|
825
|
+
`Absolute path: ${artifact.absolutePath}`,
|
|
826
|
+
`Exists: ${artifact.exists === true}`,
|
|
772
827
|
artifact.exists === false ? "not found on disk" : undefined,
|
|
773
|
-
|
|
774
|
-
|
|
828
|
+
typeof artifact.sizeBytes === "number" ? `Size: ${formatByteCount(artifact.sizeBytes)}` : undefined,
|
|
829
|
+
typeof artifact.sizeBytes === "number" ? `Size bytes: ${artifact.sizeBytes}` : undefined,
|
|
830
|
+
`Status: ${artifact.status ?? (artifact.exists === false ? "missing" : "saved")}`,
|
|
831
|
+
artifact.tempPath ? `Temp path: ${artifact.tempPath}` : undefined,
|
|
832
|
+
artifact.mediaType ? `Media type: ${artifact.mediaType}` : undefined,
|
|
833
|
+
artifact.session ? `Session: ${artifact.session}` : undefined,
|
|
834
|
+
artifact.cwd ? `CWD: ${artifact.cwd}` : undefined,
|
|
835
|
+
`Machine data: details.artifacts[${index}]`,
|
|
836
|
+
].filter((item): item is string => item !== undefined).join("\n");
|
|
775
837
|
});
|
|
776
838
|
}
|
|
777
839
|
|
|
@@ -1020,12 +1082,14 @@ function getBatchFailureDetails(steps: Array<{ details: BatchStepPresentationDet
|
|
|
1020
1082
|
|
|
1021
1083
|
async function buildBatchStepPresentation(options: {
|
|
1022
1084
|
artifactManifest?: SessionArtifactManifest;
|
|
1085
|
+
artifactRequest?: ArtifactRequestContext;
|
|
1023
1086
|
cwd: string;
|
|
1024
1087
|
index: number;
|
|
1025
1088
|
item: AgentBrowserBatchResult;
|
|
1026
1089
|
persistentArtifactStore?: PersistentSessionArtifactStore;
|
|
1090
|
+
sessionName?: string;
|
|
1027
1091
|
}): Promise<{ details: BatchStepPresentationDetails; presentation: ToolPresentation }> {
|
|
1028
|
-
const { artifactManifest, cwd, index, item, persistentArtifactStore } = options;
|
|
1092
|
+
const { artifactManifest, artifactRequest, cwd, index, item, persistentArtifactStore, sessionName } = options;
|
|
1029
1093
|
const command = isStringArray(item.command) ? item.command : undefined;
|
|
1030
1094
|
const commandText = formatBatchStepCommand(command, index);
|
|
1031
1095
|
|
|
@@ -1052,10 +1116,12 @@ async function buildBatchStepPresentation(options: {
|
|
|
1052
1116
|
|
|
1053
1117
|
const presentation = await buildToolPresentation({
|
|
1054
1118
|
artifactManifest,
|
|
1119
|
+
artifactRequest,
|
|
1055
1120
|
commandInfo: parseCommandInfo(command ?? []),
|
|
1056
1121
|
cwd,
|
|
1057
1122
|
envelope: { data: item.result, success: true },
|
|
1058
1123
|
persistentArtifactStore,
|
|
1124
|
+
sessionName,
|
|
1059
1125
|
});
|
|
1060
1126
|
const fullOutputPaths = getPresentationPaths({
|
|
1061
1127
|
primaryPath: presentation.fullOutputPath,
|
|
@@ -1090,24 +1156,28 @@ async function buildBatchStepPresentation(options: {
|
|
|
1090
1156
|
|
|
1091
1157
|
async function buildBatchPresentation(options: {
|
|
1092
1158
|
artifactManifest?: SessionArtifactManifest;
|
|
1159
|
+
artifactRequests?: Array<ArtifactRequestContext | undefined>;
|
|
1093
1160
|
cwd: string;
|
|
1094
1161
|
data: AgentBrowserBatchResult[];
|
|
1095
1162
|
persistentArtifactStore?: PersistentSessionArtifactStore;
|
|
1163
|
+
sessionName?: string;
|
|
1096
1164
|
summary: string;
|
|
1097
1165
|
}): Promise<ToolPresentation> {
|
|
1098
|
-
const { cwd, data, persistentArtifactStore, summary } = options;
|
|
1166
|
+
const { artifactRequests, cwd, data, persistentArtifactStore, sessionName, summary } = options;
|
|
1099
1167
|
const steps: Array<{ details: BatchStepPresentationDetails; presentation: ToolPresentation }> = [];
|
|
1100
1168
|
const protectedPersistentPaths: string[] = [];
|
|
1101
1169
|
let currentArtifactManifest = options.artifactManifest;
|
|
1102
1170
|
for (const [index, item] of data.entries()) {
|
|
1103
1171
|
const step = await buildBatchStepPresentation({
|
|
1104
1172
|
artifactManifest: currentArtifactManifest,
|
|
1173
|
+
artifactRequest: artifactRequests?.[index],
|
|
1105
1174
|
cwd,
|
|
1106
1175
|
index,
|
|
1107
1176
|
item,
|
|
1108
1177
|
persistentArtifactStore: persistentArtifactStore
|
|
1109
1178
|
? { ...persistentArtifactStore, protectedPaths: protectedPersistentPaths }
|
|
1110
1179
|
: undefined,
|
|
1180
|
+
sessionName,
|
|
1111
1181
|
});
|
|
1112
1182
|
steps.push(step);
|
|
1113
1183
|
currentArtifactManifest = step.presentation.artifactManifest ?? currentArtifactManifest;
|
|
@@ -1522,13 +1592,16 @@ async function compactLargePresentationOutput(options: {
|
|
|
1522
1592
|
|
|
1523
1593
|
export async function buildToolPresentation(options: {
|
|
1524
1594
|
artifactManifest?: SessionArtifactManifest;
|
|
1595
|
+
artifactRequest?: ArtifactRequestContext;
|
|
1596
|
+
batchArtifactRequests?: Array<ArtifactRequestContext | undefined>;
|
|
1525
1597
|
commandInfo: CommandInfo;
|
|
1526
1598
|
cwd: string;
|
|
1527
1599
|
envelope?: AgentBrowserEnvelope;
|
|
1528
1600
|
errorText?: string;
|
|
1529
1601
|
persistentArtifactStore?: PersistentSessionArtifactStore;
|
|
1602
|
+
sessionName?: string;
|
|
1530
1603
|
}): Promise<ToolPresentation> {
|
|
1531
|
-
const { artifactManifest, commandInfo, cwd, envelope, errorText, persistentArtifactStore } = options;
|
|
1604
|
+
const { artifactManifest, artifactRequest, commandInfo, cwd, envelope, errorText, persistentArtifactStore, sessionName } = options;
|
|
1532
1605
|
if (errorText) {
|
|
1533
1606
|
const hintedErrorText = appendSelectorRecoveryHint(redactModelFacingText(errorText));
|
|
1534
1607
|
return {
|
|
@@ -1537,14 +1610,14 @@ export async function buildToolPresentation(options: {
|
|
|
1537
1610
|
};
|
|
1538
1611
|
}
|
|
1539
1612
|
|
|
1540
|
-
const data = envelope?.data;
|
|
1541
|
-
const artifacts = await extractFileArtifacts(commandInfo, cwd, data);
|
|
1613
|
+
const data = enrichStreamStatusData(commandInfo, envelope?.data);
|
|
1614
|
+
const artifacts = await extractFileArtifacts({ artifactRequest, commandInfo, cwd, data, sessionName });
|
|
1542
1615
|
const artifactSummary = formatArtifactSummary(artifacts);
|
|
1543
1616
|
const summary = artifactSummary ?? formatSummary(commandInfo, data);
|
|
1544
1617
|
const artifactText = artifacts.length > 0 ? formatArtifactMetadataLines(artifacts).join("\n") : undefined;
|
|
1545
1618
|
const presentation =
|
|
1546
1619
|
commandInfo.command === "batch" && Array.isArray(data)
|
|
1547
|
-
? await buildBatchPresentation({ artifactManifest, cwd, data: data as AgentBrowserBatchResult[], persistentArtifactStore, summary })
|
|
1620
|
+
? await buildBatchPresentation({ artifactManifest, artifactRequests: options.batchArtifactRequests, cwd, data: data as AgentBrowserBatchResult[], persistentArtifactStore, sessionName, summary })
|
|
1548
1621
|
: commandInfo.command === "snapshot" && isRecord(data)
|
|
1549
1622
|
? await buildSnapshotPresentation(data, persistentArtifactStore, artifactManifest)
|
|
1550
1623
|
: {
|
|
@@ -1564,7 +1637,7 @@ export async function buildToolPresentation(options: {
|
|
|
1564
1637
|
}
|
|
1565
1638
|
}
|
|
1566
1639
|
|
|
1567
|
-
const imagePath = extractImagePath(commandInfo, cwd, data);
|
|
1640
|
+
const imagePath = artifactRequest?.absolutePath ?? extractImagePath(commandInfo, cwd, data);
|
|
1568
1641
|
const presentationWithImage = imagePath ? await attachInlineImage(presentation, imagePath) : presentation;
|
|
1569
1642
|
const compactedPresentation = await compactLargePresentationOutput({
|
|
1570
1643
|
artifactManifest,
|
|
@@ -21,16 +21,24 @@ export interface AgentBrowserBatchResult {
|
|
|
21
21
|
|
|
22
22
|
export type FileArtifactKind = "download" | "file" | "har" | "image" | "pdf" | "profile" | "trace" | "video";
|
|
23
23
|
|
|
24
|
+
export type FileArtifactStatus = "missing" | "repaired-from-temp" | "saved" | "upstream-temp-only";
|
|
25
|
+
|
|
24
26
|
export interface FileArtifactMetadata {
|
|
25
27
|
absolutePath: string;
|
|
28
|
+
artifactType?: FileArtifactKind;
|
|
26
29
|
command?: string;
|
|
30
|
+
cwd?: string;
|
|
27
31
|
exists?: boolean;
|
|
28
32
|
extension?: string;
|
|
29
33
|
kind: FileArtifactKind;
|
|
30
34
|
mediaType?: string;
|
|
31
35
|
path: string;
|
|
36
|
+
requestedPath?: string;
|
|
37
|
+
session?: string;
|
|
32
38
|
sizeBytes?: number;
|
|
39
|
+
status?: FileArtifactStatus;
|
|
33
40
|
subcommand?: string;
|
|
41
|
+
tempPath?: string;
|
|
34
42
|
}
|
|
35
43
|
|
|
36
44
|
export interface SavedFilePresentationDetails {
|
|
@@ -49,13 +57,16 @@ export interface SessionArtifactManifestEntry {
|
|
|
49
57
|
absolutePath?: string;
|
|
50
58
|
command?: string;
|
|
51
59
|
createdAtMs: number;
|
|
60
|
+
cwd?: string;
|
|
52
61
|
evictedAtMs?: number;
|
|
53
62
|
exists?: boolean;
|
|
54
63
|
extension?: string;
|
|
55
64
|
kind: FileArtifactKind | "spill";
|
|
56
65
|
mediaType?: string;
|
|
57
66
|
path: string;
|
|
67
|
+
requestedPath?: string;
|
|
58
68
|
retentionState: ArtifactRetentionState;
|
|
69
|
+
session?: string;
|
|
59
70
|
sizeBytes?: number;
|
|
60
71
|
storageScope: ArtifactStorageScope;
|
|
61
72
|
subcommand?: string;
|
package/package.json
CHANGED