pi-agent-browser-native 0.6.13 → 0.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +8 -6
- package/dist/extensions/agent-browser/index.js +7 -4
- package/dist/extensions/agent-browser/lib/argv-grammar.js +1 -0
- package/dist/extensions/agent-browser/lib/command-taxonomy.js +4 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/artifact-paths.js +14 -3
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.js +1 -1
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/index.js +18 -2
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/managed-session-daemon-policy.js +15 -4
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/snapshot-filter.js +3 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare.js +2 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/process-output.js +14 -3
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/session-state.js +5 -2
- package/dist/extensions/agent-browser/lib/playbook.js +1 -0
- package/dist/extensions/agent-browser/lib/recording-reservations.js +8 -1
- package/dist/extensions/agent-browser/lib/results/presentation/artifacts.js +15 -1
- package/dist/extensions/agent-browser/lib/results/presentation/batch.js +18 -9
- package/dist/extensions/agent-browser/lib/results/presentation/common.js +3 -0
- package/dist/extensions/agent-browser/lib/results/presentation/registry.js +1 -1
- package/dist/extensions/agent-browser/lib/results/presentation.js +7 -3
- package/dist/extensions/agent-browser/lib/results/snapshot-refs.js +14 -2
- package/dist/extensions/agent-browser/lib/results/snapshot.js +12 -5
- package/dist/extensions/agent-browser/lib/runtime.js +3 -1
- package/dist/extensions/agent-browser/lib/session-page-state.js +7 -3
- package/dist/extensions/agent-browser/lib/webmcp-observation.js +28 -0
- package/dist/scripts/agent-browser-target.mjs +1 -1
- package/docs/ARCHITECTURE.md +7 -1
- package/docs/COMMAND_REFERENCE.md +67 -14
- package/docs/ELECTRON.md +1 -1
- package/docs/RELEASE.md +4 -3
- package/docs/SUPPORT_MATRIX.md +12 -8
- package/docs/TOOL_CONTRACT.md +13 -3
- package/docs/platform-smoke.md +2 -2
- package/package.json +1 -1
- package/scripts/agent-browser-capability-baseline.mjs +34 -3
- package/scripts/agent-browser-target.mjs +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 0.6.14 - 2026-09-17
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- Recommend `agent-browser` 0.38.1 while retaining the stable 0.35.0 minimum and separate upstream installation.
|
|
10
|
+
- Support native delta snapshots and persistent refs without treating partial revisions as empty pages; keep native delta baselines upstream-owned.
|
|
11
|
+
- Preserve conditional screenshot options and skipped-image results without inventing files or attaching old images.
|
|
12
|
+
- Parse session pointer modes and recording cursor/contact-sheet options, verify contact-sheet images, and surface native WebMCP catalog updates. Document stateful auth login and native recording timing fixes.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- Give automatic managed-session cleanup the normal 35-second command budget, rather than aborting during native Chrome's five-second shutdown grace. Explicit cleanup-timeout overrides remain supported. Report cleanup deadlines with the phase and elapsed time instead of incorrectly reporting caller cancellation.
|
|
17
|
+
|
|
3
18
|
## 0.6.13 - 2026-09-14
|
|
4
19
|
|
|
5
20
|
### Fixed
|
package/README.md
CHANGED
|
@@ -85,7 +85,7 @@ The result is optimized for agent work:
|
|
|
85
85
|
| Dashboard scroll commands can look successful while nothing moves | Handles standard `scroll <dir> [px]` against the document first (including pages whose smooth-scroll CSS defeats upstream wheel timing), falls back upstream when the document cannot move, and samples viewport/containers around the fallback; unchanged positions fail as `upstream-error` with `details.scrollNoop`, visible recovery guidance, and exact snapshot/screenshot checks. Unsupported `scrollintoview text=...` fails before dispatch, including inside effective batch rows, and shows exact native `find text ... hover` and snapshot/ref recovery payloads; help remains native pass-through. | [`docs/TOOL_CONTRACT.md`](docs/TOOL_CONTRACT.md#details), [`docs/COMMAND_REFERENCE.md`](docs/COMMAND_REFERENCE.md#core-page-and-element-commands), `test/agent-browser.extension-validation.test.ts` |
|
|
86
86
|
| Dropdown/combobox clicks can focus or hit native option box-model errors | Adds first-class `select <selector> <value...>` paths through raw `args`, `job`, and `semanticAction`; semantic role/name or label select resolves exactly one current visible combobox/listbox ref before action. Custom combobox clicks still detect focused controls with explicit `aria-expanded` state but no visible options and return `details.comboboxFocus` plus exact recovery `nextActions` | [`docs/TOOL_CONTRACT.md`](docs/TOOL_CONTRACT.md#details), [`docs/COMMAND_REFERENCE.md`](docs/COMMAND_REFERENCE.md#core-page-and-element-commands), `extensions/agent-browser/lib/input-modes/semantic-action.ts`, `test/agent-browser.extension-input-modes.test.ts`, `test/agent-browser.extension-validation.test.ts` |
|
|
87
87
|
| Recording workflows fail late when `ffmpeg` is missing or report stale lifecycle state | After successful `record start` / `record restart`, reports `successCategory: "artifact-pending"`, returns an exact `stop-pending-recording` action, warns when older natives report pending output without `ffmpeg`, and conservatively invalidates prior page-scoped `@e…` refs on every executed start attempt and URL-bearing restart to protect older supported natives. This is not evidence of a page change: 0.37 records the active page unless a URL is supplied; FPS-only calls keep the intended tab; an unbounded transcript-backed namespace/session index reserves active destinations across aliases, serializes artifact lifecycle and explicit wait/output writes, persists cross-branch close tombstones, retires every successful close path (including every matching namespace owner for `close --all`), rejects missing/stale restart output, coalesces terminal batch state, keeps only the newest pending path per identity, rejects recording starts after a nested close, folds Unicode path aliases, and retains exact cleanup actions with visible guidance on any later same-session failure | [`docs/TOOL_CONTRACT.md`](docs/TOOL_CONTRACT.md#details), [`docs/COMMAND_REFERENCE.md`](docs/COMMAND_REFERENCE.md#diff-debug-and-streaming), `test/agent-browser.extension-validation.test.ts`, `test/agent-browser.presentation-artifacts-batch.test.ts` |
|
|
88
|
-
| Upstream CLI drift can silently invalidate wrapper behavior | Publishes a repo-readable command reference, verifies it against the recommended 0.
|
|
88
|
+
| Upstream CLI drift can silently invalidate wrapper behavior | Publishes a repo-readable command reference, verifies it against the recommended 0.38.1 target, and probes browser-backed calls once per cwd/PATH so stable versions below the 0.35.0 floor fail before browser launch with installed/expected version evidence | `npm run verify` |
|
|
89
89
|
| Pages can expose structured workflows through experimental WebMCP | Shows native positive WebMCP availability in navigation summaries and passes through `webmcp list`, `invoke`, detached `result` / `cancel`, params/frame/timeout options, and the bundled `webmcp-gen` skill; treats `--no-webmcp` as launch-scoped, keeps pending or unsuccessfully settled targets unverified with an actionable `get url` follow-up, invalidates stale refs after page tools run, and budgets effective raw or stdin batch timeouts | [`docs/COMMAND_REFERENCE.md`](docs/COMMAND_REFERENCE.md#webmcp-page-tools), `test/agent-browser.extension-ref-guards.test.ts`, `test/agent-browser.wait-timeouts.test.ts`, `test/agent-browser.real-upstream-contract.test.ts` |
|
|
90
90
|
| Desktop Electron apps need discovery, CDP attach, and safe teardown | Top-level `electron` runs host `list` / isolated `launch` (temp profile, OS-chosen debug port) / `status` / `probe` / `cleanup`, merges `launchId` plus managed `sessionName`, supports `handoff` `snapshot` / `tabs` / `connect`, and surfaces mismatch and post-command health guidance; wrapper cleanup applies only to launches it created | `extensions/agent-browser/lib/electron/discovery.ts`, `launch.ts`, `cleanup.ts`, [`docs/TOOL_CONTRACT.md`](docs/TOOL_CONTRACT.md#electron), [`docs/COMMAND_REFERENCE.md`](docs/COMMAND_REFERENCE.md#electron-desktop-apps) |
|
|
91
91
|
| Agents need bundled `skills` text and local setup/status commands without touching the live session | Treats `skills list/get/path`, local auth/profile/setup commands, `session list`, and local state lifecycle commands as sessionless reads/actions when upstream does not need a live page. Session/state rows and targets remain visible, and supported upstream state/config/path operations pass through unchanged. Browser-backed workflows still receive an implicit session only when the caller did not choose one. | [`docs/COMMAND_REFERENCE.md`](docs/COMMAND_REFERENCE.md#built-in-skills), `extensions/agent-browser/lib/command-policy.ts`, `extensions/agent-browser/lib/runtime.ts` |
|
|
@@ -108,7 +108,9 @@ Optional external tools unlock the full command surface:
|
|
|
108
108
|
| `agent-browser` | All browser automation through this extension | See upstream install docs |
|
|
109
109
|
| `ffmpeg` | Recording WebM (libvpx) or MP4 (libx264); install before `record start` / `record restart` | `brew install ffmpeg` or `brew install ffmpeg-full` |
|
|
110
110
|
|
|
111
|
-
Keep both binaries on `PATH`. This package recommends `agent-browser 0.
|
|
111
|
+
Keep both binaries on `PATH`. This package recommends `agent-browser 0.38.1` and accepts stable versions at or above the 0.35.0 floor; browser-backed calls fail fast below that floor while local inspection/setup commands remain available for diagnosis. Native 0.37 validates `ffmpeg`, the output extension and FPS before recording; older supported natives may defer failures until stop. Use `record start <path.webm|path.mp4> [url] [--fps <n>]` (1–60 fps, default 30), then verify the completed file after `record stop`. Native `doctor` checks the recording dependency and encoders.
|
|
112
|
+
|
|
113
|
+
On 0.38.1, use `snapshot --delta` for native full/unchanged/changed revisions and `snapshot --delta --full` to reset its baseline. Surviving DOM nodes keep their refs, but navigation/replaced nodes still require fresh refs. The wrapper preserves native revisions and refreshes complete refs internally for partial results. `screenshot --if-changed` (or `--threshold 0.01`) skips unchanged images without creating an artifact. `--input-mode instant|smooth|human` controls native session pointer movement; `click ... --human`, `drag ... --human`, and timed `mouse move` remain ordinary `args`. Recordings accept `--cursor` and `--contact-sheet`; the latter produces a timestamped PNG verified after stop. `auth login <name> --no-navigate` fills a prepared login page after upstream checks its origin. See the [command reference](docs/COMMAND_REFERENCE.md#upstream-0381-rebaseline) for examples and limits.
|
|
112
114
|
|
|
113
115
|
### Android / Termux
|
|
114
116
|
|
|
@@ -188,7 +190,7 @@ npm run doctor
|
|
|
188
190
|
The doctor checks:
|
|
189
191
|
|
|
190
192
|
- upstream `agent-browser` exists on `PATH`
|
|
191
|
-
- the installed upstream is a stable version at or above the supported 0.35.0 floor; 0.
|
|
193
|
+
- the installed upstream is a stable version at or above the supported 0.35.0 floor; 0.38.1 remains the recommended baseline
|
|
192
194
|
- `pi --version` meets the minimum Pi runtime floor for this release; older Pi versions are setup failures
|
|
193
195
|
- Pi settings do not point at multiple active `pi-agent-browser-native` sources
|
|
194
196
|
|
|
@@ -758,7 +760,7 @@ The opt-in real-upstream suite is separate because it drives a real browser inst
|
|
|
758
760
|
npm run verify -- real-upstream
|
|
759
761
|
```
|
|
760
762
|
|
|
761
|
-
That mode sets `PI_AGENT_BROWSER_REAL_UPSTREAM=1` and runs `test/agent-browser.real-upstream-contract.test.ts` against the real `agent-browser` on `PATH` (the stable version must meet the 0.35.0 floor; current command-reference validation targets the recommended 0.
|
|
763
|
+
That mode sets `PI_AGENT_BROWSER_REAL_UPSTREAM=1` and runs `test/agent-browser.real-upstream-contract.test.ts` against the real `agent-browser` on `PATH` (the stable version must meet the 0.35.0 floor; current command-reference validation targets the recommended 0.38.1 capability baseline). It covers inspection, skills, experimental WebMCP list/invoke/result/cancel plus `--no-webmcp`, and a broad core interaction and navigation matrix on localhost fixtures (including off-viewport click, frame-scoped selector/wait/click behavior, form command fixes, `batch` stdin, and `pushstate`), plus `vitals`, network route/requests/HAR, diff snapshot/screenshot/url, trace/profiler, console/errors/highlight, stream enable/status/disable, `cookies set --curl`, a `react tree` missing-renderer path, and `wait --download` with the on-disk caveat documented in release notes. The harness uses a throwaway temp `HOME` and dedicated socket/screenshot directories so the run does not touch your normal browser profile paths. Browser-opening or credential-dependent families such as `inspect`, `dashboard`, `chat`, provider clouds, and OS clipboard flows stay in fake-upstream or manual validation unless a safe deterministic fixture is added. For prerequisites, isolation details, and troubleshooting, see [`docs/RELEASE.md`](docs/RELEASE.md#real-upstream-contract-validation).
|
|
762
764
|
|
|
763
765
|
A deterministic host-only live-browser wrapper smoke is available without an LLM choosing tool calls:
|
|
764
766
|
|
|
@@ -791,12 +793,12 @@ npm run verify -- release
|
|
|
791
793
|
5. Screenshots, downloads, recordings, traces, profiles, and spill files are normalized as Pi-visible artifacts where possible.
|
|
792
794
|
6. Generated playbook text in docs and tool metadata stays aligned with `extensions/agent-browser/lib/playbook.ts`.
|
|
793
795
|
|
|
794
|
-
The upstream browser engine remains [`agent-browser`](https://agent-browser.dev/). This package does not bundle it. The recommended baseline is 0.
|
|
796
|
+
The upstream browser engine remains [`agent-browser`](https://agent-browser.dev/). This package does not bundle it. The recommended baseline is 0.38.1 and the stable runtime floor is 0.35.0; newer stable versions are accepted without version-specific compatibility shims.
|
|
795
797
|
|
|
796
798
|
## Current limits
|
|
797
799
|
|
|
798
800
|
- Published pre-1.0 package.
|
|
799
|
-
- Recommends upstream `agent-browser` 0.
|
|
801
|
+
- Recommends upstream `agent-browser` 0.38.1 and accepts stable runtimes at or above 0.35.0.
|
|
800
802
|
- Does not bundle `agent-browser`; users install it separately.
|
|
801
803
|
- Does not provide a human browser UI inside Pi; the primary UX is agent-invoked tool calls. `--headed` asks upstream to show a browser window, but the wrapper cannot yet prove that the window is visible on the user's desktop.
|
|
802
804
|
- Localhost means the browser host's loopback, not necessarily the shell/Pi host. If `http://localhost:<port>` or `http://127.0.0.1:<port>` fails with errors such as `ERR_EMPTY_RESPONSE`, use an environment-specific host-reachable HTTP(S) address. A `file://` fixture is supported when upstream browser launch settings allow it; use HTTP(S) only when the browser environment cannot reach the local file.
|
|
@@ -22,7 +22,7 @@ import { cleanupSecureTempArtifacts } from "./lib/temp.js";
|
|
|
22
22
|
import { AGENT_BROWSER_PARAMS } from "./lib/input-modes/params.js";
|
|
23
23
|
import { AGENT_BROWSER_SCRIPT_DEFAULT_TIMEOUT_MS, AGENT_BROWSER_SCRIPT_NAMESPACE, createAgentBrowserScriptSessionName, isAgentBrowserScriptSessionName, runAgentBrowserScript, } from "./lib/input-modes/script.js";
|
|
24
24
|
import { closeManagedSession, getSessionContextKey, runAgentBrowserTool } from "./lib/orchestration/browser-run/index.js";
|
|
25
|
-
import { canonicalizeExplicitArtifactDestination, getExplicitArtifactDestination } from "./lib/orchestration/browser-run/artifact-paths.js";
|
|
25
|
+
import { canonicalizeExplicitArtifactDestination, getExplicitArtifactDestination, getRecordContactSheetDestination } from "./lib/orchestration/browser-run/artifact-paths.js";
|
|
26
26
|
import { findElectronLaunchRecordForSession, getActiveElectronRecords } from "./lib/orchestration/browser-run/session-state.js";
|
|
27
27
|
import { parseBatchCommandArgument, parseUserBatchStdin } from "./lib/orchestration/batch-stdin.js";
|
|
28
28
|
import { ELECTRON_POST_COMMAND_STATUS_SETTLE_MS, ELECTRON_PROFILE_ISOLATION_DETAILS, cleanupActiveElectronHostLaunches, handleElectronHostInput, restoreElectronLaunchRecordsFromBranch, } from "./lib/orchestration/electron-host/index.js";
|
|
@@ -75,6 +75,8 @@ function getArtifactPreflightValidationError(options) {
|
|
|
75
75
|
for (const reservation of options.activeRecordingReservations ?? []) {
|
|
76
76
|
try {
|
|
77
77
|
activeRecordingDestinations.add(canonicalizeExplicitArtifactDestination(reservation.cwd, reservation.absolutePath));
|
|
78
|
+
if (reservation.contactSheetPath)
|
|
79
|
+
activeRecordingDestinations.add(canonicalizeExplicitArtifactDestination(reservation.cwd, reservation.contactSheetPath));
|
|
78
80
|
}
|
|
79
81
|
catch (canonicalizationError) {
|
|
80
82
|
if (!cleanupOnly)
|
|
@@ -106,8 +108,9 @@ function getArtifactPreflightValidationError(options) {
|
|
|
106
108
|
if (isCloseCommand(commandStep[0]))
|
|
107
109
|
sawBatchClose = true;
|
|
108
110
|
}
|
|
109
|
-
const artifactDestination
|
|
110
|
-
|
|
111
|
+
for (const artifactDestination of [getExplicitArtifactDestination(commandStep), getRecordContactSheetDestination(commandStep)]) {
|
|
112
|
+
if (!artifactDestination)
|
|
113
|
+
continue;
|
|
111
114
|
let canonicalDestination;
|
|
112
115
|
try {
|
|
113
116
|
canonicalDestination = canonicalizeExplicitArtifactDestination(options.cwd, artifactDestination);
|
|
@@ -1126,7 +1129,7 @@ export default function agentBrowserExtension(pi, { beforeExecute } = {}) {
|
|
|
1126
1129
|
}
|
|
1127
1130
|
for (const [key, reservation] of activeRecordingReservations) {
|
|
1128
1131
|
const restored = restoredRecordingState.active.get(key);
|
|
1129
|
-
if (restored?.absolutePath !== reservation.absolutePath || restored.cwd !== reservation.cwd || restored.recordingId !== reservation.recordingId || restored.startedAtMs !== reservation.startedAtMs)
|
|
1132
|
+
if (restored?.absolutePath !== reservation.absolutePath || restored.cwd !== reservation.cwd || restored.recordingId !== reservation.recordingId || restored.startedAtMs !== reservation.startedAtMs || restored.contactSheetPath !== reservation.contactSheetPath)
|
|
1130
1133
|
recordingReservationsDirty = true;
|
|
1131
1134
|
restoredRecordingState.active.set(key, reservation);
|
|
1132
1135
|
}
|
|
@@ -324,6 +324,10 @@ export function getRecordCommandOperands(tokens) {
|
|
|
324
324
|
// Native validates the range; bare/non-numeric --fps keeps its old literal meaning.
|
|
325
325
|
if (tokens[index] === "--fps" && /^\+?\d+$/.test(tokens[index + 1] ?? ""))
|
|
326
326
|
index += 1;
|
|
327
|
+
else if (tokens[index] === "--cursor" || tokens[index] === "--contact-sheet")
|
|
328
|
+
continue;
|
|
329
|
+
else if (tokens[index] === "--contact-sheet-threshold" && tokens[index + 1] !== undefined)
|
|
330
|
+
index += 1;
|
|
327
331
|
else
|
|
328
332
|
operands.push(tokens[index]);
|
|
329
333
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { lstatSync, readlinkSync, realpathSync, statSync } from "node:fs";
|
|
2
|
-
import { basename, dirname, join, resolve } from "node:path";
|
|
2
|
+
import { basename, dirname, extname, join, resolve } from "node:path";
|
|
3
3
|
import { foldAgentBrowserFilesystemIdentity } from "../../argv-grammar.js";
|
|
4
4
|
import { parseWaitCommandTokens } from "../../argv-descriptor.js";
|
|
5
5
|
import { getRecordCommandOperands } from "../../command-taxonomy.js";
|
|
@@ -10,14 +10,18 @@ function isSingleScreenshotPathToken(token) {
|
|
|
10
10
|
return false;
|
|
11
11
|
return explicitlyRelative || token.includes("/") || SCREENSHOT_IMAGE_EXTENSIONS.some((extension) => token.endsWith(extension));
|
|
12
12
|
}
|
|
13
|
-
function getScreenshotPositionalIndices(commandTokens) {
|
|
13
|
+
export function getScreenshotPositionalIndices(commandTokens) {
|
|
14
14
|
if (commandTokens[0] !== "screenshot")
|
|
15
15
|
return [];
|
|
16
16
|
const positionalIndices = [];
|
|
17
17
|
for (let index = 1; index < commandTokens.length; index += 1) {
|
|
18
18
|
const token = commandTokens[index];
|
|
19
|
-
if (token === "--full" || token === "-f")
|
|
19
|
+
if (token === "--full" || token === "-f" || token === "--if-changed")
|
|
20
20
|
continue;
|
|
21
|
+
if (token === "--threshold") {
|
|
22
|
+
index += 1;
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
21
25
|
positionalIndices.push(index);
|
|
22
26
|
}
|
|
23
27
|
return positionalIndices;
|
|
@@ -91,6 +95,13 @@ function canonicalizeArtifactPath(absolutePath, platform, seenSymlinks) {
|
|
|
91
95
|
export function canonicalizeExplicitArtifactDestination(cwd, destination, platform = process.platform) {
|
|
92
96
|
return canonicalizeArtifactPath(resolve(cwd, destination), platform, new Set());
|
|
93
97
|
}
|
|
98
|
+
export function getRecordContactSheetDestination(commandTokens) {
|
|
99
|
+
const path = getRecordCommandOperands(commandTokens).path;
|
|
100
|
+
if (!path || !commandTokens.some((token) => token === "--contact-sheet" || token === "--contact-sheet-threshold"))
|
|
101
|
+
return undefined;
|
|
102
|
+
const extension = extname(path);
|
|
103
|
+
return extension ? `${path.slice(0, -extension.length)}.contact-sheet.png` : undefined;
|
|
104
|
+
}
|
|
94
105
|
export function getExplicitArtifactDestination(commandTokens) {
|
|
95
106
|
const command = commandTokens[0];
|
|
96
107
|
const subcommand = commandTokens[1];
|
|
@@ -11,7 +11,7 @@ function normalizeAccessibleName(name) {
|
|
|
11
11
|
function getClickDispatchProbeTarget(commandTokens, refSnapshot) {
|
|
12
12
|
if (commandTokens[0] !== "click" || commandTokens.includes("--new-tab"))
|
|
13
13
|
return undefined;
|
|
14
|
-
const selector = commandTokens
|
|
14
|
+
const selector = commandTokens.slice(1).find((token) => token !== "--human");
|
|
15
15
|
if (!selector || selector.startsWith("-"))
|
|
16
16
|
return undefined;
|
|
17
17
|
const refId = parseRefId(selector);
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { runAgentBrowserProcess, withAttachedBrowserSessionContext, withChromeStartupArgs } from "../../process.js";
|
|
2
2
|
import { isRecord } from "../../parsing.js";
|
|
3
|
+
import { collectNativeWebMcp, getNativeWebMcpCatalog } from "../../webmcp-observation.js";
|
|
4
|
+
import { redactSensitiveValue } from "../../runtime.js";
|
|
5
|
+
import { formatWebMcpCatalogUpdate } from "../../results/presentation/common.js";
|
|
3
6
|
import { withOwnedManagedSessionContext } from "../../managed-session-restore.js";
|
|
4
7
|
import { cleanupClickDispatchProbe } from "./click-dispatch.js";
|
|
5
8
|
import { applyBrowserRunStatePatch, getSessionContextKey } from "./session-state.js";
|
|
@@ -9,8 +12,21 @@ import { processBrowserOutput } from "./process-output.js";
|
|
|
9
12
|
export { closeManagedSession } from "./managed-session-daemon-policy.js";
|
|
10
13
|
export { getSessionContextKey } from "./session-state.js";
|
|
11
14
|
export async function runAgentBrowserTool(options) {
|
|
12
|
-
const
|
|
13
|
-
const
|
|
15
|
+
const observed = await collectNativeWebMcp(() => withChromeStartupArgs(options.input.persistentChromeArgs, () => withAttachedBrowserSessionContext(options.preserveAttachedBrowserSession === true, () => runAgentBrowserToolInContext(options))));
|
|
16
|
+
const result = observed.result;
|
|
17
|
+
let details = isRecord(result.details) ? result.details : undefined;
|
|
18
|
+
if (observed.catalog) {
|
|
19
|
+
const catalog = redactSensitiveValue(observed.catalog);
|
|
20
|
+
details = { ...details, webMcpCatalog: catalog };
|
|
21
|
+
result.details = details;
|
|
22
|
+
if (!options.input.toolArgs.includes("--json") && JSON.stringify(getNativeWebMcpCatalog(details.data)) !== JSON.stringify(catalog)) {
|
|
23
|
+
const notice = formatWebMcpCatalogUpdate(catalog);
|
|
24
|
+
if (result.content[0]?.type === "text")
|
|
25
|
+
result.content[0] = { ...result.content[0], text: `${result.content[0].text}\n\n${notice}` };
|
|
26
|
+
else
|
|
27
|
+
result.content.push({ type: "text", text: notice });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
14
30
|
const sessionKey = getSessionContextKey(typeof details?.sessionName === "string" ? details.sessionName : undefined, typeof details?.namespace === "string" ? details.namespace : undefined);
|
|
15
31
|
const page = options.state.sessionPageState.get(sessionKey);
|
|
16
32
|
return page.tabReopenPending === undefined ? result : {
|
package/dist/extensions/agent-browser/lib/orchestration/browser-run/managed-session-daemon-policy.js
CHANGED
|
@@ -142,7 +142,13 @@ export async function acquireOwnedManagedSessionDaemonPolicy(options) {
|
|
|
142
142
|
}
|
|
143
143
|
export async function closeManagedSession(options) {
|
|
144
144
|
const controller = new AbortController();
|
|
145
|
-
|
|
145
|
+
let phase = "policy coordination";
|
|
146
|
+
let phaseStartedAt = Date.now();
|
|
147
|
+
let timeoutError;
|
|
148
|
+
const timer = setTimeout(() => {
|
|
149
|
+
timeoutError = `Managed-session cleanup timed out after ${options.timeoutMs} ms during ${phase} (${Date.now() - phaseStartedAt} ms in this phase).`;
|
|
150
|
+
controller.abort();
|
|
151
|
+
}, options.timeoutMs);
|
|
146
152
|
let stdoutSpillPath;
|
|
147
153
|
const closeArgs = [...(options.namespace !== undefined ? ["--namespace", options.namespace] : []), "--session", options.sessionName, "close"];
|
|
148
154
|
const policyLock = options.policyLock ?? await acquireManagedSessionPolicyLock({
|
|
@@ -153,9 +159,11 @@ export async function closeManagedSession(options) {
|
|
|
153
159
|
});
|
|
154
160
|
if (!policyLock) {
|
|
155
161
|
clearTimeout(timer);
|
|
156
|
-
return "Managed-session policy coordination is unavailable or busy; cleanup did not run. Retry after the current operation finishes or repair the private policy-lock directory.";
|
|
162
|
+
return timeoutError ?? "Managed-session policy coordination is unavailable or busy; cleanup did not run. Retry after the current operation finishes or repair the private policy-lock directory.";
|
|
157
163
|
}
|
|
158
164
|
try {
|
|
165
|
+
phase = "daemon inspection";
|
|
166
|
+
phaseStartedAt = Date.now();
|
|
159
167
|
const daemon = await inspectManagedSessionDaemon({
|
|
160
168
|
cwd: options.cwd,
|
|
161
169
|
headedManagedAutosaveInterval: options.headedManagedAutosaveInterval,
|
|
@@ -170,6 +178,8 @@ export async function closeManagedSession(options) {
|
|
|
170
178
|
const daemonRestoreKey = options.restoreState.getDaemonRestoreKey(options.sessionName, options.namespace);
|
|
171
179
|
const ownedRestoreKey = !options.restoreState.isDisabled(options.sessionName, options.namespace)
|
|
172
180
|
&& isManagedSessionRestoreKey(daemonRestoreKey) ? daemonRestoreKey : null;
|
|
181
|
+
phase = "native close";
|
|
182
|
+
phaseStartedAt = Date.now();
|
|
173
183
|
const processResult = await runAgentBrowserProcess({
|
|
174
184
|
args: closeArgs,
|
|
175
185
|
cwd: options.cwd,
|
|
@@ -179,6 +189,7 @@ export async function closeManagedSession(options) {
|
|
|
179
189
|
preserveAttachedBrowserSession: options.preserveAttachedBrowserSession,
|
|
180
190
|
signal: controller.signal,
|
|
181
191
|
});
|
|
192
|
+
clearTimeout(timer);
|
|
182
193
|
stdoutSpillPath = processResult.stdoutSpillPath;
|
|
183
194
|
if (!processResult.aborted && !processResult.spawnError && processResult.exitCode === 0) {
|
|
184
195
|
const parsed = await parseAgentBrowserEnvelope({ stdout: processResult.stdout, stdoutPath: processResult.stdoutSpillPath });
|
|
@@ -191,7 +202,7 @@ export async function closeManagedSession(options) {
|
|
|
191
202
|
statePath: typeof data?.statePath === "string" ? data.statePath : undefined,
|
|
192
203
|
});
|
|
193
204
|
}
|
|
194
|
-
return getAgentBrowserErrorText({
|
|
205
|
+
return timeoutError ?? getAgentBrowserErrorText({
|
|
195
206
|
aborted: processResult.aborted,
|
|
196
207
|
command: "close",
|
|
197
208
|
effectiveArgs: redactInvocationArgs(closeArgs),
|
|
@@ -204,7 +215,7 @@ export async function closeManagedSession(options) {
|
|
|
204
215
|
});
|
|
205
216
|
}
|
|
206
217
|
catch (error) {
|
|
207
|
-
return error instanceof Error ? error.message : String(error);
|
|
218
|
+
return timeoutError ?? (error instanceof Error ? error.message : String(error));
|
|
208
219
|
}
|
|
209
220
|
finally {
|
|
210
221
|
clearTimeout(timer);
|
package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/snapshot-filter.js
CHANGED
|
@@ -13,6 +13,9 @@ function parseSnapshotFilterRequest(commandTokens) {
|
|
|
13
13
|
let search;
|
|
14
14
|
for (let index = 0; index < commandTokens.length; index += 1) {
|
|
15
15
|
const token = commandTokens[index];
|
|
16
|
+
// Wrapper filters need a full native tree, not a revision-relative patch.
|
|
17
|
+
if (token === "--delta" || token === "--full")
|
|
18
|
+
continue;
|
|
16
19
|
if (token === "--viewport")
|
|
17
20
|
continue;
|
|
18
21
|
if (token === "--diff")
|
|
@@ -172,6 +172,8 @@ export async function prepareAgentBrowserArgs(args, stdin, cwd) {
|
|
|
172
172
|
}
|
|
173
173
|
async function repairScreenshotData(options) {
|
|
174
174
|
const { cwd, data, request } = options;
|
|
175
|
+
if (data.changed === false)
|
|
176
|
+
return { data };
|
|
175
177
|
const reportedPath = typeof data.path === "string" ? data.path : undefined;
|
|
176
178
|
const reportedAbsolutePath = reportedPath ? resolve(cwd, reportedPath) : undefined;
|
|
177
179
|
let status = await pathExists(request.absolutePath) ? "saved" : "missing";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { rm } from "node:fs/promises";
|
|
2
|
+
import { observeNativeWebMcp } from "../../webmcp-observation.js";
|
|
2
3
|
import { parseArgvDescriptor } from "../../argv-descriptor.js";
|
|
3
4
|
import { isBrowserIndependentRead, needsManagedSession } from "../../command-policy.js";
|
|
4
5
|
import { deleteIdentityKeysInNamespace, getAgentBrowserSessionIdentityKey, isAgentBrowserSessionIdentityKeyInNamespace } from "../../argv-grammar.js";
|
|
@@ -25,7 +26,7 @@ import { isManagedSessionRestoreKey } from "../../managed-session-storage.js";
|
|
|
25
26
|
import { createFreshSessionName, extractUpstreamCommandTokens, resolveManagedSessionState } from "../../runtime.js";
|
|
26
27
|
import { getUpstreamEffectiveBatchSteps } from "../batch-stdin.js";
|
|
27
28
|
import { closeManagedSession, inspectManagedSessionDaemon } from "./managed-session-daemon-policy.js";
|
|
28
|
-
import { applyOpenResultTabCorrection, buildAboutBlankRecoveryHint, buildAboutBlankWarning, buildElectronPostCommandHealthDiagnostic, buildElectronRefFreshnessDiagnostic, buildElectronSessionMismatch, buildManagedSessionOutcome, collectOpenResultTabCorrection, collectSessionTabSelection, commandChoosesSessionTabTarget, extractNavigationSummaryFromData, extractStringResultField, findElectronLaunchRecordForSession, formatElectronPostCommandHealthText, formatElectronSessionMismatchText, getSessionContextKey, getStaleRefArgs, mergeNavigationSummaryIntoData, shouldCaptureNavigationSummary, shouldCorrectSessionTabAfterCommand, shouldInspectElectronPostCommandHealth, updateTraceOwnerState, } from "./session-state.js";
|
|
29
|
+
import { applyOpenResultTabCorrection, buildAboutBlankRecoveryHint, buildAboutBlankWarning, buildElectronPostCommandHealthDiagnostic, buildElectronRefFreshnessDiagnostic, buildElectronSessionMismatch, buildManagedSessionOutcome, collectOpenResultTabCorrection, collectSessionTabSelection, commandChoosesSessionTabTarget, extractNavigationSummaryFromData, extractStringResultField, findElectronLaunchRecordForSession, formatElectronPostCommandHealthText, formatElectronSessionMismatchText, getSessionContextKey, getStaleRefArgs, mergeNavigationSummaryIntoData, runSessionCommandData, shouldCaptureNavigationSummary, shouldCorrectSessionTabAfterCommand, shouldInspectElectronPostCommandHealth, updateTraceOwnerState, } from "./session-state.js";
|
|
29
30
|
import { collectClickDispatchDiagnostic } from "./click-dispatch.js";
|
|
30
31
|
import { buildScrollNoopDiagnostic, collectComboboxFocusDiagnostic, collectElectronBroadGetTextScopeDiagnostics, collectElectronHandoff, collectFillVerificationDiagnostic, collectNavigationSummary, collectOverlayBlockerDiagnostic, collectQaAttachedTarget, collectSnapshotOverlayBlockerDiagnostic, collectRecordingDependencyWarning, collectScrollPositionSnapshot, collectSelectorTextVisibilityDiagnostics, collectTimeoutPartialProgress, sleepMs, formatQaAttachedTargetText, getArtifactCleanupGuidance, getEvalResultWarning, getEvalStdinHint, getSourceLookupElectronContext, } from "./diagnostics.js";
|
|
31
32
|
import { repairScreenshotData } from "./prepare.js";
|
|
@@ -163,6 +164,7 @@ export async function processBrowserOutput(input) {
|
|
|
163
164
|
// Native upgrade prints text even with --json; all other command shapes keep strict JSON parsing.
|
|
164
165
|
const plainTextUpgrade = !prepared.executionPlan.plainTextInspection && prepared.executionPlan.commandInfo.command === "upgrade" && !needsManagedSession(parseArgvDescriptor(prepared.runtimeToolArgs));
|
|
165
166
|
const parsed = await parseAgentBrowserEnvelope({ stdout: processResult.stdout, stdoutPath: processResult.stdoutSpillPath, plainText: plainTextUpgrade });
|
|
167
|
+
observeNativeWebMcp(parsed.envelope?.data);
|
|
166
168
|
let parseError = parsed.parseError;
|
|
167
169
|
const recordingStopRecovery = await recoverRecordingStop({
|
|
168
170
|
artifactManifest, artifactRunStartedAtMs: input.artifactRunStartedAtMs, commandTokens: prepared.commandTokens, cwd,
|
|
@@ -459,13 +461,22 @@ export async function processBrowserOutput(input) {
|
|
|
459
461
|
else if (processResult.agentBrowserStarted && (resultingPageState.pageUrlUnknown || resultingPageState.pageTargetMayHaveChanged) && !(prepared.commandTokens[0] === "session" && prepared.commandTokens[1] === "info")) {
|
|
460
462
|
sessionPageState.markTabTargetUnknown({ sessionName: sessionStateKey, update: sessionPageStateUpdate });
|
|
461
463
|
}
|
|
462
|
-
|
|
464
|
+
let refSnapshot = unsettledWebMcpMutation
|
|
463
465
|
? undefined
|
|
464
466
|
: prepared.executionPlan.commandInfo.command === "batch"
|
|
465
467
|
? batchRefSnapshotState?.snapshot
|
|
466
468
|
: succeeded
|
|
467
469
|
? prepared.executionPlan.commandInfo.command === "snapshot" ? extractRefSnapshotFromData(presentationEnvelope?.data) : prepared.resolvedSemanticActionRefSnapshot ?? overlayBlockerDiagnostic?.snapshot
|
|
468
470
|
: undefined;
|
|
471
|
+
const refreshArgs = batchRefSnapshotState?.refreshArgs ?? (succeeded && prepared.commandTokens[0] === "snapshot" && isRecord(presentationEnvelope?.data) && isRecord(presentationEnvelope.data.snapshot) && !refSnapshot
|
|
472
|
+
? prepared.commandTokens.filter((token) => token !== "--delta" && token !== "--full") : undefined);
|
|
473
|
+
// Native owns delta baselines. Read complete refs without advancing that baseline
|
|
474
|
+
// rather than maintaining a second revision/tree cache in the wrapper.
|
|
475
|
+
if (refreshArgs && !unsettledWebMcpMutation) {
|
|
476
|
+
refSnapshot = extractRefSnapshotFromData(await runSessionCommandData({ args: refreshArgs, cwd, namespace: prepared.executionPlan.namespace, sessionName: prepared.executionPlan.sessionName, signal }));
|
|
477
|
+
if (!refSnapshot)
|
|
478
|
+
sessionPageState.applyRefSnapshotInvalidation({ invalidation: buildPageTransitionRefSnapshotInvalidation("The native snapshot delta did not include complete refs and the full ref read failed. Run snapshot -i before using refs."), sessionName: sessionStateKey, update: sessionPageStateUpdate });
|
|
479
|
+
}
|
|
469
480
|
if (refSnapshot) {
|
|
470
481
|
const refUpdate = sessionPageState.applyRefSnapshot({ fallbackTarget: currentSessionTabTarget, sessionName: sessionStateKey, snapshot: refSnapshot, update: sessionPageStateUpdate });
|
|
471
482
|
currentRefSnapshot = refUpdate.refSnapshot;
|
|
@@ -670,7 +681,7 @@ export async function processBrowserOutput(input) {
|
|
|
670
681
|
}
|
|
671
682
|
if (plainTextUpgrade && errorText)
|
|
672
683
|
presentationEnvelope = { ...presentationEnvelope, success: false, error: errorText };
|
|
673
|
-
let presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text", text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : recordingStopRecovery && !recordingStopRecovery.batch ? recordingStopRecovery.presentation : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactMaxUpdatedAtMs: Date.now(), artifactMinUpdatedAtMs: input.artifactRunStartedAtMs, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd, envelope: presentationEnvelope, errorText, namespace: prepared.executionPlan.namespace, networkRouteDiagnostics, networkRoutes: activeNetworkRoutes, persistentArtifactStore, piCleanupOwnership: sessionStateKey && (state.ownedManagedSessions.has(sessionStateKey) || prepared.executionPlan.managedSessionName !== undefined) ? "wrapper-managed" : "caller-owned", sessionName: prepared.executionPlan.sessionName });
|
|
684
|
+
let presentation = plainTextInspection ? { artifacts: undefined, batchFailure: undefined, batchSteps: undefined, content: [{ type: "text", text: inspectionText ?? "" }], data: undefined, fullOutputPath: undefined, fullOutputPaths: undefined, imagePath: undefined, imagePaths: undefined, savedFile: undefined, savedFilePath: undefined, summary: `${prepared.redactedArgs.join(" ")} completed` } : recordingStopRecovery && !recordingStopRecovery.batch ? recordingStopRecovery.presentation : await buildToolPresentation({ args: prepared.redactedProcessArgs, artifactManifest, artifactMaxUpdatedAtMs: Date.now(), artifactMinUpdatedAtMs: input.artifactRunStartedAtMs, artifactRequest: screenshotArtifactRequest, batchArtifactRequests: batchScreenshotArtifactRequests, commandInfo: prepared.executionPlan.commandInfo, compiledSemanticAction: prepared.compiledSemanticAction, cwd, envelope: presentationEnvelope, errorText, namespace: prepared.executionPlan.namespace, networkRouteDiagnostics, networkRoutes: activeNetworkRoutes, persistentArtifactStore, previousRecordingContactSheetPath: sessionStateKey ? state.activeRecordingReservations?.get(sessionStateKey)?.contactSheetPath : undefined, piCleanupOwnership: sessionStateKey && (state.ownedManagedSessions.has(sessionStateKey) || prepared.executionPlan.managedSessionName !== undefined) ? "wrapper-managed" : "caller-owned", sessionName: prepared.executionPlan.sessionName });
|
|
674
685
|
if (recordingStopRecovery)
|
|
675
686
|
presentation = mergeRecordingRecoveryPresentation(presentation, recordingStopRecovery);
|
|
676
687
|
const confirmation = readConfirmationEvent ?? prepared.readConfirmation;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { rm } from "node:fs/promises";
|
|
2
|
+
import { getScreenshotPositionalIndices } from "./artifact-paths.js";
|
|
3
|
+
import { observeNativeWebMcp } from "../../webmcp-observation.js";
|
|
2
4
|
import { getAgentBrowserSessionIdentityKey } from "../../argv-grammar.js";
|
|
3
5
|
import { parseArgvDescriptor } from "../../argv-descriptor.js";
|
|
4
6
|
import { isBrowserIndependentRead, needsManagedSession } from "../../command-policy.js";
|
|
@@ -287,7 +289,7 @@ function collectRefsFromTokens(tokens) {
|
|
|
287
289
|
let selectors;
|
|
288
290
|
switch (tokens[0]) {
|
|
289
291
|
case "click":
|
|
290
|
-
selectors = [tokens.slice(1).find((token) => token !== "--new-tab")];
|
|
292
|
+
selectors = [tokens.slice(1).find((token) => token !== "--new-tab" && token !== "--human")];
|
|
291
293
|
break;
|
|
292
294
|
case "drag":
|
|
293
295
|
selectors = tokens.slice(1, 3);
|
|
@@ -299,7 +301,7 @@ function collectRefsFromTokens(tokens) {
|
|
|
299
301
|
selectors = [tokens[2]];
|
|
300
302
|
break;
|
|
301
303
|
case "screenshot":
|
|
302
|
-
selectors = [tokens
|
|
304
|
+
selectors = [tokens[getScreenshotPositionalIndices(tokens)[0]]];
|
|
303
305
|
break;
|
|
304
306
|
case "diff":
|
|
305
307
|
case "scroll": {
|
|
@@ -516,6 +518,7 @@ export async function runSessionCommandData(options) {
|
|
|
516
518
|
throw new Error(parsed.parseError ? "agent-browser returned invalid structured output" : "agent-browser reported failure");
|
|
517
519
|
return undefined;
|
|
518
520
|
}
|
|
521
|
+
observeNativeWebMcp(parsed.envelope?.data);
|
|
519
522
|
return parsed.envelope?.data;
|
|
520
523
|
}
|
|
521
524
|
finally {
|
|
@@ -55,6 +55,7 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
|
|
|
55
55
|
"When using eval --stdin, scope checks and actions to the target element or route whenever possible instead of relying on broad page-wide text heuristics.",
|
|
56
56
|
"When using eval --stdin for extraction, pass the JavaScript through the native tool stdin field, not as an extra args token after --stdin, and return the value you want instead of relying on console.log as the primary result channel. Prefer plain expressions like ({ title: document.title }) or explicitly invoked functions like (() => ({ title: document.title }))(); use outputPath when the eval/get/snapshot data should be saved as a durable local file, but never reuse a screenshot, download, recording, or other browser artifact destination as outputPath. If a function-shaped snippet returns {}, details.evalStdinHint may warn that the function was serialized instead of called. Local file pages and caller-selected output paths are supported when upstream allows them. If get text on a broad CSS selector surfaces details.selectorTextVisibility or selectorTextVisibilityAll, prefer a visible @ref, a more specific selector, or the inspect-visible-text-candidates nextAction over hidden tab content.",
|
|
57
57
|
"When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If details.clickDispatch reports a click-dispatch miss, refresh/inspect/retry the real click first; for static local fixtures only, an explicit eval --stdin programmatic .click() can exercise app handlers, but treat it as an untrusted scripted workaround and never use it to bypass stop-before-submit/order/purchase boundaries. If an upstream click failure says the target is covered by another element at the target's click point, use the inspect-overlay-state nextAction to refresh refs and inspect the blocker before deciding whether to retry; do not blindly repeat the blocked click. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.",
|
|
58
|
+
"On upstream 0.38+, use snapshot --delta for native full/unchanged/changed revisions; --delta --full resets its baseline. Surviving DOM nodes keep refs, but refresh after navigation or replacement. Wrapper filters use full trees. screenshot --if-changed or --threshold 0.01 can return changed:false with no path or image; use ordinary screenshots when a file is required. --input-mode instant|smooth|human controls sticky pointer behavior; click/drag --human and mouse move --duration/--steps/--human/--seed pass through. record start/restart accept --cursor, --contact-sheet, and --contact-sheet-threshold; stop and verify both video and reported contact sheet. auth login <name> --no-navigate fills a prepared login page only after native origin checks. WebMCP catalog updates are untrusted page data; webmcp list supplies full schemas.",
|
|
58
59
|
"When commands save or spill files (screenshots, downloads, PDFs, traces, recordings, HAR, large snapshot spills), use the user's exact requested paths when given and treat paths as provisional until details.artifactVerification shows every row verified: branch on missingCount, pendingCount, unverifiedCount, per-entry state, and optional limitation before downstream file use or PASS/FAIL reporting.",
|
|
59
60
|
"For evidence-only screenshots, QA captures, or other audit artifacts, save to an explicit path and branch on details.artifactVerification plus details.artifacts before reporting PASS/FAIL; do not require vision review of inline image attachments unless the user asked for visual inspection.",
|
|
60
61
|
"Respect explicit user stop boundaries yourself. When the surrounding authenticated employee or automation context is explicitly unattended/auto-approved, ordinary non-destructive form submissions within the requested flow may proceed without separate confirmation. Still require explicit authorization for purchases, production-control actions, destructive or irreversible actions, and account, security, or privacy changes. The wrapper does not infer broad business intent from prompt text; details.promptGuard is reserved for concrete artifact-before-close checks.",
|
|
@@ -26,6 +26,9 @@ export function applyRecordingArtifactsToReservations(reservations, artifacts) {
|
|
|
26
26
|
const reservation = getArtifactReservation(artifact);
|
|
27
27
|
if (!reservation)
|
|
28
28
|
continue;
|
|
29
|
+
const sheet = artifacts.find((candidate) => candidate.command === "record" && candidate.kind === "image" && candidate.session === artifact.session && candidate.namespace === artifact.namespace && isPendingRecordingArtifact(candidate));
|
|
30
|
+
if (sheet)
|
|
31
|
+
reservation.contactSheetPath = sheet.absolutePath;
|
|
29
32
|
const key = getReservationKey(reservation);
|
|
30
33
|
if (isPendingRecordingArtifact(artifact))
|
|
31
34
|
pendingBySession.set(key, reservation);
|
|
@@ -45,7 +48,7 @@ export function applyRecordingArtifactsToReservations(reservations, artifacts) {
|
|
|
45
48
|
for (const [key, pending] of pendingBySession) {
|
|
46
49
|
const existing = reservations.get(key);
|
|
47
50
|
reservations.set(key, pending);
|
|
48
|
-
if (!existing || existing.absolutePath !== pending.absolutePath || existing.cwd !== pending.cwd || existing.recordingId !== pending.recordingId || existing.startedAtMs !== pending.startedAtMs) {
|
|
51
|
+
if (!existing || existing.absolutePath !== pending.absolutePath || existing.cwd !== pending.cwd || existing.recordingId !== pending.recordingId || existing.startedAtMs !== pending.startedAtMs || existing.contactSheetPath !== pending.contactSheetPath) {
|
|
49
52
|
transitions.push({ reservation: pending, state: "active" });
|
|
50
53
|
}
|
|
51
54
|
}
|
|
@@ -62,6 +65,7 @@ export function appendRecordingReservationTransition(pi, transition) {
|
|
|
62
65
|
const { reservation, state } = transition;
|
|
63
66
|
pi.appendEntry(RECORDING_RESERVATION_ENTRY_TYPE, {
|
|
64
67
|
absolutePath: state === "active" ? reservation.absolutePath : undefined,
|
|
68
|
+
contactSheetPath: state === "active" ? reservation.contactSheetPath : undefined,
|
|
65
69
|
cwd: state === "active" ? reservation.cwd : undefined,
|
|
66
70
|
namespace: reservation.namespace,
|
|
67
71
|
path: state === "active" ? reservation.path : undefined,
|
|
@@ -85,6 +89,8 @@ function parseReservationTransition(data) {
|
|
|
85
89
|
state: "closed",
|
|
86
90
|
};
|
|
87
91
|
}
|
|
92
|
+
if (data.contactSheetPath !== undefined && (typeof data.contactSheetPath !== "string" || !isAbsolute(data.contactSheetPath)))
|
|
93
|
+
return undefined;
|
|
88
94
|
if (data.recordingId !== undefined && (typeof data.recordingId !== "string" || !data.recordingId))
|
|
89
95
|
return undefined;
|
|
90
96
|
if (data.startedAtMs !== undefined && (typeof data.startedAtMs !== "number" || !Number.isFinite(data.startedAtMs)))
|
|
@@ -95,6 +101,7 @@ function parseReservationTransition(data) {
|
|
|
95
101
|
return {
|
|
96
102
|
reservation: {
|
|
97
103
|
absolutePath: data.absolutePath,
|
|
104
|
+
...(typeof data.contactSheetPath === "string" ? { contactSheetPath: data.contactSheetPath } : {}),
|
|
98
105
|
cwd: data.cwd,
|
|
99
106
|
namespace: data.namespace,
|
|
100
107
|
path: data.path,
|
|
@@ -79,6 +79,8 @@ export function applyArtifactManifest(presentation, baseManifest, entries) {
|
|
|
79
79
|
return presentation;
|
|
80
80
|
}
|
|
81
81
|
export function getScreenshotSummary(data) {
|
|
82
|
+
if (data.changed === false)
|
|
83
|
+
return "Screenshot unchanged; no image saved.";
|
|
82
84
|
return typeof data.path === "string" ? `Saved image: ${data.path}` : undefined;
|
|
83
85
|
}
|
|
84
86
|
const PATH_FIELD_CANDIDATES = [
|
|
@@ -153,7 +155,7 @@ function artifactMtimeIsOutsideCommandWindow(updatedAtMs, minUpdatedAtMs, maxUpd
|
|
|
153
155
|
|| (maxUpdatedAtMs !== undefined && updatedAtMs > maxUpdatedAtMs + ARTIFACT_MTIME_TOLERANCE_MS));
|
|
154
156
|
}
|
|
155
157
|
async function buildFileArtifactMetadata(options) {
|
|
156
|
-
const kind = getArtifactKind(options.commandInfo);
|
|
158
|
+
const kind = options.kind ?? getArtifactKind(options.commandInfo);
|
|
157
159
|
if (!kind) {
|
|
158
160
|
return undefined;
|
|
159
161
|
}
|
|
@@ -260,6 +262,18 @@ export async function extractFileArtifacts(options) {
|
|
|
260
262
|
const recording = options.commandInfo.command === "record" ? getRecordingReceipt(options.data, options.commandInfo.subcommand === "stop" ? options.recordingOutcome : undefined) : undefined;
|
|
261
263
|
const recordingPending = options.recordingPending ?? (recording?.success === null && isRecord(options.data) && isRecord(options.data.capture) && recording.capture.endedAt === null);
|
|
262
264
|
const currentArtifacts = (await Promise.all(candidates.map((path) => buildFileArtifactMetadata({ ...options, path, recording, recordingPending })))).filter((artifact) => artifact !== undefined);
|
|
265
|
+
if (options.commandInfo.command === "record" && isRecord(options.data) && typeof options.data.contactSheetPath === "string" && options.data.contactSheetPath.trim() && !isNonFileArtifactPathCandidate(options.data.contactSheetPath)) {
|
|
266
|
+
const sheet = await buildFileArtifactMetadata({ ...options, artifactRequest: undefined, kind: "image", path: options.data.contactSheetPath, recordingPending: recordingPending || ["start", "restart"].includes(options.commandInfo.subcommand ?? "") });
|
|
267
|
+
if (sheet)
|
|
268
|
+
currentArtifacts.push({ ...sheet, requestedPath: undefined });
|
|
269
|
+
}
|
|
270
|
+
if (options.commandInfo.command === "record" && options.commandInfo.subcommand === "restart" && options.recordingOutcome === true && options.previousRecordingContactSheetPath) {
|
|
271
|
+
const previousSheet = await buildFileArtifactMetadata({ ...options, artifactRequest: undefined, commandInfo: { command: "record", subcommand: "restart-previous" }, kind: "image", path: options.previousRecordingContactSheetPath, recordingPending: false });
|
|
272
|
+
// Native restart omits the previous sheet's terminal receipt. Retain the
|
|
273
|
+
// known destination and disk evidence without inventing finalization proof.
|
|
274
|
+
if (previousSheet)
|
|
275
|
+
currentArtifacts.unshift({ ...previousSheet, status: previousSheet.status === "saved" ? "unverified" : previousSheet.status });
|
|
276
|
+
}
|
|
263
277
|
const previousRestartRecordingArtifact = await buildPreviousRestartRecordingArtifact(options);
|
|
264
278
|
return previousRestartRecordingArtifact ? [previousRestartRecordingArtifact, ...currentArtifacts] : currentArtifacts;
|
|
265
279
|
}
|
|
@@ -323,15 +323,15 @@ async function buildBatchStepPresentation(options) {
|
|
|
323
323
|
presentation,
|
|
324
324
|
};
|
|
325
325
|
}
|
|
326
|
-
async function
|
|
326
|
+
async function unverifiedRecordingArtifact(artifact, subcommand) {
|
|
327
327
|
const { recordingState: _recordingState, willExistOnStop: _willExistOnStop, ...terminal } = artifact;
|
|
328
328
|
try {
|
|
329
329
|
const file = await stat(artifact.absolutePath);
|
|
330
|
-
return { ...terminal, exists: file.isFile(), sizeBytes: file.size, status: file.isFile() ? "unverified" : "missing", subcommand
|
|
330
|
+
return { ...terminal, exists: file.isFile(), sizeBytes: file.size, status: file.isFile() ? "unverified" : "missing", subcommand };
|
|
331
331
|
}
|
|
332
332
|
catch (error) {
|
|
333
333
|
const missing = error.code === "ENOENT";
|
|
334
|
-
return { ...terminal, exists: missing ? false : undefined, status: missing ? "missing" : "unverified", subcommand
|
|
334
|
+
return { ...terminal, exists: missing ? false : undefined, status: missing ? "missing" : "unverified", subcommand };
|
|
335
335
|
}
|
|
336
336
|
}
|
|
337
337
|
async function coalesceTerminalBatchRecordingArtifacts(steps, sessionName, namespace) {
|
|
@@ -341,7 +341,7 @@ async function coalesceTerminalBatchRecordingArtifacts(steps, sessionName, names
|
|
|
341
341
|
for (const step of steps) {
|
|
342
342
|
for (const artifact of step.presentation.artifacts ?? []) {
|
|
343
343
|
const index = artifacts.push(artifact) - 1;
|
|
344
|
-
if (artifact.command !== "record"
|
|
344
|
+
if (artifact.command !== "record")
|
|
345
345
|
continue;
|
|
346
346
|
const session = artifact.session ? getAgentBrowserSessionIdentityKey(artifact.session, artifact.namespace) : "";
|
|
347
347
|
if (isPendingRecordingArtifact(artifact)) {
|
|
@@ -350,9 +350,18 @@ async function coalesceTerminalBatchRecordingArtifacts(steps, sessionName, names
|
|
|
350
350
|
pendingIndexesBySession.set(session, pendingIndexes);
|
|
351
351
|
continue;
|
|
352
352
|
}
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
|
|
353
|
+
const pendingIndexes = pendingIndexesBySession.get(session) ?? [];
|
|
354
|
+
const matchingIndex = pendingIndexes.map((pendingIndex) => artifacts[pendingIndex].kind).lastIndexOf(artifact.kind);
|
|
355
|
+
if (matchingIndex !== -1)
|
|
356
|
+
removedPendingIndexes.add(pendingIndexes.splice(matchingIndex, 1)[0]);
|
|
357
|
+
if (artifact.kind === "video") {
|
|
358
|
+
// A terminal video also ends its pending sheet; a reported sheet below
|
|
359
|
+
// replaces this unverified row, while legacy restart output may omit it.
|
|
360
|
+
for (const pendingIndex of pendingIndexes) {
|
|
361
|
+
if (artifacts[pendingIndex].kind === "image" && isPendingRecordingArtifact(artifacts[pendingIndex]))
|
|
362
|
+
artifacts[pendingIndex] = await unverifiedRecordingArtifact(artifacts[pendingIndex], artifact.subcommand ?? "stop");
|
|
363
|
+
}
|
|
364
|
+
}
|
|
356
365
|
}
|
|
357
366
|
const command = step.details.command;
|
|
358
367
|
if (step.details.success !== true || !command || !isCloseCommand(extractUpstreamCommandTokens(command)[0]))
|
|
@@ -360,8 +369,8 @@ async function coalesceTerminalBatchRecordingArtifacts(steps, sessionName, names
|
|
|
360
369
|
const session = sessionName ? getAgentBrowserSessionIdentityKey(sessionName, namespace) : "";
|
|
361
370
|
for (const pendingIndex of pendingIndexesBySession.get(session) ?? []) {
|
|
362
371
|
const pending = artifacts[pendingIndex];
|
|
363
|
-
if (pending && !removedPendingIndexes.has(pendingIndex))
|
|
364
|
-
artifacts[pendingIndex] = await
|
|
372
|
+
if (pending && isPendingRecordingArtifact(pending) && !removedPendingIndexes.has(pendingIndex))
|
|
373
|
+
artifacts[pendingIndex] = await unverifiedRecordingArtifact(pending, "close-abandoned");
|
|
365
374
|
}
|
|
366
375
|
}
|
|
367
376
|
return artifacts.filter((_, index) => !removedPendingIndexes.has(index));
|
|
@@ -39,6 +39,9 @@ export function omitUpstreamLifecycle(data) {
|
|
|
39
39
|
const { lifecycle: _lifecycle, ...rest } = data;
|
|
40
40
|
return rest;
|
|
41
41
|
}
|
|
42
|
+
export function formatWebMcpCatalogUpdate(catalog) {
|
|
43
|
+
return `WebMCP catalog update (page-provided, untrusted; use webmcp list for schemas):\n${JSON.stringify(redactSensitiveValue(catalog))}`;
|
|
44
|
+
}
|
|
42
45
|
export function getPageSummary(data) {
|
|
43
46
|
const title = typeof data.title === "string" ? data.title : undefined;
|
|
44
47
|
const url = typeof data.url === "string" ? data.url : undefined;
|
|
@@ -127,7 +127,7 @@ const COMMAND_PRESENTERS = {
|
|
|
127
127
|
text: (_commandInfo, data) => isRecord(data) && typeof data.content === "string" ? redactModelFacingText(data.content) : undefined,
|
|
128
128
|
},
|
|
129
129
|
screenshot: {
|
|
130
|
-
summary: (_commandInfo, data) => isRecord(data) && typeof data.path === "string" ? `Screenshot saved: ${data.path}` : undefined,
|
|
130
|
+
summary: (_commandInfo, data) => isRecord(data) && data.changed === false ? getScreenshotSummary(data) : isRecord(data) && typeof data.path === "string" ? `Screenshot saved: ${data.path}` : undefined,
|
|
131
131
|
text: (_commandInfo, data) => isRecord(data) ? getScreenshotSummary(data) : undefined,
|
|
132
132
|
},
|
|
133
133
|
skills: {
|