pi-agent-browser-native 0.6.6 → 0.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -1
- package/README.md +21 -6
- package/dist/extensions/agent-browser/index.js +138 -75
- package/dist/extensions/agent-browser/lib/command-taxonomy.js +6 -5
- package/dist/extensions/agent-browser/lib/electron/cleanup.js +10 -1
- package/dist/extensions/agent-browser/lib/input-modes/params.js +20 -7
- package/dist/extensions/agent-browser/lib/launch-scoped-flags.js +0 -1
- package/dist/extensions/agent-browser/lib/managed-session-restore.js +13 -12
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.js +6 -25
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.js +2 -2
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/final-result.js +2 -3
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/index.js +10 -3
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/managed-session-daemon-policy.js +3 -0
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare/wait-timeouts.js +1 -1
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/prepare.js +104 -117
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/process-output.js +64 -36
- package/dist/extensions/agent-browser/lib/orchestration/browser-run/session-state.js +100 -135
- package/dist/extensions/agent-browser/lib/orchestration/electron-host/index.js +3 -1
- package/dist/extensions/agent-browser/lib/orchestration/input-plan.js +3 -1
- package/dist/extensions/agent-browser/lib/page-target-validation.js +10 -10
- package/dist/extensions/agent-browser/lib/parsing.js +7 -0
- package/dist/extensions/agent-browser/lib/playbook.js +6 -9
- package/dist/extensions/agent-browser/lib/process.js +28 -113
- package/dist/extensions/agent-browser/lib/recording-reservations.js +3 -1
- package/dist/extensions/agent-browser/lib/results/action-recommendations.js +5 -2
- package/dist/extensions/agent-browser/lib/results/envelope.js +9 -4
- package/dist/extensions/agent-browser/lib/results/next-actions.js +8 -0
- package/dist/extensions/agent-browser/lib/results/presentation/artifacts.js +45 -43
- package/dist/extensions/agent-browser/lib/results/presentation/batch.js +2 -1
- package/dist/extensions/agent-browser/lib/results/presentation/errors.js +10 -2
- package/dist/extensions/agent-browser/lib/results/presentation.js +9 -2
- package/dist/extensions/agent-browser/lib/results/recovery-actions.js +4 -4
- package/dist/extensions/agent-browser/lib/runtime.js +18 -2
- package/dist/extensions/agent-browser/lib/session-page-state.js +29 -10
- package/docs/ARCHITECTURE.md +8 -5
- package/docs/COMMAND_REFERENCE.md +25 -16
- package/docs/ELECTRON.md +6 -6
- package/docs/RELEASE.md +14 -5
- package/docs/REQUIREMENTS.md +1 -1
- package/docs/SUPPORT_MATRIX.md +18 -5
- package/docs/TOOL_CONTRACT.md +38 -25
- package/package.json +5 -1
|
@@ -13,7 +13,7 @@ export const QUICK_START_GUIDELINES = [
|
|
|
13
13
|
"Common first calls (first-call recipe): { args: [\"open\", \"<url>\"] } → { args: [\"snapshot\", \"-i\"] } → { args: [\"click\", \"@eN\"] } or { args: [\"fill\", \"@eN\", \"<text>\"] } using @refs and visible labels from that snapshot, then { args: [\"snapshot\", \"-i\"] } after navigation or DOM changes. On https://example.com/ the main link label is Learn more (use exact snapshot text, not guessed link copy).",
|
|
14
14
|
"Locator-first clicks/fills and native select changes without hand-building argv: { semanticAction: { action: \"click\", locator: \"text\", value: \"Close\" } }, { semanticAction: { action: \"fill\", locator: \"label\", value: \"Email\", text: \"user@example.com\" } }, direct current targets such as { semanticAction: { action: \"fill\", selector: \"@e1\", text: \"prompt\" } }, or { semanticAction: { action: \"select\", selector: \"#flavor\", value: \"chocolate\" } }; add semanticAction.session when targeting a named upstream browser session; details.compiledSemanticAction shows the semantic target, while details.effectiveArgs may show a resolved current @ref for active-session role/name click/check/fill actions to avoid hidden duplicate matches; semanticAction does not expose uncheck while upstream find ... uncheck is not runtime-supported, so use raw uncheck with a stable selector or current ref; selector-not-found failures may append bounded click try-*-candidate next actions or, for fill misses with current editable refs, details.richInputRecovery with focus/click actions that do not copy fill text; stale-ref failures can return retry-semantic-action-after-stale-ref for compiled find actions when retry safety is provable.",
|
|
15
15
|
`Common advanced calls: { args: ["batch", "--bail"], stdin: "[[\"open\",\"https://example.com\"],[\"snapshot\",\"-i\"]]" }, { job: { steps: [{ action: "open", url: "https://example.com" }, { action: "assertText", text: "Example Domain" }, { action: "screenshot", path: ".dogfood/example.png" }] } }, { qa: { url: "https://example.com", expectedText: "Example Domain", screenshotPath: ".dogfood/qa-example.png" } } (example.com smoke only; elsewhere match exact visible text from snapshot -i), { electron: { action: "list", query: "code" } }, { electron: { action: "launch", appName: "Visual Studio Code", handoff: "snapshot" } }, { electron: { action: "probe" } }, { qa: { attached: true, expectedText: "Explorer" } }, { args: ["eval", "--stdin"], stdin: "document.title", outputPath: "logs/page-title.json" }, { args: ["auth", "save", "name", "--password-stdin"], stdin: "<password from user-approved secret source>" }, { args: ["--profile", "Default", "open", "https://example.com/account"], sessionMode: "fresh" }, and { args: ["open", "--enable", "react-devtools", "https://example.com"], sessionMode: "fresh" }. For app pages with a native dropdown, job steps can include { action: "select", selector: "#flavor", value: "chocolate" } before the dependent assertion; for locator-friendly pages, job click/fill steps can use semantic locator fields such as { action: "fill", locator: "role", role: "searchbox", name: "Search", text: "agent browser" }; for human-paced input, job type steps can use { action: "type", selector: "#prompt", text: "hello", delayMs: 20, press: "Enter" }; delayed typing is capped at 200 characters per step, and generated per-character rows are compacted in visible batch prose while full rows remain in details.batchSteps.`,
|
|
16
|
-
"Constrained job navigation is explicit only: click (and select/submit flows that may navigate) does not prove the next page loaded; add an assertUrl that does not already match the starting page and/or assertText for new page state after navigation-prone steps before screenshot or later interactions. Keep jobs short around navigation, click, and rerender boundaries on dynamic React/product apps; avoid a whole checkout in one job. If a long job times out and details.timeoutPartialProgress shows a mutating incomplete step, inspect current page state and continue with a shorter job or single action instead of blindly retrying the mutating step. Example: { job: { steps: [{ action: \"open\", url: \"https://shop.example/checkout\" }, { action: \"fill\", selector: \"#email\", text: \"user@example.com\" }, { action: \"click\", selector: \"#continue\" }, { action: \"assertUrl\", url: \"**/shipping\" }, { action: \"assertText\", text: \"Shipping address\" }, { action: \"screenshot\", path: \".dogfood/shipping.png\" }] } }. Top-level click may add pageChangeSummary hints, but job never auto-inserts post-click asserts.",
|
|
16
|
+
"Constrained job navigation is explicit only: click (and select/submit flows that may navigate) does not prove the next page loaded; add an assertUrl that does not already match the starting page and/or assertText for new page state after navigation-prone steps before screenshot or later interactions. assertText takes only text, not selector or locator fields. Clicks can stale subsequent @refs: split the job and re-snapshot before using those refs. Keep jobs short around navigation, click, and rerender boundaries on dynamic React/product apps; avoid a whole checkout in one job. If a long job times out and details.timeoutPartialProgress shows a mutating incomplete step, inspect current page state and continue with a shorter job or single action instead of blindly retrying the mutating step. Example: { job: { steps: [{ action: \"open\", url: \"https://shop.example/checkout\" }, { action: \"fill\", selector: \"#email\", text: \"user@example.com\" }, { action: \"click\", selector: \"#continue\" }, { action: \"assertUrl\", url: \"**/shipping\" }, { action: \"assertText\", text: \"Shipping address\" }, { action: \"screenshot\", path: \".dogfood/shipping.png\" }] } }. Top-level click may add pageChangeSummary hints, but job never auto-inserts post-click asserts.",
|
|
17
17
|
"High-value command reference: click <selector> --new-tab opens link-like targets in a new tab; select <selector> <value...> changes native dropdown values; wrapper-handled scroll <dir> [px|percent] and scroll to end/top target document scrolling before upstream fallback, while scroll <selector> <dir> [px|percent] targets nested scrollers; download <selector> <path> saves a file triggered by a click; read [url] returns agent-readable text (explicit URLs prefer markdown without requiring a Chrome page; omit the URL for rendered active-tab DOM); get title/url need no selector; get text/html/value/count <selector> and get attr <selector> <name> read elements/page state (use body for whole-page text/html); screenshot [selector] [path] captures a page or element image; pdf <path> saves a PDF; tab list and tab <tab-id-or-label> inspect or recover the active tab; react tree, react inspect <fiberId>, react renders start/stop, and react suspense introspect React after --enable react-devtools; vitals [url] measures Core Web Vitals; pushstate <url> performs SPA navigation; tap <selector> and swipe <direction> [distance] support iOS/provider touch flows.",
|
|
18
18
|
"For artifact-producing commands, read the visible artifact block and details.artifactVerification before using files: check requested path, absolute path, existence, size bytes, artifact kind, optional mediaType, status, optional limitation, and verified/missing/pending/unverified counts. details.artifacts contains per-file metadata; record start rows are pending/openRecording until record stop writes the target. Upstream record start uses a fresh active page for video capture, so prior in-page DOM and JavaScript state does not carry over; the wrapper blocks prior @e… refs as stale-ref even when the start fails as already-active, and record restart with a URL navigates and invalidates the same way (plain record restart keeps the page), so take a fresh snapshot before continuing. The wrapper creates parent directories for direct artifact paths and can save simple loopback HTTP(S) anchor downloads directly to the requested path before upstream download fallback. Browser close does not delete explicit saved files; if close reports details.artifactCleanup, use host file tools to remove paths listed in explicitArtifactPaths (when non-empty) after inspection. If close fails with details.promptGuard.reason=requested-artifacts-missing-before-close, save the exact required artifact path before closing. A bare inbound image/video path is not a requested output artifact and does not block close. For annotated screenshots inside batch, put --annotate in top-level args (for example { args: [\"--annotate\", \"batch\"], stdin: \"[[\\\"screenshot\\\",\\\"/tmp/page.png\\\"]]\" }) rather than inside the screenshot step; if annotation labels crowd a dense page, use a scoped or non-annotated screenshot plus snapshot refs instead.",
|
|
19
19
|
"When failure output shows Next actions, prefer those exact native agent_browser follow-up payloads over guessed commands. The same actions are available in details.nextActions to callers that expose structured details; short stdin is shown inline, while long stdin stays details-only.",
|
|
@@ -46,14 +46,14 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
|
|
|
46
46
|
"For Electron desktop apps, prefer top-level electron for wrapper-owned discovery, isolated launch, status, compact probe, and cleanup: list first, treat likely-sensitive annotations as hints rather than enforcement, launch with the default snapshot handoff unless handoff: \"tabs\" is the safer diagnostic starting point, use electron.probe or snapshot -i/qa.attached for current-session state, and always cleanup the returned launchId when done. electron.launch uses an isolated temporary profile; it does not reuse the app's normal signed-in profile or attach to an already-running authenticated app. For signed-in local app state, host-launch the normal app with --remote-debugging-port when appropriate, then use raw args connect <port|url>; after connect, run get url to verify the active target before page-content reads, inspect tab list, select the stable tab id such as tab t2, verify it again with get url, then run a condition wait or snapshot -i before using refs. close commands (`close`, `quit`, or `exit`) only close the browser/CDP session; leave manually launched app shutdown, profile cleanup, and explicit artifacts to the host owner.",
|
|
47
47
|
"For provider or specialized app workflows, load version-matched upstream guidance with skills get agentcore|electron|slack|dogfood|vercel-sandbox|derive-client through the native tool; add --full when you need references/templates, and use skills get --all only for broad skill audits. Use derive-client when recording HAR traffic to generate a standalone API client; prefer network har start (text bodies by default) or network har start --content all|none before multi-step capture. For accessibility audits use a11y or a11y --tags wcag2a,wcag2aa (CDP browsers only). Hosted sandbox workflows should use upstream @agent-browser/sandbox helpers outside this wrapper. Provider launches such as -p ios, --provider browserbase/kernel/browseruse/browserless/agentcore, and iOS --device are upstream-owned setup paths; use sessionMode fresh when switching providers and expect external credentials or local Appium/Xcode setup to be required.",
|
|
48
48
|
"For dialogs and frames, use dialog status/accept/dismiss and frame <selector|main> through native args; dialog commands and eval snippets that look like alert/confirm/prompt/dialog triggers are shorter-bounded than normal browser calls, and timed-out dialog-like interactions may add inspect-dialog-after-timeout, dismiss-dialog-after-timeout, or recover-fresh-session-after-dialog-timeout nextActions. When --confirm-actions produces a pending confirmation, use details.nextActions or exact confirm <id> / deny <id> calls instead of inventing ids.",
|
|
49
|
-
"If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For headed demos, put --headed on the first launch with sessionMode=fresh and verify with screenshot/tab/get-url evidence because tool success cannot prove the OS window is visible to the user. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; for disappearance checks, use wait --fn predicates instead of stale upstream-help examples like wait <selector> --state hidden. Use electron.probe/status for wrapper-owned launch health or target mismatch. Fixed waits are a last resort:
|
|
49
|
+
"If a session lands on the wrong page or tab, an interaction changes origin unexpectedly, or an open call returns blocked, blank, or otherwise unexpected results, use tab list / tab <tab-id-or-label> / snapshot -i to recover state before retrying different URLs or fallback strategies. For headed demos, put --headed on the first launch with sessionMode=fresh and verify with screenshot/tab/get-url evidence because tool success cannot prove the OS window is visible to the user. For desktop readiness, prefer real conditions first: wait --text, wait --url, wait --fn, wait --load <state>, wait --download, or qa.attached; for disappearance checks, use wait --fn predicates instead of stale upstream-help examples like wait <selector> --state hidden. Use electron.probe/status for wrapper-owned launch health or target mismatch. Fixed waits are a last resort: their duration is positional (wait <ms>, not wait --time <ms>). Use explicit --timeout or top-level timeoutMs for legitimately slow waits, and treat a successful payload like \"waited\":\"timeout\" as elapsed time only—verify completion with an observed condition, fresh snapshot, or screenshot.",
|
|
50
50
|
"For feed, timeline, or inbox reading tasks, focus on the main timeline/list region and read the first item there rather than unrelated composer or sidebar content.",
|
|
51
51
|
"For read-only browsing tasks, use read <url> for documentation or other unstructured text without requiring a Chrome page, or read with no URL for rendered active-tab DOM. Prefer the current snapshot, structured ref labels, getters, or scoped eval --stdin when you need interactive structure or targeted page state. Only click into media viewers, detail routes, or new pages when the current view does not contain the needed information.",
|
|
52
52
|
"For downloads, prefer download <selector> <path> when an element click should save a file; simple loopback anchor downloads are saved to the requested path when the wrapper can resolve an HTTP(S) href. Do not rely on click alone when you need the downloaded file on disk.",
|
|
53
|
-
"On dashboards with nested scroll containers, verify scroll with a screenshot or fresh snapshot -i; if the viewport did not move, details.data.scrolled may be false/noMovement true and you should prefer scrollintoview <@ref> or target the actual scrollable region with scroll <selector> <dir> [px|percent]. For native selects, use select <selector> <value...> (or semanticAction/job select) instead of clicking option refs; for custom comboboxes, a click/semanticAction may only focus the field, so re-snapshot and
|
|
53
|
+
"On dashboards with nested scroll containers, verify scroll with a screenshot or fresh snapshot -i; if the viewport did not move, details.data.scrolled may be false/noMovement true and you should prefer scrollintoview <@ref> or target the actual scrollable region with scroll <selector> <dir> [px|percent]. For native selects, use select <selector> <value...> (or semanticAction/job select) instead of clicking option refs; for custom comboboxes, a click/semanticAction may only focus the field, so re-snapshot and use keyboard type <text> for focused input, press ArrowDown or press Enter, or visible option refs. Raw type requires both <selector> and <text>.",
|
|
54
54
|
"When using eval --stdin, scope checks and actions to the target element or route whenever possible instead of relying on broad page-wide text heuristics.",
|
|
55
55
|
"When using eval --stdin for extraction, pass the JavaScript through the native tool stdin field, not as an extra args token after --stdin, and return the value you want instead of relying on console.log as the primary result channel. Prefer plain expressions like ({ title: document.title }) or explicitly invoked functions like (() => ({ title: document.title }))(); use outputPath when the eval/get/snapshot data should be saved as a durable local file, but never reuse a screenshot, download, recording, or other browser artifact destination as outputPath. If a function-shaped snippet returns {}, details.evalStdinHint may warn that the function was serialized instead of called. Local file pages and caller-selected output paths are supported when upstream allows them. If get text on a broad CSS selector surfaces details.selectorTextVisibility or selectorTextVisibilityAll, prefer a visible @ref, a more specific selector, or the inspect-visible-text-candidates nextAction over hidden tab content.",
|
|
56
|
-
"When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If details.clickDispatch reports a click-dispatch miss, refresh/inspect/retry the real click first; for static local fixtures only, an explicit eval --stdin programmatic .click() can exercise app handlers, but treat it as an untrusted scripted workaround and never use it to bypass stop-before-submit/order/purchase boundaries. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.",
|
|
56
|
+
"When details.pageChangeSummary is present, use changeType and summary as a compact signal for navigation, DOM mutation, confirmations, or artifacts; when nextActionIds is set, match those ids to entries in details.nextActions (or per-step nextActions inside batch) for concrete follow-up payloads instead of inferring from prose alone. If details.clickDispatch reports a click-dispatch miss, refresh/inspect/retry the real click first; for static local fixtures only, an explicit eval --stdin programmatic .click() can exercise app handlers, but treat it as an untrusted scripted workaround and never use it to bypass stop-before-submit/order/purchase boundaries. If an upstream click failure says the target is covered by another element at the target's click point, use the inspect-overlay-state nextAction to refresh refs and inspect the blocker before deciding whether to retry; do not blindly repeat the blocked click. If a no-navigation click surfaces details.overlayBlockers, inspect the fresh snapshot evidence before using a close/dismiss candidate nextAction; ordinary page chrome without dialog/alertdialog evidence should not trigger this diagnostic.",
|
|
57
57
|
"When commands save or spill files (screenshots, downloads, PDFs, traces, recordings, HAR, large snapshot spills), use the user's exact requested paths when given and treat paths as provisional until details.artifactVerification shows every row verified: branch on missingCount, pendingCount, unverifiedCount, per-entry state, and optional limitation before downstream file use or PASS/FAIL reporting.",
|
|
58
58
|
"For evidence-only screenshots, QA captures, or other audit artifacts, save to an explicit path and branch on details.artifactVerification plus details.artifacts before reporting PASS/FAIL; do not require vision review of inline image attachments unless the user asked for visual inspection.",
|
|
59
59
|
"Respect explicit user stop boundaries yourself. When the surrounding authenticated employee or automation context is explicitly unattended/auto-approved, ordinary non-destructive form submissions within the requested flow may proceed without separate confirmation. Still require explicit authorization for purchases, production-control actions, destructive or irreversible actions, and account, security, or privacy changes. The wrapper does not infer broad business intent from prompt text; details.promptGuard is reserved for concrete artifact-before-close checks.",
|
|
@@ -62,10 +62,6 @@ export const SHARED_BROWSER_PLAYBOOK_GUIDELINES = [
|
|
|
62
62
|
];
|
|
63
63
|
export const TOOL_PROMPT_GUIDELINES_SUFFIX = [
|
|
64
64
|
"Prefer agent_browser over bash, osascript, AppleScript, or generic browser shell for sites, docs, clicks, fills, screenshots, eval, and batch.",
|
|
65
|
-
"Pass exact agent-browser CLI arguments in agent_browser args when you are not using script, semanticAction, job, or qa, excluding the binary name and --json (agent_browser injects --json automatically).",
|
|
66
|
-
"Use top-level agent_browser stdin only for eval --stdin, batch, auth save --password-stdin, or wrapper-generated job/qa batches instead of shell heredocs or password args; script puts any inner stdin on browser({ stdin }), and other command/stdin combinations are rejected before launch.",
|
|
67
|
-
`Let the agent_browser extension-managed session handle the common path unless you explicitly need a fresh launch for launch-scoped flags (${LAUNCH_SCOPED_FLAG_LABEL}).`,
|
|
68
|
-
"Use agent_browser sessionMode=fresh when switching from an existing implicit session to a new profile/browser executable/debug/init-script/provider launch without inventing a fixed explicit session name; later auto calls will follow that new session.",
|
|
69
65
|
];
|
|
70
66
|
export const INSPECTION_TOOL_CALL_EXAMPLES = [
|
|
71
67
|
'{ "args": ["--help"] }',
|
|
@@ -73,7 +69,8 @@ export const INSPECTION_TOOL_CALL_EXAMPLES = [
|
|
|
73
69
|
];
|
|
74
70
|
export const WRAPPER_TAB_RECOVERY_BEHAVIOR = [
|
|
75
71
|
"After open/goto/navigate calls with --profile, --restore, --session-name, or --state, agent_browser best-effort re-selects the tab whose URL matches the returned page when restored tabs steal focus during launch or reconnect.",
|
|
76
|
-
"After
|
|
72
|
+
"After confirmed shutdown of an automatically restored managed session, the wrapper retains its complete recorded URL, including the fragment, until the first current-page operation (including get url and reload). Non-page calls such as tab list or read <url> may start a daemon without fulfilling that reopen. The wrapper uses native open once, verifies the observed tab, and discards old refs/frame scope; it does not restore unsaved forms, JavaScript memory, or history. Explicit navigation, caller-owned/attached sessions, and restore-disabled sessions are not auto-reopened.",
|
|
73
|
+
"For a still-live browser after tab drift or resume, the wrapper verifies/selects the intended tab before ref/semantic helpers and page commands; failed selection stops the call without navigating. Local commands, read <url>, URL a11y/vitals, diff url, window new, and explicit tab/navigation/connection/state recovery do not require the prior tab. Batch checks follow effective rows past non-page prefixes and stop at explicit context changes, preserving caller argv/stdin and continue-on-error behavior. Same-tab reselection is avoided because it clears refs. Use exact batch --bail for fail-fast, not --bail=<value>. Routine same-session calls skip tab-list preflights.",
|
|
77
74
|
"For sessions with observed tab-drift risk, after a successful command on a known target tab, agent_browser also best-effort restores that intended tab if a restored/background tab steals focus after the command completes. Routine same-session commands skip this post-command tab-list probe.",
|
|
78
75
|
"If a known session target unexpectedly reports about:blank, agent_browser best-effort re-selects the prior intended target when it still exists; if recovery fails, it records the observed about:blank target and reports exact recovery guidance instead of treating the prior page as active.",
|
|
79
76
|
"If upstream reports tab_gone, the pinned bound tab is gone; use details.nextActions (tab list / tab new) instead of assuming another tab is yours.",
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
2
|
import { spawn } from "node:child_process";
|
|
3
|
-
import { lstat, mkdir, readdir } from "node:fs/promises";
|
|
3
|
+
import { lstat, mkdir, readdir, readlink, stat } from "node:fs/promises";
|
|
4
4
|
import { dirname, isAbsolute, join } from "node:path";
|
|
5
5
|
import { env as processEnv, platform as processPlatform } from "node:process";
|
|
6
|
+
import { spawn as crossSpawn } from "cross-spawn";
|
|
6
7
|
import { parseArgvDescriptor } from "./argv-descriptor.js";
|
|
7
|
-
import {
|
|
8
|
-
import { extractExplicitSessionName, getFlagName, GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES, GLOBAL_VALUE_FLAGS, optionalGlobalValueFlagConsumesNext, resolveAgentBrowserNamespace, } from "./argv-grammar.js";
|
|
8
|
+
import { extractExplicitSessionName, resolveAgentBrowserNamespace } from "./argv-grammar.js";
|
|
9
9
|
import { commitManagedSessionRestoreSuppression, getManagedSessionRestoreEnv, getManagedSessionRestoreProtectedEnv, getOwnedManagedSessionCompatibilityEnv, getOwnedManagedSessionNamespaceEnv, isOwnedManagedSessionTarget, validateManagedSessionRestoreContextForSpawn, } from "./managed-session-restore.js";
|
|
10
10
|
import { getPageTargetValidationError, } from "./page-target-validation.js";
|
|
11
11
|
import { getImplicitSessionIdleTimeoutMs } from "./runtime.js";
|
|
@@ -26,114 +26,19 @@ export const SAFE_AGENT_BROWSER_OPERATION_TIMEOUT_MS = 25_000;
|
|
|
26
26
|
const DEFAULT_AGENT_BROWSER_PROCESS_TIMEOUT_MS = 35_000;
|
|
27
27
|
/** Grace period after `exit` before resolving when `close` is delayed by inherited stdio handles. */
|
|
28
28
|
const EXIT_STDIO_GRACE_MS = 100;
|
|
29
|
-
const WINDOWS_AGENT_BROWSER_MISSING_MARKER = "PI_AGENT_BROWSER_COMMAND_NOT_FOUND:agent-browser.cmd";
|
|
30
29
|
const attachedBrowserSessionContext = new AsyncLocalStorage();
|
|
31
|
-
const WINDOWS_COMMANDS_WITH_ADJACENT_SUBCOMMAND = new Set([
|
|
32
|
-
"auth", "clipboard", "cookies", "dashboard", "device", "dialog", "diff", "find", "get", "is", "keyboard",
|
|
33
|
-
"mouse", "network", "plugin", "profiler", "react", "record", "session", "set", "skills", "state", "storage",
|
|
34
|
-
"stream", "tab", "trace", "webmcp", "window",
|
|
35
|
-
]);
|
|
36
30
|
export function withAttachedBrowserSessionContext(preserve, run) {
|
|
37
31
|
return attachedBrowserSessionContext.run(preserve || attachedBrowserSessionContext.getStore() === true, run);
|
|
38
32
|
}
|
|
39
|
-
export function getWindowsExplicitDefaultNamespaceEnv(args, parentNamespace, platform = processPlatform) {
|
|
40
|
-
return platform === "win32" && resolveAgentBrowserNamespace(args, parentNamespace) === ""
|
|
41
|
-
? { AGENT_BROWSER_NAMESPACE: "" }
|
|
42
|
-
: {};
|
|
43
|
-
}
|
|
44
33
|
function appendTail(text, addition, maxChars) {
|
|
45
34
|
const combined = text + addition;
|
|
46
35
|
return combined.length <= maxChars ? combined : combined.slice(combined.length - maxChars);
|
|
47
36
|
}
|
|
48
|
-
function quoteWindowsPowerShellArg(value) {
|
|
49
|
-
return `'${value.replace(/'/g, "''")}'`;
|
|
50
|
-
}
|
|
51
|
-
/** Exported for unit tests that lock Windows launcher argv ordering. */
|
|
52
|
-
export function reorderWindowsLeadingGlobalArgs(args) {
|
|
53
|
-
const leadingGlobals = [];
|
|
54
|
-
for (let index = 0; index < args.length; index += 1) {
|
|
55
|
-
const token = args[index];
|
|
56
|
-
if (isKnownCommandToken(token)) {
|
|
57
|
-
if (index === 0)
|
|
58
|
-
return args;
|
|
59
|
-
const firstPositional = args[index + 1];
|
|
60
|
-
return WINDOWS_COMMANDS_WITH_ADJACENT_SUBCOMMAND.has(token) && firstPositional && !firstPositional.startsWith("-")
|
|
61
|
-
? [token, firstPositional, ...leadingGlobals, ...args.slice(index + 2)]
|
|
62
|
-
: [token, ...leadingGlobals, ...args.slice(index + 1)];
|
|
63
|
-
}
|
|
64
|
-
if (!token.startsWith("-"))
|
|
65
|
-
return args;
|
|
66
|
-
if (token.startsWith("--restore=")) {
|
|
67
|
-
leadingGlobals.push(token);
|
|
68
|
-
continue;
|
|
69
|
-
}
|
|
70
|
-
if (token === "--restore") {
|
|
71
|
-
const value = args[index + 1];
|
|
72
|
-
if (optionalGlobalValueFlagConsumesNext(token, value)) {
|
|
73
|
-
leadingGlobals.push(`--restore=${value}`);
|
|
74
|
-
index += 1;
|
|
75
|
-
}
|
|
76
|
-
else {
|
|
77
|
-
leadingGlobals.push(token);
|
|
78
|
-
}
|
|
79
|
-
continue;
|
|
80
|
-
}
|
|
81
|
-
if (token.includes("="))
|
|
82
|
-
return args;
|
|
83
|
-
const flag = getFlagName(token);
|
|
84
|
-
if (GLOBAL_BOOLEAN_FLAGS_WITH_OPTIONAL_VALUES.has(flag)) {
|
|
85
|
-
leadingGlobals.push(token);
|
|
86
|
-
if (["true", "false"].includes(args[index + 1] ?? "")) {
|
|
87
|
-
leadingGlobals.push(args[index + 1]);
|
|
88
|
-
index += 1;
|
|
89
|
-
}
|
|
90
|
-
continue;
|
|
91
|
-
}
|
|
92
|
-
if (GLOBAL_VALUE_FLAGS.includes(flag)) {
|
|
93
|
-
const value = args[index + 1];
|
|
94
|
-
if (value === undefined)
|
|
95
|
-
return args;
|
|
96
|
-
// PowerShell -> .cmd drops empty argv values. Planning rejects empty
|
|
97
|
-
// caller --args; keep this defensive skip so an unexpected empty value
|
|
98
|
-
// cannot turn the next flag into its accidental value on native Windows.
|
|
99
|
-
if (value === "" && (flag === "--args" || flag === "--namespace")) {
|
|
100
|
-
index += 1;
|
|
101
|
-
continue;
|
|
102
|
-
}
|
|
103
|
-
leadingGlobals.push(token, value);
|
|
104
|
-
index += 1;
|
|
105
|
-
continue;
|
|
106
|
-
}
|
|
107
|
-
return args;
|
|
108
|
-
}
|
|
109
|
-
return args;
|
|
110
|
-
}
|
|
111
37
|
export function prepareAgentBrowserSpawnArgs(args, wrapperCompatibilityUserAgent, preserveAttachedBrowserSession = false) {
|
|
112
38
|
if (preserveAttachedBrowserSession || !wrapperCompatibilityUserAgent)
|
|
113
39
|
return args;
|
|
114
40
|
return ["--args", `--user-agent=${wrapperCompatibilityUserAgent.replaceAll(/[\r\n,]/g, "")}`, ...args];
|
|
115
41
|
}
|
|
116
|
-
export function buildAgentBrowserSpawnCommand(args, platform = processPlatform) {
|
|
117
|
-
if (platform !== "win32") {
|
|
118
|
-
return { command: "agent-browser", args };
|
|
119
|
-
}
|
|
120
|
-
const invocationArgs = reorderWindowsLeadingGlobalArgs(args).map(quoteWindowsPowerShellArg).join(" ");
|
|
121
|
-
const commandLine = [
|
|
122
|
-
"$agentBrowser = Get-Command agent-browser.cmd -ErrorAction SilentlyContinue;",
|
|
123
|
-
`if (-not $agentBrowser) { [Console]::Error.WriteLine('${WINDOWS_AGENT_BROWSER_MISSING_MARKER}'); exit 127 };`,
|
|
124
|
-
`& $agentBrowser.Source ${invocationArgs}`.trimEnd(),
|
|
125
|
-
].join(" ");
|
|
126
|
-
return { command: "powershell.exe", args: ["-NoLogo", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", commandLine] };
|
|
127
|
-
}
|
|
128
|
-
export function isWindowsAgentBrowserCommandMissing(stderr) {
|
|
129
|
-
const normalized = stderr.toLowerCase();
|
|
130
|
-
return normalized.includes(WINDOWS_AGENT_BROWSER_MISSING_MARKER.toLowerCase()) || (normalized.includes("agent-browser.cmd") && (normalized.includes("commandnotfoundexception") ||
|
|
131
|
-
normalized.includes("not recognized as the name of a cmdlet") ||
|
|
132
|
-
normalized.includes("not recognized as an internal or external command")));
|
|
133
|
-
}
|
|
134
|
-
export function shouldCommitManagedRestoreAfterWindowsProcess(input) {
|
|
135
|
-
return !input.spawnError && !(input.exitCode !== 0 && isWindowsAgentBrowserCommandMissing(input.stderr));
|
|
136
|
-
}
|
|
137
42
|
function terminateSpawnedChild(child, signal) {
|
|
138
43
|
if (processPlatform === "win32" && child.pid) {
|
|
139
44
|
const killer = spawn("taskkill.exe", ["/PID", String(child.pid), "/T", "/F"], { stdio: "ignore" });
|
|
@@ -252,13 +157,30 @@ export function isTrustedSocketDirAncestor(metadata, uid, platform = processPlat
|
|
|
252
157
|
return (mode & 0o022) === 0;
|
|
253
158
|
return metadata.uid === 0 && ((mode & 0o022) === 0 || (mode & 0o1000) !== 0);
|
|
254
159
|
}
|
|
255
|
-
async function hasTrustedSocketDirAncestry(socketDir, uid) {
|
|
160
|
+
async function hasTrustedSocketDirAncestry(socketDir, uid, visited = new Set()) {
|
|
256
161
|
for (let current = dirname(socketDir);;) {
|
|
162
|
+
current = current.replace(/\/+$/, "") || "/";
|
|
163
|
+
if (visited.has(current))
|
|
164
|
+
return true;
|
|
165
|
+
visited.add(current);
|
|
257
166
|
const metadata = await lstat(current);
|
|
167
|
+
// The operating environment supplies /; its reported owner may be unmapped in a user namespace.
|
|
168
|
+
if (current === "/" && metadata.isDirectory() && (metadata.mode & 0o022) === 0)
|
|
169
|
+
return true;
|
|
258
170
|
if (isTrustedAndroidAppDataRoot(current, metadata, uid))
|
|
259
171
|
return true;
|
|
260
172
|
if (!isTrustedSocketDirAncestor(metadata, uid))
|
|
261
173
|
return false;
|
|
174
|
+
if (metadata.isSymbolicLink()) {
|
|
175
|
+
// Native stat rejects broken/cyclic links before walking their destination ancestry.
|
|
176
|
+
if (!isTrustedSocketDirAncestor(await stat(current), uid))
|
|
177
|
+
return false;
|
|
178
|
+
const target = await readlink(current);
|
|
179
|
+
const targetPath = isAbsolute(target) ? target : `${dirname(current)}/${target}`;
|
|
180
|
+
// Keep '..' after symlinks intact; '/.' includes the target itself in the parent walk.
|
|
181
|
+
if (!await hasTrustedSocketDirAncestry(`${targetPath}/.`, uid, visited))
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
262
184
|
const parent = dirname(current);
|
|
263
185
|
if (parent === current)
|
|
264
186
|
return true;
|
|
@@ -364,7 +286,7 @@ export function buildAgentBrowserProcessEnv(baseEnv = processEnv, overrides = un
|
|
|
364
286
|
clampUpstreamDefaultTimeout(childEnv);
|
|
365
287
|
return childEnv;
|
|
366
288
|
}
|
|
367
|
-
function getManagedPreSpawnPolicyError(options, currentPageUrl, pageUrlUnknown = false
|
|
289
|
+
function getManagedPreSpawnPolicyError(options, currentPageUrl, pageUrlUnknown = false) {
|
|
368
290
|
if (!validateManagedSessionRestoreContextForSpawn(options)) {
|
|
369
291
|
return "Managed session restore policy, storage, or checkout identity changed after planning; refusing to start agent-browser.";
|
|
370
292
|
}
|
|
@@ -373,11 +295,10 @@ function getManagedPreSpawnPolicyError(options, currentPageUrl, pageUrlUnknown =
|
|
|
373
295
|
currentPageUrl,
|
|
374
296
|
pageUrlUnknown,
|
|
375
297
|
stdin: options.stdin,
|
|
376
|
-
trustedFirstBatchTabSelection,
|
|
377
298
|
});
|
|
378
299
|
}
|
|
379
300
|
export async function runAgentBrowserProcess(options) {
|
|
380
|
-
const { cwd, env, managedSessionRestoreState, managedStateCurrentPageUrl, managedStatePageUrlUnknown, signal, stdin
|
|
301
|
+
const { cwd, env, managedSessionRestoreState, managedStateCurrentPageUrl, managedStatePageUrlUnknown, signal, stdin } = options;
|
|
381
302
|
const preserveAttachedBrowserSession = options.preserveAttachedBrowserSession === true || attachedBrowserSessionContext.getStore() === true;
|
|
382
303
|
const ownedManagedSession = options.ownedManagedSession === true || isOwnedManagedSessionTarget(options.args);
|
|
383
304
|
const args = options.args;
|
|
@@ -395,7 +316,7 @@ export async function runAgentBrowserProcess(options) {
|
|
|
395
316
|
restoreState: managedSessionRestoreState,
|
|
396
317
|
stdin,
|
|
397
318
|
};
|
|
398
|
-
const planningPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown
|
|
319
|
+
const planningPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown);
|
|
399
320
|
if (planningPolicyError) {
|
|
400
321
|
return {
|
|
401
322
|
aborted: false,
|
|
@@ -415,7 +336,6 @@ export async function runAgentBrowserProcess(options) {
|
|
|
415
336
|
...env,
|
|
416
337
|
...getManagedSessionRestoreProtectedEnv(managedSessionRestoreOptions, managedSessionRestoreEnv),
|
|
417
338
|
...getOwnedManagedSessionNamespaceEnv(managedSessionRestoreOptions),
|
|
418
|
-
...getWindowsExplicitDefaultNamespaceEnv(args, parentEnv.AGENT_BROWSER_NAMESPACE),
|
|
419
339
|
...ownedManagedSessionCompatibilityEnv,
|
|
420
340
|
};
|
|
421
341
|
const explicitSocketDir = processOverrides[AGENT_BROWSER_SOCKET_DIR_ENV];
|
|
@@ -520,13 +440,8 @@ export async function runAgentBrowserProcess(options) {
|
|
|
520
440
|
if (stdoutSpillHandle) {
|
|
521
441
|
await stdoutSpillHandle.close().catch(() => undefined);
|
|
522
442
|
}
|
|
523
|
-
|
|
524
|
-
if (processPlatform === "win32" && !windowsMissingBinary && !spawnError)
|
|
443
|
+
if (processPlatform === "win32" && !spawnError) {
|
|
525
444
|
agentBrowserStarted = true;
|
|
526
|
-
if (windowsMissingBinary && !spawnError) {
|
|
527
|
-
spawnError = Object.assign(new Error("spawn agent-browser ENOENT"), { code: "ENOENT" });
|
|
528
|
-
}
|
|
529
|
-
else if (processPlatform === "win32" && shouldCommitManagedRestoreAfterWindowsProcess({ exitCode, spawnError, stderr })) {
|
|
530
445
|
commitManagedSessionRestoreSuppression(managedSessionRestoreOptions);
|
|
531
446
|
}
|
|
532
447
|
if (!spawnError && stdoutSpillError) {
|
|
@@ -548,13 +463,13 @@ export async function runAgentBrowserProcess(options) {
|
|
|
548
463
|
});
|
|
549
464
|
};
|
|
550
465
|
const childEnv = buildAgentBrowserProcessEnv(parentEnv, effectiveEnv);
|
|
551
|
-
const spawnPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown
|
|
466
|
+
const spawnPolicyError = getManagedPreSpawnPolicyError(managedSessionRestoreOptions, managedStateCurrentPageUrl, managedStatePageUrlUnknown);
|
|
552
467
|
if (spawnPolicyError) {
|
|
553
468
|
resolve({ aborted: false, agentBrowserStarted: false, exitCode: 1, spawnError: new Error(spawnPolicyError), stderr: "", stdout: "", timedOut: false });
|
|
554
469
|
return;
|
|
555
470
|
}
|
|
556
|
-
const
|
|
557
|
-
const child =
|
|
471
|
+
const spawnBrowser = processPlatform === "win32" ? crossSpawn : spawn;
|
|
472
|
+
const child = spawnBrowser("agent-browser", prepareAgentBrowserSpawnArgs(args, ownedManagedSessionCompatibilityEnv.AGENT_BROWSER_USER_AGENT, preserveAttachedBrowserSession), {
|
|
558
473
|
cwd,
|
|
559
474
|
env: childEnv,
|
|
560
475
|
stdio: ["pipe", "pipe", "pipe"],
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isAbsolute } from "node:path";
|
|
1
2
|
import { getAgentBrowserSessionIdentityKey } from "./argv-grammar.js";
|
|
2
3
|
import { isRecord } from "./parsing.js";
|
|
3
4
|
import { isPendingRecordingArtifact } from "./results/artifact-manifest.js";
|
|
@@ -80,7 +81,8 @@ function parseReservationTransition(data) {
|
|
|
80
81
|
state: "closed",
|
|
81
82
|
};
|
|
82
83
|
}
|
|
83
|
-
if (typeof data.absolutePath !== "string" ||
|
|
84
|
+
if (typeof data.absolutePath !== "string" || !isAbsolute(data.absolutePath)
|
|
85
|
+
|| typeof data.cwd !== "string" || !isAbsolute(data.cwd) || typeof data.path !== "string")
|
|
84
86
|
return undefined;
|
|
85
87
|
return {
|
|
86
88
|
reservation: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { isOpenNavigationCommand, isPageMutationCommand } from "../command-taxonomy.js";
|
|
2
2
|
import { isPendingRecordingArtifact } from "./artifact-manifest.js";
|
|
3
|
-
import { applySessionToNextActions, buildNextToolAction } from "./next-actions.js";
|
|
3
|
+
import { applySessionToNextActions, buildInspectOverlayStateAction, buildNextToolAction } from "./next-actions.js";
|
|
4
4
|
import { AGENT_BROWSER_RECOVERY_NEXT_ACTION_IDS, buildRecoveryNextActions, } from "./recovery-actions.js";
|
|
5
5
|
function buildArtifactAction(path) {
|
|
6
6
|
return {
|
|
@@ -223,7 +223,10 @@ export function buildAgentBrowserNextActions(options) {
|
|
|
223
223
|
}
|
|
224
224
|
break;
|
|
225
225
|
case "upstream-error":
|
|
226
|
-
if (
|
|
226
|
+
if (options.overlayBlockedClick) {
|
|
227
|
+
actions.push(buildInspectOverlayStateAction(options.sessionName));
|
|
228
|
+
}
|
|
229
|
+
else if (isOpenNavigationCommand(options.command)) {
|
|
227
230
|
actions.push(buildNextToolAction({
|
|
228
231
|
args: ["get", "url"],
|
|
229
232
|
id: "inspect-page-after-navigation-error",
|
|
@@ -17,7 +17,7 @@ async function readEnvelopeSource(options) {
|
|
|
17
17
|
throw new Error(`agent-browser output spill file could not be read: ${message}`);
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
|
-
function extractEnvelopeErrorText(error) {
|
|
20
|
+
export function extractEnvelopeErrorText(error) {
|
|
21
21
|
if (typeof error === "string") {
|
|
22
22
|
return error.trim() || undefined;
|
|
23
23
|
}
|
|
@@ -48,7 +48,8 @@ export async function parseAgentBrowserEnvelope(options) {
|
|
|
48
48
|
return { parseError: error instanceof Error ? error.message : String(error) };
|
|
49
49
|
}
|
|
50
50
|
const trimmed = stdout.trim();
|
|
51
|
-
|
|
51
|
+
const plainText = typeof options !== "string" && options.plainText === true;
|
|
52
|
+
if (trimmed.length === 0 && !plainText) {
|
|
52
53
|
return { parseError: "agent-browser returned no JSON output." };
|
|
53
54
|
}
|
|
54
55
|
try {
|
|
@@ -81,6 +82,8 @@ export async function parseAgentBrowserEnvelope(options) {
|
|
|
81
82
|
return { envelope: parsed };
|
|
82
83
|
}
|
|
83
84
|
catch (error) {
|
|
85
|
+
if (plainText)
|
|
86
|
+
return { envelope: { success: true, data: trimmed } };
|
|
84
87
|
const message = error instanceof Error ? error.message : String(error);
|
|
85
88
|
return { parseError: `agent-browser returned invalid JSON: ${message}` };
|
|
86
89
|
}
|
|
@@ -149,10 +152,12 @@ export function getAgentBrowserErrorText(options) {
|
|
|
149
152
|
if (parseError)
|
|
150
153
|
return parseError;
|
|
151
154
|
if (envelope?.success === false) {
|
|
152
|
-
|
|
155
|
+
const explicitErrorText = extractEnvelopeErrorText(envelope.error);
|
|
156
|
+
if ((hasStructuredBatchStepFailure(envelope.data) || detectConfirmationRequired(envelope.data)) && explicitErrorText === undefined) {
|
|
153
157
|
return undefined;
|
|
154
158
|
}
|
|
155
|
-
const envelopeErrorText =
|
|
159
|
+
const envelopeErrorText = explicitErrorText
|
|
160
|
+
?? extractEnvelopeErrorText(typeof envelope.data === "string" ? envelope.data : isRecord(envelope.data) ? envelope.data.error : undefined);
|
|
156
161
|
if (envelopeErrorText && isUpstreamIpcReadTimeoutMessage(envelopeErrorText)) {
|
|
157
162
|
return buildUpstreamIpcReadTimeoutMessage();
|
|
158
163
|
}
|
|
@@ -44,6 +44,14 @@ export function buildNextToolAction(options) {
|
|
|
44
44
|
tool: "agent_browser",
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
|
+
export function buildInspectOverlayStateAction(sessionName) {
|
|
48
|
+
return buildNextToolAction({
|
|
49
|
+
args: withOptionalSessionArgs(sessionName, ["snapshot", "-i"]),
|
|
50
|
+
id: "inspect-overlay-state",
|
|
51
|
+
reason: "Refresh interactive refs and inspect whether an overlay, banner, modal, or dialog is blocking the intended click.",
|
|
52
|
+
safety: "Read-only inspection; do not blindly retry the blocked click, and use current refs from this snapshot before interacting.",
|
|
53
|
+
});
|
|
54
|
+
}
|
|
47
55
|
export function appendUniqueAgentBrowserNextActions(target, additions) {
|
|
48
56
|
if (!additions || additions.length === 0)
|
|
49
57
|
return target;
|
|
@@ -1,22 +1,43 @@
|
|
|
1
|
-
import { readFile, stat } from "node:fs/promises";
|
|
1
|
+
import { open, readFile, stat } from "node:fs/promises";
|
|
2
2
|
import { extname, resolve } from "node:path";
|
|
3
3
|
import { getAgentBrowserSessionIdentityKey } from "../../argv-grammar.js";
|
|
4
|
+
import { getExplicitArtifactDestination } from "../../orchestration/browser-run/artifact-paths.js";
|
|
4
5
|
import { isRecord, parsePositiveInteger } from "../../parsing.js";
|
|
6
|
+
import { extractUpstreamCommandTokens } from "../../runtime.js";
|
|
5
7
|
import { formatSessionArtifactRetentionSummary, getSessionArtifactManifestEntryKey, isPendingRecordingArtifact, isPendingRecordingCommand, mergeSessionArtifactManifest, } from "../artifact-manifest.js";
|
|
6
8
|
import { classifyAgentBrowserSuccessCategory } from "../categories.js";
|
|
7
|
-
const
|
|
8
|
-
".gif": "image/gif",
|
|
9
|
-
".jpeg": "image/jpeg",
|
|
10
|
-
".jpg": "image/jpeg",
|
|
11
|
-
".png": "image/png",
|
|
12
|
-
".webp": "image/webp",
|
|
13
|
-
};
|
|
9
|
+
const PNG_HEADER = Buffer.from("89504e470d0a1a0a0000000d49484452", "hex");
|
|
14
10
|
const INLINE_IMAGE_MAX_BYTES_ENV = "PI_AGENT_BROWSER_INLINE_IMAGE_MAX_BYTES";
|
|
15
11
|
const DEFAULT_INLINE_IMAGE_MAX_BYTES = 5 * 1_024 * 1_024;
|
|
16
12
|
const ARTIFACT_MTIME_TOLERANCE_MS = 2_000;
|
|
17
|
-
function getImageMimeType(
|
|
18
|
-
|
|
19
|
-
|
|
13
|
+
function getImageMimeType(bytes) {
|
|
14
|
+
if (bytes.length < 16)
|
|
15
|
+
return undefined;
|
|
16
|
+
if (bytes.subarray(0, 16).equals(PNG_HEADER))
|
|
17
|
+
return "image/png";
|
|
18
|
+
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff && bytes[3] !== 0xf7)
|
|
19
|
+
return "image/jpeg";
|
|
20
|
+
if (["GIF87a", "GIF89a"].includes(bytes.toString("utf8", 0, 6)))
|
|
21
|
+
return "image/gif";
|
|
22
|
+
if (bytes.toString("utf8", 0, 4) === "RIFF" && bytes.toString("utf8", 8, 12) === "WEBP")
|
|
23
|
+
return "image/webp";
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
async function getFileImageMimeType(path) {
|
|
27
|
+
try {
|
|
28
|
+
const file = await open(path, "r");
|
|
29
|
+
try {
|
|
30
|
+
const bytes = Buffer.alloc(16);
|
|
31
|
+
const { bytesRead } = await file.read(bytes, 0, bytes.length, 0);
|
|
32
|
+
return getImageMimeType(bytes.subarray(0, bytesRead));
|
|
33
|
+
}
|
|
34
|
+
finally {
|
|
35
|
+
await file.close();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
20
41
|
}
|
|
21
42
|
function getInlineImageMaxBytes(env = process.env) {
|
|
22
43
|
return parsePositiveInteger(env[INLINE_IMAGE_MAX_BYTES_ENV]) ?? DEFAULT_INLINE_IMAGE_MAX_BYTES;
|
|
@@ -74,17 +95,6 @@ const PATH_FIELD_CANDIDATES = [
|
|
|
74
95
|
"profilePath",
|
|
75
96
|
"videoPath",
|
|
76
97
|
];
|
|
77
|
-
const ARTIFACT_EXTENSION_TO_MEDIA_TYPE = {
|
|
78
|
-
".cpuprofile": "application/json",
|
|
79
|
-
".har": "application/json",
|
|
80
|
-
".html": "text/html",
|
|
81
|
-
".json": "application/json",
|
|
82
|
-
".pdf": "application/pdf",
|
|
83
|
-
".txt": "text/plain",
|
|
84
|
-
".webm": "video/webm",
|
|
85
|
-
".zip": "application/zip",
|
|
86
|
-
...IMAGE_EXTENSION_TO_MIME_TYPE,
|
|
87
|
-
};
|
|
88
98
|
function isDownloadWaitSubcommand(subcommand) {
|
|
89
99
|
return subcommand === "--download" || subcommand === "-d";
|
|
90
100
|
}
|
|
@@ -153,6 +163,7 @@ async function buildFileArtifactMetadata(options) {
|
|
|
153
163
|
const pendingRecording = isPendingRecordingCommand(options.commandInfo.command, options.commandInfo.subcommand, kind);
|
|
154
164
|
let exists;
|
|
155
165
|
let sizeBytes;
|
|
166
|
+
let mediaType;
|
|
156
167
|
let stale = false;
|
|
157
168
|
let updatedAtMs;
|
|
158
169
|
if (!pendingRecording) {
|
|
@@ -161,6 +172,7 @@ async function buildFileArtifactMetadata(options) {
|
|
|
161
172
|
exists = true;
|
|
162
173
|
sizeBytes = fileStats.size;
|
|
163
174
|
updatedAtMs = fileStats.mtimeMs;
|
|
175
|
+
mediaType = fileStats.isFile() ? await getFileImageMimeType(absolutePath) : undefined;
|
|
164
176
|
const commandCreatesArtifact = !(options.commandInfo.command === "wait" && isDownloadWaitSubcommand(options.commandInfo.subcommand));
|
|
165
177
|
stale = commandCreatesArtifact && artifactMtimeIsOutsideCommandWindow(updatedAtMs, options.artifactMinUpdatedAtMs, options.artifactMaxUpdatedAtMs);
|
|
166
178
|
}
|
|
@@ -176,11 +188,11 @@ async function buildFileArtifactMetadata(options) {
|
|
|
176
188
|
exists,
|
|
177
189
|
extension,
|
|
178
190
|
kind,
|
|
179
|
-
mediaType
|
|
191
|
+
mediaType,
|
|
180
192
|
namespace: options.namespace,
|
|
181
193
|
path: displayPath,
|
|
182
194
|
recordingState: pendingRecording ? "openRecording" : undefined,
|
|
183
|
-
requestedPath: options.artifactRequest?.path,
|
|
195
|
+
requestedPath: options.artifactRequest?.path ?? getExplicitArtifactDestination(extractUpstreamCommandTokens(options.commandInfo.commandTokens ?? [])),
|
|
184
196
|
session: options.sessionName,
|
|
185
197
|
sizeBytes,
|
|
186
198
|
status: pendingRecording ? "pending" : exists === false ? "missing" : stale ? "stale" : options.artifactRequest?.status ?? "saved",
|
|
@@ -212,7 +224,7 @@ async function buildPreviousRestartRecordingArtifact(options) {
|
|
|
212
224
|
exists: true,
|
|
213
225
|
extension: previousRecording.extension ?? (extname(absolutePath).toLowerCase() || undefined),
|
|
214
226
|
kind: "video",
|
|
215
|
-
mediaType:
|
|
227
|
+
mediaType: fileStats.isFile() ? await getFileImageMimeType(absolutePath) : undefined,
|
|
216
228
|
namespace: previousRecording.namespace ?? options.namespace,
|
|
217
229
|
path: previousRecording.path,
|
|
218
230
|
requestedPath: previousRecording.requestedPath,
|
|
@@ -232,7 +244,6 @@ async function buildPreviousRestartRecordingArtifact(options) {
|
|
|
232
244
|
exists: false,
|
|
233
245
|
extension: previousRecording.extension ?? (extname(absolutePath).toLowerCase() || undefined),
|
|
234
246
|
kind: "video",
|
|
235
|
-
mediaType: previousRecording.mediaType,
|
|
236
247
|
namespace: previousRecording.namespace ?? options.namespace,
|
|
237
248
|
path: previousRecording.path,
|
|
238
249
|
requestedPath: previousRecording.requestedPath,
|
|
@@ -449,13 +460,12 @@ export function formatArtifactMetadataLines(artifacts) {
|
|
|
449
460
|
return [
|
|
450
461
|
`${formatArtifactLabel(artifact)}: ${artifact.path}`,
|
|
451
462
|
`Artifact type: ${artifact.kind}`,
|
|
452
|
-
`Requested path: ${artifact.requestedPath
|
|
463
|
+
artifact.requestedPath ? `Requested path: ${artifact.requestedPath}` : undefined,
|
|
453
464
|
`Absolute path: ${artifact.absolutePath}`,
|
|
454
465
|
"Exists: pending until record stop",
|
|
455
466
|
`Status: ${artifact.status ?? "pending"}`,
|
|
456
467
|
`Recording state: ${artifact.recordingState ?? "openRecording"}`,
|
|
457
468
|
`Will exist on stop: ${artifact.willExistOnStop !== false}`,
|
|
458
|
-
artifact.subcommand === "start" ? "Page state: record start uses a fresh active page for video capture; prior in-page DOM and JavaScript state does not carry over. Take a fresh snapshot before continuing." : undefined,
|
|
459
469
|
artifact.session ? `Session: ${artifact.session}` : undefined,
|
|
460
470
|
artifact.cwd ? `CWD: ${artifact.cwd}` : undefined,
|
|
461
471
|
`Machine data: details.artifacts[${index}]`,
|
|
@@ -464,14 +474,14 @@ export function formatArtifactMetadataLines(artifacts) {
|
|
|
464
474
|
return [
|
|
465
475
|
`${formatArtifactLabel(artifact)}: ${artifact.path}`,
|
|
466
476
|
`Artifact type: ${artifact.kind}`,
|
|
467
|
-
`Requested path: ${artifact.requestedPath
|
|
477
|
+
artifact.requestedPath ? `Requested path: ${artifact.requestedPath}` : undefined,
|
|
468
478
|
`Absolute path: ${artifact.absolutePath}`,
|
|
469
479
|
`Exists: ${artifact.exists === true}`,
|
|
470
480
|
artifact.exists === false ? "not found on disk" : undefined,
|
|
471
481
|
typeof artifact.sizeBytes === "number" ? `Size: ${formatByteCount(artifact.sizeBytes)}` : undefined,
|
|
472
482
|
typeof artifact.sizeBytes === "number" ? `Size bytes: ${artifact.sizeBytes}` : undefined,
|
|
473
483
|
`Status: ${artifact.status ?? (artifact.exists === false ? "missing" : "saved")}`,
|
|
474
|
-
artifact.tempPath ? `
|
|
484
|
+
artifact.tempPath ? `Reported path: ${artifact.tempPath}` : undefined,
|
|
475
485
|
artifact.mediaType ? `Media type: ${artifact.mediaType}` : undefined,
|
|
476
486
|
artifact.session ? `Session: ${artifact.session}` : undefined,
|
|
477
487
|
artifact.cwd ? `CWD: ${artifact.cwd}` : undefined,
|
|
@@ -519,21 +529,10 @@ export function extractImagePath(commandInfo, cwd, data) {
|
|
|
519
529
|
if (!isTrustedScreenshotOutput(commandInfo)) {
|
|
520
530
|
return undefined;
|
|
521
531
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
return mimeType ? resolve(cwd, data) : undefined;
|
|
525
|
-
}
|
|
526
|
-
if (!isRecord(data) || typeof data.path !== "string") {
|
|
527
|
-
return undefined;
|
|
528
|
-
}
|
|
529
|
-
const mimeType = getImageMimeType(data.path);
|
|
530
|
-
return mimeType ? resolve(cwd, data.path) : undefined;
|
|
532
|
+
const path = typeof data === "string" ? data : isRecord(data) && typeof data.path === "string" ? data.path : undefined;
|
|
533
|
+
return path?.trim() && !isNonFileArtifactPathCandidate(path) ? resolve(cwd, path) : undefined;
|
|
531
534
|
}
|
|
532
535
|
export async function attachInlineImage(presentation, imagePath) {
|
|
533
|
-
const mimeType = getImageMimeType(imagePath);
|
|
534
|
-
if (!mimeType) {
|
|
535
|
-
return presentation;
|
|
536
|
-
}
|
|
537
536
|
try {
|
|
538
537
|
const fileStats = await stat(imagePath);
|
|
539
538
|
const inlineImageMaxBytes = getInlineImageMaxBytes();
|
|
@@ -543,6 +542,9 @@ export async function attachInlineImage(presentation, imagePath) {
|
|
|
543
542
|
return presentation;
|
|
544
543
|
}
|
|
545
544
|
const file = await readFile(imagePath);
|
|
545
|
+
const mimeType = getImageMimeType(file);
|
|
546
|
+
if (!mimeType)
|
|
547
|
+
return presentation;
|
|
546
548
|
presentation.content.push({ type: "image", data: file.toString("base64"), mimeType });
|
|
547
549
|
presentation.imagePath = imagePath;
|
|
548
550
|
return presentation;
|
|
@@ -13,7 +13,7 @@ import { extractAgentBrowserLifecycle, stringifyModelFacing } from "./common.js"
|
|
|
13
13
|
import { buildArtifactVerificationSummary, classifyPresentationSuccessCategory, manifestHasNewNoticeWorthyEntries } from "./artifacts.js";
|
|
14
14
|
import { formatBatchStepCommand, getPresentationImages, getPresentationPaths, getPresentationText, isStringArray } from "./content.js";
|
|
15
15
|
import { buildPageChangeSummary } from "./navigation.js";
|
|
16
|
-
import { appendSelectorRecoveryHint, getClipboardWritePayloadCandidates, redactClipboardPermissionErrorValue } from "./errors.js";
|
|
16
|
+
import { appendSelectorRecoveryHint, getClipboardWritePayloadCandidates, isOverlayBlockedClickError, redactClipboardPermissionErrorValue } from "./errors.js";
|
|
17
17
|
export function isAgentBrowserBatchResultArray(value) {
|
|
18
18
|
return Array.isArray(value) && value.every(isRecord);
|
|
19
19
|
}
|
|
@@ -208,6 +208,7 @@ async function buildBatchStepPresentation(options) {
|
|
|
208
208
|
command: command?.[0],
|
|
209
209
|
confirmationId: confirmationRequired?.id,
|
|
210
210
|
failureCategory,
|
|
211
|
+
overlayBlockedClick: isOverlayBlockedClickError(command?.[0], errorText, command),
|
|
211
212
|
resultCategory: "failure",
|
|
212
213
|
sessionName,
|
|
213
214
|
subcommand: command?.[1],
|