replicas-cli 0.2.355 → 0.2.357

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.mjs +1353 -164
  2. package/package.json +1 -1
package/dist/index.mjs CHANGED
@@ -7878,7 +7878,7 @@ var SANDBOX_PATHS = {
7878
7878
  HOME_DIR: "/home/user",
7879
7879
  WORKSPACES_DIR: "/home/user/workspaces",
7880
7880
  REPLICAS_DIR: "/home/user/.replicas",
7881
- REPLICAS_FILES_DIR: "/home/user/.replicas/files",
7881
+ REPLICAS_CANVAS_DIR: "/home/user/.replicas/canvas",
7882
7882
  REPOS_PREPARED_MARKER: "/home/user/.replicas/repos-prepared-for-engine-init",
7883
7883
  REPLICAS_RUNTIME_ENV_FILE: "/home/user/.replicas/runtime-env.sh"
7884
7884
  };
@@ -8010,13 +8010,12 @@ replicas computer info
8010
8010
  # 2) Launch a browser on the workspace display.
8011
8011
  replicas computer launch chrome https://news.ycombinator.com
8012
8012
 
8013
- # 3) Observe the settled screen and browser tab state before clicking.
8014
- replicas computer observe /tmp/state.png
8015
- replicas computer browser --snapshot
8013
+ # 3) Capture the settled viewport and native accessibility tree together.
8014
+ replicas computer browser-state /tmp/state.png --full
8016
8015
 
8017
- # 4) Drive the UI.
8018
- replicas computer click 521 700 # click coordinates from the screenshot
8019
- replicas computer scroll down --amount 5
8016
+ # 4) Batch actions that use refs from the same state, then capture the semantic diff.
8017
+ replicas computer browser-batch '[{"action":"fill","ref":17,"value":"hello"},{"action":"click","ref":42}]'
8018
+ replicas computer browser-state /tmp/after-click.png
8020
8019
 
8021
8020
  # 5) (Optional) Record a screencap to share back.
8022
8021
  replicas computer record start /tmp/demo.mp4 --fps 60
@@ -8045,21 +8044,33 @@ Waits briefly for the screen to stop changing, saves a 1:1 screenshot, and print
8045
8044
 
8046
8045
  Use this instead of hand-written \`sleep && screenshot\` loops after clicks, navigation, typing, or page loads. By default it saves a 100px coordinate-grid screenshot and waits up to 3s for 600ms of visual stability. Pass \`--raw\` if you need an unannotated 1:1 screenshot.
8047
8046
 
8048
- ### \`replicas computer browser\`
8049
- Prints JSON for Chrome tabs launched through Replicas, including page titles and URLs. Pass \`--snapshot\` to include visible page text and interactive controls with DOM viewport bounding boxes.
8047
+ ### \`replicas computer browser-state <path> [--full] [--timeout MS] [--stable-ms MS]\`
8048
+ Waits for the selected Chrome page to settle, saves its viewport screenshot, and prints a compact native Chrome accessibility tree in the same coordinate space. Actionable nodes have DOM-backed refs such as \`[ref=42] button "Save"\`; the tree includes accessible roles, names, values, states, visible bounds, and iframe content.
8049
+
8050
+ The first call returns a full tree. Later calls for the same target and document return semantic changes; navigation or reload forces a new full tree. Pass \`--full\` to reset the baseline. Refs are valid for the current document.
8051
+
8052
+ ### \`replicas computer browser [--snapshot]\`
8053
+ Lists Chrome tabs. \`--snapshot\` returns the compatibility accessibility snapshot. Use \`browser-state\` for the primary action/observe loop.
8054
+
8055
+ ### \`replicas computer browser-batch '<actions-json>'\`
8056
+ Runs up to 100 ordered browser actions through one Chrome session and stops at the first failure. Supported actions are \`click\`, \`fill\`, \`key\`, \`type\`, \`scroll\`, and \`wait\`. Prefer this when several actions use refs from the same \`browser-state\`; it avoids repeated process, tab-selection, and connection setup. Do not carry refs across navigation\u2014capture fresh state and start a new batch.
8057
+
8058
+ ### \`replicas computer browser-click --ref ID [--button left|middle|right] [--double]\`
8059
+ Scrolls the referenced node into view and sends trusted Chrome mouse input at its current visible bounds. Prefer refs from \`browser-state\`. \`browser-click <text> [--exact] [--index N]\` remains a fallback when no fresh ref is available.
8050
8060
 
8051
- Use this alongside \`observe\` when testing web apps so you do not infer navigation, page state, or click targets from pixels alone. Snapshot coordinates are DOM viewport coordinates, not desktop click coordinates.
8061
+ ### \`replicas computer browser-fill --ref ID <value>\`
8062
+ Focuses the referenced field, fills it with trusted Chrome input, and verifies the resulting value. Select controls use their semantic option value. \`browser-fill <field> <value> [--exact] [--index N]\` remains a text-matching fallback.
8052
8063
 
8053
- ### \`replicas computer browser-click <text> [--exact] [--index N]\`
8054
- Clicks the first visible Chrome control whose text, label, placeholder, or href matches \`<text>\`. Use this for web buttons and links found via \`browser --snapshot\`; it is faster and less error-prone than converting DOM coordinates to desktop pixels.
8064
+ ### \`replicas computer browser-scroll <up|down|left|right> [--ref ID] [--amount PX]\`
8065
+ Sends trusted wheel input at a referenced element or the center of the viewport.
8055
8066
 
8056
- ### \`replicas computer browser-fill <field> <value> [--exact] [--index N]\`
8057
- Fills the first visible Chrome field whose label, placeholder, name, or text matches \`<field>\`, then dispatches input/change events. Use this for web forms instead of clicking a field and typing through the desktop.
8067
+ ### \`replicas computer browser-key <combo>\` / \`browser-type <text>\`
8068
+ Sends trusted keyboard input to the focused element in the selected Chrome target. Prefer \`browser-fill\` when setting a known field value.
8058
8069
 
8059
8070
  ### \`replicas computer browser-wait <text> [--mode any|text|title|url|control] [--exact] [--timeout MS]\`
8060
8071
  Waits until the active Chrome page matches text in the title, URL, body text, or visible controls. Use this after \`browser-click\` / \`browser-fill\` when you need web app state to settle without screenshot polling.
8061
8072
 
8062
- For \`browser-click\`, \`browser-fill\`, and \`browser-wait\`, pass \`--id <id>\`, \`--title <text>\`, \`--url <text>\`, or \`--page <n>\` when multiple Chrome tabs are open. Run \`replicas computer browser\` first to list tabs. Prefer \`--id\` when a click may change the page title or URL.
8073
+ For browser state and actions, pass \`--target-id <id>\`, \`--title <text>\`, \`--url <text>\`, or \`--page <n>\` when multiple Chrome tabs are open. Prefer \`--target-id\` because titles and URLs can change after an action. The matched tab is brought to the foreground before each action, so driving multiple tabs or recording the screen switches which tab is visible.
8063
8074
 
8064
8075
  ### \`replicas computer click <x> <y> [--button N] [--double] [--modifiers ctrl+shift]\`
8065
8076
  Move to (x, y) and click. Coordinates can be absolute pixels or percentages such as \`50%\` \`50%\`. Default is left-click (button 1); pass \`--button 3\` for right-click. \`--modifiers\` holds keys during the click (e.g. ctrl-click a link to open in a new tab).
@@ -8092,26 +8103,29 @@ Spawns an app on the workspace display. Built-in aliases:
8092
8103
  Anything else gets \`exec\`'d verbatim, so \`replicas computer launch xeyes\` works if xeyes is installed.
8093
8104
  When opening a known page, prefer passing the URL directly: \`replicas computer launch chrome http://localhost:3000/\`.
8094
8105
 
8106
+ Chrome launches do not return until the requested page is controllable. The output includes its page ID; carry that ID with \`--target-id\` into \`browser-state\` and browser actions so unrelated tabs cannot steal task context.
8107
+
8095
8108
  ### \`replicas computer record start <path> [--fps N]\`
8096
- Starts an ffmpeg screen recorder. Output is post-processed when stopped: action windows stay at normal speed, idle gaps accelerate, click moments get eased camera zoom, and a synthetic cursor animates between logged mouse positions. The raw capture is fragmented MP4 while active (still playable if the workspace dies mid-record). Default 60fps; drop to 30 if the workspace is CPU-constrained.
8109
+ Starts an ffmpeg screen recorder and returns only after ffmpeg has written the first bytes. Output is post-processed when stopped: action windows stay at normal speed, idle gaps accelerate, click moments get eased camera zoom, and a synthetic cursor animates between logged mouse positions. The raw capture is fragmented MP4 while active (still playable if the workspace dies mid-record). Default 60fps; drop to 30 if the workspace is CPU-constrained.
8097
8110
 
8098
8111
  Only one recording at a time. Re-running \`start\` while one is active fails - call \`stop\` first.
8099
8112
 
8100
8113
  ### \`replicas computer record stop\`
8101
- SIGINTs ffmpeg, waits for it to finalize the MP4, prints the output path. Upload it with \`replicas media upload <path>\` to share it.
8114
+ SIGINTs ffmpeg, verifies that the process has finalized the MP4, then prints the output path. A finalization timeout preserves the recording state so \`stop\` can be retried safely. Upload it with \`replicas media upload <path>\` to share it.
8102
8115
 
8103
8116
  ## Patterns
8104
8117
 
8105
- ### Action / observe loop
8106
- You are blind between tool calls. After any action that changes the screen, observe before deciding the next coordinate:
8118
+ ### Browser action / state loop
8119
+ For Chrome, use the native accessibility state before interpreting pixels or guessing coordinates:
8107
8120
 
8108
8121
  \`\`\`bash
8109
- replicas computer click 521 700
8110
- replicas computer observe /tmp/after-click.png
8111
- # read /tmp/after-click.png and the JSON output, decide next click
8122
+ replicas computer launch chrome https://example.com
8123
+ replicas computer browser-state /tmp/before.png --full --target-id <page-id>
8124
+ replicas computer browser-batch '[{"action":"fill","ref":17,"value":"hello"},{"action":"click","ref":23}]' --target-id <page-id>
8125
+ replicas computer browser-state /tmp/after.png --target-id <page-id>
8112
8126
  \`\`\`
8113
8127
 
8114
- The JSON \`stable\`, \`frames\`, and \`changes\` fields tell you whether something changed while you were waiting. If \`stable\` is false, observe again or increase \`--timeout\` before acting on coordinates.
8128
+ Read the screenshot and tree as one state. Batch ordered actions whose refs all come from that state, then inspect the next semantic diff. A batch is fail-fast and validates each ref against the cached document. Split at navigation or whenever a later action depends on newly rendered refs. Fall back to \`observe\` plus desktop coordinates for canvas content, browser chrome, non-Chrome apps, or controls missing from the accessibility tree.
8115
8129
 
8116
8130
  ### Typing into an address bar
8117
8131
  \`\`\`bash
@@ -8120,10 +8134,9 @@ replicas computer observe /tmp/browser-open.png
8120
8134
  replicas computer key ctrl+l
8121
8135
  replicas computer type "https://example.com"
8122
8136
  replicas computer key Return
8123
- replicas computer observe /tmp/loaded.png
8124
- replicas computer browser --snapshot
8125
- replicas computer browser-fill "Search" "replicas"
8126
- replicas computer browser-click "More information" --title "Example"
8137
+ replicas computer browser-state /tmp/loaded.png --full
8138
+ replicas computer browser-fill --ref 17 "replicas"
8139
+ replicas computer browser-click --ref 23 --title "Example"
8127
8140
  replicas computer browser-wait "Example Domain" --mode title --title "Example"
8128
8141
  \`\`\`
8129
8142
 
@@ -9679,7 +9692,7 @@ var HOOK_EXEC_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
9679
9692
  var REPLICAS_CONFIG_FILENAMES = ["replicas.json", "replicas.yaml", "replicas.yml"];
9680
9693
 
9681
9694
  // ../shared/src/cli-version.ts
9682
- var CLI_VERSION = "0.2.355";
9695
+ var CLI_VERSION = "0.2.357";
9683
9696
 
9684
9697
  // ../shared/src/version.ts
9685
9698
  function compareVersions(v1, v2) {
@@ -9734,8 +9747,6 @@ function workspaceConfigWithCapabilities(config2, capabilities = {}) {
9734
9747
  function workspaceConfigWithPrFollowups(config2, prFollowups = config2?.capabilities?.pr_followups === true) {
9735
9748
  return workspaceConfigWithCapabilities(config2, { pr_followups: prFollowups });
9736
9749
  }
9737
- var WORKSPACE_FILE_UPLOAD_MAX_SIZE_BYTES = 20 * 1024 * 1024;
9738
- var WORKSPACE_FILE_CONTENT_MAX_SIZE_BYTES = 1 * 1024 * 1024;
9739
9750
  var WORKSPACE_SIDEBAR_VIEWS = ["owned", "shared", "team", "automated"];
9740
9751
  var WORKSPACE_STATUS_FILTERS = WORKSPACE_STATUSES.map((status) => status === "error" ? "failed" : status);
9741
9752
  var WORKSPACE_SORT_CHOICES = [
@@ -15035,7 +15046,7 @@ var sleep = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
15035
15046
 
15036
15047
  // src/commands/computer/recording.ts
15037
15048
  import { spawn as spawn3 } from "child_process";
15038
- import { appendFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, rmSync as rmSync2, writeFileSync as writeFileSync2 } from "fs";
15049
+ import { appendFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, rmSync as rmSync2, statSync, writeFileSync as writeFileSync2 } from "fs";
15039
15050
  import { dirname as dirname2 } from "path";
15040
15051
 
15041
15052
  // src/commands/computer/recording/render.ts
@@ -15516,6 +15527,18 @@ var RECORD_STARTED_AT_FILE = `${STATE_DIR}/recording-started-at.txt`;
15516
15527
  var RECORD_FPS_FILE = `${STATE_DIR}/recording-fps.txt`;
15517
15528
  var RECORD_DIMENSIONS_FILE = `${STATE_DIR}/recording-dimensions.json`;
15518
15529
  var RECORD_ACTIONS_FILE = `${STATE_DIR}/recording-actions.jsonl`;
15530
+ var RECORD_STATE_FILES = [
15531
+ RECORD_PID_FILE,
15532
+ RECORD_PATH_FILE,
15533
+ RECORD_RAW_PATH_FILE,
15534
+ RECORD_STARTED_AT_FILE,
15535
+ RECORD_FPS_FILE,
15536
+ RECORD_DIMENSIONS_FILE,
15537
+ RECORD_ACTIONS_FILE
15538
+ ];
15539
+ function clearRecordingState() {
15540
+ for (const file of RECORD_STATE_FILES) rmSync2(file, { force: true });
15541
+ }
15519
15542
  function recordingStartedAt() {
15520
15543
  if (!existsSync2(RECORD_STARTED_AT_FILE)) return null;
15521
15544
  const startedAt = Number.parseInt(readFileSync2(RECORD_STARTED_AT_FILE, "utf8").trim(), 10);
@@ -15573,13 +15596,13 @@ async function computerRecordStartCommand(path6, options) {
15573
15596
  if (existsSync2(RECORD_PID_FILE)) {
15574
15597
  const pid = parseInt(readFileSync2(RECORD_PID_FILE, "utf8").trim(), 10);
15575
15598
  if (Number.isFinite(pid)) {
15576
- let alive = false;
15599
+ let alive2 = false;
15577
15600
  try {
15578
15601
  process.kill(pid, 0);
15579
- alive = true;
15602
+ alive2 = true;
15580
15603
  } catch {
15581
15604
  }
15582
- if (alive) fail(`recording already in progress (pid ${pid}). run \`replicas computer record stop\` first.`);
15605
+ if (alive2) fail(`recording already in progress (pid ${pid}). run \`replicas computer record stop\` first.`);
15583
15606
  }
15584
15607
  }
15585
15608
  const target = resolvePath(path6);
@@ -15626,7 +15649,29 @@ async function computerRecordStartCommand(path6, options) {
15626
15649
  writeFileSync2(RECORD_STARTED_AT_FILE, String(Date.now()));
15627
15650
  writeFileSync2(RECORD_FPS_FILE, String(fps));
15628
15651
  writeFileSync2(RECORD_DIMENSIONS_FILE, JSON.stringify({ width, height }));
15629
- console.log(target);
15652
+ const startedAt = Date.now();
15653
+ while (Date.now() - startedAt < 5e3) {
15654
+ try {
15655
+ process.kill(child.pid, 0);
15656
+ if (existsSync2(rawTarget) && statSync(rawTarget).size > 0) {
15657
+ console.log(`${target} (recording ready in ${Date.now() - startedAt}ms)`);
15658
+ return;
15659
+ }
15660
+ } catch {
15661
+ break;
15662
+ }
15663
+ await sleep(100);
15664
+ }
15665
+ let alive = false;
15666
+ try {
15667
+ process.kill(child.pid, 0);
15668
+ alive = true;
15669
+ } catch {
15670
+ }
15671
+ if (alive) fail("ffmpeg is running but did not produce recording output within 5 seconds; run `replicas computer record stop` to finalize or retry");
15672
+ clearRecordingState();
15673
+ rmSync2(rawTarget, { force: true });
15674
+ fail("ffmpeg exited before screen recording became ready");
15630
15675
  }
15631
15676
  async function computerRecordStopCommand() {
15632
15677
  if (!existsSync2(RECORD_PID_FILE) && !existsSync2(RECORD_PATH_FILE)) fail("no recording in progress");
@@ -15637,15 +15682,17 @@ async function computerRecordStopCommand() {
15637
15682
  process.kill(pid, "SIGINT");
15638
15683
  } catch {
15639
15684
  }
15640
- for (let i = 0; i < 30; i++) {
15685
+ let alive = true;
15686
+ for (let i = 0; i < 150; i++) {
15641
15687
  try {
15642
15688
  process.kill(pid, 0);
15643
15689
  } catch {
15690
+ alive = false;
15644
15691
  break;
15645
15692
  }
15646
- await new Promise((r) => setTimeout(r, 200));
15693
+ await sleep(200);
15647
15694
  }
15648
- rmSync2(RECORD_PID_FILE, { force: true });
15695
+ if (alive) fail(`ffmpeg did not finalize recording within 30 seconds (pid ${pid})`);
15649
15696
  }
15650
15697
  if (existsSync2(RECORD_PATH_FILE)) {
15651
15698
  const target = readFileSync2(RECORD_PATH_FILE, "utf8").trim();
@@ -15658,13 +15705,8 @@ async function computerRecordStopCommand() {
15658
15705
  rmSync2(rawPath, { force: true });
15659
15706
  }
15660
15707
  console.log(target);
15661
- rmSync2(RECORD_PATH_FILE, { force: true });
15662
- rmSync2(RECORD_RAW_PATH_FILE, { force: true });
15663
- rmSync2(RECORD_STARTED_AT_FILE, { force: true });
15664
- rmSync2(RECORD_FPS_FILE, { force: true });
15665
- rmSync2(RECORD_DIMENSIONS_FILE, { force: true });
15666
- rmSync2(RECORD_ACTIONS_FILE, { force: true });
15667
15708
  }
15709
+ clearRecordingState();
15668
15710
  }
15669
15711
 
15670
15712
  // src/commands/computer/index.ts
@@ -16003,8 +16045,8 @@ async function getChromePages() {
16003
16045
  async function selectChromePage(options = {}) {
16004
16046
  const pages = await getChromePages();
16005
16047
  let matches = pages;
16006
- if (options.id) {
16007
- matches = matches.filter((page2) => page2.id === options.id);
16048
+ if (options.targetId) {
16049
+ matches = matches.filter((page2) => page2.id === options.targetId);
16008
16050
  }
16009
16051
  if (options.title) {
16010
16052
  const needle = options.title.toLowerCase();
@@ -16023,121 +16065,528 @@ async function selectChromePage(options = {}) {
16023
16065
  `No matching debuggable Chrome page found (${matches.length} match${matches.length === 1 ? "" : "es"}, index ${index}). Use \`replicas computer browser\` to list pages.`
16024
16066
  );
16025
16067
  }
16068
+ await sendChromeCommand(webSocketDebuggerUrl, "Page.bringToFront", {});
16026
16069
  return { ...page, webSocketDebuggerUrl };
16027
16070
  }
16028
- function buildBrowserSnapshotExpression(textLimit, elementLimit) {
16029
- return `(() => {
16030
- const clip = (value, limit) => String(value || '').replace(/\\s+/g, ' ').trim().slice(0, limit);
16031
- const escape = (value) => globalThis.CSS && CSS.escape ? CSS.escape(value) : String(value).replace(/["\\\\]/g, '\\\\$&');
16071
+ var VISIBLE_JS = `
16032
16072
  const visible = (el) => {
16033
16073
  const rect = el.getBoundingClientRect();
16034
16074
  const style = getComputedStyle(el);
16035
16075
  return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
16036
- };
16037
- const labels = (el) => {
16038
- const values = [];
16039
- const id = el.getAttribute('id');
16040
- if (id) values.push(...Array.from(document.querySelectorAll('label[for="' + escape(id) + '"]')).map((label) => label.innerText));
16041
- const wrappingLabel = el.closest('label');
16042
- if (wrappingLabel) values.push(wrappingLabel.innerText);
16043
- values.push(el.innerText, el.value, el.getAttribute('aria-label'), el.getAttribute('title'), el.getAttribute('placeholder'), el.href, el.id, el.name, el.tagName);
16044
- return values.map((value) => clip(value, 160)).filter(Boolean);
16045
- };
16046
- const name = (el) => labels(el)[0] || clip(el.tagName, 160);
16047
- const selector = 'a,button,input,textarea,select,[role="button"],[role="link"],[role="textbox"],[contenteditable="true"]';
16048
- const controls = Array.from(document.querySelectorAll(selector))
16049
- .filter(visible)
16050
- .slice(0, ${elementLimit})
16051
- .map((el) => {
16052
- const rect = el.getBoundingClientRect();
16053
- return {
16054
- tag: el.tagName.toLowerCase(),
16055
- role: el.getAttribute('role') || null,
16056
- type: el.getAttribute('type') || null,
16057
- text: name(el),
16058
- href: el.href || null,
16059
- disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
16060
- rect: {
16061
- x: Math.round(rect.x),
16062
- y: Math.round(rect.y),
16063
- width: Math.round(rect.width),
16064
- height: Math.round(rect.height),
16065
- centerX: Math.round(rect.x + rect.width / 2),
16066
- centerY: Math.round(rect.y + rect.height / 2),
16067
- },
16068
- };
16069
- });
16070
- return {
16071
- title: document.title,
16072
- url: location.href,
16073
- text: clip(document.body ? document.body.innerText : '', ${textLimit}),
16074
- controls,
16075
- };
16076
- })()`;
16076
+ };`;
16077
+ async function sendChromeSessionCommand(session, method, params) {
16078
+ return chromeResult(await session.send({ method, params }), method);
16077
16079
  }
16078
- async function evaluateChromeTarget(webSocketDebuggerUrl, expression) {
16080
+ async function withChromeSession(webSocketDebuggerUrl, callback) {
16079
16081
  return await new Promise((resolve2, reject) => {
16080
16082
  const ws = new WebSocket(webSocketDebuggerUrl);
16083
+ const pending = /* @__PURE__ */ new Map();
16084
+ let nextId = 1;
16081
16085
  let settled = false;
16082
- const finish = (callback) => {
16086
+ const failSession = (error) => {
16083
16087
  if (settled) return;
16084
16088
  settled = true;
16085
- clearTimeout(timeout);
16089
+ for (const request of pending.values()) {
16090
+ clearTimeout(request.timeout);
16091
+ request.reject(error);
16092
+ }
16093
+ pending.clear();
16086
16094
  ws.close();
16087
- callback();
16095
+ reject(error);
16088
16096
  };
16089
- const timeout = setTimeout(() => {
16090
- finish(() => reject(new Error("Chrome DevTools evaluation timed out")));
16091
- }, 5e3);
16092
- ws.addEventListener("open", () => {
16093
- ws.send(JSON.stringify({
16094
- id: 1,
16095
- method: "Runtime.evaluate",
16096
- params: {
16097
- expression,
16098
- awaitPromise: true,
16099
- returnByValue: true
16100
- }
16101
- }));
16097
+ const finishSession = (value) => {
16098
+ if (settled) return;
16099
+ settled = true;
16100
+ for (const request of pending.values()) clearTimeout(request.timeout);
16101
+ pending.clear();
16102
+ ws.close();
16103
+ resolve2(value);
16104
+ };
16105
+ const session = {
16106
+ send: ({ method, params }) => new Promise((resolveCommand, rejectCommand) => {
16107
+ const id = nextId++;
16108
+ const timeout = setTimeout(() => {
16109
+ pending.delete(id);
16110
+ rejectCommand(new Error(`Chrome DevTools ${method} timed out`));
16111
+ }, 1e4);
16112
+ pending.set(id, { method, resolve: resolveCommand, reject: rejectCommand, timeout });
16113
+ ws.send(JSON.stringify({ id, method, params }));
16114
+ })
16115
+ };
16116
+ ws.addEventListener("open", async () => {
16117
+ try {
16118
+ finishSession(await callback(session));
16119
+ } catch (error) {
16120
+ failSession(error instanceof Error ? error : new Error(String(error)));
16121
+ }
16102
16122
  });
16103
16123
  ws.addEventListener("message", (event) => {
16104
16124
  const data = typeof event.data === "string" ? event.data : Buffer.isBuffer(event.data) ? event.data.toString("utf8") : Buffer.from(event.data).toString("utf8");
16105
- let parsed;
16125
+ let message;
16106
16126
  try {
16107
- parsed = JSON.parse(data);
16127
+ message = JSON.parse(data);
16108
16128
  } catch (error) {
16109
- const reason = error instanceof Error ? error.message : String(error);
16110
- finish(() => reject(new Error(`Chrome DevTools returned invalid WebSocket JSON: ${reason}`)));
16111
- return;
16112
- }
16113
- if (typeof parsed !== "object" || parsed === null) {
16114
- finish(() => reject(new Error("Chrome DevTools returned an invalid WebSocket message")));
16115
- return;
16116
- }
16117
- const message = parsed;
16118
- if (message.id !== 1) return;
16119
- if (message.error) {
16120
- finish(() => reject(new Error(message.error?.message || "Chrome DevTools evaluation failed")));
16121
- return;
16122
- }
16123
- if (message.result?.exceptionDetails) {
16124
- finish(() => reject(new Error(message.result.exceptionDetails.text || "Chrome DevTools evaluation threw")));
16129
+ failSession(new Error(`Chrome DevTools returned invalid WebSocket JSON: ${error instanceof Error ? error.message : String(error)}`));
16125
16130
  return;
16126
16131
  }
16127
- finish(() => resolve2(message.result?.result?.value ?? null));
16132
+ if (!isRecord(message) || typeof message.id !== "number") return;
16133
+ const request = pending.get(message.id);
16134
+ if (!request) return;
16135
+ clearTimeout(request.timeout);
16136
+ pending.delete(message.id);
16137
+ const detail = isRecord(message.error) && typeof message.error.message === "string" ? message.error.message : void 0;
16138
+ request.resolve(message.error ? { error: { message: detail } } : { result: isRecord(message.result) ? message.result : {} });
16128
16139
  });
16129
- ws.addEventListener("error", () => {
16130
- finish(() => reject(new Error("Chrome DevTools WebSocket failed")));
16140
+ ws.addEventListener("error", () => failSession(new Error("Chrome DevTools WebSocket failed")));
16141
+ ws.addEventListener("close", () => {
16142
+ if (!settled) failSession(new Error("Chrome DevTools WebSocket closed unexpectedly"));
16131
16143
  });
16132
16144
  });
16133
16145
  }
16146
+ async function sendChromeCommands(webSocketDebuggerUrl, commands) {
16147
+ return await withChromeSession(webSocketDebuggerUrl, async (session) => await Promise.all(commands.map((command) => session.send(command))));
16148
+ }
16149
+ async function sendChromeCommand(webSocketDebuggerUrl, method, params) {
16150
+ const response = (await sendChromeCommands(webSocketDebuggerUrl, [{ method, params }]))[0];
16151
+ if (response.error) throw new Error(response.error.message || `Chrome DevTools ${method} failed`);
16152
+ return response.result ?? {};
16153
+ }
16154
+ async function callFunctionOnChromeNodeInSession(session, backendNodeId, functionDeclaration, args = []) {
16155
+ const resolved = await sendChromeSessionCommand(session, "DOM.resolveNode", { backendNodeId });
16156
+ const objectId = isRecord(resolved.object) ? resolved.object.objectId : void 0;
16157
+ if (typeof objectId !== "string") throw new Error(`Could not resolve browser element ref ${backendNodeId}`);
16158
+ return await sendChromeSessionCommand(session, "Runtime.callFunctionOn", {
16159
+ objectId,
16160
+ functionDeclaration,
16161
+ arguments: args.map((value) => ({ value })),
16162
+ returnByValue: true
16163
+ });
16164
+ }
16165
+ function chromeCallValue(response) {
16166
+ return isRecord(response.result) ? response.result.value : void 0;
16167
+ }
16168
+ async function evaluateChromeTarget(webSocketDebuggerUrl, expression) {
16169
+ return await withChromeSession(webSocketDebuggerUrl, async (session) => await evaluateChromeSession(session, expression));
16170
+ }
16171
+ async function evaluateChromeSession(session, expression) {
16172
+ const response = await sendChromeSessionCommand(session, "Runtime.evaluate", {
16173
+ expression,
16174
+ awaitPromise: true,
16175
+ returnByValue: true
16176
+ });
16177
+ const exception = response.exceptionDetails;
16178
+ if (exception) {
16179
+ const text = typeof exception === "object" && exception !== null && "text" in exception && typeof exception.text === "string" ? exception.text : "";
16180
+ throw new Error(text || "Chrome DevTools evaluation threw");
16181
+ }
16182
+ const evaluated = response.result;
16183
+ return typeof evaluated === "object" && evaluated !== null && "value" in evaluated ? evaluated.value ?? null : null;
16184
+ }
16185
+ function isRawAXNode(value) {
16186
+ return isRecord(value);
16187
+ }
16188
+ function isDOMSnapshotDocument(value) {
16189
+ return isRecord(value);
16190
+ }
16191
+ function chromeVisualViewport(layout) {
16192
+ const viewport = isRecord(layout.cssVisualViewport) ? layout.cssVisualViewport : {};
16193
+ return {
16194
+ width: typeof viewport.clientWidth === "number" ? viewport.clientWidth : 0,
16195
+ height: typeof viewport.clientHeight === "number" ? viewport.clientHeight : 0
16196
+ };
16197
+ }
16198
+ function roundBrowserRect(rect) {
16199
+ return {
16200
+ x: Math.round(rect.x),
16201
+ y: Math.round(rect.y),
16202
+ width: Math.round(rect.width),
16203
+ height: Math.round(rect.height),
16204
+ centerX: Math.round(rect.centerX),
16205
+ centerY: Math.round(rect.centerY)
16206
+ };
16207
+ }
16208
+ function browserStateProperties(node) {
16209
+ const properties = [];
16210
+ for (const { name, value } of node.properties ?? []) {
16211
+ const state = value?.value;
16212
+ if (!name || !BROWSER_STATE_PROPERTIES.has(name) || typeof state !== "string" && typeof state !== "number" && typeof state !== "boolean") continue;
16213
+ properties.push([name, name === "url" ? clippedText(state, 300) : state]);
16214
+ }
16215
+ return Object.fromEntries(properties);
16216
+ }
16217
+ function readBrowserStateCache(path6) {
16218
+ let value;
16219
+ try {
16220
+ value = JSON.parse(readFileSync3(path6, "utf8"));
16221
+ } catch {
16222
+ return null;
16223
+ }
16224
+ if (!isRecord(value) || !Array.isArray(value.entries)) return null;
16225
+ const entries = [];
16226
+ for (const entry of value.entries) {
16227
+ if (!isRecord(entry) || typeof entry.key !== "string" || typeof entry.semantic !== "string" || typeof entry.line !== "string") return null;
16228
+ entries.push({ key: entry.key, semantic: entry.semantic, line: entry.line });
16229
+ }
16230
+ return {
16231
+ url: typeof value.url === "string" ? value.url : void 0,
16232
+ documentId: typeof value.documentId === "string" || value.documentId === null ? value.documentId : void 0,
16233
+ entries
16234
+ };
16235
+ }
16236
+ function chromeResult(response, method) {
16237
+ if (response.error) throw new Error(response.error.message || `Chrome DevTools ${method} failed`);
16238
+ return response.result ?? {};
16239
+ }
16240
+ function collectFrameIds(frameTree) {
16241
+ const ids = [];
16242
+ const visit = (value) => {
16243
+ if (!isRecord(value)) return;
16244
+ const frame = isRecord(value.frame) ? value.frame : {};
16245
+ if (typeof frame.id === "string") ids.push(frame.id);
16246
+ if (Array.isArray(value.childFrames)) value.childFrames.forEach(visit);
16247
+ };
16248
+ visit(frameTree);
16249
+ return ids;
16250
+ }
16251
+ function browserDocumentId(frameTree) {
16252
+ if (!isRecord(frameTree) || !isRecord(frameTree.frame)) return null;
16253
+ const loaderId = frameTree.frame.loaderId;
16254
+ return typeof loaderId === "string" ? loaderId : null;
16255
+ }
16256
+ function intersectRects(a, b) {
16257
+ const x = Math.max(a.x, b.x);
16258
+ const y = Math.max(a.y, b.y);
16259
+ const right = Math.min(a.x + a.width, b.x + b.width);
16260
+ const bottom = Math.min(a.y + a.height, b.y + b.height);
16261
+ if (right <= x || bottom <= y) return null;
16262
+ return {
16263
+ x,
16264
+ y,
16265
+ width: right - x,
16266
+ height: bottom - y,
16267
+ centerX: x + (right - x) / 2,
16268
+ centerY: y + (bottom - y) / 2
16269
+ };
16270
+ }
16271
+ function browserContentQuadRects(quads, viewportRect) {
16272
+ if (!Array.isArray(quads)) return [];
16273
+ return quads.filter((quad) => Array.isArray(quad) && quad.length >= 8 && quad.every(Number.isFinite)).map((quad) => {
16274
+ const xs = [quad[0], quad[2], quad[4], quad[6]];
16275
+ const ys = [quad[1], quad[3], quad[5], quad[7]];
16276
+ const x = Math.min(...xs);
16277
+ const y = Math.min(...ys);
16278
+ const width = Math.max(...xs) - x;
16279
+ const height = Math.max(...ys) - y;
16280
+ return { x, y, width, height, centerX: x + width / 2, centerY: y + height / 2 };
16281
+ }).map((rect) => intersectRects(rect, viewportRect)).filter((rect) => !!rect).sort((a, b) => b.width * b.height - a.width * a.height);
16282
+ }
16283
+ function buildBrowserBounds(result, viewport, targetId) {
16284
+ const strings = Array.isArray(result.strings) ? result.strings : [];
16285
+ const documents = Array.isArray(result.documents) ? result.documents.filter(isDOMSnapshotDocument) : [];
16286
+ const localRects = documents.map((document) => {
16287
+ const rects = /* @__PURE__ */ new Map();
16288
+ const nodeIndexes = document.layout?.nodeIndex ?? [];
16289
+ const bounds = document.layout?.bounds ?? [];
16290
+ const scrollX = document.scrollOffsetX ?? 0;
16291
+ const scrollY = document.scrollOffsetY ?? 0;
16292
+ nodeIndexes.forEach((nodeIndex, index) => {
16293
+ const bound = bounds[index];
16294
+ if (!Array.isArray(bound) || bound.length < 4 || !bound.every(Number.isFinite)) return;
16295
+ const [rawX, rawY, width, height] = bound;
16296
+ rects.set(nodeIndex, {
16297
+ x: rawX - scrollX,
16298
+ y: rawY - scrollY,
16299
+ width,
16300
+ height,
16301
+ centerX: rawX - scrollX + width / 2,
16302
+ centerY: rawY - scrollY + height / 2
16303
+ });
16304
+ });
16305
+ return rects;
16306
+ });
16307
+ const parents = /* @__PURE__ */ new Map();
16308
+ documents.forEach((document, documentIndex) => {
16309
+ const sparse = document.nodes?.contentDocumentIndex;
16310
+ sparse?.index?.forEach((nodeIndex, index) => {
16311
+ const childDocumentIndex = sparse.value?.[index];
16312
+ if (typeof childDocumentIndex === "number") parents.set(childDocumentIndex, { documentIndex, nodeIndex });
16313
+ });
16314
+ });
16315
+ const targetDocumentIndex = documents.findIndex(
16316
+ (document) => typeof document.frameId === "number" && strings[document.frameId] === targetId
16317
+ );
16318
+ const rootIndex = targetDocumentIndex >= 0 ? targetDocumentIndex : documents.findIndex((_, index) => !parents.has(index));
16319
+ const viewportRect = {
16320
+ x: 0,
16321
+ y: 0,
16322
+ width: viewport.width,
16323
+ height: viewport.height,
16324
+ centerX: viewport.width / 2,
16325
+ centerY: viewport.height / 2
16326
+ };
16327
+ const origins = /* @__PURE__ */ new Map();
16328
+ origins.set(rootIndex < 0 ? 0 : rootIndex, { x: 0, y: 0, clip: viewportRect });
16329
+ for (let pass = 0; pass < documents.length; pass++) {
16330
+ let changed = false;
16331
+ for (const [childIndex, parent] of parents) {
16332
+ if (origins.has(childIndex)) continue;
16333
+ const parentOrigin = origins.get(parent.documentIndex);
16334
+ const frameRect = localRects[parent.documentIndex]?.get(parent.nodeIndex);
16335
+ if (!parentOrigin || !frameRect) continue;
16336
+ const globalFrameRect = {
16337
+ ...frameRect,
16338
+ x: parentOrigin.x + frameRect.x,
16339
+ y: parentOrigin.y + frameRect.y,
16340
+ centerX: parentOrigin.x + frameRect.centerX,
16341
+ centerY: parentOrigin.y + frameRect.centerY
16342
+ };
16343
+ const childDocument = documents[childIndex];
16344
+ const insetX = Math.max(0, (globalFrameRect.width - (childDocument?.contentWidth ?? globalFrameRect.width)) / 2);
16345
+ const insetY = Math.max(0, (globalFrameRect.height - (childDocument?.contentHeight ?? globalFrameRect.height)) / 2);
16346
+ const contentRect = {
16347
+ x: globalFrameRect.x + insetX,
16348
+ y: globalFrameRect.y + insetY,
16349
+ width: Math.max(0, globalFrameRect.width - insetX * 2),
16350
+ height: Math.max(0, globalFrameRect.height - insetY * 2),
16351
+ centerX: globalFrameRect.centerX,
16352
+ centerY: globalFrameRect.centerY
16353
+ };
16354
+ const clip = intersectRects(parentOrigin.clip, contentRect);
16355
+ if (!clip) continue;
16356
+ origins.set(childIndex, { x: contentRect.x, y: contentRect.y, clip });
16357
+ changed = true;
16358
+ }
16359
+ if (!changed) break;
16360
+ }
16361
+ const byBackendNodeId = /* @__PURE__ */ new Map();
16362
+ documents.forEach((document, documentIndex) => {
16363
+ const origin = origins.get(documentIndex);
16364
+ if (!origin) return;
16365
+ const backendNodeIds = document.nodes?.backendNodeId ?? [];
16366
+ for (const [nodeIndex, local] of localRects[documentIndex] ?? []) {
16367
+ const backendNodeId = backendNodeIds[nodeIndex];
16368
+ if (!Number.isInteger(backendNodeId)) continue;
16369
+ const rect = {
16370
+ ...local,
16371
+ x: origin.x + local.x,
16372
+ y: origin.y + local.y,
16373
+ centerX: origin.x + local.centerX,
16374
+ centerY: origin.y + local.centerY
16375
+ };
16376
+ if (intersectRects(rect, origin.clip)) byBackendNodeId.set(backendNodeId, { rect, clip: origin.clip });
16377
+ }
16378
+ });
16379
+ return byBackendNodeId;
16380
+ }
16381
+ var BROWSER_INTERACTIVE_ROLES = /* @__PURE__ */ new Set([
16382
+ "button",
16383
+ "checkbox",
16384
+ "combobox",
16385
+ "link",
16386
+ "listbox",
16387
+ "menuitem",
16388
+ "menuitemcheckbox",
16389
+ "menuitemradio",
16390
+ "option",
16391
+ "radio",
16392
+ "scrollbar",
16393
+ "searchbox",
16394
+ "slider",
16395
+ "spinbutton",
16396
+ "switch",
16397
+ "tab",
16398
+ "textbox",
16399
+ "treeitem"
16400
+ ]);
16401
+ var BROWSER_STRUCTURAL_ROLES = /* @__PURE__ */ new Set([
16402
+ "alert",
16403
+ "cell",
16404
+ "columnheader",
16405
+ "dialog",
16406
+ "figure",
16407
+ "gridcell",
16408
+ "heading",
16409
+ "image",
16410
+ "listitem",
16411
+ "main",
16412
+ "paragraph",
16413
+ "region",
16414
+ "row",
16415
+ "rowheader",
16416
+ "status",
16417
+ "StaticText"
16418
+ ]);
16419
+ var BROWSER_STATE_PROPERTIES = /* @__PURE__ */ new Set([
16420
+ "autocomplete",
16421
+ "checked",
16422
+ "disabled",
16423
+ "editable",
16424
+ "expanded",
16425
+ "focusable",
16426
+ "focused",
16427
+ "haspopup",
16428
+ "invalid",
16429
+ "level",
16430
+ "multiselectable",
16431
+ "orientation",
16432
+ "pressed",
16433
+ "readonly",
16434
+ "protected",
16435
+ "required",
16436
+ "selected",
16437
+ "url"
16438
+ ]);
16439
+ function clippedText(value, limit = 240) {
16440
+ return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
16441
+ }
16442
+ function browserTargetOutput(page) {
16443
+ return {
16444
+ targetId: page.id ?? null,
16445
+ title: (page.title ?? "").slice(0, 1e3),
16446
+ url: (page.url ?? "").slice(0, 2e3)
16447
+ };
16448
+ }
16449
+ function browserNodeActionable(role, states) {
16450
+ return BROWSER_INTERACTIVE_ROLES.has(role) || states.editable === true || states.focusable === true && role !== "RootWebArea";
16451
+ }
16452
+ async function captureBrowserSnapshot(page, options) {
16453
+ const initialCommands = [
16454
+ { method: "Page.getFrameTree", params: {} },
16455
+ { method: "Page.getLayoutMetrics", params: {} },
16456
+ { method: "DOMSnapshot.captureSnapshot", params: { computedStyles: [], includeDOMRects: true, includePaintOrder: true } }
16457
+ ];
16458
+ const initial = await sendChromeCommands(page.webSocketDebuggerUrl, initialCommands);
16459
+ const frameTree = chromeResult(initial[0], initialCommands[0].method).frameTree;
16460
+ const layout = chromeResult(initial[1], initialCommands[1].method);
16461
+ const domSnapshot = chromeResult(initial[2], initialCommands[2].method);
16462
+ const viewport = chromeVisualViewport(layout);
16463
+ const frameIds = collectFrameIds(frameTree);
16464
+ if (frameIds.length === 0 && page.id) frameIds.push(page.id);
16465
+ const axCommands = frameIds.map((frameId) => ({
16466
+ method: "Accessibility.getFullAXTree",
16467
+ params: { frameId }
16468
+ }));
16469
+ const axResults = await sendChromeCommands(page.webSocketDebuggerUrl, axCommands);
16470
+ const nodes = [];
16471
+ axResults.forEach((response, index) => {
16472
+ if (response.error) return;
16473
+ const responseNodes = response.result?.nodes;
16474
+ if (!Array.isArray(responseNodes)) return;
16475
+ for (const node of responseNodes) {
16476
+ if (!isRawAXNode(node)) continue;
16477
+ nodes.push({ ...node, sourceFrameId: frameIds[index] });
16478
+ }
16479
+ });
16480
+ const bounds = buildBrowserBounds(domSnapshot, viewport, page.id);
16481
+ const nodesByKey = new Map(nodes.map((node) => [`${node.sourceFrameId}:${node.nodeId}`, node]));
16482
+ const propertyMap = browserStateProperties;
16483
+ const roleOf = (node) => clippedText(node.role?.value, 80);
16484
+ const nameOf = (node) => clippedText(node.name?.value);
16485
+ const valueOf = (node) => {
16486
+ if (propertyMap(node).protected === true) return null;
16487
+ const value = node.value?.value;
16488
+ if (typeof value === "number" || typeof value === "boolean") return value;
16489
+ const text = clippedText(value);
16490
+ if (/^[•●*]+$/.test(text)) return "[protected]";
16491
+ return text || null;
16492
+ };
16493
+ const meaningful = nodes.filter((node) => !node.ignored && Number.isInteger(node.backendDOMNodeId)).map((node) => ({
16494
+ node,
16495
+ role: roleOf(node),
16496
+ bound: typeof node.backendDOMNodeId === "number" ? bounds.get(node.backendDOMNodeId) : void 0
16497
+ })).filter(({ node, role, bound }) => {
16498
+ if (!bound || bound.rect.width <= 0 || bound.rect.height <= 0) return false;
16499
+ if (role === "InlineTextBox" || role === "none" || role === "generic") return false;
16500
+ if (BROWSER_INTERACTIVE_ROLES.has(role) || BROWSER_STRUCTURAL_ROLES.has(role)) return true;
16501
+ return !!nameOf(node) || valueOf(node) !== null || propertyMap(node).focusable === true;
16502
+ }).filter(({ node, role }) => {
16503
+ if (role !== "StaticText") return true;
16504
+ const parent = node.parentId ? nodesByKey.get(`${node.sourceFrameId}:${node.parentId}`) : void 0;
16505
+ return !parent || nameOf(parent) !== nameOf(node);
16506
+ }).sort((a, b) => a.bound.rect.y - b.bound.rect.y || a.bound.rect.x - b.bound.rect.x);
16507
+ const elements = [];
16508
+ const entries = [];
16509
+ const meaningfulKeys = new Set(meaningful.map(({ node }) => `${node.sourceFrameId}:${node.nodeId}`));
16510
+ const visibleText = [];
16511
+ let visibleTextLength = 0;
16512
+ let treeLength = 0;
16513
+ const treeLimit = Math.min(5e4, Math.max(4e3, options.textLimit * 2));
16514
+ for (const { node, role, bound } of meaningful) {
16515
+ const name = nameOf(node);
16516
+ const description = clippedText(node.description?.value) || null;
16517
+ const value = valueOf(node);
16518
+ const states = propertyMap(node);
16519
+ const actionable = browserNodeActionable(role, states);
16520
+ const ref = actionable && elements.length < options.elementLimit ? String(node.backendDOMNodeId) : null;
16521
+ const visibleRect = role === "RootWebArea" && node.sourceFrameId === page.id ? { x: 0, y: 0, width: viewport.width, height: viewport.height, centerX: viewport.width / 2, centerY: viewport.height / 2 } : intersectRects(bound.rect, bound.clip) ?? bound.rect;
16522
+ const rect = roundBrowserRect(visibleRect);
16523
+ const actions = [
16524
+ ...actionable ? ["click"] : [],
16525
+ ...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
16526
+ "scroll"
16527
+ ];
16528
+ if (ref) {
16529
+ elements.push({
16530
+ ref,
16531
+ role,
16532
+ name: name || null,
16533
+ description,
16534
+ value,
16535
+ states,
16536
+ actions,
16537
+ rect,
16538
+ frameId: node.sourceFrameId
16539
+ });
16540
+ }
16541
+ const stateText = Object.entries(states).filter(([key]) => key !== "url" && key !== "focusable").map(([key, state]) => `${key}=${JSON.stringify(state)}`);
16542
+ const url = typeof states.url === "string" ? clippedText(states.url, 300) : "";
16543
+ const semantic = [role, name && JSON.stringify(name), value !== null && `value=${JSON.stringify(value)}`, description && `description=${JSON.stringify(description)}`, url && `url=${JSON.stringify(url)}`, ...stateText].filter(Boolean).join(" ");
16544
+ let depth = 0;
16545
+ let parentId = node.parentId;
16546
+ const visited = /* @__PURE__ */ new Set();
16547
+ while (parentId && depth < 8) {
16548
+ const parentKey = `${node.sourceFrameId}:${parentId}`;
16549
+ if (visited.has(parentKey)) break;
16550
+ visited.add(parentKey);
16551
+ if (meaningfulKeys.has(parentKey)) depth++;
16552
+ parentId = nodesByKey.get(parentKey)?.parentId;
16553
+ }
16554
+ const line = `${" ".repeat(depth)}${ref ? `[ref=${ref}]` : "-"} ${semantic} (${rect.x},${rect.y} ${rect.width}x${rect.height})`;
16555
+ if (treeLength + line.length + 1 <= treeLimit) {
16556
+ const key = Number.isInteger(node.backendDOMNodeId) ? `dom:${node.sourceFrameId}:${node.backendDOMNodeId}` : `ax:${node.sourceFrameId}:${node.nodeId}`;
16557
+ entries.push({ key, semantic, line });
16558
+ treeLength += line.length + 1;
16559
+ }
16560
+ if (name && visibleTextLength < options.textLimit && (role === "StaticText" || role === "heading" || BROWSER_INTERACTIVE_ROLES.has(role))) {
16561
+ const remaining = options.textLimit - visibleTextLength;
16562
+ visibleText.push(name.slice(0, remaining));
16563
+ visibleTextLength += Math.min(name.length, remaining) + 1;
16564
+ }
16565
+ }
16566
+ const revision = createHash2("sha256").update(JSON.stringify([page.url, entries.map(({ key, semantic }) => [key, semantic])])).digest("hex").slice(0, 16);
16567
+ const snapshot = {
16568
+ title: (page.title ?? "").slice(0, 1e3),
16569
+ url: (page.url ?? "").slice(0, 2e3),
16570
+ documentId: browserDocumentId(frameTree),
16571
+ revision,
16572
+ viewport,
16573
+ text: visibleText.join(" ").slice(0, options.textLimit),
16574
+ tree: entries.map(({ line }) => line).join("\n"),
16575
+ elements,
16576
+ controls: elements,
16577
+ nodeCount: entries.length,
16578
+ elementCount: elements.length,
16579
+ truncated: entries.length < meaningful.length || elements.length < meaningful.filter(({ node, role }) => browserNodeActionable(role, propertyMap(node))).length
16580
+ };
16581
+ return { snapshot, entries };
16582
+ }
16134
16583
  async function computerBrowserCommand(options = {}) {
16135
16584
  const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 4e3;
16136
16585
  const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 80;
16137
16586
  if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
16138
16587
  if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
16139
- const expression = buildBrowserSnapshotExpression(textLimit, elementLimit);
16140
- const pages = (await getChromePages()).map((target) => ({
16588
+ const targets = options.targetId || options.page || options.title || options.url ? [await selectChromePage(options)] : await getChromePages();
16589
+ const pages = targets.map((target) => ({
16141
16590
  target,
16142
16591
  page: {
16143
16592
  id: target.id ?? null,
@@ -16150,7 +16599,11 @@ async function computerBrowserCommand(options = {}) {
16150
16599
  for (const { target, page } of pages) {
16151
16600
  if (options.snapshot && target.webSocketDebuggerUrl) {
16152
16601
  try {
16153
- pageResults.push({ ...page, snapshot: await evaluateChromeTarget(target.webSocketDebuggerUrl, expression) });
16602
+ const { snapshot } = await captureBrowserSnapshot(
16603
+ { ...target, webSocketDebuggerUrl: target.webSocketDebuggerUrl },
16604
+ { textLimit, elementLimit }
16605
+ );
16606
+ pageResults.push({ ...page, snapshot });
16154
16607
  } catch (error) {
16155
16608
  pageResults.push({ ...page, snapshotError: error instanceof Error ? error.message : "snapshot failed" });
16156
16609
  }
@@ -16164,6 +16617,188 @@ async function computerBrowserCommand(options = {}) {
16164
16617
  pages: pageResults
16165
16618
  }, null, 2));
16166
16619
  }
16620
+ function buildBrowserStabilityExpression() {
16621
+ return `(() => {
16622
+ const key = '__replicasComputerUseObserver';
16623
+ if (!globalThis[key]) {
16624
+ const state = { revision: 0 };
16625
+ const observer = new MutationObserver(() => state.revision++);
16626
+ globalThis[key] = { state, observer, observing: false };
16627
+ }
16628
+ if (document.documentElement && !globalThis[key].observing) {
16629
+ globalThis[key].observer.observe(document.documentElement, {
16630
+ subtree: true,
16631
+ childList: true,
16632
+ attributes: true,
16633
+ characterData: true,
16634
+ });
16635
+ globalThis[key].observing = true;
16636
+ }
16637
+ const controls = Array.from(document.querySelectorAll('input,textarea,select,[contenteditable="true"],[role="checkbox"],[role="combobox"],[role="slider"],[role="switch"],[role="textbox"]')).slice(0, 200);
16638
+ const active = document.activeElement;
16639
+ const controlState = controls.map((element, index) => {
16640
+ const value = 'value' in element
16641
+ ? element instanceof HTMLInputElement && element.type === 'password'
16642
+ ? String(element.value).length
16643
+ : String(element.value)
16644
+ : element.textContent;
16645
+ return [
16646
+ index,
16647
+ element === active,
16648
+ value,
16649
+ 'checked' in element ? element.checked : null,
16650
+ 'selectedIndex' in element ? element.selectedIndex : null,
16651
+ 'disabled' in element ? element.disabled : null,
16652
+ element.getAttribute('aria-checked'),
16653
+ element.getAttribute('aria-expanded'),
16654
+ element.getAttribute('aria-selected'),
16655
+ ].join('|');
16656
+ }).join('\\u001f');
16657
+ let controlRevision = 2166136261;
16658
+ for (let index = 0; index < controlState.length; index++) {
16659
+ controlRevision ^= controlState.charCodeAt(index);
16660
+ controlRevision = Math.imul(controlRevision, 16777619);
16661
+ }
16662
+ return {
16663
+ revision: globalThis[key].state.revision,
16664
+ controlRevision: (controlRevision >>> 0).toString(16),
16665
+ readyState: document.readyState,
16666
+ busy: !!document.querySelector('[aria-busy="true"]'),
16667
+ url: location.href.slice(0, 2000),
16668
+ scrollX: Math.round(scrollX),
16669
+ scrollY: Math.round(scrollY),
16670
+ innerWidth,
16671
+ innerHeight,
16672
+ devicePixelRatio,
16673
+ };
16674
+ })()`;
16675
+ }
16676
+ async function waitForBrowserStability(page, options) {
16677
+ const startedAt = Date.now();
16678
+ let lastSignature = null;
16679
+ let lastChangeAt = startedAt;
16680
+ let samples = 0;
16681
+ let changes = 0;
16682
+ let state = null;
16683
+ while (Date.now() - startedAt <= options.timeoutMs) {
16684
+ try {
16685
+ state = await evaluateChromeTarget(page.webSocketDebuggerUrl, buildBrowserStabilityExpression());
16686
+ samples++;
16687
+ const signature = JSON.stringify(state);
16688
+ const now = Date.now();
16689
+ if (lastSignature === null || signature !== lastSignature) {
16690
+ if (lastSignature !== null) changes++;
16691
+ lastSignature = signature;
16692
+ lastChangeAt = now;
16693
+ }
16694
+ const ready = typeof state === "object" && state !== null && "readyState" in state && state.readyState !== "loading" && "busy" in state && state.busy !== true;
16695
+ if (samples > 1 && ready && now - lastChangeAt >= options.stableMs) {
16696
+ return { stable: true, elapsedMs: now - startedAt, samples, changes, state };
16697
+ }
16698
+ } catch {
16699
+ }
16700
+ await sleep(options.pollMs);
16701
+ }
16702
+ return { stable: false, elapsedMs: Date.now() - startedAt, samples, changes, state };
16703
+ }
16704
+ function browserStateCachePath(targetId) {
16705
+ return `${STATE_DIR}/browser-state-${targetId.replace(/[^a-zA-Z0-9_-]/g, "_")}.json`;
16706
+ }
16707
+ function browserStateDiff(previous, current) {
16708
+ const before = new Map(previous.map((entry) => [entry.key, entry]));
16709
+ const after = new Map(current.map((entry) => [entry.key, entry]));
16710
+ const lines = [];
16711
+ let added = 0;
16712
+ let changed = 0;
16713
+ let removed = 0;
16714
+ for (const entry of current) {
16715
+ const old = before.get(entry.key);
16716
+ if (!old) {
16717
+ added++;
16718
+ lines.push(`+ ${entry.line}`);
16719
+ } else if (old.semantic !== entry.semantic) {
16720
+ changed++;
16721
+ lines.push(`~ ${old.line}
16722
+ ${entry.line}`);
16723
+ }
16724
+ }
16725
+ for (const entry of previous) {
16726
+ if (after.has(entry.key)) continue;
16727
+ removed++;
16728
+ lines.push(`- ${entry.line}`);
16729
+ }
16730
+ return { tree: lines.join("\n") || "No accessibility changes.", added, changed, removed };
16731
+ }
16732
+ async function computerBrowserStateCommand(path6, options = {}) {
16733
+ const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 8e3;
16734
+ const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 120;
16735
+ const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 5e3;
16736
+ const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 500;
16737
+ const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
16738
+ if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
16739
+ if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
16740
+ if (timeoutMs < 0) fail("--timeout must be >= 0");
16741
+ if (stableMs < 0) fail("--stable-ms must be >= 0");
16742
+ if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
16743
+ const page = await selectChromePage(options);
16744
+ const stability = await waitForBrowserStability(page, { timeoutMs, stableMs, pollMs });
16745
+ const target = resolvePath(path6);
16746
+ mkdirSync3(dirname3(target), { recursive: true });
16747
+ const [{ snapshot, entries }, screenshotResult] = await Promise.all([
16748
+ captureBrowserSnapshot(page, { textLimit, elementLimit }),
16749
+ sendChromeCommand(page.webSocketDebuggerUrl, "Page.captureScreenshot", {
16750
+ format: "png",
16751
+ fromSurface: true,
16752
+ captureBeyondViewport: false
16753
+ })
16754
+ ]);
16755
+ const data = screenshotResult.data;
16756
+ if (typeof data !== "string") fail("Chrome did not return screenshot data");
16757
+ writeFileSync3(target, Buffer.from(data, "base64"));
16758
+ const screenshot = readPngDimensions(target);
16759
+ const targetId = page.id ?? "unknown";
16760
+ const cachePath = browserStateCachePath(targetId);
16761
+ let mode = "full";
16762
+ let tree = snapshot.tree;
16763
+ let changes = { added: snapshot.nodeCount, changed: 0, removed: 0 };
16764
+ if (!options.full && existsSync3(cachePath)) {
16765
+ const cached = readBrowserStateCache(cachePath);
16766
+ if (cached?.url === snapshot.url && cached.documentId === snapshot.documentId) {
16767
+ mode = "diff";
16768
+ const diff = browserStateDiff(cached.entries, entries);
16769
+ tree = diff.tree;
16770
+ changes = { added: diff.added, changed: diff.changed, removed: diff.removed };
16771
+ }
16772
+ }
16773
+ mkdirSync3(STATE_DIR, { recursive: true });
16774
+ writeFileSync3(cachePath, JSON.stringify({ url: snapshot.url, documentId: snapshot.documentId, entries }));
16775
+ const stableState = isRecord(stability.state) ? stability.state : {};
16776
+ const state = {
16777
+ title: snapshot.title,
16778
+ url: snapshot.url,
16779
+ documentId: snapshot.documentId,
16780
+ revision: snapshot.revision,
16781
+ viewport: snapshot.viewport,
16782
+ text: snapshot.text,
16783
+ tree,
16784
+ nodeCount: snapshot.nodeCount,
16785
+ elementCount: snapshot.elementCount,
16786
+ truncated: snapshot.truncated,
16787
+ mode,
16788
+ changes
16789
+ };
16790
+ console.log(JSON.stringify({
16791
+ targetId,
16792
+ screenshot: {
16793
+ path: target,
16794
+ width: screenshot.width,
16795
+ height: screenshot.height,
16796
+ devicePixelRatio: typeof stableState.devicePixelRatio === "number" ? stableState.devicePixelRatio : 1
16797
+ },
16798
+ stability,
16799
+ state
16800
+ }, null, 2));
16801
+ }
16167
16802
  var DESKTOP_POINT_JS = `
16168
16803
  const desktopPoint = (rect) => {
16169
16804
  const borderX = Math.max(0, (window.outerWidth - window.innerWidth) / 2);
@@ -16173,17 +16808,95 @@ var DESKTOP_POINT_JS = `
16173
16808
  y: Math.round(window.screenY + topChrome + rect.y + rect.height / 2),
16174
16809
  };
16175
16810
  };`;
16811
+ function parseBrowserRef(value) {
16812
+ const ref = parseCoord(value.replace(/^ref=/, ""), "--ref");
16813
+ if (ref <= 0) fail("--ref must be a positive backend DOM node ID from the latest browser state");
16814
+ return ref;
16815
+ }
16816
+ function readBrowserRefCache(page) {
16817
+ const targetId = page.id;
16818
+ if (!targetId) fail("Chrome target has no stable ID. Capture fresh browser state and retry.");
16819
+ const cachePath = browserStateCachePath(targetId);
16820
+ if (!existsSync3(cachePath)) fail(`No browser state is cached for target ${targetId}. Run replicas computer browser-state <path> --target-id ${targetId} first.`);
16821
+ const cached = readBrowserStateCache(cachePath);
16822
+ if (!cached) fail(`Browser state for target ${targetId} is invalid. Capture fresh browser state and retry.`);
16823
+ if (cached.url !== (page.url ?? "").slice(0, 2e3)) fail("Cached browser state belongs to a different URL. Capture fresh browser state and retry.");
16824
+ return { targetId, cached };
16825
+ }
16826
+ function browserViewportRect(layout) {
16827
+ const viewport = chromeVisualViewport(layout);
16828
+ return {
16829
+ x: 0,
16830
+ y: 0,
16831
+ width: viewport.width,
16832
+ height: viewport.height,
16833
+ centerX: viewport.width / 2,
16834
+ centerY: viewport.height / 2
16835
+ };
16836
+ }
16837
+ async function prepareBrowserElement(page, refValue) {
16838
+ const backendNodeId = parseBrowserRef(refValue);
16839
+ const { cached } = readBrowserRefCache(page);
16840
+ return await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
16841
+ const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
16842
+ const rect = await prepareBrowserRefInSession(session, backendNodeId, cached, browserViewportRect(layout));
16843
+ const axNodes = (await sendChromeSessionCommand(session, "Accessibility.getPartialAXTree", {
16844
+ backendNodeId,
16845
+ fetchRelatives: false
16846
+ })).nodes;
16847
+ const rawNode = Array.isArray(axNodes) && isRawAXNode(axNodes[0]) ? axNodes[0] : null;
16848
+ const states = rawNode ? browserStateProperties(rawNode) : {};
16849
+ const role = clippedText(rawNode?.role?.value, 80);
16850
+ const element = rawNode ? {
16851
+ ref: String(backendNodeId),
16852
+ role,
16853
+ name: clippedText(rawNode.name?.value) || null,
16854
+ description: clippedText(rawNode.description?.value) || null,
16855
+ value: typeof rawNode.value?.value === "string" ? clippedText(rawNode.value.value) : typeof rawNode.value?.value === "number" || typeof rawNode.value?.value === "boolean" ? rawNode.value.value : null,
16856
+ states,
16857
+ actions: [
16858
+ "click",
16859
+ ...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
16860
+ "scroll"
16861
+ ]
16862
+ } : null;
16863
+ const desktopPoint = await evaluateChromeSession(session, `(() => {
16864
+ ${DESKTOP_POINT_JS}
16865
+ return desktopPoint({ x: ${rect.centerX}, y: ${rect.centerY}, width: 0, height: 0 });
16866
+ })()`);
16867
+ return {
16868
+ ref: String(backendNodeId),
16869
+ rect,
16870
+ element,
16871
+ desktopPoint: typeof desktopPoint === "object" && desktopPoint !== null && "x" in desktopPoint && typeof desktopPoint.x === "number" && "y" in desktopPoint && typeof desktopPoint.y === "number" ? { x: desktopPoint.x, y: desktopPoint.y } : null
16872
+ };
16873
+ });
16874
+ }
16875
+ async function prepareBrowserRefInSession(session, backendNodeId, cached, viewportRect) {
16876
+ if (!cached.entries.some(({ line }) => line.trimStart().startsWith(`[ref=${backendNodeId}]`))) {
16877
+ fail(`Browser element ref ${backendNodeId} is not in the latest state. Capture fresh browser state and retry.`);
16878
+ }
16879
+ const frameTree = (await sendChromeSessionCommand(session, "Page.getFrameTree", {})).frameTree;
16880
+ if (cached.documentId !== browserDocumentId(frameTree)) {
16881
+ fail(`Browser element ref ${backendNodeId} belongs to a previous document. Capture fresh browser state and retry.`);
16882
+ }
16883
+ try {
16884
+ await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId });
16885
+ } catch (error) {
16886
+ fail(`Browser element ref ${backendNodeId} is stale or cannot be scrolled into view: ${error instanceof Error ? error.message : String(error)}`);
16887
+ }
16888
+ const quads = (await sendChromeSessionCommand(session, "DOM.getContentQuads", { backendNodeId })).quads;
16889
+ const candidates = browserContentQuadRects(quads, viewportRect);
16890
+ if (!candidates[0]) fail(`Browser element ref ${backendNodeId} has no visible content quad. Fetch fresh browser state and retry.`);
16891
+ return roundBrowserRect(candidates[0]);
16892
+ }
16176
16893
  function buildBrowserClickExpression(query, options) {
16177
16894
  return `(() => {
16178
16895
  const query = ${JSON.stringify(query)};
16179
16896
  const exact = ${JSON.stringify(options.exact)};
16180
16897
  const index = ${options.index};
16181
16898
  const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
16182
- const visible = (el) => {
16183
- const rect = el.getBoundingClientRect();
16184
- const style = getComputedStyle(el);
16185
- return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
16186
- };
16899
+ ${VISIBLE_JS}
16187
16900
  const label = (el) => normalize(el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('placeholder') || el.href || el.id || el.name || el.tagName);
16188
16901
  ${DESKTOP_POINT_JS}
16189
16902
  const matches = (text) => {
@@ -16208,8 +16921,7 @@ ${DESKTOP_POINT_JS}
16208
16921
  const { el, text } = candidate;
16209
16922
  el.scrollIntoView({ block: 'center', inline: 'center' });
16210
16923
  const rect = el.getBoundingClientRect();
16211
- if (typeof el.focus === 'function') el.focus();
16212
- el.click();
16924
+ if (typeof el.focus === 'function') el.focus({ preventScroll: true });
16213
16925
  return {
16214
16926
  clicked: true,
16215
16927
  query,
@@ -16232,6 +16944,15 @@ ${DESKTOP_POINT_JS}
16232
16944
  };
16233
16945
  })()`;
16234
16946
  }
16947
+ function browserActionRectCenter(result) {
16948
+ if (typeof result !== "object" || result === null || !("rect" in result)) return null;
16949
+ const rect = result.rect;
16950
+ if (typeof rect !== "object" || rect === null) return null;
16951
+ const x = "centerX" in rect ? rect.centerX : void 0;
16952
+ const y = "centerY" in rect ? rect.centerY : void 0;
16953
+ if (typeof x !== "number" || typeof y !== "number" || !Number.isFinite(x) || !Number.isFinite(y)) return null;
16954
+ return { x, y };
16955
+ }
16235
16956
  function browserActionPoint(result) {
16236
16957
  if (typeof result !== "object" || result === null) return null;
16237
16958
  const action = "result" in result ? result.result : result;
@@ -16247,19 +16968,149 @@ function browserActionPoint(result) {
16247
16968
  y: clamp(Math.round(y), 0, dimensions.height)
16248
16969
  };
16249
16970
  }
16971
+ async function dispatchChromeClick(page, x, y, button, clicks) {
16972
+ await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
16973
+ await dispatchChromeClickInSession(session, x, y, button, clicks);
16974
+ });
16975
+ }
16976
+ async function dispatchChromeClickInSession(session, x, y, button, clicks) {
16977
+ for (let clickCount = 1; clickCount <= clicks; clickCount++) {
16978
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
16979
+ type: "mousePressed",
16980
+ x,
16981
+ y,
16982
+ button,
16983
+ clickCount
16984
+ });
16985
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
16986
+ type: "mouseReleased",
16987
+ x,
16988
+ y,
16989
+ button,
16990
+ clickCount
16991
+ });
16992
+ }
16993
+ }
16994
+ async function replaceChromeText(page, text) {
16995
+ await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
16996
+ await replaceChromeTextInSession(session, text);
16997
+ });
16998
+ }
16999
+ async function replaceChromeTextInSession(session, text) {
17000
+ await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
17001
+ type: "rawKeyDown",
17002
+ key: "a",
17003
+ code: "KeyA",
17004
+ windowsVirtualKeyCode: 65,
17005
+ nativeVirtualKeyCode: 65,
17006
+ modifiers: 2
17007
+ });
17008
+ await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
17009
+ type: "keyUp",
17010
+ key: "a",
17011
+ code: "KeyA",
17012
+ windowsVirtualKeyCode: 65,
17013
+ nativeVirtualKeyCode: 65,
17014
+ modifiers: 2
17015
+ });
17016
+ await sendChromeSessionCommand(session, "Input.insertText", { text });
17017
+ }
17018
+ var BROWSER_DIRECT_VALUE_INPUT_TYPES = /* @__PURE__ */ new Set(["number", "range", "date", "datetime-local", "month", "week", "time", "color"]);
17019
+ function browserFillInvalid(actual, expected, fieldType, semanticSelection = false, selectionMatched = false) {
17020
+ if (semanticSelection) return !selectionMatched;
17021
+ if (["number", "range"].includes(fieldType)) {
17022
+ const actualNumber = typeof actual === "number" ? actual : Number(actual);
17023
+ const expectedNumber = Number(expected);
17024
+ return !Number.isFinite(actualNumber) || !Number.isFinite(expectedNumber) || actualNumber !== expectedNumber;
17025
+ }
17026
+ if (fieldType === "color") return String(actual).toLowerCase() !== expected.toLowerCase();
17027
+ return actual !== expected;
17028
+ }
17029
+ async function fillChromeNodeInSession(session, backendNodeId, value) {
17030
+ await sendChromeSessionCommand(session, "DOM.focus", { backendNodeId });
17031
+ const directResult = await callFunctionOnChromeNodeInSession(
17032
+ session,
17033
+ backendNodeId,
17034
+ `function(value) {
17035
+ const emit = () => {
17036
+ this.dispatchEvent(new Event('input', { bubbles: true }));
17037
+ this.dispatchEvent(new Event('change', { bubbles: true }));
17038
+ };
17039
+ if (this instanceof HTMLSelectElement) {
17040
+ const option = Array.from(this.options).find((item) => item.value === value || item.text.trim() === value);
17041
+ if (option) this.value = option.value;
17042
+ else this.value = value;
17043
+ emit();
17044
+ return { handled: true, semanticSelection: true, matched: !!option, value: this.value };
17045
+ }
17046
+ if (this instanceof HTMLInputElement && ${JSON.stringify([...BROWSER_DIRECT_VALUE_INPUT_TYPES])}.includes(this.type.toLowerCase())) {
17047
+ const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
17048
+ if (!setter) return { handled: false };
17049
+ setter.call(this, value);
17050
+ emit();
17051
+ return { handled: true, semanticSelection: false, matched: true, value: this.value };
17052
+ }
17053
+ return { handled: false };
17054
+ }`,
17055
+ [value]
17056
+ );
17057
+ const directValue = chromeCallValue(directResult);
17058
+ const directHandled = isRecord(directValue) && directValue.handled === true;
17059
+ if (!directHandled) await replaceChromeTextInSession(session, value);
17060
+ const verification = await callFunctionOnChromeNodeInSession(
17061
+ session,
17062
+ backendNodeId,
17063
+ `function() {
17064
+ const value = this instanceof HTMLSelectElement || 'value' in this ? this.value : this.textContent;
17065
+ const type = this instanceof HTMLSelectElement
17066
+ ? 'select'
17067
+ : this instanceof HTMLInputElement
17068
+ ? this.type.toLowerCase()
17069
+ : this.isContentEditable
17070
+ ? 'contenteditable'
17071
+ : 'text';
17072
+ return { value, valueLength: String(value ?? '').length, type };
17073
+ }`
17074
+ );
17075
+ const actual = chromeCallValue(verification);
17076
+ if (!isRecord(actual)) fail(`Chrome did not return a value for browser element ref ${backendNodeId}.`);
17077
+ const fieldType = typeof actual.type === "string" ? actual.type : "";
17078
+ const semanticSelection = isRecord(directValue) && directValue.semanticSelection === true;
17079
+ const selectionMatched = isRecord(directValue) && directValue.matched === true;
17080
+ if (browserFillInvalid(actual.value, value, fieldType, semanticSelection, selectionMatched)) {
17081
+ fail(`Chrome reported that ref ${backendNodeId} contains ${JSON.stringify(actual.value)} after filling, expected ${JSON.stringify(value)}.`);
17082
+ }
17083
+ return { value: actual.value, valueLength: actual.valueLength, type: fieldType };
17084
+ }
16250
17085
  async function computerBrowserClickCommand(query, options = {}) {
16251
17086
  const page = await selectChromePage(options);
17087
+ if (options.ref) {
17088
+ const target = await prepareBrowserElement(page, options.ref);
17089
+ const buttonNames = { "1": "left", "2": "middle", "3": "right", left: "left", middle: "middle", right: "right" };
17090
+ const button = buttonNames[(options.button ?? "left").toLowerCase()];
17091
+ if (!button) fail("--button must be one of left|middle|right|1|2|3");
17092
+ const clicks = options.double ? 2 : 1;
17093
+ await dispatchChromeClick(page, target.rect.centerX, target.rect.centerY, button, clicks);
17094
+ if (target.desktopPoint) logRecordingAction({ type: "click", ...target.desktopPoint });
17095
+ console.log(JSON.stringify({
17096
+ ...browserTargetOutput(page),
17097
+ result: { clicked: true, ref: target.ref, button, clickCount: clicks, rect: target.rect, element: target.element }
17098
+ }, null, 2));
17099
+ return;
17100
+ }
17101
+ if (!query) fail("Provide control text or --ref <id> from the latest browser state");
16252
17102
  const index = options.index ? parseCoord(options.index, "--index") : 0;
16253
17103
  if (index < 0) fail("--index must be >= 0");
16254
17104
  const result = await evaluateChromeTarget(
16255
17105
  page.webSocketDebuggerUrl,
16256
17106
  buildBrowserClickExpression(query, { exact: !!options.exact, index })
16257
17107
  );
17108
+ const center = browserActionRectCenter(result);
17109
+ if (center) await dispatchChromeClick(page, center.x, center.y, "left", 1);
16258
17110
  const point = browserActionPoint(result);
16259
17111
  if (point) logRecordingAction({ type: "click", ...point });
16260
17112
  console.log(JSON.stringify({
16261
- title: page.title ?? "",
16262
- url: page.url ?? "",
17113
+ ...browserTargetOutput(page),
16263
17114
  result
16264
17115
  }, null, 2));
16265
17116
  }
@@ -16271,11 +17122,7 @@ function buildBrowserFillExpression(query, value, options) {
16271
17122
  const index = ${options.index};
16272
17123
  const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
16273
17124
  const escape = (text) => globalThis.CSS && CSS.escape ? CSS.escape(text) : String(text).replace(/["\\\\]/g, '\\\\$&');
16274
- const visible = (el) => {
16275
- const rect = el.getBoundingClientRect();
16276
- const style = getComputedStyle(el);
16277
- return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
16278
- };
17125
+ ${VISIBLE_JS}
16279
17126
  const matches = (text) => {
16280
17127
  const haystack = text.toLowerCase();
16281
17128
  const needle = query.toLowerCase();
@@ -16320,18 +17167,42 @@ ${DESKTOP_POINT_JS}
16320
17167
  }
16321
17168
  const { el, text } = candidate;
16322
17169
  el.scrollIntoView({ block: 'center', inline: 'center' });
16323
- if (typeof el.focus === 'function') el.focus();
17170
+ if (typeof el.focus === 'function') el.focus({ preventScroll: true });
17171
+ const fieldType = el.tagName.toLowerCase() === 'select' ? 'select' : String(el.getAttribute('type') || 'text').toLowerCase();
17172
+ let nativeInputRequired = false;
17173
+ let semanticSelection = false;
17174
+ let selectionMatched = false;
16324
17175
  if (el.tagName.toLowerCase() === 'select') {
16325
17176
  const option = Array.from(el.options).find((item) => item.value === value || normalize(item.text) === value);
16326
17177
  if (option) el.value = option.value;
16327
17178
  else el.value = value;
17179
+ semanticSelection = true;
17180
+ selectionMatched = !!option;
17181
+ el.dispatchEvent(new Event('input', { bubbles: true }));
17182
+ el.dispatchEvent(new Event('change', { bubbles: true }));
17183
+ } else if (el instanceof HTMLInputElement && ${JSON.stringify([...BROWSER_DIRECT_VALUE_INPUT_TYPES])}.includes(fieldType)) {
17184
+ const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
17185
+ if (setter) {
17186
+ setter.call(el, value);
17187
+ el.dispatchEvent(new Event('input', { bubbles: true }));
17188
+ el.dispatchEvent(new Event('change', { bubbles: true }));
17189
+ } else {
17190
+ nativeInputRequired = true;
17191
+ }
16328
17192
  } else if (el.isContentEditable) {
16329
- el.textContent = value;
17193
+ const selection = getSelection();
17194
+ const range = document.createRange();
17195
+ range.selectNodeContents(el);
17196
+ selection.removeAllRanges();
17197
+ selection.addRange(range);
17198
+ nativeInputRequired = true;
17199
+ } else if (typeof el.select === 'function') {
17200
+ el.select();
17201
+ nativeInputRequired = true;
16330
17202
  } else {
16331
- el.value = value;
17203
+ el.setSelectionRange(0, String(el.value || '').length);
17204
+ nativeInputRequired = true;
16332
17205
  }
16333
- el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: value }));
16334
- el.dispatchEvent(new Event('change', { bubbles: true }));
16335
17206
  const rect = el.getBoundingClientRect();
16336
17207
  return {
16337
17208
  filled: true,
@@ -16340,9 +17211,13 @@ ${DESKTOP_POINT_JS}
16340
17211
  index,
16341
17212
  tag: el.tagName.toLowerCase(),
16342
17213
  role: el.getAttribute('role') || null,
16343
- type: el.getAttribute('type') || null,
17214
+ type: fieldType,
16344
17215
  text,
16345
- value: el.isContentEditable ? el.textContent : el.value,
17216
+ actualValue: el.isContentEditable ? el.textContent : 'value' in el ? el.value : null,
17217
+ valueLength: String(el.isContentEditable ? el.textContent : 'value' in el ? el.value : '').length,
17218
+ nativeInputRequired,
17219
+ semanticSelection,
17220
+ selectionMatched,
16346
17221
  desktopPoint: desktopPoint(rect),
16347
17222
  rect: {
16348
17223
  x: Math.round(rect.x),
@@ -16357,6 +17232,18 @@ ${DESKTOP_POINT_JS}
16357
17232
  }
16358
17233
  async function computerBrowserFillCommand(query, value, options = {}) {
16359
17234
  const page = await selectChromePage(options);
17235
+ if (options.ref) {
17236
+ const target = await prepareBrowserElement(page, options.ref);
17237
+ const backendNodeId = parseBrowserRef(options.ref);
17238
+ const actual = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await fillChromeNodeInSession(session, backendNodeId, value));
17239
+ if (target.desktopPoint) logRecordingAction({ type: "type", ...target.desktopPoint });
17240
+ console.log(JSON.stringify({
17241
+ ...browserTargetOutput(page),
17242
+ result: { filled: true, ref: target.ref, valueLength: actual.valueLength, rect: target.rect, element: target.element }
17243
+ }, null, 2));
17244
+ return;
17245
+ }
17246
+ if (!query) fail("Provide field text or --ref <id> from the latest browser state");
16360
17247
  const index = options.index ? parseCoord(options.index, "--index") : 0;
16361
17248
  if (index < 0) fail("--index must be >= 0");
16362
17249
  const result = await evaluateChromeTarget(
@@ -16364,10 +17251,28 @@ async function computerBrowserFillCommand(query, value, options = {}) {
16364
17251
  buildBrowserFillExpression(query, value, { exact: !!options.exact, index })
16365
17252
  );
16366
17253
  const point = browserActionPoint(result);
17254
+ if (typeof result === "object" && result !== null && "nativeInputRequired" in result && result.nativeInputRequired === true) {
17255
+ await replaceChromeText(page, value);
17256
+ const actualValue = await evaluateChromeTarget(page.webSocketDebuggerUrl, `(() => {
17257
+ const el = document.activeElement;
17258
+ return el && el.isContentEditable ? el.textContent : el && 'value' in el ? el.value : null;
17259
+ })()`);
17260
+ const fieldType = "type" in result && typeof result.type === "string" ? result.type.toLowerCase() : "";
17261
+ if (browserFillInvalid(actualValue, value, fieldType)) {
17262
+ fail(`Chrome reported that ${JSON.stringify(query)} contains ${JSON.stringify(actualValue)} after filling, expected ${JSON.stringify(value)}.`);
17263
+ }
17264
+ if (isRecord(result)) {
17265
+ Object.assign(result, { actualValue, valueLength: String(actualValue ?? "").length });
17266
+ }
17267
+ } else if (isRecord(result) && result.filled === true) {
17268
+ const fieldType = typeof result.type === "string" ? result.type : "";
17269
+ if (browserFillInvalid(result.actualValue, value, fieldType, result.semanticSelection === true, result.selectionMatched === true)) {
17270
+ fail(`Chrome reported that ${JSON.stringify(query)} contains ${JSON.stringify(result.actualValue)} after filling, expected ${JSON.stringify(value)}.`);
17271
+ }
17272
+ }
16367
17273
  if (point) logRecordingAction({ type: "type", ...point });
16368
17274
  console.log(JSON.stringify({
16369
- title: page.title ?? "",
16370
- url: page.url ?? "",
17275
+ ...browserTargetOutput(page),
16371
17276
  result
16372
17277
  }, null, 2));
16373
17278
  }
@@ -16377,11 +17282,7 @@ function buildBrowserWaitExpression(query, options) {
16377
17282
  const mode = ${JSON.stringify(options.mode)};
16378
17283
  const exact = ${JSON.stringify(options.exact)};
16379
17284
  const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
16380
- const visible = (el) => {
16381
- const rect = el.getBoundingClientRect();
16382
- const style = getComputedStyle(el);
16383
- return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
16384
- };
17285
+ ${VISIBLE_JS}
16385
17286
  const matches = (text) => {
16386
17287
  const haystack = normalize(text).toLowerCase();
16387
17288
  const needle = query.toLowerCase();
@@ -16436,6 +17337,241 @@ async function computerBrowserWaitCommand(query, options = {}) {
16436
17337
  console.log(JSON.stringify({ ok: false, elapsedMs: Date.now() - start, attempts, result: lastResult }, null, 2));
16437
17338
  process.exitCode = 1;
16438
17339
  }
17340
+ async function computerBrowserScrollCommand(direction, options = {}) {
17341
+ const page = await selectChromePage(options);
17342
+ const normalized = direction.toLowerCase();
17343
+ if (!["up", "down", "left", "right"].includes(normalized)) fail("direction must be one of up|down|left|right");
17344
+ const amount = options.amount ? parseCoord(options.amount, "--amount") : 600;
17345
+ if (amount <= 0 || amount > 1e4) fail("--amount must be between 1 and 10000 CSS pixels");
17346
+ const layout = await sendChromeCommand(page.webSocketDebuggerUrl, "Page.getLayoutMetrics", {});
17347
+ const viewport = chromeVisualViewport(layout);
17348
+ const target = options.ref ? await prepareBrowserElement(page, options.ref) : null;
17349
+ const x = target?.rect.centerX ?? viewport.width / 2;
17350
+ const y = target?.rect.centerY ?? viewport.height / 2;
17351
+ const deltaX = normalized === "left" ? -amount : normalized === "right" ? amount : 0;
17352
+ const deltaY = normalized === "up" ? -amount : normalized === "down" ? amount : 0;
17353
+ await sendChromeCommand(page.webSocketDebuggerUrl, "Input.dispatchMouseEvent", {
17354
+ type: "mouseWheel",
17355
+ x,
17356
+ y,
17357
+ deltaX,
17358
+ deltaY
17359
+ });
17360
+ logRecordingAction({ type: "scroll", ...target?.desktopPoint ?? {} });
17361
+ console.log(JSON.stringify({
17362
+ ...browserTargetOutput(page),
17363
+ result: { scrolled: true, direction: normalized, amount, ref: target?.ref ?? null, x: Math.round(x), y: Math.round(y) }
17364
+ }, null, 2));
17365
+ }
17366
+ function browserKeyDetails(combo) {
17367
+ const tokens = combo.split("+").map((token) => token.trim()).filter(Boolean);
17368
+ if (tokens.length === 0) fail("key combination cannot be empty");
17369
+ let modifiers = 0;
17370
+ for (const token of tokens.slice(0, -1)) {
17371
+ const normalized2 = token.toLowerCase();
17372
+ if (["alt", "option"].includes(normalized2)) modifiers |= 1;
17373
+ else if (["ctrl", "control"].includes(normalized2)) modifiers |= 2;
17374
+ else if (["meta", "cmd", "command", "super"].includes(normalized2)) modifiers |= 4;
17375
+ else if (normalized2 === "shift") modifiers |= 8;
17376
+ else fail(`Unsupported browser key modifier ${JSON.stringify(token)}`);
17377
+ }
17378
+ const requested = tokens[tokens.length - 1];
17379
+ const aliases = {
17380
+ enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 },
17381
+ return: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 },
17382
+ tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9 },
17383
+ escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
17384
+ esc: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
17385
+ backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 },
17386
+ delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 },
17387
+ space: { key: " ", code: "Space", windowsVirtualKeyCode: 32 },
17388
+ up: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
17389
+ arrowup: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
17390
+ down: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
17391
+ arrowdown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
17392
+ left: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
17393
+ arrowleft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
17394
+ right: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
17395
+ arrowright: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
17396
+ home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 },
17397
+ end: { key: "End", code: "End", windowsVirtualKeyCode: 35 },
17398
+ pageup: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 },
17399
+ pagedown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 }
17400
+ };
17401
+ const normalized = requested.toLowerCase();
17402
+ const alias = aliases[normalized];
17403
+ if (alias) return { ...alias, modifiers, ...alias.key === " " && modifiers === 0 ? { text: " " } : {} };
17404
+ if (/^[a-z]$/i.test(requested)) {
17405
+ const upper = requested.toUpperCase();
17406
+ const key = modifiers & 8 ? upper : requested.toLowerCase();
17407
+ return { key, code: `Key${upper}`, windowsVirtualKeyCode: upper.charCodeAt(0), modifiers, ...modifiers === 0 ? { text: key } : {} };
17408
+ }
17409
+ if (/^[0-9]$/.test(requested)) {
17410
+ return { key: requested, code: `Digit${requested}`, windowsVirtualKeyCode: requested.charCodeAt(0), modifiers, ...modifiers === 0 ? { text: requested } : {} };
17411
+ }
17412
+ fail(`Unsupported browser key ${JSON.stringify(requested)}`);
17413
+ }
17414
+ async function computerBrowserKeyCommand(combo, options = {}) {
17415
+ const page = await selectChromePage(options);
17416
+ const key = browserKeyDetails(combo);
17417
+ await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
17418
+ await dispatchChromeKeyInSession(session, key);
17419
+ });
17420
+ logRecordingAction({ type: "key" });
17421
+ console.log(JSON.stringify({ targetId: page.id ?? null, result: { pressed: combo } }, null, 2));
17422
+ }
17423
+ async function dispatchChromeKeyInSession(session, key) {
17424
+ await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
17425
+ type: "keyDown",
17426
+ ...key,
17427
+ nativeVirtualKeyCode: key.windowsVirtualKeyCode
17428
+ });
17429
+ await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
17430
+ type: "keyUp",
17431
+ key: key.key,
17432
+ code: key.code,
17433
+ modifiers: key.modifiers,
17434
+ windowsVirtualKeyCode: key.windowsVirtualKeyCode,
17435
+ nativeVirtualKeyCode: key.windowsVirtualKeyCode
17436
+ });
17437
+ }
17438
+ async function computerBrowserTypeCommand(text, options = {}) {
17439
+ const page = await selectChromePage(options);
17440
+ await sendChromeCommand(page.webSocketDebuggerUrl, "Input.insertText", { text });
17441
+ logRecordingAction({ type: "type" });
17442
+ console.log(JSON.stringify({ targetId: page.id ?? null, result: { typed: true, length: text.length } }, null, 2));
17443
+ }
17444
+ function parseBrowserBatchActions(value) {
17445
+ let input;
17446
+ try {
17447
+ input = JSON.parse(value);
17448
+ } catch (error) {
17449
+ fail(`Batch actions must be valid JSON: ${error instanceof Error ? error.message : String(error)}`);
17450
+ }
17451
+ if (!Array.isArray(input) || input.length === 0 || input.length > 100) {
17452
+ fail("Batch actions must be a JSON array containing 1 to 100 actions");
17453
+ }
17454
+ return input.map((item, index) => {
17455
+ if (!isRecord(item) || typeof item.action !== "string") fail(`Batch action ${index} must be an object with an action`);
17456
+ if (item.action === "click") {
17457
+ if (typeof item.ref !== "string" && typeof item.ref !== "number") fail(`Batch click ${index} requires ref`);
17458
+ const buttons = { "1": "left", "2": "middle", "3": "right", left: "left", middle: "middle", right: "right" };
17459
+ const requestedButton = item.button === void 0 ? "left" : String(item.button).toLowerCase();
17460
+ const button = buttons[requestedButton];
17461
+ if (!button) fail(`Batch click ${index} button must be left, middle, or right`);
17462
+ return { action: "click", ref: String(item.ref), button, clicks: item.double === true ? 2 : 1 };
17463
+ }
17464
+ if (item.action === "fill") {
17465
+ if (typeof item.ref !== "string" && typeof item.ref !== "number" || typeof item.value !== "string") {
17466
+ fail(`Batch fill ${index} requires ref and string value`);
17467
+ }
17468
+ return { action: "fill", ref: String(item.ref), value: item.value };
17469
+ }
17470
+ if (item.action === "key") {
17471
+ if (typeof item.combo !== "string") fail(`Batch key ${index} requires combo`);
17472
+ return { action: "key", combo: item.combo };
17473
+ }
17474
+ if (item.action === "type") {
17475
+ if (typeof item.text !== "string") fail(`Batch type ${index} requires text`);
17476
+ return { action: "type", text: item.text };
17477
+ }
17478
+ if (item.action === "scroll") {
17479
+ const directions = { up: "up", down: "down", left: "left", right: "right" };
17480
+ const direction = typeof item.direction === "string" ? directions[item.direction] : void 0;
17481
+ if (!direction) {
17482
+ fail(`Batch scroll ${index} direction must be up, down, left, or right`);
17483
+ }
17484
+ const amount = item.amount === void 0 ? 600 : Number(item.amount);
17485
+ if (!Number.isFinite(amount) || amount <= 0 || amount > 1e4) fail(`Batch scroll ${index} amount must be between 1 and 10000`);
17486
+ if (item.ref !== void 0 && typeof item.ref !== "string" && typeof item.ref !== "number") fail(`Batch scroll ${index} ref is invalid`);
17487
+ return {
17488
+ action: "scroll",
17489
+ direction,
17490
+ amount,
17491
+ ...item.ref === void 0 ? {} : { ref: String(item.ref) }
17492
+ };
17493
+ }
17494
+ if (item.action === "wait") {
17495
+ if (typeof item.text !== "string") fail(`Batch wait ${index} requires text`);
17496
+ const mode = item.mode === void 0 ? "any" : String(item.mode);
17497
+ if (!["any", "text", "title", "url", "control"].includes(mode)) fail(`Batch wait ${index} mode is invalid`);
17498
+ const timeoutMs = item.timeoutMs === void 0 ? 1e4 : Number(item.timeoutMs);
17499
+ const pollMs = item.pollMs === void 0 ? 250 : Number(item.pollMs);
17500
+ if (!Number.isFinite(timeoutMs) || timeoutMs < 0) fail(`Batch wait ${index} timeoutMs must be >= 0`);
17501
+ if (!Number.isFinite(pollMs) || pollMs < 50 || pollMs > 2e3) fail(`Batch wait ${index} pollMs must be between 50 and 2000`);
17502
+ return { action: "wait", text: item.text, mode, exact: item.exact === true, timeoutMs, pollMs };
17503
+ }
17504
+ fail(`Batch action ${index} has unsupported action ${JSON.stringify(item.action)}`);
17505
+ });
17506
+ }
17507
+ async function computerBrowserBatchCommand(actionsJson, options = {}) {
17508
+ const actions = parseBrowserBatchActions(actionsJson);
17509
+ const page = await selectChromePage(options);
17510
+ const { targetId, cached } = readBrowserRefCache(page);
17511
+ const startedAt = Date.now();
17512
+ const completed = [];
17513
+ try {
17514
+ await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
17515
+ const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
17516
+ const viewportRect = browserViewportRect(layout);
17517
+ for (const [index, action] of actions.entries()) {
17518
+ const actionStartedAt = Date.now();
17519
+ if (action.action === "click") {
17520
+ const ref = parseBrowserRef(action.ref);
17521
+ const rect = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
17522
+ await dispatchChromeClickInSession(session, rect.centerX, rect.centerY, action.button, action.clicks);
17523
+ logRecordingAction({ type: "click" });
17524
+ completed.push({ index, action: action.action, ref: String(ref), elapsedMs: Date.now() - actionStartedAt });
17525
+ } else if (action.action === "fill") {
17526
+ const ref = parseBrowserRef(action.ref);
17527
+ await prepareBrowserRefInSession(session, ref, cached, viewportRect);
17528
+ const result = await fillChromeNodeInSession(session, ref, action.value);
17529
+ logRecordingAction({ type: "type" });
17530
+ completed.push({ index, action: action.action, ref: String(ref), valueLength: result.valueLength, elapsedMs: Date.now() - actionStartedAt });
17531
+ } else if (action.action === "key") {
17532
+ await dispatchChromeKeyInSession(session, browserKeyDetails(action.combo));
17533
+ logRecordingAction({ type: "key" });
17534
+ completed.push({ index, action: action.action, elapsedMs: Date.now() - actionStartedAt });
17535
+ } else if (action.action === "type") {
17536
+ await sendChromeSessionCommand(session, "Input.insertText", { text: action.text });
17537
+ logRecordingAction({ type: "type" });
17538
+ completed.push({ index, action: action.action, length: action.text.length, elapsedMs: Date.now() - actionStartedAt });
17539
+ } else if (action.action === "scroll") {
17540
+ const ref = action.ref ? parseBrowserRef(action.ref) : null;
17541
+ const rect = ref ? await prepareBrowserRefInSession(session, ref, cached, viewportRect) : viewportRect;
17542
+ const deltaX = action.direction === "left" ? -action.amount : action.direction === "right" ? action.amount : 0;
17543
+ const deltaY = action.direction === "up" ? -action.amount : action.direction === "down" ? action.amount : 0;
17544
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
17545
+ type: "mouseWheel",
17546
+ x: rect.centerX,
17547
+ y: rect.centerY,
17548
+ deltaX,
17549
+ deltaY
17550
+ });
17551
+ logRecordingAction({ type: "scroll" });
17552
+ completed.push({ index, action: action.action, direction: action.direction, amount: action.amount, elapsedMs: Date.now() - actionStartedAt });
17553
+ } else {
17554
+ const expression = buildBrowserWaitExpression(action.text, { mode: action.mode, exact: action.exact });
17555
+ let attempts = 0;
17556
+ let result = null;
17557
+ while (Date.now() - actionStartedAt <= action.timeoutMs) {
17558
+ attempts++;
17559
+ result = await evaluateChromeSession(session, expression);
17560
+ if (isRecord(result) && result.matched === true) break;
17561
+ await sleep(action.pollMs);
17562
+ }
17563
+ if (!isRecord(result) || result.matched !== true) fail(`Batch wait ${index} timed out after ${Date.now() - actionStartedAt}ms`);
17564
+ completed.push({ index, action: action.action, attempts, elapsedMs: Date.now() - actionStartedAt });
17565
+ }
17566
+ }
17567
+ });
17568
+ } catch (error) {
17569
+ const message = error instanceof Error ? error.message : String(error);
17570
+ console.log(JSON.stringify({ ok: false, targetId, elapsedMs: Date.now() - startedAt, completed, failedIndex: completed.length, error: message }, null, 2));
17571
+ throw error;
17572
+ }
17573
+ console.log(JSON.stringify({ ok: true, targetId, elapsedMs: Date.now() - startedAt, completed }, null, 2));
17574
+ }
16439
17575
  async function computerClickCommand(xStr, yStr, options) {
16440
17576
  const dimensions = getDisplayDimensions();
16441
17577
  const x = parseScreenCoord(xStr, "x", dimensions.width);
@@ -16558,12 +17694,55 @@ async function computerLaunchCommand(app, args) {
16558
17694
  const baseArgs = app === "chrome" && !existsSync3(CHROME_WRAPPER) ? LEGACY_CHROME_ALIAS : APP_ALIASES[app] ?? [app];
16559
17695
  const bin = baseArgs[0];
16560
17696
  const fullArgs = [...baseArgs.slice(1), ...args];
17697
+ const expectsNewChromePage = args.some((arg) => !arg.startsWith("-"));
17698
+ const existingChromePageIds = /* @__PURE__ */ new Set();
17699
+ if (app === "chrome") {
17700
+ try {
17701
+ for (const page of await getChromePages()) {
17702
+ if (page.id) existingChromePageIds.add(page.id);
17703
+ }
17704
+ } catch {
17705
+ }
17706
+ }
16561
17707
  const child = spawn4(bin, fullArgs, {
16562
17708
  env: withDisplay(),
16563
17709
  detached: true,
16564
17710
  stdio: "ignore"
16565
17711
  });
16566
17712
  child.unref();
17713
+ if (app === "chrome") {
17714
+ const startedAt = Date.now();
17715
+ let pageCount = 0;
17716
+ let readyPage = null;
17717
+ while (Date.now() - startedAt < 15e3) {
17718
+ let pages = [];
17719
+ try {
17720
+ pages = await getChromePages();
17721
+ } catch {
17722
+ }
17723
+ pageCount = pages.length;
17724
+ const candidates = expectsNewChromePage && existingChromePageIds.size > 0 ? pages.filter((page) => page.id && !existingChromePageIds.has(page.id)) : pages;
17725
+ for (const page of candidates) {
17726
+ if (!page.webSocketDebuggerUrl || !page.url || page.url === "about:blank") continue;
17727
+ const readyState = await evaluateChromeTarget(page.webSocketDebuggerUrl, "document.readyState");
17728
+ if (readyState === "interactive" || readyState === "complete") {
17729
+ readyPage = page;
17730
+ break;
17731
+ }
17732
+ }
17733
+ if (readyPage) break;
17734
+ try {
17735
+ if (child.pid) process.kill(child.pid, 0);
17736
+ } catch {
17737
+ fail(`Chrome exited before its desktop control channel became ready.`);
17738
+ }
17739
+ await sleep(100);
17740
+ }
17741
+ if (!readyPage?.webSocketDebuggerUrl) fail(`Chrome launched but its requested page was not controllable after 15 seconds.`);
17742
+ await sendChromeCommand(readyPage.webSocketDebuggerUrl, "Page.bringToFront", {});
17743
+ console.log(`launched ${bin} (pid ${child.pid}, ready in ${Date.now() - startedAt}ms, page ${readyPage.id}, ${pageCount} open)`);
17744
+ return;
17745
+ }
16567
17746
  console.log(`launched ${bin} (pid ${child.pid})`);
16568
17747
  }
16569
17748
 
@@ -20990,10 +22169,20 @@ if (isAgentMode()) {
20990
22169
  computer.command("status").description("Show which desktop services are running and the active preview URL").action(wrap(() => computerStatusCommand()));
20991
22170
  computer.command("screenshot <path>").description("Capture the current desktop to a PNG file. Use --raw or --grid for agent click planning; omit both for a branded shareable image.").option("--raw", "Save a 1:1 desktop capture with no branding, padding, or rounded corners").option("--grid [px]", "Overlay a coordinate grid on a 1:1 desktop capture. Default 100px.").action(wrap((path6, options) => computerScreenshotCommand(path6, options)));
20992
22171
  computer.command("observe <path>").description("Wait briefly for the desktop to settle, save a 1:1 screenshot, and print JSON screen context for agents.").option("--raw", "Save a 1:1 desktop capture with no coordinate grid").option("--grid [px]", "Overlay a coordinate grid on the 1:1 desktop capture. Default 100px.").option("--timeout <ms>", "Maximum time to wait for visual stability. Default 3000.").option("--stable-ms <ms>", "Required unchanged time before the screen is considered stable. Default 600.").option("--poll-ms <ms>", "Screenshot polling interval while waiting. Default 200.").action(wrap((path6, options) => computerObserveCommand(path6, options)));
20993
- computer.command("browser").description("Print JSON for Chrome tabs launched through Replicas, including page titles and URLs.").option("--snapshot", "Include visible page text and interactive controls with bounding boxes").option("--limit <chars>", "Maximum body text characters per page snapshot. Default 4000.").option("--element-limit <n>", "Maximum controls per page snapshot. Default 80.").action(wrap((options) => computerBrowserCommand(options)));
20994
- computer.command("browser-click <text>").description("Click the first visible Chrome control whose text, label, placeholder, or href matches <text>.").option("--exact", "Require an exact text match instead of substring matching").option("--index <n>", "When multiple controls match, click the nth match. Default 0.").option("--id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserClickCommand(text, options)));
20995
- computer.command("browser-fill <field> <value>").description("Fill the first visible Chrome field whose label, placeholder, name, or text matches <field>.").option("--exact", "Require an exact field match instead of substring matching").option("--index <n>", "When multiple fields match, fill the nth match. Default 0.").option("--id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((field, value, options) => computerBrowserFillCommand(field, value, options)));
20996
- computer.command("browser-wait <text>").description("Wait until Chrome page title, URL, body text, or controls match <text>.").option("--mode <mode>", "Where to match: any, text, title, url, or control. Default any.").option("--exact", "Require an exact match instead of substring matching").option("--timeout <ms>", "Maximum time to wait. Default 10000.").option("--poll-ms <ms>", "Polling interval. Default 250.").option("--id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserWaitCommand(text, options)));
22172
+ computer.command("browser").description("Print JSON for Chrome tabs launched through Replicas, including page titles and URLs.").option("--snapshot", "Include visible page text and interactive controls with bounding boxes").option("--limit <chars>", "Maximum body text characters per page snapshot. Default 4000.").option("--element-limit <n>", "Maximum controls per page snapshot. Default 80.").option("--target-id <id>", "Only include the Chrome page with this target id").option("--page <n>", "When multiple pages match, include the nth page. Default 0.").option("--title <text>", "Only include pages whose title contains text").option("--url <text>", "Only include pages whose URL contains text").action(wrap((options) => computerBrowserCommand(options)));
22173
+ computer.command("browser-state <path>").description("Wait for a Chrome page to settle, capture its viewport, and print a bounded accessibility state with actionable refs.").option("--full", "Return the full accessibility tree instead of a diff from the previous state").option("--limit <chars>", "Maximum visible text characters. Default 8000.").option("--element-limit <n>", "Maximum actionable elements. Default 120.").option("--timeout <ms>", "Maximum time to wait for state stability. Default 5000.").option("--stable-ms <ms>", "Required unchanged time before capture. Default 500.").option("--poll-ms <ms>", "State polling interval. Default 100.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((path6, options) => computerBrowserStateCommand(path6, options)));
22174
+ computer.command("browser-batch <actions>").description("Run an ordered, fail-fast JSON array of browser actions through one Chrome session.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((actions, options) => computerBrowserBatchCommand(actions, options)));
22175
+ computer.command("browser-click [text]").description("Click a Chrome element by ref from browser-state, or fall back to visible control text.").option("--ref <id>", "Click the backend DOM node ref from the latest browser state").option("--exact", "Require an exact text match instead of substring matching").option("--index <n>", "When multiple controls match, click the nth match. Default 0.").option("--button <button>", "Mouse button for ref clicks: left, middle, or right. Default left.").option("--double", "Double-click a ref target").option("--target-id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserClickCommand(text, options)));
22176
+ computer.command("browser-fill [field] [value]").description("Fill a Chrome field by ref from browser-state, or fall back to visible field text.").option("--ref <id>", "Fill the backend DOM node ref from the latest browser state").option("--exact", "Require an exact field match instead of substring matching").option("--index <n>", "When multiple fields match, fill the nth match. Default 0.").option("--target-id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((field, value, options) => {
22177
+ const resolvedValue = value ?? (options.ref ? field : void 0);
22178
+ const resolvedField = value === void 0 && options.ref ? void 0 : field;
22179
+ if (resolvedValue === void 0) throw new Error("Provide <field> <value>, or --ref <id> <value>");
22180
+ return computerBrowserFillCommand(resolvedField, resolvedValue, options);
22181
+ }));
22182
+ computer.command("browser-scroll <direction>").description("Send trusted wheel input to a Chrome element ref or the center of the target page.").option("--ref <id>", "Scroll from the backend DOM node ref from the latest browser state").option("--amount <px>", "Scroll distance in CSS pixels. Default 600.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((direction, options) => computerBrowserScrollCommand(direction, options)));
22183
+ computer.command("browser-key <combo>").description("Press a key or modifier chord in the target Chrome page with trusted browser input.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((combo, options) => computerBrowserKeyCommand(combo, options)));
22184
+ computer.command("browser-type <text>").description("Type literal text into the focused field in the target Chrome page with trusted browser input.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserTypeCommand(text, options)));
22185
+ computer.command("browser-wait <text>").description("Wait until Chrome page title, URL, body text, or controls match <text>.").option("--mode <mode>", "Where to match: any, text, title, url, or control. Default any.").option("--exact", "Require an exact match instead of substring matching").option("--timeout <ms>", "Maximum time to wait. Default 10000.").option("--poll-ms <ms>", "Polling interval. Default 250.").option("--target-id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserWaitCommand(text, options)));
20997
22186
  computer.command("click <x> <y>").description("Move to (x, y) and click. Coordinates are pixels or percentages like 50% 50%.").option("-b, --button <n>", "Mouse button (1=left, 2=middle, 3=right). Default 1.").option("--double", "Double-click instead of single-click").option("--modifiers <mods>", "Hold modifier keys during the click, e.g. ctrl or ctrl+shift").action(wrap((x, y, options) => computerClickCommand(x, y, options)));
20998
22187
  computer.command("move <x> <y>").description("Move the mouse to (x, y) without clicking. Coordinates are pixels or percentages like 50% 50%.").action(wrap((x, y) => computerMoveCommand(x, y)));
20999
22188
  computer.command("type <text>").description("Type a literal string into the focused field. Use `key` for key combos like ctrl+l.").option("--delay <ms>", "Per-character delay in ms (default 12 \u2248 80 wpm)").action(wrap((text, options) => computerTypeCommand(text, options)));