replicas-computer 0.3.22 → 0.3.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +422 -284
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -191,21 +191,18 @@ import { dirname as dirname4 } from "path";
191
191
  // src/commands/capture.ts
192
192
  import { spawnSync as spawnSync2 } from "child_process";
193
193
  import { createHash } from "crypto";
194
- import { closeSync, copyFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, openSync, readdirSync, readFileSync as readFileSync2, readSync, rmSync, writeFileSync } from "fs";
194
+ import { copyFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readdirSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
195
195
  import { dirname as dirname2 } from "path";
196
196
  var PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
197
- function readPngDimensions(filePath) {
198
- const fd = openSync(filePath, "r");
199
- try {
200
- const buf = Buffer.alloc(24);
201
- const bytesRead = readSync(fd, buf, 0, 24, 0);
202
- if (bytesRead < 24 || !buf.subarray(0, 8).equals(PNG_SIGNATURE)) {
203
- fail(`${filePath} is not a valid PNG (read ${bytesRead}/24 bytes)`);
197
+ function readImageDimensions(filePath) {
198
+ const data = readFileSync2(filePath);
199
+ if (data.length >= 24 && data.subarray(0, 8).equals(PNG_SIGNATURE)) return { width: data.readUInt32BE(16), height: data.readUInt32BE(20) };
200
+ if (data[0] === 255 && data[1] === 216) {
201
+ for (let offset = 2; offset + 9 < data.length && data[offset] === 255; offset += 2 + data.readUInt16BE(offset + 2)) {
202
+ if (data[offset + 1] >= 192 && data[offset + 1] <= 195) return { width: data.readUInt16BE(offset + 7), height: data.readUInt16BE(offset + 5) };
204
203
  }
205
- return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };
206
- } finally {
207
- closeSync(fd);
208
204
  }
205
+ fail(`${filePath} is not a valid PNG or JPEG image`);
209
206
  }
210
207
  function currentWallpaper() {
211
208
  const recorded = `${STATE_DIR}/wallpaper-path`;
@@ -225,6 +222,12 @@ var SHADOW_OFFSET_Y_FRACTION = 0.013;
225
222
  var SHADOW_ALPHA = 0.6;
226
223
  var SCREENSHOT_MASK_TEMPLATE = `<svg width="__W__" height="__H__" xmlns="http://www.w3.org/2000/svg"><rect width="__W__" height="__H__" rx="__R__" ry="__R__" fill="white"/></svg>`;
227
224
  var SHADOW_MASK_TEMPLATE = `<svg width="__SW__" height="__SH__" xmlns="http://www.w3.org/2000/svg"><rect x="__M__" y="__M__" width="__W__" height="__H__" rx="__R__" ry="__R__" fill="white"/></svg>`;
225
+ var isJpeg = (path) => /\.jpe?g$/i.test(path);
226
+ var imageExtension = (path) => isJpeg(path) ? "jpg" : "png";
227
+ var jpegQuality = (path) => isJpeg(path) ? ["-q:v", "2"] : [];
228
+ function captureScreen(path) {
229
+ runDisplayCmd("scrot", ["-o", ...isJpeg(path) ? ["-q", "90"] : [], path]);
230
+ }
228
231
  function parseGridSize(value) {
229
232
  if (value === void 0 || value === false) return null;
230
233
  const size = value === true ? 100 : parseCoord(value, "--grid");
@@ -269,6 +272,7 @@ function overlayGrid(rawPath, target, width2, height2, gridSize, gridPath) {
269
272
  "1",
270
273
  "-update",
271
274
  "1",
275
+ ...jpegQuality(target),
272
276
  target
273
277
  ],
274
278
  { stdio: "pipe" }
@@ -281,13 +285,13 @@ async function computerScreenshotCommand(path, options = {}) {
281
285
  const target = resolvePath(path);
282
286
  mkdirSync2(dirname2(target), { recursive: true });
283
287
  const stamp = `${process.pid}-${Date.now()}`;
284
- const rawPath = `/tmp/replicas-screenshot-${stamp}.raw.png`;
288
+ const rawPath = `/tmp/replicas-screenshot-${stamp}.raw.${imageExtension(target)}`;
285
289
  const maskPath = `/tmp/replicas-screenshot-${stamp}.mask.svg`;
286
290
  const shadowPath = `/tmp/replicas-screenshot-${stamp}.shadow.svg`;
287
291
  const gridPath = `/tmp/replicas-screenshot-${stamp}.grid.svg`;
288
292
  try {
289
- runDisplayCmd("scrot", ["-o", rawPath]);
290
- const { width: width2, height: height2 } = readPngDimensions(rawPath);
293
+ captureScreen(rawPath);
294
+ const { width: width2, height: height2 } = getDisplayDimensions();
291
295
  const gridSize = parseGridSize(options.grid);
292
296
  if (options.raw && gridSize === null) {
293
297
  copyFileSync(rawPath, target);
@@ -339,6 +343,7 @@ async function computerScreenshotCommand(path, options = {}) {
339
343
  "1",
340
344
  "-update",
341
345
  "1",
346
+ ...jpegQuality(target),
342
347
  target
343
348
  ],
344
349
  { stdio: "pipe" }
@@ -357,22 +362,18 @@ async function computerScreenshotCommand(path, options = {}) {
357
362
  function hashFile(path) {
358
363
  return createHash("sha256").update(readFileSync2(path)).digest("hex");
359
364
  }
360
- async function captureStableRawScreenshot(target, options) {
365
+ async function captureStableScreenshot(target, options) {
361
366
  const start = Date.now();
362
- const framePath = `${target}.frame.png`;
367
+ const framePath = `${target}.frame.jpg`;
363
368
  let lastHash = null;
364
369
  let lastChangeAt = start;
365
370
  let frames = 0;
366
371
  let changes = 0;
367
- let width2 = 0;
368
- let height2 = 0;
372
+ let stable = false;
369
373
  try {
370
374
  while (Date.now() - start <= options.timeoutMs) {
371
- runDisplayCmd("scrot", ["-o", framePath]);
375
+ captureScreen(framePath);
372
376
  frames++;
373
- const dimensions = readPngDimensions(framePath);
374
- width2 = dimensions.width;
375
- height2 = dimensions.height;
376
377
  const hash = hashFile(framePath);
377
378
  const now = Date.now();
378
379
  if (lastHash === null) {
@@ -382,13 +383,16 @@ async function captureStableRawScreenshot(target, options) {
382
383
  lastChangeAt = now;
383
384
  }
384
385
  lastHash = hash;
385
- copyFileSync(framePath, target);
386
386
  if (frames > 1 && now - lastChangeAt >= options.stableMs) {
387
- return { width: width2, height: height2, stable: true, elapsedMs: now - start, frames, changes };
387
+ stable = true;
388
+ break;
388
389
  }
389
390
  await sleep(options.pollMs);
390
391
  }
391
- return { width: width2, height: height2, stable: false, elapsedMs: Date.now() - start, frames, changes };
392
+ const elapsedMs = Date.now() - start;
393
+ if (isJpeg(target)) copyFileSync(framePath, target);
394
+ else captureScreen(target);
395
+ return { stable, elapsedMs, frames, changes };
392
396
  } finally {
393
397
  rmSync(framePath, { force: true });
394
398
  }
@@ -405,26 +409,27 @@ async function computerObserveCommand(path, options = {}) {
405
409
  mkdirSync2(dirname2(target), { recursive: true });
406
410
  const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 3e3;
407
411
  const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 600;
408
- const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 200;
412
+ const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
409
413
  if (timeoutMs < 0) fail("--timeout must be >= 0");
410
414
  if (stableMs < 0) fail("--stable-ms must be >= 0");
411
415
  if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
412
416
  const stamp = `${process.pid}-${Date.now()}`;
413
- const rawPath = `/tmp/replicas-observe-${stamp}.raw.png`;
417
+ const rawPath = `/tmp/replicas-observe-${stamp}.raw.${imageExtension(target)}`;
414
418
  const gridPath = `/tmp/replicas-observe-${stamp}.grid.svg`;
415
419
  try {
416
420
  ensureServicesRunning();
417
- const capture = await captureStableRawScreenshot(rawPath, { timeoutMs, stableMs, pollMs });
418
- const gridSize = options.raw ? null : parseGridSize(options.grid ?? true);
421
+ const capture = await captureStableScreenshot(rawPath, { timeoutMs, stableMs, pollMs });
422
+ const { width: width2, height: height2 } = getDisplayDimensions();
423
+ const gridSize = options.raw ? null : parseGridSize(options.grid);
419
424
  if (gridSize === null) {
420
425
  copyFileSync(rawPath, target);
421
426
  } else {
422
- overlayGrid(rawPath, target, capture.width, capture.height, gridSize, gridPath);
427
+ overlayGrid(rawPath, target, width2, height2, gridSize, gridPath);
423
428
  }
424
429
  console.log(JSON.stringify({
425
430
  screenshot: target,
426
- width: capture.width,
427
- height: capture.height,
431
+ width: width2,
432
+ height: height2,
428
433
  stable: capture.stable,
429
434
  elapsedMs: capture.elapsedMs,
430
435
  frames: capture.frames,
@@ -1115,6 +1120,7 @@ function isRecord(value) {
1115
1120
  async function openChromeSession(webSocketDebuggerUrl) {
1116
1121
  const socket = new WebSocket(webSocketDebuggerUrl);
1117
1122
  const pending = /* @__PURE__ */ new Map();
1123
+ const listeners = /* @__PURE__ */ new Map();
1118
1124
  let nextId = 1;
1119
1125
  let closed = false;
1120
1126
  const fail2 = (error) => {
@@ -1137,6 +1143,7 @@ async function openChromeSession(webSocketDebuggerUrl) {
1137
1143
  return;
1138
1144
  }
1139
1145
  if (!isRecord(message)) return;
1146
+ if (typeof message.method === "string" && isRecord(message.params)) listeners.get(message.method)?.(message.params);
1140
1147
  if (typeof message.id !== "number") return;
1141
1148
  const request = pending.get(message.id);
1142
1149
  if (!request) return;
@@ -1175,6 +1182,7 @@ async function openChromeSession(webSocketDebuggerUrl) {
1175
1182
  pending.set(id, { resolve: resolve5, reject, timeout });
1176
1183
  socket.send(JSON.stringify({ id, method, params }));
1177
1184
  }),
1185
+ on: (method, listener) => listeners.set(method, listener),
1178
1186
  close: () => fail2(new Error("Chrome DevTools connection closed"))
1179
1187
  };
1180
1188
  }
@@ -1233,7 +1241,11 @@ async function selectChromePage(options = {}) {
1233
1241
  `No matching debuggable Chrome page found (${matches.length} match${matches.length === 1 ? "" : "es"}, index ${index}). Use \`replicas computer browser\` to list pages.`
1234
1242
  );
1235
1243
  }
1236
- await sendChromeCommand(webSocketDebuggerUrl, "Page.bringToFront", {});
1244
+ const [front] = await sendChromeCommands(webSocketDebuggerUrl, [
1245
+ { method: "Page.bringToFront", params: {} },
1246
+ { method: "Runtime.evaluate", params: { expression: BROWSER_QUIET_JS, returnByValue: true } }
1247
+ ]);
1248
+ chromeResult(front, "Page.bringToFront");
1237
1249
  return { ...page, webSocketDebuggerUrl };
1238
1250
  }
1239
1251
  var VISIBLE_JS = `
@@ -1332,7 +1344,9 @@ function readBrowserStateCache(path) {
1332
1344
  return {
1333
1345
  url: typeof value.url === "string" ? value.url : void 0,
1334
1346
  documentId: typeof value.documentId === "string" || value.documentId === null ? value.documentId : void 0,
1335
- entries
1347
+ entries,
1348
+ textLimit: typeof value.textLimit === "number" ? value.textLimit : void 0,
1349
+ elementLimit: typeof value.elementLimit === "number" ? value.elementLimit : void 0
1336
1350
  };
1337
1351
  }
1338
1352
  function chromeResult(response, method) {
@@ -1385,6 +1399,22 @@ function browserContentQuadRects(quads, viewportRect) {
1385
1399
  function buildBrowserBounds(result, viewport, targetId) {
1386
1400
  const strings = Array.isArray(result.strings) ? result.strings : [];
1387
1401
  const documents = Array.isArray(result.documents) ? result.documents.filter(isDOMSnapshotDocument) : [];
1402
+ const pointerNodes = documents.map((document) => {
1403
+ const cursors = /* @__PURE__ */ new Map();
1404
+ document.layout?.nodeIndex?.forEach((nodeIndex, index) => {
1405
+ const cursor = strings[document.layout?.styles?.[index]?.[0] ?? -1];
1406
+ if (typeof cursor === "string") cursors.set(nodeIndex, cursor);
1407
+ });
1408
+ const parentIndex = document.nodes?.parentIndex ?? [];
1409
+ const starts = /* @__PURE__ */ new Set();
1410
+ for (const [nodeIndex, cursor] of cursors) {
1411
+ if (!BROWSER_POINTER_CURSORS.has(cursor)) continue;
1412
+ let parent = parentIndex[nodeIndex] ?? -1;
1413
+ while (parent >= 0 && !cursors.has(parent)) parent = parentIndex[parent] ?? -1;
1414
+ if (cursors.get(parent) !== cursor) starts.add(nodeIndex);
1415
+ }
1416
+ return starts;
1417
+ });
1388
1418
  const localRects = documents.map((document) => {
1389
1419
  const rects = /* @__PURE__ */ new Map();
1390
1420
  const nodeIndexes = document.layout?.nodeIndex ?? [];
@@ -1475,7 +1505,7 @@ function buildBrowserBounds(result, viewport, targetId) {
1475
1505
  centerX: origin.x + local.centerX,
1476
1506
  centerY: origin.y + local.centerY
1477
1507
  };
1478
- if (intersectRects(rect, origin.clip)) byBackendNodeId.set(backendNodeId, { rect, clip: origin.clip });
1508
+ if (intersectRects(rect, origin.clip)) byBackendNodeId.set(backendNodeId, { rect, clip: origin.clip, pointer: pointerNodes[documentIndex].has(nodeIndex) });
1479
1509
  }
1480
1510
  });
1481
1511
  return byBackendNodeId;
@@ -1518,6 +1548,10 @@ var BROWSER_STRUCTURAL_ROLES = /* @__PURE__ */ new Set([
1518
1548
  "status",
1519
1549
  "StaticText"
1520
1550
  ]);
1551
+ var BROWSER_HIDDEN_STATES = /* @__PURE__ */ new Set(["focusable", "url"]);
1552
+ var BROWSER_POINTER_CURSORS = /* @__PURE__ */ new Set(["pointer", "grab", "move"]);
1553
+ var BROWSER_UNREFERENCED_ROLES = /* @__PURE__ */ new Set(["ListMarker", "RootWebArea", "StaticText"]);
1554
+ var BROWSER_TOGGLE_STATES = /* @__PURE__ */ new Set(["checked", "expanded", "pressed", "selected"]);
1521
1555
  var BROWSER_STATE_PROPERTIES = /* @__PURE__ */ new Set([
1522
1556
  "autocomplete",
1523
1557
  "checked",
@@ -1541,34 +1575,23 @@ var BROWSER_STATE_PROPERTIES = /* @__PURE__ */ new Set([
1541
1575
  function clippedText(value, limit = 240) {
1542
1576
  return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
1543
1577
  }
1544
- function browserTargetOutput(page) {
1545
- return {
1546
- targetId: page.id ?? null,
1547
- title: (page.title ?? "").slice(0, 1e3),
1548
- url: (page.url ?? "").slice(0, 2e3)
1549
- };
1550
- }
1551
1578
  function browserNodeActionable(role, states) {
1552
1579
  return BROWSER_INTERACTIVE_ROLES.has(role) || states.editable === true || states.focusable === true && role !== "RootWebArea";
1553
1580
  }
1554
- async function captureBrowserSnapshot(page, options) {
1555
- const initialCommands = [
1556
- { method: "Page.getFrameTree", params: {} },
1557
- { method: "Page.getLayoutMetrics", params: {} },
1558
- { method: "DOMSnapshot.captureSnapshot", params: { computedStyles: [], includeDOMRects: true, includePaintOrder: true } }
1559
- ];
1560
- const initial = await sendChromeCommands(page.webSocketDebuggerUrl, initialCommands);
1561
- const frameTree = chromeResult(initial[0], initialCommands[0].method).frameTree;
1562
- const layout = chromeResult(initial[1], initialCommands[1].method);
1563
- const domSnapshot = chromeResult(initial[2], initialCommands[2].method);
1581
+ async function captureBrowserSnapshot(session, page, options) {
1582
+ const [frameTreeResult, layoutResult, domSnapshotResult, mainAxResult] = await Promise.all([
1583
+ session.send({ method: "Page.getFrameTree", params: {} }),
1584
+ session.send({ method: "Page.getLayoutMetrics", params: {} }),
1585
+ session.send({ method: "DOMSnapshot.captureSnapshot", params: { computedStyles: ["cursor"] } }),
1586
+ session.send({ method: "Accessibility.getFullAXTree", params: {} })
1587
+ ]);
1588
+ const frameTree = chromeResult(frameTreeResult, "Page.getFrameTree").frameTree;
1589
+ const layout = chromeResult(layoutResult, "Page.getLayoutMetrics");
1590
+ const domSnapshot = chromeResult(domSnapshotResult, "DOMSnapshot.captureSnapshot");
1564
1591
  const viewport = chromeVisualViewport(layout);
1565
1592
  const frameIds = collectFrameIds(frameTree);
1566
1593
  if (frameIds.length === 0 && page.id) frameIds.push(page.id);
1567
- const axCommands = frameIds.map((frameId) => ({
1568
- method: "Accessibility.getFullAXTree",
1569
- params: { frameId }
1570
- }));
1571
- const axResults = await sendChromeCommands(page.webSocketDebuggerUrl, axCommands);
1594
+ const axResults = [mainAxResult, ...await Promise.all(frameIds.slice(1).map((frameId) => session.send({ method: "Accessibility.getFullAXTree", params: { frameId } })))];
1572
1595
  const nodes = [];
1573
1596
  axResults.forEach((response, index) => {
1574
1597
  if (response.error) return;
@@ -1584,6 +1607,8 @@ async function captureBrowserSnapshot(page, options) {
1584
1607
  const propertyMap = browserStateProperties;
1585
1608
  const roleOf = (node) => clippedText(node.role?.value, 80);
1586
1609
  const nameOf = (node) => clippedText(node.name?.value);
1610
+ const textOf = (node, depth = 0) => depth > 6 ? "" : (node.childIds ?? []).map((id) => nodesByKey.get(`${node.sourceFrameId}:${id}`)).map((child) => !child ? "" : roleOf(child) === "StaticText" ? nameOf(child) : textOf(child, depth + 1)).filter(Boolean).join(" ");
1611
+ const displayNameOf = (node) => nameOf(node) || (bounds.get(node.backendDOMNodeId ?? -1)?.pointer ? clippedText(textOf(node), 120) : "");
1587
1612
  const valueOf = (node) => {
1588
1613
  if (propertyMap(node).protected === true) return null;
1589
1614
  const value = node.value?.value;
@@ -1592,20 +1617,20 @@ async function captureBrowserSnapshot(page, options) {
1592
1617
  if (/^[•●*]+$/.test(text)) return "[protected]";
1593
1618
  return text || null;
1594
1619
  };
1595
- const meaningful = nodes.filter((node) => !node.ignored && Number.isInteger(node.backendDOMNodeId)).map((node) => ({
1596
- node,
1597
- role: roleOf(node),
1598
- bound: typeof node.backendDOMNodeId === "number" ? bounds.get(node.backendDOMNodeId) : void 0
1599
- })).filter((entry) => {
1620
+ const meaningful = nodes.filter((node) => !node.ignored && Number.isInteger(node.backendDOMNodeId)).map((node) => {
1621
+ const bound = typeof node.backendDOMNodeId === "number" ? bounds.get(node.backendDOMNodeId) : void 0;
1622
+ const role = roleOf(node);
1623
+ return { node, role: role === "generic" && bound?.pointer ? "clickable" : role, bound };
1624
+ }).filter((entry) => {
1600
1625
  const { node, role, bound } = entry;
1601
1626
  if (!bound || bound.rect.width <= 0 || bound.rect.height <= 0) return false;
1602
1627
  if (role === "InlineTextBox" || role === "none" || role === "generic") return false;
1603
- if (BROWSER_INTERACTIVE_ROLES.has(role) || BROWSER_STRUCTURAL_ROLES.has(role)) return true;
1628
+ if (bound.pointer || BROWSER_INTERACTIVE_ROLES.has(role) || BROWSER_STRUCTURAL_ROLES.has(role)) return true;
1604
1629
  return !!nameOf(node) || valueOf(node) !== null || propertyMap(node).focusable === true;
1605
1630
  }).filter(({ node, role }) => {
1606
1631
  if (role !== "StaticText") return true;
1607
1632
  const parent = node.parentId ? nodesByKey.get(`${node.sourceFrameId}:${node.parentId}`) : void 0;
1608
- return !parent || nameOf(parent) !== nameOf(node);
1633
+ return !parent || displayNameOf(parent) !== nameOf(node);
1609
1634
  }).sort((a, b) => a.bound.rect.y - b.bound.rect.y || a.bound.rect.x - b.bound.rect.x);
1610
1635
  const elements = [];
1611
1636
  const entries = [];
@@ -1615,12 +1640,12 @@ async function captureBrowserSnapshot(page, options) {
1615
1640
  let treeLength = 0;
1616
1641
  const treeLimit = Math.min(5e4, Math.max(4e3, options.textLimit * 2));
1617
1642
  for (const { node, role, bound } of meaningful) {
1618
- const name = nameOf(node);
1643
+ const name = displayNameOf(node);
1619
1644
  const description = clippedText(node.description?.value) || null;
1620
1645
  const value = valueOf(node);
1621
1646
  const states = propertyMap(node);
1622
- const actionable = browserNodeActionable(role, states);
1623
- const ref = actionable && elements.length < options.elementLimit ? String(node.backendDOMNodeId) : null;
1647
+ const actionable = browserNodeActionable(role, states) || bound.pointer;
1648
+ const ref = (actionable ? elements.length < options.elementLimit : !BROWSER_UNREFERENCED_ROLES.has(role)) ? String(node.backendDOMNodeId) : null;
1624
1649
  const visibleRect = role === "RootWebArea" && node.sourceFrameId === page.id ? { x: 0, y: 0, width: viewport.width, height: viewport.height, centerX: viewport.width / 2, centerY: viewport.height / 2 } : intersectRects(bound.rect, bound.clip) ?? bound.rect;
1625
1650
  const rect = roundBrowserRect(visibleRect);
1626
1651
  const actions = [
@@ -1628,7 +1653,7 @@ async function captureBrowserSnapshot(page, options) {
1628
1653
  ...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
1629
1654
  "scroll"
1630
1655
  ];
1631
- if (ref) {
1656
+ if (ref && actionable) {
1632
1657
  elements.push({
1633
1658
  ref,
1634
1659
  role,
@@ -1641,7 +1666,7 @@ async function captureBrowserSnapshot(page, options) {
1641
1666
  frameId: node.sourceFrameId
1642
1667
  });
1643
1668
  }
1644
- const stateText = Object.entries(states).filter(([key]) => key !== "url" && key !== "focusable").map(([key, state]) => `${key}=${JSON.stringify(state)}`);
1669
+ const stateText = Object.entries(states).filter(([key, state]) => !BROWSER_HIDDEN_STATES.has(key) && (BROWSER_TOGGLE_STATES.has(key) || state !== false && state !== "false")).filter(([key]) => key !== "editable" || !["textbox", "searchbox"].includes(role)).map(([key, state]) => `${key}=${JSON.stringify(state)}`);
1645
1670
  const url = typeof states.url === "string" ? clippedText(states.url, 300) : "";
1646
1671
  const semantic = [role, name && JSON.stringify(name), value !== null && `value=${JSON.stringify(value)}`, description && `description=${JSON.stringify(description)}`, url && `url=${JSON.stringify(url)}`, ...stateText].filter(Boolean).join(" ");
1647
1672
  let depth = 0;
@@ -1667,9 +1692,11 @@ async function captureBrowserSnapshot(page, options) {
1667
1692
  }
1668
1693
  }
1669
1694
  const revision = createHash2("sha256").update(JSON.stringify([page.url, entries.map(({ key, semantic }) => [key, semantic])])).digest("hex").slice(0, 16);
1695
+ const mainFrame = isRecord(frameTree) && isRecord(frameTree.frame) ? frameTree.frame : {};
1696
+ const frameUrl = typeof mainFrame.url === "string" ? `${mainFrame.url}${typeof mainFrame.urlFragment === "string" ? mainFrame.urlFragment : ""}` : page.url;
1670
1697
  const snapshot = {
1671
- title: (page.title ?? "").slice(0, 1e3),
1672
- url: (page.url ?? "").slice(0, 2e3),
1698
+ title: page.title.slice(0, 1e3),
1699
+ url: frameUrl.slice(0, 2e3),
1673
1700
  documentId: browserDocumentId(frameTree),
1674
1701
  revision,
1675
1702
  viewport,
@@ -1679,7 +1706,7 @@ async function captureBrowserSnapshot(page, options) {
1679
1706
  controls: elements,
1680
1707
  nodeCount: entries.length,
1681
1708
  elementCount: elements.length,
1682
- truncated: entries.length < meaningful.length || elements.length < meaningful.filter(({ node, role }) => browserNodeActionable(role, propertyMap(node))).length
1709
+ truncated: entries.length < meaningful.length || elements.length < meaningful.filter(({ node, role, bound }) => browserNodeActionable(role, propertyMap(node)) || bound.pointer).length
1683
1710
  };
1684
1711
  return { snapshot, entries };
1685
1712
  }
@@ -1702,10 +1729,11 @@ async function computerBrowserCommand(options = {}) {
1702
1729
  for (const { target, page } of pages) {
1703
1730
  if (options.snapshot && target.webSocketDebuggerUrl) {
1704
1731
  try {
1705
- const { snapshot } = await captureBrowserSnapshot(
1706
- { ...target, webSocketDebuggerUrl: target.webSocketDebuggerUrl },
1732
+ const { snapshot } = await withChromeSession(target.webSocketDebuggerUrl, async (session) => await captureBrowserSnapshot(
1733
+ session,
1734
+ { id: target.id, url: target.url ?? "", title: target.title ?? "" },
1707
1735
  { textLimit, elementLimit }
1708
- );
1736
+ ));
1709
1737
  pageResults.push({ ...page, snapshot });
1710
1738
  } catch (error) {
1711
1739
  pageResults.push({ ...page, snapshotError: error instanceof Error ? error.message : "snapshot failed" });
@@ -1720,89 +1748,36 @@ async function computerBrowserCommand(options = {}) {
1720
1748
  pages: pageResults
1721
1749
  }, null, 2));
1722
1750
  }
1723
- function buildBrowserStabilityExpression() {
1724
- return `(() => {
1725
- const key = '__replicasComputerUseObserver';
1726
- if (!globalThis[key]) {
1727
- const state = { revision: 0 };
1728
- const observer = new MutationObserver(() => state.revision++);
1729
- globalThis[key] = { state, observer, observing: false };
1730
- }
1731
- if (document.documentElement && !globalThis[key].observing) {
1732
- globalThis[key].observer.observe(document.documentElement, {
1733
- subtree: true,
1734
- childList: true,
1735
- attributes: true,
1736
- characterData: true,
1737
- });
1738
- globalThis[key].observing = true;
1739
- }
1740
- const controls = Array.from(document.querySelectorAll('input,textarea,select,[contenteditable="true"],[role="checkbox"],[role="combobox"],[role="slider"],[role="switch"],[role="textbox"]')).slice(0, 200);
1741
- const active = document.activeElement;
1742
- const controlState = controls.map((element, index) => {
1743
- const value = 'value' in element
1744
- ? element instanceof HTMLInputElement && element.type === 'password'
1745
- ? String(element.value).length
1746
- : String(element.value)
1747
- : element.textContent;
1748
- return [
1749
- index,
1750
- element === active,
1751
- value,
1752
- 'checked' in element ? element.checked : null,
1753
- 'selectedIndex' in element ? element.selectedIndex : null,
1754
- 'disabled' in element ? element.disabled : null,
1755
- element.getAttribute('aria-checked'),
1756
- element.getAttribute('aria-expanded'),
1757
- element.getAttribute('aria-selected'),
1758
- ].join('|');
1759
- }).join('\\u001f');
1760
- let controlRevision = 2166136261;
1761
- for (let index = 0; index < controlState.length; index++) {
1762
- controlRevision ^= controlState.charCodeAt(index);
1763
- controlRevision = Math.imul(controlRevision, 16777619);
1764
- }
1765
- return {
1766
- revision: globalThis[key].state.revision,
1767
- controlRevision: (controlRevision >>> 0).toString(16),
1768
- readyState: document.readyState,
1769
- busy: !!document.querySelector('[aria-busy="true"]'),
1770
- url: location.href.slice(0, 2000),
1771
- scrollX: Math.round(scrollX),
1772
- scrollY: Math.round(scrollY),
1773
- innerWidth,
1774
- innerHeight,
1775
- devicePixelRatio,
1776
- };
1777
- })()`;
1778
- }
1779
- async function waitForBrowserStability(page, options) {
1751
+ var BROWSER_QUIET_JS = `(() => {
1752
+ const key = '__replicasComputerUseQuiet';
1753
+ if (!globalThis[key]) {
1754
+ const tracker = { lastChange: performance.now() };
1755
+ const touch = () => { tracker.lastChange = performance.now(); };
1756
+ new MutationObserver(touch).observe(document, { subtree: true, childList: true, attributes: true, characterData: true });
1757
+ for (const type of ['input', 'change', 'scroll']) addEventListener(type, touch, { capture: true, passive: true });
1758
+ globalThis[key] = tracker;
1759
+ }
1760
+ return {
1761
+ quietMs: Math.round(performance.now() - globalThis[key].lastChange),
1762
+ readyState: document.readyState,
1763
+ busy: !!document.querySelector('[aria-busy="true"]'),
1764
+ animating: document.getAnimations().some((animation) => animation.playState === 'running' && animation.effect?.getTiming().iterations !== Infinity),
1765
+ url: location.href.slice(0, 2000),
1766
+ title: document.title.slice(0, 1000),
1767
+ devicePixelRatio,
1768
+ };
1769
+ })()`;
1770
+ async function waitForBrowserQuiet(session, options) {
1780
1771
  const startedAt = Date.now();
1781
- let lastSignature = null;
1782
- let lastChangeAt = startedAt;
1783
- let samples = 0;
1784
- let changes = 0;
1785
- let state = null;
1786
- while (Date.now() - startedAt <= options.timeoutMs) {
1787
- try {
1788
- state = await evaluateChromeTarget(page.webSocketDebuggerUrl, buildBrowserStabilityExpression());
1789
- samples++;
1790
- const signature = JSON.stringify(state);
1791
- const now = Date.now();
1792
- if (lastSignature === null || signature !== lastSignature) {
1793
- if (lastSignature !== null) changes++;
1794
- lastSignature = signature;
1795
- lastChangeAt = now;
1796
- }
1797
- const ready = typeof state === "object" && state !== null && "readyState" in state && state.readyState !== "loading" && "busy" in state && state.busy !== true;
1798
- if (samples > 1 && ready && now - lastChangeAt >= options.stableMs) {
1799
- return { stable: true, elapsedMs: now - startedAt, samples, changes, state };
1800
- }
1801
- } catch {
1802
- }
1772
+ let state;
1773
+ while (true) {
1774
+ const value = await evaluateChromeSession(session, BROWSER_QUIET_JS).catch(() => null);
1775
+ state = isRecord(value) ? value : {};
1776
+ const now = Date.now();
1777
+ const quiet = isRecord(value) && typeof value.quietMs === "number" && value.quietMs >= options.stableMs && value.readyState !== "loading" && value.busy !== true && value.animating !== true && now - (options.actedAt ?? 0) >= options.stableMs;
1778
+ if (quiet || now - startedAt >= options.timeoutMs) return { stable: quiet, elapsedMs: now - startedAt, state };
1803
1779
  await sleep(options.pollMs);
1804
1780
  }
1805
- return { stable: false, elapsedMs: Date.now() - startedAt, samples, changes, state };
1806
1781
  }
1807
1782
  function browserStateCachePath(targetId) {
1808
1783
  return `${STATE_DIR}/browser-state-${targetId.replace(/[^a-zA-Z0-9_-]/g, "_")}.json`;
@@ -1832,35 +1807,30 @@ function browserStateDiff(previous, current) {
1832
1807
  }
1833
1808
  return { tree: lines.join("\n") || "No accessibility changes.", added, changed, removed };
1834
1809
  }
1835
- async function computerBrowserStateCommand(path, options = {}) {
1836
- const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 8e3;
1837
- const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 120;
1838
- const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 5e3;
1839
- const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 500;
1840
- const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
1841
- if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
1842
- if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
1843
- if (timeoutMs < 0) fail("--timeout must be >= 0");
1844
- if (stableMs < 0) fail("--stable-ms must be >= 0");
1845
- if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
1846
- const page = await selectChromePage(options);
1847
- const stability = await waitForBrowserStability(page, { timeoutMs, stableMs, pollMs });
1848
- const target = resolvePath(path);
1849
- mkdirSync4(dirname4(target), { recursive: true });
1850
- const [{ snapshot, entries }, screenshotResult] = await Promise.all([
1851
- captureBrowserSnapshot(page, { textLimit, elementLimit }),
1852
- sendChromeCommand(page.webSocketDebuggerUrl, "Page.captureScreenshot", {
1853
- format: "png",
1854
- fromSurface: true,
1855
- captureBeyondViewport: false
1856
- })
1810
+ var BROWSER_ACTION_OBSERVATION = { textLimit: 8e3, elementLimit: 120, timeoutMs: 2e3, stableMs: 300, pollMs: 50 };
1811
+ async function observeBrowserPage(session, page, options) {
1812
+ const settled = await waitForBrowserQuiet(session, options);
1813
+ const live = {
1814
+ id: page.id,
1815
+ url: typeof settled.state.url === "string" ? settled.state.url : page.url ?? "",
1816
+ title: typeof settled.state.title === "string" ? settled.state.title : page.title ?? ""
1817
+ };
1818
+ const [{ snapshot, entries }, capture] = await Promise.all([
1819
+ captureBrowserSnapshot(session, live, options),
1820
+ options.screenshotPath ? sendChromeSessionCommand(session, "Page.captureScreenshot", { format: isJpeg(options.screenshotPath) ? "jpeg" : "png", fromSurface: true, captureBeyondViewport: false }) : null
1857
1821
  ]);
1858
- const data = screenshotResult.data;
1859
- if (typeof data !== "string") fail("Chrome did not return screenshot data");
1860
- writeFileSync4(target, Buffer.from(data, "base64"));
1861
- const screenshot = readPngDimensions(target);
1862
- const targetId = page.id ?? "unknown";
1863
- const cachePath = browserStateCachePath(targetId);
1822
+ let screenshot = null;
1823
+ if (options.screenshotPath && capture) {
1824
+ if (typeof capture.data !== "string") fail("Chrome did not return screenshot data");
1825
+ mkdirSync4(dirname4(options.screenshotPath), { recursive: true });
1826
+ writeFileSync4(options.screenshotPath, Buffer.from(capture.data, "base64"));
1827
+ screenshot = {
1828
+ path: options.screenshotPath,
1829
+ ...readImageDimensions(options.screenshotPath),
1830
+ devicePixelRatio: typeof settled.state.devicePixelRatio === "number" ? settled.state.devicePixelRatio : 1
1831
+ };
1832
+ }
1833
+ const cachePath = browserStateCachePath(page.id ?? "unknown");
1864
1834
  let mode = "full";
1865
1835
  let tree = snapshot.tree;
1866
1836
  let changes = { added: snapshot.nodeCount, changed: 0, removed: 0 };
@@ -1874,33 +1844,62 @@ async function computerBrowserStateCommand(path, options = {}) {
1874
1844
  }
1875
1845
  }
1876
1846
  mkdirSync4(STATE_DIR, { recursive: true });
1877
- writeFileSync4(cachePath, JSON.stringify({ url: snapshot.url, documentId: snapshot.documentId, entries }));
1878
- const stableState = isRecord(stability.state) ? stability.state : {};
1879
- const state = {
1880
- title: snapshot.title,
1881
- url: snapshot.url,
1882
- documentId: snapshot.documentId,
1883
- revision: snapshot.revision,
1884
- viewport: snapshot.viewport,
1885
- text: snapshot.text,
1886
- tree,
1887
- nodeCount: snapshot.nodeCount,
1888
- elementCount: snapshot.elementCount,
1889
- truncated: snapshot.truncated,
1890
- mode,
1891
- changes
1847
+ writeFileSync4(cachePath, JSON.stringify({ url: snapshot.url, documentId: snapshot.documentId, entries, textLimit: options.textLimit, elementLimit: options.elementLimit }));
1848
+ return {
1849
+ screenshot,
1850
+ stability: { stable: settled.stable, elapsedMs: settled.elapsedMs },
1851
+ state: {
1852
+ title: snapshot.title,
1853
+ url: snapshot.url,
1854
+ documentId: snapshot.documentId,
1855
+ revision: snapshot.revision,
1856
+ viewport: snapshot.viewport,
1857
+ tree,
1858
+ nodeCount: snapshot.nodeCount,
1859
+ elementCount: snapshot.elementCount,
1860
+ truncated: snapshot.truncated,
1861
+ mode,
1862
+ changes
1863
+ }
1892
1864
  };
1893
- console.log(JSON.stringify({
1894
- targetId,
1895
- screenshot: {
1896
- path: target,
1897
- width: screenshot.width,
1898
- height: screenshot.height,
1899
- devicePixelRatio: typeof stableState.devicePixelRatio === "number" ? stableState.devicePixelRatio : 1
1900
- },
1901
- stability,
1902
- state
1903
- }, null, 2));
1865
+ }
1866
+ async function observeAfterBrowserAction(page, actedAt) {
1867
+ const cachePath = browserStateCachePath(page.id ?? "unknown");
1868
+ const cached = existsSync4(cachePath) ? readBrowserStateCache(cachePath) : null;
1869
+ try {
1870
+ const { stability, state } = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await observeBrowserPage(session, page, {
1871
+ ...BROWSER_ACTION_OBSERVATION,
1872
+ textLimit: cached?.textLimit ?? BROWSER_ACTION_OBSERVATION.textLimit,
1873
+ elementLimit: cached?.elementLimit ?? BROWSER_ACTION_OBSERVATION.elementLimit,
1874
+ actedAt
1875
+ }));
1876
+ return { stability, state };
1877
+ } catch (error) {
1878
+ return { error: `Action completed but the page could not be observed: ${error instanceof Error ? error.message : String(error)}` };
1879
+ }
1880
+ }
1881
+ async function computerBrowserStateCommand(path, options = {}) {
1882
+ const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 8e3;
1883
+ const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 120;
1884
+ const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 5e3;
1885
+ const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 500;
1886
+ const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
1887
+ if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
1888
+ if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
1889
+ if (timeoutMs < 0) fail("--timeout must be >= 0");
1890
+ if (stableMs < 0) fail("--stable-ms must be >= 0");
1891
+ if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
1892
+ const page = await selectChromePage(options);
1893
+ const observation = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await observeBrowserPage(session, page, {
1894
+ full: options.full,
1895
+ textLimit,
1896
+ elementLimit,
1897
+ timeoutMs,
1898
+ stableMs,
1899
+ pollMs,
1900
+ screenshotPath: path === void 0 ? void 0 : resolvePath(path)
1901
+ }));
1902
+ console.log(JSON.stringify({ targetId: page.id ?? "unknown", ...observation }, null, 2));
1904
1903
  }
1905
1904
  var DESKTOP_POINT_JS = `
1906
1905
  const desktopPoint = (rect) => {
@@ -1920,10 +1919,9 @@ function readBrowserRefCache(page) {
1920
1919
  const targetId = page.id;
1921
1920
  if (!targetId) fail("Chrome target has no stable ID. Capture fresh browser state and retry.");
1922
1921
  const cachePath = browserStateCachePath(targetId);
1923
- if (!existsSync4(cachePath)) fail(`No browser state is cached for target ${targetId}. Run replicas computer browser-state <path> --target-id ${targetId} first.`);
1922
+ if (!existsSync4(cachePath)) fail(`No browser state is cached for target ${targetId}. Run replicas computer browser-state --target-id ${targetId} first.`);
1924
1923
  const cached = readBrowserStateCache(cachePath);
1925
1924
  if (!cached) fail(`Browser state for target ${targetId} is invalid. Capture fresh browser state and retry.`);
1926
- if (cached.url !== (page.url ?? "").slice(0, 2e3)) fail("Cached browser state belongs to a different URL. Capture fresh browser state and retry.");
1927
1925
  return { targetId, cached };
1928
1926
  }
1929
1927
  function browserViewportRect(layout) {
@@ -1938,11 +1936,10 @@ function browserViewportRect(layout) {
1938
1936
  };
1939
1937
  }
1940
1938
  async function prepareBrowserElement(page, refValue) {
1941
- const backendNodeId = parseBrowserRef(refValue);
1942
1939
  const { cached } = readBrowserRefCache(page);
1943
1940
  return await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
1944
1941
  const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
1945
- const rect = await prepareBrowserRefInSession(session, backendNodeId, cached, browserViewportRect(layout));
1942
+ const { backendNodeId, rect } = await prepareBrowserRefInSession(session, parseBrowserRef(refValue), cached, browserViewportRect(layout));
1946
1943
  const axNodes = (await sendChromeSessionCommand(session, "Accessibility.getPartialAXTree", {
1947
1944
  backendNodeId,
1948
1945
  fetchRelatives: false
@@ -1975,23 +1972,35 @@ ${DESKTOP_POINT_JS}
1975
1972
  };
1976
1973
  });
1977
1974
  }
1978
- async function prepareBrowserRefInSession(session, backendNodeId, cached, viewportRect) {
1979
- if (!cached.entries.some(({ line }) => line.trimStart().startsWith(`[ref=${backendNodeId}]`))) {
1980
- fail(`Browser element ref ${backendNodeId} is not in the latest state. Capture fresh browser state and retry.`);
1981
- }
1975
+ var browserRefIdentity = (semantic) => semantic.match(/^\S+ "(?:[^"\\]|\\.)+"/)?.[0] ?? null;
1976
+ async function findReplacedBrowserRef(session, semantic, cached) {
1977
+ const identity = browserRefIdentity(semantic);
1978
+ if (!identity || cached.entries.filter((entry) => browserRefIdentity(entry.semantic) === identity).length !== 1) return null;
1979
+ const { entries } = await captureBrowserSnapshot(session, { url: "", title: "" }, { textLimit: 5e4, elementLimit: 500 });
1980
+ const matches = entries.filter((entry) => entry.line.trimStart().startsWith("[ref=") && browserRefIdentity(entry.semantic) === identity);
1981
+ return matches.length === 1 ? Number(matches[0].key.split(":").pop()) : null;
1982
+ }
1983
+ async function prepareBrowserRefInSession(session, ref, cached, viewportRect) {
1984
+ const entry = cached.entries.find(({ line }) => line.trimStart().startsWith(`[ref=${ref}]`));
1985
+ if (!entry) fail(`Browser element ref ${ref} is not in the latest state. Capture fresh browser state and retry.`);
1982
1986
  const frameTree = (await sendChromeSessionCommand(session, "Page.getFrameTree", {})).frameTree;
1983
1987
  if (cached.documentId !== browserDocumentId(frameTree)) {
1984
- fail(`Browser element ref ${backendNodeId} belongs to a previous document. Capture fresh browser state and retry.`);
1985
- }
1986
- try {
1987
- await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId });
1988
- } catch (error) {
1989
- fail(`Browser element ref ${backendNodeId} is stale or cannot be scrolled into view: ${error instanceof Error ? error.message : String(error)}`);
1988
+ fail(`Browser element ref ${ref} belongs to a previous document. Capture fresh browser state and retry.`);
1990
1989
  }
1990
+ const backendNodeId = await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId: ref }).then(
1991
+ () => ref,
1992
+ async (error) => {
1993
+ const message = error instanceof Error ? error.message : String(error);
1994
+ const replacement = /detached|No node/i.test(message) ? await findReplacedBrowserRef(session, entry.semantic, cached) : null;
1995
+ if (replacement === null) fail(`Browser element ref ${ref} is stale or cannot be scrolled into view: ${message}`);
1996
+ await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId: replacement });
1997
+ return replacement;
1998
+ }
1999
+ );
1991
2000
  const quads = (await sendChromeSessionCommand(session, "DOM.getContentQuads", { backendNodeId })).quads;
1992
2001
  const candidates = browserContentQuadRects(quads, viewportRect);
1993
- if (!candidates[0]) fail(`Browser element ref ${backendNodeId} has no visible content quad. Fetch fresh browser state and retry.`);
1994
- return roundBrowserRect(candidates[0]);
2002
+ if (!candidates[0]) fail(`Browser element ref ${ref} has no visible content quad. Fetch fresh browser state and retry.`);
2003
+ return { backendNodeId, rect: roundBrowserRect(candidates[0]) };
1995
2004
  }
1996
2005
  function buildBrowserClickExpression(query, options) {
1997
2006
  return `(() => {
@@ -2094,6 +2103,62 @@ async function dispatchChromeClickInSession(session, x, y, button, clicks) {
2094
2103
  });
2095
2104
  }
2096
2105
  }
2106
+ async function dragChromeInSession(session, from, to) {
2107
+ const intercepted = new Promise((resolve5) => session.on("Input.dragIntercepted", (params) => resolve5(params.data)));
2108
+ await sendChromeSessionCommand(session, "Input.setInterceptDrags", { enabled: true });
2109
+ let released = false;
2110
+ try {
2111
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseMoved", x: from.centerX, y: from.centerY });
2112
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mousePressed", x: from.centerX, y: from.centerY, button: "left", buttons: 1, clickCount: 1 });
2113
+ const steps = 10;
2114
+ for (let step = 1; step <= steps; step++) {
2115
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
2116
+ type: "mouseMoved",
2117
+ x: from.centerX + (to.centerX - from.centerX) * step / steps,
2118
+ y: from.centerY + (to.centerY - from.centerY) * step / steps,
2119
+ button: "left",
2120
+ buttons: 1
2121
+ });
2122
+ }
2123
+ const dragData = await Promise.race([intercepted, sleep(100).then(() => void 0)]);
2124
+ if (dragData) {
2125
+ for (const type of ["dragEnter", "dragOver", "drop"]) {
2126
+ await sendChromeSessionCommand(session, "Input.dispatchDragEvent", { type, x: to.centerX, y: to.centerY, data: dragData });
2127
+ }
2128
+ }
2129
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseReleased", x: to.centerX, y: to.centerY, button: "left", buttons: 0, clickCount: 1 });
2130
+ released = true;
2131
+ } finally {
2132
+ if (!released) {
2133
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseReleased", x: to.centerX, y: to.centerY, button: "left", buttons: 0, clickCount: 1 }).catch(() => void 0);
2134
+ }
2135
+ await sendChromeSessionCommand(session, "Input.setInterceptDrags", { enabled: false }).catch(() => void 0);
2136
+ }
2137
+ }
2138
+ async function prepareBrowserDragInSession(session, refs, cached, viewportRect) {
2139
+ const to = await prepareBrowserRefInSession(session, refs.to, cached, viewportRect);
2140
+ const from = await prepareBrowserRefInSession(session, refs.from, cached, viewportRect);
2141
+ const [fromQuad, toQuad] = await Promise.all([from, to].map(async ({ backendNodeId }) => browserContentQuadRects((await sendChromeSessionCommand(session, "DOM.getContentQuads", { backendNodeId })).quads, viewportRect)[0]));
2142
+ if (!fromQuad || !toQuad) fail(`Browser element refs ${refs.from} and ${refs.to} are not visible together. Scroll so both are in view and retry.`);
2143
+ return { from: roundBrowserRect(fromQuad), to: roundBrowserRect(toQuad), refs: { from: from.backendNodeId, to: to.backendNodeId } };
2144
+ }
2145
+ async function computerBrowserDragCommand(options) {
2146
+ const page = await selectChromePage(options);
2147
+ const { cached } = readBrowserRefCache(page);
2148
+ const refs = { from: parseBrowserRef(options.ref), to: parseBrowserRef(options.toRef) };
2149
+ const rects = await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
2150
+ const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
2151
+ const prepared = await prepareBrowserDragInSession(session, refs, cached, browserViewportRect(layout));
2152
+ await dragChromeInSession(session, prepared.from, prepared.to);
2153
+ return prepared;
2154
+ });
2155
+ logRecordingAction({ type: "drag" });
2156
+ console.log(JSON.stringify({
2157
+ targetId: page.id ?? null,
2158
+ result: { dragged: true, ref: String(rects.refs.from), toRef: String(rects.refs.to), from: rects.from, to: rects.to },
2159
+ observation: await observeAfterBrowserAction(page, Date.now())
2160
+ }, null, 2));
2161
+ }
2097
2162
  async function replaceChromeText(page, text) {
2098
2163
  await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
2099
2164
  await replaceChromeTextInSession(session, text);
@@ -2129,8 +2194,14 @@ function browserFillInvalid(actual, expected, fieldType, semanticSelection = fal
2129
2194
  if (fieldType === "color") return String(actual).toLowerCase() !== expected.toLowerCase();
2130
2195
  return actual !== expected;
2131
2196
  }
2132
- async function fillChromeNodeInSession(session, backendNodeId, value) {
2197
+ async function fillBrowserRefInSession(session, ref, cached, viewportRect, value) {
2198
+ let { backendNodeId } = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
2133
2199
  await sendChromeSessionCommand(session, "DOM.focus", { backendNodeId });
2200
+ const connected = await callFunctionOnChromeNodeInSession(session, backendNodeId, "function() { return this.isConnected; }").then(chromeCallValue, () => false);
2201
+ if (connected !== true) {
2202
+ ({ backendNodeId } = await prepareBrowserRefInSession(session, ref, cached, viewportRect));
2203
+ await sendChromeSessionCommand(session, "DOM.focus", { backendNodeId });
2204
+ }
2134
2205
  const directResult = await callFunctionOnChromeNodeInSession(
2135
2206
  session,
2136
2207
  backendNodeId,
@@ -2183,7 +2254,7 @@ async function fillChromeNodeInSession(session, backendNodeId, value) {
2183
2254
  if (browserFillInvalid(actual.value, value, fieldType, semanticSelection, selectionMatched)) {
2184
2255
  fail(`Chrome reported that ref ${backendNodeId} contains ${JSON.stringify(actual.value)} after filling, expected ${JSON.stringify(value)}.`);
2185
2256
  }
2186
- return { value: actual.value, valueLength: actual.valueLength, type: fieldType };
2257
+ return { backendNodeId, value: actual.value, valueLength: actual.valueLength, type: fieldType };
2187
2258
  }
2188
2259
  async function computerBrowserClickCommand(query, options = {}) {
2189
2260
  const page = await selectChromePage(options);
@@ -2196,8 +2267,9 @@ async function computerBrowserClickCommand(query, options = {}) {
2196
2267
  await dispatchChromeClick(page, target.rect.centerX, target.rect.centerY, button, clicks);
2197
2268
  if (target.desktopPoint) logRecordingAction({ type: "click", ...target.desktopPoint });
2198
2269
  console.log(JSON.stringify({
2199
- ...browserTargetOutput(page),
2200
- result: { clicked: true, ref: target.ref, button, clickCount: clicks, rect: target.rect, element: target.element }
2270
+ targetId: page.id ?? null,
2271
+ result: { clicked: true, ref: target.ref, button, clickCount: clicks, rect: target.rect, element: target.element },
2272
+ observation: await observeAfterBrowserAction(page, Date.now())
2201
2273
  }, null, 2));
2202
2274
  return;
2203
2275
  }
@@ -2213,8 +2285,9 @@ async function computerBrowserClickCommand(query, options = {}) {
2213
2285
  const point = browserActionPoint(result);
2214
2286
  if (point) logRecordingAction({ type: "click", ...point });
2215
2287
  console.log(JSON.stringify({
2216
- ...browserTargetOutput(page),
2217
- result
2288
+ targetId: page.id ?? null,
2289
+ result,
2290
+ observation: await observeAfterBrowserAction(page, Date.now())
2218
2291
  }, null, 2));
2219
2292
  }
2220
2293
  function buildBrowserFillExpression(query, value, options) {
@@ -2336,13 +2409,18 @@ ${DESKTOP_POINT_JS}
2336
2409
  async function computerBrowserFillCommand(query, value, options = {}) {
2337
2410
  const page = await selectChromePage(options);
2338
2411
  if (options.ref) {
2412
+ const ref = parseBrowserRef(options.ref);
2339
2413
  const target = await prepareBrowserElement(page, options.ref);
2340
- const backendNodeId = parseBrowserRef(options.ref);
2341
- const actual = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await fillChromeNodeInSession(session, backendNodeId, value));
2414
+ const { cached } = readBrowserRefCache(page);
2415
+ const actual = await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
2416
+ const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
2417
+ return await fillBrowserRefInSession(session, ref, cached, browserViewportRect(layout), value);
2418
+ });
2342
2419
  if (target.desktopPoint) logRecordingAction({ type: "type", ...target.desktopPoint });
2343
2420
  console.log(JSON.stringify({
2344
- ...browserTargetOutput(page),
2345
- result: { filled: true, ref: target.ref, valueLength: actual.valueLength, rect: target.rect, element: target.element }
2421
+ targetId: page.id ?? null,
2422
+ result: { filled: true, ref: String(actual.backendNodeId), valueLength: actual.valueLength, rect: target.rect, element: target.element },
2423
+ observation: await observeAfterBrowserAction(page, Date.now())
2346
2424
  }, null, 2));
2347
2425
  return;
2348
2426
  }
@@ -2375,8 +2453,9 @@ async function computerBrowserFillCommand(query, value, options = {}) {
2375
2453
  }
2376
2454
  if (point) logRecordingAction({ type: "type", ...point });
2377
2455
  console.log(JSON.stringify({
2378
- ...browserTargetOutput(page),
2379
- result
2456
+ targetId: page.id ?? null,
2457
+ result,
2458
+ observation: await observeAfterBrowserAction(page, Date.now())
2380
2459
  }, null, 2));
2381
2460
  }
2382
2461
  function buildBrowserWaitExpression(query, options) {
@@ -2438,7 +2517,34 @@ async function computerBrowserWaitCommand(query, options = {}) {
2438
2517
  await sleep(pollMs);
2439
2518
  }
2440
2519
  console.log(JSON.stringify({ ok: false, elapsedMs: Date.now() - start, attempts, result: lastResult }, null, 2));
2441
- process.exitCode = 1;
2520
+ fail(`Timed out after ${timeoutMs}ms waiting for ${JSON.stringify(query)}`);
2521
+ }
2522
+ async function browserScrollPointInSession(session, viewportRect, deltaX, deltaY) {
2523
+ const point = await evaluateChromeSession(session, `(() => {
2524
+ const vertical = ${deltaY !== 0};
2525
+ const root = document.scrollingElement || document.documentElement;
2526
+ if (vertical ? root.scrollHeight > innerHeight : root.scrollWidth > innerWidth) return null;
2527
+ let best = null;
2528
+ let bestArea = 0;
2529
+ for (const el of document.querySelectorAll('*')) {
2530
+ const style = getComputedStyle(el);
2531
+ const scrollable = vertical
2532
+ ? /(auto|scroll)/.test(style.overflowY) && el.scrollHeight > el.clientHeight
2533
+ : /(auto|scroll)/.test(style.overflowX) && el.scrollWidth > el.clientWidth;
2534
+ if (!scrollable) continue;
2535
+ const rect = el.getBoundingClientRect();
2536
+ const left = Math.max(rect.left, 0);
2537
+ const top = Math.max(rect.top, 0);
2538
+ const width = Math.min(rect.right, innerWidth) - left;
2539
+ const height = Math.min(rect.bottom, innerHeight) - top;
2540
+ if (width > 0 && height > 0 && width * height > bestArea) {
2541
+ bestArea = width * height;
2542
+ best = { x: left + width / 2, y: top + height / 2 };
2543
+ }
2544
+ }
2545
+ return best;
2546
+ })()`);
2547
+ return isRecord(point) && typeof point.x === "number" && typeof point.y === "number" ? { x: point.x, y: point.y } : { x: viewportRect.centerX, y: viewportRect.centerY };
2442
2548
  }
2443
2549
  async function computerBrowserScrollCommand(direction, options = {}) {
2444
2550
  const page = await selectChromePage(options);
@@ -2446,25 +2552,54 @@ async function computerBrowserScrollCommand(direction, options = {}) {
2446
2552
  if (!["up", "down", "left", "right"].includes(normalized)) fail("direction must be one of up|down|left|right");
2447
2553
  const amount = options.amount ? parseCoord(options.amount, "--amount") : 600;
2448
2554
  if (amount <= 0 || amount > 1e4) fail("--amount must be between 1 and 10000 CSS pixels");
2449
- const layout = await sendChromeCommand(page.webSocketDebuggerUrl, "Page.getLayoutMetrics", {});
2450
- const viewport = chromeVisualViewport(layout);
2555
+ const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 1e4;
2451
2556
  const target = options.ref ? await prepareBrowserElement(page, options.ref) : null;
2452
- const x = target?.rect.centerX ?? viewport.width / 2;
2453
- const y = target?.rect.centerY ?? viewport.height / 2;
2454
2557
  const deltaX = normalized === "left" ? -amount : normalized === "right" ? amount : 0;
2455
2558
  const deltaY = normalized === "up" ? -amount : normalized === "down" ? amount : 0;
2456
- await sendChromeCommand(page.webSocketDebuggerUrl, "Input.dispatchMouseEvent", {
2457
- type: "mouseWheel",
2458
- x,
2459
- y,
2460
- deltaX,
2461
- deltaY
2559
+ const startedAt = Date.now();
2560
+ const { point, scrolls, search } = await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
2561
+ const point2 = target ? { x: target.rect.centerX, y: target.rect.centerY } : await browserScrollPointInSession(session, browserViewportRect(await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {})), deltaX, deltaY);
2562
+ const wheel = () => sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseWheel", ...point2, deltaX, deltaY });
2563
+ if (options.until === void 0) {
2564
+ await wheel();
2565
+ return { point: point2, scrolls: 1, search: null };
2566
+ }
2567
+ const expression = `(() => {
2568
+ const needle = ${JSON.stringify(options.until)}.replace(/\\s+/g, ' ').trim().toLowerCase();
2569
+ const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
2570
+ const has = (el) => normalize(el.innerText).toLowerCase().includes(needle);
2571
+ ${VISIBLE_JS}
2572
+ const offsets = [scrollX, scrollY, document.getElementsByTagName('*').length];
2573
+ for (let el = document.elementFromPoint(${point2.x}, ${point2.y}); el; el = el.parentElement) offsets.push(el.scrollLeft, el.scrollTop);
2574
+ if (!document.body || !has(document.body)) return { match: null, signature: offsets.join(',') };
2575
+ let el = document.body;
2576
+ for (let child = el; child; child = Array.from(el.children).find((item) => visible(item) && has(item))) el = child;
2577
+ el.scrollIntoView({ block: 'center', inline: 'nearest' });
2578
+ return { match: normalize(el.innerText).slice(0, 300), signature: null };
2579
+ })()`;
2580
+ let scrolls2 = 0;
2581
+ let unchanged = 0;
2582
+ let previous = null;
2583
+ while (true) {
2584
+ const found = await evaluateChromeSession(session, expression);
2585
+ if (!isRecord(found)) fail("Chrome did not return a text search result");
2586
+ if (typeof found.match === "string") return { point: point2, scrolls: scrolls2, search: { found: true, match: found.match } };
2587
+ unchanged = found.signature === previous ? unchanged + 1 : 0;
2588
+ previous = found.signature;
2589
+ if (unchanged >= 2) return { point: point2, scrolls: scrolls2, search: { found: false, reason: "Reached the end of the scrollable content" } };
2590
+ if (Date.now() - startedAt >= timeoutMs) return { point: point2, scrolls: scrolls2, search: { found: false, reason: `Timed out after ${timeoutMs}ms` } };
2591
+ await wheel();
2592
+ scrolls2++;
2593
+ await waitForBrowserQuiet(session, { timeoutMs: 2e3, stableMs: 300, pollMs: 50, actedAt: Date.now() });
2594
+ }
2462
2595
  });
2463
2596
  logRecordingAction({ type: "scroll", ...target?.desktopPoint ?? {} });
2464
2597
  console.log(JSON.stringify({
2465
- ...browserTargetOutput(page),
2466
- result: { scrolled: true, direction: normalized, amount, ref: target?.ref ?? null, x: Math.round(x), y: Math.round(y) }
2598
+ targetId: page.id ?? null,
2599
+ result: { scrolled: true, direction: normalized, amount, scrolls, ref: target?.ref ?? null, x: Math.round(point.x), y: Math.round(point.y), ...search && { until: options.until, ...search } },
2600
+ observation: await observeAfterBrowserAction(page, Date.now())
2467
2601
  }, null, 2));
2602
+ if (search?.found === false) fail(`${search.reason} without finding ${JSON.stringify(options.until)}`);
2468
2603
  }
2469
2604
  function browserKeyDetails(combo) {
2470
2605
  const tokens = combo.split("+").map((token) => token.trim()).filter(Boolean);
@@ -2521,7 +2656,7 @@ async function computerBrowserKeyCommand(combo, options = {}) {
2521
2656
  await dispatchChromeKeyInSession(session, key);
2522
2657
  });
2523
2658
  logRecordingAction({ type: "key" });
2524
- console.log(JSON.stringify({ targetId: page.id ?? null, result: { pressed: combo } }, null, 2));
2659
+ console.log(JSON.stringify({ targetId: page.id ?? null, result: { pressed: combo }, observation: await observeAfterBrowserAction(page, Date.now()) }, null, 2));
2525
2660
  }
2526
2661
  async function dispatchChromeKeyInSession(session, key) {
2527
2662
  await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
@@ -2542,7 +2677,7 @@ async function computerBrowserTypeCommand(text, options = {}) {
2542
2677
  const page = await selectChromePage(options);
2543
2678
  await sendChromeCommand(page.webSocketDebuggerUrl, "Input.insertText", { text });
2544
2679
  logRecordingAction({ type: "type" });
2545
- console.log(JSON.stringify({ targetId: page.id ?? null, result: { typed: true, length: text.length } }, null, 2));
2680
+ console.log(JSON.stringify({ targetId: page.id ?? null, result: { typed: true, length: text.length }, observation: await observeAfterBrowserAction(page, Date.now()) }, null, 2));
2546
2681
  }
2547
2682
  function parseBrowserBatchActions(value) {
2548
2683
  let input;
@@ -2570,6 +2705,12 @@ function parseBrowserBatchActions(value) {
2570
2705
  }
2571
2706
  return { action: "fill", ref: String(item.ref), value: item.value };
2572
2707
  }
2708
+ if (item.action === "drag") {
2709
+ if (typeof item.ref !== "string" && typeof item.ref !== "number" || typeof item.toRef !== "string" && typeof item.toRef !== "number") {
2710
+ fail(`Batch drag ${index} requires ref and toRef`);
2711
+ }
2712
+ return { action: "drag", ref: String(item.ref), toRef: String(item.toRef) };
2713
+ }
2573
2714
  if (item.action === "key") {
2574
2715
  if (typeof item.combo !== "string") fail(`Batch key ${index} requires combo`);
2575
2716
  return { action: "key", combo: item.combo };
@@ -2621,16 +2762,20 @@ async function computerBrowserBatchCommand(actionsJson, options = {}) {
2621
2762
  const actionStartedAt = Date.now();
2622
2763
  if (action.action === "click") {
2623
2764
  const ref = parseBrowserRef(action.ref);
2624
- const rect = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
2765
+ const { backendNodeId, rect } = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
2625
2766
  await dispatchChromeClickInSession(session, rect.centerX, rect.centerY, action.button, action.clicks);
2626
2767
  logRecordingAction({ type: "click" });
2627
- completed.push({ index, action: action.action, ref: String(ref), elapsedMs: Date.now() - actionStartedAt });
2768
+ completed.push({ index, action: action.action, ref: String(backendNodeId), elapsedMs: Date.now() - actionStartedAt });
2628
2769
  } else if (action.action === "fill") {
2629
- const ref = parseBrowserRef(action.ref);
2630
- await prepareBrowserRefInSession(session, ref, cached, viewportRect);
2631
- const result = await fillChromeNodeInSession(session, ref, action.value);
2770
+ const result = await fillBrowserRefInSession(session, parseBrowserRef(action.ref), cached, viewportRect, action.value);
2632
2771
  logRecordingAction({ type: "type" });
2633
- completed.push({ index, action: action.action, ref: String(ref), valueLength: result.valueLength, elapsedMs: Date.now() - actionStartedAt });
2772
+ completed.push({ index, action: action.action, ref: String(result.backendNodeId), valueLength: result.valueLength, elapsedMs: Date.now() - actionStartedAt });
2773
+ } else if (action.action === "drag") {
2774
+ const refs = { from: parseBrowserRef(action.ref), to: parseBrowserRef(action.toRef) };
2775
+ const rects = await prepareBrowserDragInSession(session, refs, cached, viewportRect);
2776
+ await dragChromeInSession(session, rects.from, rects.to);
2777
+ logRecordingAction({ type: "drag" });
2778
+ completed.push({ index, action: action.action, ref: String(rects.refs.from), toRef: String(rects.refs.to), elapsedMs: Date.now() - actionStartedAt });
2634
2779
  } else if (action.action === "key") {
2635
2780
  await dispatchChromeKeyInSession(session, browserKeyDetails(action.combo));
2636
2781
  logRecordingAction({ type: "key" });
@@ -2641,16 +2786,11 @@ async function computerBrowserBatchCommand(actionsJson, options = {}) {
2641
2786
  completed.push({ index, action: action.action, length: action.text.length, elapsedMs: Date.now() - actionStartedAt });
2642
2787
  } else if (action.action === "scroll") {
2643
2788
  const ref = action.ref ? parseBrowserRef(action.ref) : null;
2644
- const rect = ref ? await prepareBrowserRefInSession(session, ref, cached, viewportRect) : viewportRect;
2645
2789
  const deltaX = action.direction === "left" ? -action.amount : action.direction === "right" ? action.amount : 0;
2646
2790
  const deltaY = action.direction === "up" ? -action.amount : action.direction === "down" ? action.amount : 0;
2647
- await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
2648
- type: "mouseWheel",
2649
- x: rect.centerX,
2650
- y: rect.centerY,
2651
- deltaX,
2652
- deltaY
2653
- });
2791
+ const rect = ref ? (await prepareBrowserRefInSession(session, ref, cached, viewportRect)).rect : null;
2792
+ const point = rect ? { x: rect.centerX, y: rect.centerY } : await browserScrollPointInSession(session, viewportRect, deltaX, deltaY);
2793
+ await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseWheel", ...point, deltaX, deltaY });
2654
2794
  logRecordingAction({ type: "scroll" });
2655
2795
  completed.push({ index, action: action.action, direction: action.direction, amount: action.amount, elapsedMs: Date.now() - actionStartedAt });
2656
2796
  } else {
@@ -2670,10 +2810,11 @@ async function computerBrowserBatchCommand(actionsJson, options = {}) {
2670
2810
  });
2671
2811
  } catch (error) {
2672
2812
  const message = error instanceof Error ? error.message : String(error);
2673
- console.log(JSON.stringify({ ok: false, targetId, elapsedMs: Date.now() - startedAt, completed, failedIndex: completed.length, error: message }, null, 2));
2813
+ const observation = await observeAfterBrowserAction(page, Date.now());
2814
+ console.log(JSON.stringify({ ok: false, targetId, elapsedMs: Date.now() - startedAt, completed, failedIndex: completed.length, error: message, observation }, null, 2));
2674
2815
  throw error;
2675
2816
  }
2676
- console.log(JSON.stringify({ ok: true, targetId, elapsedMs: Date.now() - startedAt, completed }, null, 2));
2817
+ console.log(JSON.stringify({ ok: true, targetId, elapsedMs: Date.now() - startedAt, completed, observation: await observeAfterBrowserAction(page, Date.now()) }, null, 2));
2677
2818
  }
2678
2819
  var APP_ALIASES = {
2679
2820
  chrome: [CHROME_WRAPPER],
@@ -2730,8 +2871,8 @@ async function computerLaunchCommand(app, args) {
2730
2871
  const candidates = expectsNewChromePage && existingChromePageIds.size > 0 ? pages.filter((page) => page.id && !existingChromePageIds.has(page.id)) : pages;
2731
2872
  for (const page of candidates) {
2732
2873
  if (!page.webSocketDebuggerUrl || !page.url || page.url === "about:blank") continue;
2733
- const readyState = await evaluateChromeTarget(page.webSocketDebuggerUrl, "document.readyState");
2734
- if (readyState === "interactive" || readyState === "complete") {
2874
+ const quiet = await evaluateChromeTarget(page.webSocketDebuggerUrl, BROWSER_QUIET_JS);
2875
+ if (isRecord(quiet) && (quiet.readyState === "interactive" || quiet.readyState === "complete")) {
2735
2876
  readyPage = page;
2736
2877
  break;
2737
2878
  }
@@ -2764,7 +2905,7 @@ async function computerClickCommand(xStr, yStr, options) {
2764
2905
  const x = parseScreenCoord(xStr, "x", dimensions.width);
2765
2906
  const y = parseScreenCoord(yStr, "y", dimensions.height);
2766
2907
  const button = options.button ?? "1";
2767
- const args = ["mousemove", "--sync", String(x), String(y)];
2908
+ const args = ["mousemove", String(x), String(y)];
2768
2909
  if (options.modifiers) {
2769
2910
  for (const mod of options.modifiers.split("+")) {
2770
2911
  args.push("keydown", mod);
@@ -2788,7 +2929,7 @@ async function computerMoveCommand(xStr, yStr) {
2788
2929
  const dimensions = getDisplayDimensions();
2789
2930
  const x = parseScreenCoord(xStr, "x", dimensions.width);
2790
2931
  const y = parseScreenCoord(yStr, "y", dimensions.height);
2791
- runDesktopInputCmd(["mousemove", "--sync", String(x), String(y)]);
2932
+ runDesktopInputCmd(["mousemove", String(x), String(y)]);
2792
2933
  logRecordingAction({ type: "move", x, y });
2793
2934
  console.log(`moved to (${x},${y})`);
2794
2935
  }
@@ -2819,12 +2960,7 @@ async function computerScrollCommand(direction, options) {
2819
2960
  x: parseScreenCoord(options.x, "--x", dimensions.width),
2820
2961
  y: parseScreenCoord(options.y, "--y", dimensions.height)
2821
2962
  };
2822
- args.push(
2823
- "mousemove",
2824
- "--sync",
2825
- String(hoverPosition.x),
2826
- String(hoverPosition.y)
2827
- );
2963
+ args.push("mousemove", String(hoverPosition.x), String(hoverPosition.y));
2828
2964
  }
2829
2965
  args.push("click", "--repeat", String(amount), "--delay", "30", button);
2830
2966
  runDesktopInputCmd(args);
@@ -2842,7 +2978,6 @@ async function computerDragCommand(fx, fy, tx, ty) {
2842
2978
  const steps = clamp(Math.ceil(distance / 70), 8, 24);
2843
2979
  const args = [
2844
2980
  "mousemove",
2845
- "--sync",
2846
2981
  String(fromX),
2847
2982
  String(fromY),
2848
2983
  "mousedown",
@@ -2851,8 +2986,10 @@ async function computerDragCommand(fx, fy, tx, ty) {
2851
2986
  for (let i = 1; i <= steps; i++) {
2852
2987
  const x = Math.round(fromX + (toX - fromX) * i / steps);
2853
2988
  const y = Math.round(fromY + (toY - fromY) * i / steps);
2854
- args.push("mousemove", "--sync", String(x), String(y), "sleep", "0.018");
2989
+ args.push("mousemove", String(x), String(y), "sleep", "0.018");
2855
2990
  }
2991
+ const nudgeX = toX > 0 ? toX - 1 : toX + 1;
2992
+ for (const x of [nudgeX, toX, nudgeX, toX]) args.push("mousemove", String(x), String(toY), "sleep", "0.018");
2856
2993
  args.push("mouseup", "1");
2857
2994
  runDesktopInputCmd(args);
2858
2995
  logRecordingAction({ type: "drag", x: fromX, y: fromY, toX, toY });
@@ -2932,7 +3069,7 @@ async function computerStatusCommand() {
2932
3069
  }
2933
3070
 
2934
3071
  // src/version.ts
2935
- var COMPUTER_USE_VERSION = "0.3.22";
3072
+ var COMPUTER_USE_VERSION = "0.3.23";
2936
3073
  var COMPUTER_USE_PROTOCOL_VERSION = 1;
2937
3074
 
2938
3075
  // src/program.ts
@@ -2947,9 +3084,9 @@ function createProgram() {
2947
3084
  program.command("info").description("Print the live noVNC viewer URL").action(() => computerInfoCommand());
2948
3085
  program.command("status").description("Show desktop services and the active preview URL").action(() => computerStatusCommand());
2949
3086
  program.command("screenshot <path>").description("Capture the current desktop to a PNG file").option("--raw", "Save a 1:1 desktop capture").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").action((path, options) => computerScreenshotCommand(path, options));
2950
- program.command("observe <path>").description("Wait for the desktop to settle and capture screen context").option("--raw", "Save a 1:1 desktop capture without a grid").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").option("--timeout <ms>", "Maximum stability wait. Default 3000.").option("--stable-ms <ms>", "Required unchanged time. Default 600.").option("--poll-ms <ms>", "Polling interval. Default 200.").action((path, options) => computerObserveCommand(path, options));
3087
+ program.command("observe <path>").description("Wait for the desktop to settle and capture screen context").option("--raw", "Save a 1:1 desktop capture without a grid (default)").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").option("--timeout <ms>", "Maximum stability wait. Default 3000.").option("--stable-ms <ms>", "Required unchanged time. Default 600.").option("--poll-ms <ms>", "Polling interval. Default 100.").action((path, options) => computerObserveCommand(path, options));
2951
3088
  program.command("browser").description("Print JSON for controllable Chrome pages").option("--snapshot", "Include visible text and controls").option("--limit <chars>", "Maximum body text characters. Default 4000.").option("--element-limit <n>", "Maximum controls. Default 80.").option("--target-id <id>", "Only include this Chrome target").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((options) => computerBrowserCommand(options));
2952
- program.command("browser-state <path>").description("Capture a Chrome screenshot and accessibility state").option("--full", "Return the full tree instead of a diff").option("--limit <chars>", "Maximum visible text characters. Default 8000.").option("--element-limit <n>", "Maximum actionable elements. Default 120.").option("--timeout <ms>", "Maximum stability wait. Default 5000.").option("--stable-ms <ms>", "Required unchanged time. Default 500.").option("--poll-ms <ms>", "Polling interval. Default 100.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((path, options) => computerBrowserStateCommand(path, options));
3089
+ program.command("browser-state [path]").description("Capture Chrome accessibility state, and a viewport screenshot when a path is given").option("--full", "Return the full tree instead of a diff").option("--limit <chars>", "Maximum visible text characters. Default 8000.").option("--element-limit <n>", "Maximum actionable elements. Default 120.").option("--timeout <ms>", "Maximum stability wait. Default 5000.").option("--stable-ms <ms>", "Required unchanged time. Default 500.").option("--poll-ms <ms>", "Polling interval. Default 100.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((path, options) => computerBrowserStateCommand(path, options));
2953
3090
  program.command("browser-batch <actions>").description("Run an ordered, fail-fast JSON array of browser actions").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((actions, options) => computerBrowserBatchCommand(actions, options));
2954
3091
  program.command("browser-click [text]").description("Click a Chrome element by ref or visible text").option("--ref <id>", "Backend DOM node ref from browser-state").option("--exact", "Require an exact text match").option("--index <n>", "Select the nth match. Default 0.").option("--button <button>", "left, middle, or right").option("--double", "Double-click").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserClickCommand(text, options));
2955
3092
  program.command("browser-fill [field] [value]").description("Fill a Chrome field by ref or visible text").option("--ref <id>", "Backend DOM node ref from browser-state").option("--exact", "Require an exact field match").option("--index <n>", "Select the nth match. Default 0.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((field, value, options) => {
@@ -2958,7 +3095,8 @@ function createProgram() {
2958
3095
  if (resolvedValue === void 0) throw new Error("Provide <field> <value>, or --ref <id> <value>");
2959
3096
  return computerBrowserFillCommand(resolvedField, resolvedValue, options);
2960
3097
  });
2961
- program.command("browser-scroll <direction>").description("Send trusted wheel input to Chrome").option("--ref <id>", "Scroll from a backend DOM node ref").option("--amount <px>", "Scroll distance. Default 600.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((direction, options) => computerBrowserScrollCommand(direction, options));
3098
+ program.command("browser-drag").description("Drag a Chrome element onto another by ref").requiredOption("--ref <id>", "Element to drag, from the latest browser state").requiredOption("--to-ref <id>", "Element to drop it on").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((options) => computerBrowserDragCommand(options));
3099
+ program.command("browser-scroll <direction>").description("Send trusted wheel input to Chrome").option("--ref <id>", "Scroll from a backend DOM node ref").option("--amount <px>", "Scroll distance. Default 600.").option("--until <text>", "Keep scrolling until this text is on the page, then center it").option("--timeout <ms>", "Give up on --until after this long. Default 10000.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((direction, options) => computerBrowserScrollCommand(direction, options));
2962
3100
  program.command("browser-key <combo>").description("Press a key chord in Chrome").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((combo, options) => computerBrowserKeyCommand(combo, options));
2963
3101
  program.command("browser-type <text>").description("Type literal text in Chrome").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserTypeCommand(text, options));
2964
3102
  program.command("browser-wait <text>").description("Wait for Chrome title, URL, text, or controls").option("--mode <mode>", "any, text, title, url, or control").option("--exact", "Require an exact match").option("--timeout <ms>", "Maximum wait. Default 10000.").option("--poll-ms <ms>", "Polling interval. Default 250.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserWaitCommand(text, options));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "replicas-computer",
3
- "version": "0.3.22",
3
+ "version": "0.3.24",
4
4
  "description": "Replicas workspace computer-use runtime",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",