replicas-computer 0.3.22 → 0.3.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +422 -284
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -191,21 +191,18 @@ import { dirname as dirname4 } from "path";
|
|
|
191
191
|
// src/commands/capture.ts
|
|
192
192
|
import { spawnSync as spawnSync2 } from "child_process";
|
|
193
193
|
import { createHash } from "crypto";
|
|
194
|
-
import {
|
|
194
|
+
import { copyFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readdirSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
195
195
|
import { dirname as dirname2 } from "path";
|
|
196
196
|
var PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
|
|
197
|
-
function
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
fail(`${filePath} is not a valid PNG (read ${bytesRead}/24 bytes)`);
|
|
197
|
+
function readImageDimensions(filePath) {
|
|
198
|
+
const data = readFileSync2(filePath);
|
|
199
|
+
if (data.length >= 24 && data.subarray(0, 8).equals(PNG_SIGNATURE)) return { width: data.readUInt32BE(16), height: data.readUInt32BE(20) };
|
|
200
|
+
if (data[0] === 255 && data[1] === 216) {
|
|
201
|
+
for (let offset = 2; offset + 9 < data.length && data[offset] === 255; offset += 2 + data.readUInt16BE(offset + 2)) {
|
|
202
|
+
if (data[offset + 1] >= 192 && data[offset + 1] <= 195) return { width: data.readUInt16BE(offset + 7), height: data.readUInt16BE(offset + 5) };
|
|
204
203
|
}
|
|
205
|
-
return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };
|
|
206
|
-
} finally {
|
|
207
|
-
closeSync(fd);
|
|
208
204
|
}
|
|
205
|
+
fail(`${filePath} is not a valid PNG or JPEG image`);
|
|
209
206
|
}
|
|
210
207
|
function currentWallpaper() {
|
|
211
208
|
const recorded = `${STATE_DIR}/wallpaper-path`;
|
|
@@ -225,6 +222,12 @@ var SHADOW_OFFSET_Y_FRACTION = 0.013;
|
|
|
225
222
|
var SHADOW_ALPHA = 0.6;
|
|
226
223
|
var SCREENSHOT_MASK_TEMPLATE = `<svg width="__W__" height="__H__" xmlns="http://www.w3.org/2000/svg"><rect width="__W__" height="__H__" rx="__R__" ry="__R__" fill="white"/></svg>`;
|
|
227
224
|
var SHADOW_MASK_TEMPLATE = `<svg width="__SW__" height="__SH__" xmlns="http://www.w3.org/2000/svg"><rect x="__M__" y="__M__" width="__W__" height="__H__" rx="__R__" ry="__R__" fill="white"/></svg>`;
|
|
225
|
+
var isJpeg = (path) => /\.jpe?g$/i.test(path);
|
|
226
|
+
var imageExtension = (path) => isJpeg(path) ? "jpg" : "png";
|
|
227
|
+
var jpegQuality = (path) => isJpeg(path) ? ["-q:v", "2"] : [];
|
|
228
|
+
function captureScreen(path) {
|
|
229
|
+
runDisplayCmd("scrot", ["-o", ...isJpeg(path) ? ["-q", "90"] : [], path]);
|
|
230
|
+
}
|
|
228
231
|
function parseGridSize(value) {
|
|
229
232
|
if (value === void 0 || value === false) return null;
|
|
230
233
|
const size = value === true ? 100 : parseCoord(value, "--grid");
|
|
@@ -269,6 +272,7 @@ function overlayGrid(rawPath, target, width2, height2, gridSize, gridPath) {
|
|
|
269
272
|
"1",
|
|
270
273
|
"-update",
|
|
271
274
|
"1",
|
|
275
|
+
...jpegQuality(target),
|
|
272
276
|
target
|
|
273
277
|
],
|
|
274
278
|
{ stdio: "pipe" }
|
|
@@ -281,13 +285,13 @@ async function computerScreenshotCommand(path, options = {}) {
|
|
|
281
285
|
const target = resolvePath(path);
|
|
282
286
|
mkdirSync2(dirname2(target), { recursive: true });
|
|
283
287
|
const stamp = `${process.pid}-${Date.now()}`;
|
|
284
|
-
const rawPath = `/tmp/replicas-screenshot-${stamp}.raw
|
|
288
|
+
const rawPath = `/tmp/replicas-screenshot-${stamp}.raw.${imageExtension(target)}`;
|
|
285
289
|
const maskPath = `/tmp/replicas-screenshot-${stamp}.mask.svg`;
|
|
286
290
|
const shadowPath = `/tmp/replicas-screenshot-${stamp}.shadow.svg`;
|
|
287
291
|
const gridPath = `/tmp/replicas-screenshot-${stamp}.grid.svg`;
|
|
288
292
|
try {
|
|
289
|
-
|
|
290
|
-
const { width: width2, height: height2 } =
|
|
293
|
+
captureScreen(rawPath);
|
|
294
|
+
const { width: width2, height: height2 } = getDisplayDimensions();
|
|
291
295
|
const gridSize = parseGridSize(options.grid);
|
|
292
296
|
if (options.raw && gridSize === null) {
|
|
293
297
|
copyFileSync(rawPath, target);
|
|
@@ -339,6 +343,7 @@ async function computerScreenshotCommand(path, options = {}) {
|
|
|
339
343
|
"1",
|
|
340
344
|
"-update",
|
|
341
345
|
"1",
|
|
346
|
+
...jpegQuality(target),
|
|
342
347
|
target
|
|
343
348
|
],
|
|
344
349
|
{ stdio: "pipe" }
|
|
@@ -357,22 +362,18 @@ async function computerScreenshotCommand(path, options = {}) {
|
|
|
357
362
|
function hashFile(path) {
|
|
358
363
|
return createHash("sha256").update(readFileSync2(path)).digest("hex");
|
|
359
364
|
}
|
|
360
|
-
async function
|
|
365
|
+
async function captureStableScreenshot(target, options) {
|
|
361
366
|
const start = Date.now();
|
|
362
|
-
const framePath = `${target}.frame.
|
|
367
|
+
const framePath = `${target}.frame.jpg`;
|
|
363
368
|
let lastHash = null;
|
|
364
369
|
let lastChangeAt = start;
|
|
365
370
|
let frames = 0;
|
|
366
371
|
let changes = 0;
|
|
367
|
-
let
|
|
368
|
-
let height2 = 0;
|
|
372
|
+
let stable = false;
|
|
369
373
|
try {
|
|
370
374
|
while (Date.now() - start <= options.timeoutMs) {
|
|
371
|
-
|
|
375
|
+
captureScreen(framePath);
|
|
372
376
|
frames++;
|
|
373
|
-
const dimensions = readPngDimensions(framePath);
|
|
374
|
-
width2 = dimensions.width;
|
|
375
|
-
height2 = dimensions.height;
|
|
376
377
|
const hash = hashFile(framePath);
|
|
377
378
|
const now = Date.now();
|
|
378
379
|
if (lastHash === null) {
|
|
@@ -382,13 +383,16 @@ async function captureStableRawScreenshot(target, options) {
|
|
|
382
383
|
lastChangeAt = now;
|
|
383
384
|
}
|
|
384
385
|
lastHash = hash;
|
|
385
|
-
copyFileSync(framePath, target);
|
|
386
386
|
if (frames > 1 && now - lastChangeAt >= options.stableMs) {
|
|
387
|
-
|
|
387
|
+
stable = true;
|
|
388
|
+
break;
|
|
388
389
|
}
|
|
389
390
|
await sleep(options.pollMs);
|
|
390
391
|
}
|
|
391
|
-
|
|
392
|
+
const elapsedMs = Date.now() - start;
|
|
393
|
+
if (isJpeg(target)) copyFileSync(framePath, target);
|
|
394
|
+
else captureScreen(target);
|
|
395
|
+
return { stable, elapsedMs, frames, changes };
|
|
392
396
|
} finally {
|
|
393
397
|
rmSync(framePath, { force: true });
|
|
394
398
|
}
|
|
@@ -405,26 +409,27 @@ async function computerObserveCommand(path, options = {}) {
|
|
|
405
409
|
mkdirSync2(dirname2(target), { recursive: true });
|
|
406
410
|
const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 3e3;
|
|
407
411
|
const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 600;
|
|
408
|
-
const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") :
|
|
412
|
+
const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
|
|
409
413
|
if (timeoutMs < 0) fail("--timeout must be >= 0");
|
|
410
414
|
if (stableMs < 0) fail("--stable-ms must be >= 0");
|
|
411
415
|
if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
|
|
412
416
|
const stamp = `${process.pid}-${Date.now()}`;
|
|
413
|
-
const rawPath = `/tmp/replicas-observe-${stamp}.raw
|
|
417
|
+
const rawPath = `/tmp/replicas-observe-${stamp}.raw.${imageExtension(target)}`;
|
|
414
418
|
const gridPath = `/tmp/replicas-observe-${stamp}.grid.svg`;
|
|
415
419
|
try {
|
|
416
420
|
ensureServicesRunning();
|
|
417
|
-
const capture = await
|
|
418
|
-
const
|
|
421
|
+
const capture = await captureStableScreenshot(rawPath, { timeoutMs, stableMs, pollMs });
|
|
422
|
+
const { width: width2, height: height2 } = getDisplayDimensions();
|
|
423
|
+
const gridSize = options.raw ? null : parseGridSize(options.grid);
|
|
419
424
|
if (gridSize === null) {
|
|
420
425
|
copyFileSync(rawPath, target);
|
|
421
426
|
} else {
|
|
422
|
-
overlayGrid(rawPath, target,
|
|
427
|
+
overlayGrid(rawPath, target, width2, height2, gridSize, gridPath);
|
|
423
428
|
}
|
|
424
429
|
console.log(JSON.stringify({
|
|
425
430
|
screenshot: target,
|
|
426
|
-
width:
|
|
427
|
-
height:
|
|
431
|
+
width: width2,
|
|
432
|
+
height: height2,
|
|
428
433
|
stable: capture.stable,
|
|
429
434
|
elapsedMs: capture.elapsedMs,
|
|
430
435
|
frames: capture.frames,
|
|
@@ -1115,6 +1120,7 @@ function isRecord(value) {
|
|
|
1115
1120
|
async function openChromeSession(webSocketDebuggerUrl) {
|
|
1116
1121
|
const socket = new WebSocket(webSocketDebuggerUrl);
|
|
1117
1122
|
const pending = /* @__PURE__ */ new Map();
|
|
1123
|
+
const listeners = /* @__PURE__ */ new Map();
|
|
1118
1124
|
let nextId = 1;
|
|
1119
1125
|
let closed = false;
|
|
1120
1126
|
const fail2 = (error) => {
|
|
@@ -1137,6 +1143,7 @@ async function openChromeSession(webSocketDebuggerUrl) {
|
|
|
1137
1143
|
return;
|
|
1138
1144
|
}
|
|
1139
1145
|
if (!isRecord(message)) return;
|
|
1146
|
+
if (typeof message.method === "string" && isRecord(message.params)) listeners.get(message.method)?.(message.params);
|
|
1140
1147
|
if (typeof message.id !== "number") return;
|
|
1141
1148
|
const request = pending.get(message.id);
|
|
1142
1149
|
if (!request) return;
|
|
@@ -1175,6 +1182,7 @@ async function openChromeSession(webSocketDebuggerUrl) {
|
|
|
1175
1182
|
pending.set(id, { resolve: resolve5, reject, timeout });
|
|
1176
1183
|
socket.send(JSON.stringify({ id, method, params }));
|
|
1177
1184
|
}),
|
|
1185
|
+
on: (method, listener) => listeners.set(method, listener),
|
|
1178
1186
|
close: () => fail2(new Error("Chrome DevTools connection closed"))
|
|
1179
1187
|
};
|
|
1180
1188
|
}
|
|
@@ -1233,7 +1241,11 @@ async function selectChromePage(options = {}) {
|
|
|
1233
1241
|
`No matching debuggable Chrome page found (${matches.length} match${matches.length === 1 ? "" : "es"}, index ${index}). Use \`replicas computer browser\` to list pages.`
|
|
1234
1242
|
);
|
|
1235
1243
|
}
|
|
1236
|
-
await
|
|
1244
|
+
const [front] = await sendChromeCommands(webSocketDebuggerUrl, [
|
|
1245
|
+
{ method: "Page.bringToFront", params: {} },
|
|
1246
|
+
{ method: "Runtime.evaluate", params: { expression: BROWSER_QUIET_JS, returnByValue: true } }
|
|
1247
|
+
]);
|
|
1248
|
+
chromeResult(front, "Page.bringToFront");
|
|
1237
1249
|
return { ...page, webSocketDebuggerUrl };
|
|
1238
1250
|
}
|
|
1239
1251
|
var VISIBLE_JS = `
|
|
@@ -1332,7 +1344,9 @@ function readBrowserStateCache(path) {
|
|
|
1332
1344
|
return {
|
|
1333
1345
|
url: typeof value.url === "string" ? value.url : void 0,
|
|
1334
1346
|
documentId: typeof value.documentId === "string" || value.documentId === null ? value.documentId : void 0,
|
|
1335
|
-
entries
|
|
1347
|
+
entries,
|
|
1348
|
+
textLimit: typeof value.textLimit === "number" ? value.textLimit : void 0,
|
|
1349
|
+
elementLimit: typeof value.elementLimit === "number" ? value.elementLimit : void 0
|
|
1336
1350
|
};
|
|
1337
1351
|
}
|
|
1338
1352
|
function chromeResult(response, method) {
|
|
@@ -1385,6 +1399,22 @@ function browserContentQuadRects(quads, viewportRect) {
|
|
|
1385
1399
|
function buildBrowserBounds(result, viewport, targetId) {
|
|
1386
1400
|
const strings = Array.isArray(result.strings) ? result.strings : [];
|
|
1387
1401
|
const documents = Array.isArray(result.documents) ? result.documents.filter(isDOMSnapshotDocument) : [];
|
|
1402
|
+
const pointerNodes = documents.map((document) => {
|
|
1403
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
1404
|
+
document.layout?.nodeIndex?.forEach((nodeIndex, index) => {
|
|
1405
|
+
const cursor = strings[document.layout?.styles?.[index]?.[0] ?? -1];
|
|
1406
|
+
if (typeof cursor === "string") cursors.set(nodeIndex, cursor);
|
|
1407
|
+
});
|
|
1408
|
+
const parentIndex = document.nodes?.parentIndex ?? [];
|
|
1409
|
+
const starts = /* @__PURE__ */ new Set();
|
|
1410
|
+
for (const [nodeIndex, cursor] of cursors) {
|
|
1411
|
+
if (!BROWSER_POINTER_CURSORS.has(cursor)) continue;
|
|
1412
|
+
let parent = parentIndex[nodeIndex] ?? -1;
|
|
1413
|
+
while (parent >= 0 && !cursors.has(parent)) parent = parentIndex[parent] ?? -1;
|
|
1414
|
+
if (cursors.get(parent) !== cursor) starts.add(nodeIndex);
|
|
1415
|
+
}
|
|
1416
|
+
return starts;
|
|
1417
|
+
});
|
|
1388
1418
|
const localRects = documents.map((document) => {
|
|
1389
1419
|
const rects = /* @__PURE__ */ new Map();
|
|
1390
1420
|
const nodeIndexes = document.layout?.nodeIndex ?? [];
|
|
@@ -1475,7 +1505,7 @@ function buildBrowserBounds(result, viewport, targetId) {
|
|
|
1475
1505
|
centerX: origin.x + local.centerX,
|
|
1476
1506
|
centerY: origin.y + local.centerY
|
|
1477
1507
|
};
|
|
1478
|
-
if (intersectRects(rect, origin.clip)) byBackendNodeId.set(backendNodeId, { rect, clip: origin.clip });
|
|
1508
|
+
if (intersectRects(rect, origin.clip)) byBackendNodeId.set(backendNodeId, { rect, clip: origin.clip, pointer: pointerNodes[documentIndex].has(nodeIndex) });
|
|
1479
1509
|
}
|
|
1480
1510
|
});
|
|
1481
1511
|
return byBackendNodeId;
|
|
@@ -1518,6 +1548,10 @@ var BROWSER_STRUCTURAL_ROLES = /* @__PURE__ */ new Set([
|
|
|
1518
1548
|
"status",
|
|
1519
1549
|
"StaticText"
|
|
1520
1550
|
]);
|
|
1551
|
+
var BROWSER_HIDDEN_STATES = /* @__PURE__ */ new Set(["focusable", "url"]);
|
|
1552
|
+
var BROWSER_POINTER_CURSORS = /* @__PURE__ */ new Set(["pointer", "grab", "move"]);
|
|
1553
|
+
var BROWSER_UNREFERENCED_ROLES = /* @__PURE__ */ new Set(["ListMarker", "RootWebArea", "StaticText"]);
|
|
1554
|
+
var BROWSER_TOGGLE_STATES = /* @__PURE__ */ new Set(["checked", "expanded", "pressed", "selected"]);
|
|
1521
1555
|
var BROWSER_STATE_PROPERTIES = /* @__PURE__ */ new Set([
|
|
1522
1556
|
"autocomplete",
|
|
1523
1557
|
"checked",
|
|
@@ -1541,34 +1575,23 @@ var BROWSER_STATE_PROPERTIES = /* @__PURE__ */ new Set([
|
|
|
1541
1575
|
function clippedText(value, limit = 240) {
|
|
1542
1576
|
return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
|
|
1543
1577
|
}
|
|
1544
|
-
function browserTargetOutput(page) {
|
|
1545
|
-
return {
|
|
1546
|
-
targetId: page.id ?? null,
|
|
1547
|
-
title: (page.title ?? "").slice(0, 1e3),
|
|
1548
|
-
url: (page.url ?? "").slice(0, 2e3)
|
|
1549
|
-
};
|
|
1550
|
-
}
|
|
1551
1578
|
function browserNodeActionable(role, states) {
|
|
1552
1579
|
return BROWSER_INTERACTIVE_ROLES.has(role) || states.editable === true || states.focusable === true && role !== "RootWebArea";
|
|
1553
1580
|
}
|
|
1554
|
-
async function captureBrowserSnapshot(page, options) {
|
|
1555
|
-
const
|
|
1556
|
-
{ method: "Page.getFrameTree", params: {} },
|
|
1557
|
-
{ method: "Page.getLayoutMetrics", params: {} },
|
|
1558
|
-
{ method: "DOMSnapshot.captureSnapshot", params: { computedStyles: []
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
const frameTree = chromeResult(
|
|
1562
|
-
const layout = chromeResult(
|
|
1563
|
-
const domSnapshot = chromeResult(
|
|
1581
|
+
async function captureBrowserSnapshot(session, page, options) {
|
|
1582
|
+
const [frameTreeResult, layoutResult, domSnapshotResult, mainAxResult] = await Promise.all([
|
|
1583
|
+
session.send({ method: "Page.getFrameTree", params: {} }),
|
|
1584
|
+
session.send({ method: "Page.getLayoutMetrics", params: {} }),
|
|
1585
|
+
session.send({ method: "DOMSnapshot.captureSnapshot", params: { computedStyles: ["cursor"] } }),
|
|
1586
|
+
session.send({ method: "Accessibility.getFullAXTree", params: {} })
|
|
1587
|
+
]);
|
|
1588
|
+
const frameTree = chromeResult(frameTreeResult, "Page.getFrameTree").frameTree;
|
|
1589
|
+
const layout = chromeResult(layoutResult, "Page.getLayoutMetrics");
|
|
1590
|
+
const domSnapshot = chromeResult(domSnapshotResult, "DOMSnapshot.captureSnapshot");
|
|
1564
1591
|
const viewport = chromeVisualViewport(layout);
|
|
1565
1592
|
const frameIds = collectFrameIds(frameTree);
|
|
1566
1593
|
if (frameIds.length === 0 && page.id) frameIds.push(page.id);
|
|
1567
|
-
const
|
|
1568
|
-
method: "Accessibility.getFullAXTree",
|
|
1569
|
-
params: { frameId }
|
|
1570
|
-
}));
|
|
1571
|
-
const axResults = await sendChromeCommands(page.webSocketDebuggerUrl, axCommands);
|
|
1594
|
+
const axResults = [mainAxResult, ...await Promise.all(frameIds.slice(1).map((frameId) => session.send({ method: "Accessibility.getFullAXTree", params: { frameId } })))];
|
|
1572
1595
|
const nodes = [];
|
|
1573
1596
|
axResults.forEach((response, index) => {
|
|
1574
1597
|
if (response.error) return;
|
|
@@ -1584,6 +1607,8 @@ async function captureBrowserSnapshot(page, options) {
|
|
|
1584
1607
|
const propertyMap = browserStateProperties;
|
|
1585
1608
|
const roleOf = (node) => clippedText(node.role?.value, 80);
|
|
1586
1609
|
const nameOf = (node) => clippedText(node.name?.value);
|
|
1610
|
+
const textOf = (node, depth = 0) => depth > 6 ? "" : (node.childIds ?? []).map((id) => nodesByKey.get(`${node.sourceFrameId}:${id}`)).map((child) => !child ? "" : roleOf(child) === "StaticText" ? nameOf(child) : textOf(child, depth + 1)).filter(Boolean).join(" ");
|
|
1611
|
+
const displayNameOf = (node) => nameOf(node) || (bounds.get(node.backendDOMNodeId ?? -1)?.pointer ? clippedText(textOf(node), 120) : "");
|
|
1587
1612
|
const valueOf = (node) => {
|
|
1588
1613
|
if (propertyMap(node).protected === true) return null;
|
|
1589
1614
|
const value = node.value?.value;
|
|
@@ -1592,20 +1617,20 @@ async function captureBrowserSnapshot(page, options) {
|
|
|
1592
1617
|
if (/^[•●*]+$/.test(text)) return "[protected]";
|
|
1593
1618
|
return text || null;
|
|
1594
1619
|
};
|
|
1595
|
-
const meaningful = nodes.filter((node) => !node.ignored && Number.isInteger(node.backendDOMNodeId)).map((node) =>
|
|
1596
|
-
node
|
|
1597
|
-
role
|
|
1598
|
-
|
|
1599
|
-
})
|
|
1620
|
+
const meaningful = nodes.filter((node) => !node.ignored && Number.isInteger(node.backendDOMNodeId)).map((node) => {
|
|
1621
|
+
const bound = typeof node.backendDOMNodeId === "number" ? bounds.get(node.backendDOMNodeId) : void 0;
|
|
1622
|
+
const role = roleOf(node);
|
|
1623
|
+
return { node, role: role === "generic" && bound?.pointer ? "clickable" : role, bound };
|
|
1624
|
+
}).filter((entry) => {
|
|
1600
1625
|
const { node, role, bound } = entry;
|
|
1601
1626
|
if (!bound || bound.rect.width <= 0 || bound.rect.height <= 0) return false;
|
|
1602
1627
|
if (role === "InlineTextBox" || role === "none" || role === "generic") return false;
|
|
1603
|
-
if (BROWSER_INTERACTIVE_ROLES.has(role) || BROWSER_STRUCTURAL_ROLES.has(role)) return true;
|
|
1628
|
+
if (bound.pointer || BROWSER_INTERACTIVE_ROLES.has(role) || BROWSER_STRUCTURAL_ROLES.has(role)) return true;
|
|
1604
1629
|
return !!nameOf(node) || valueOf(node) !== null || propertyMap(node).focusable === true;
|
|
1605
1630
|
}).filter(({ node, role }) => {
|
|
1606
1631
|
if (role !== "StaticText") return true;
|
|
1607
1632
|
const parent = node.parentId ? nodesByKey.get(`${node.sourceFrameId}:${node.parentId}`) : void 0;
|
|
1608
|
-
return !parent ||
|
|
1633
|
+
return !parent || displayNameOf(parent) !== nameOf(node);
|
|
1609
1634
|
}).sort((a, b) => a.bound.rect.y - b.bound.rect.y || a.bound.rect.x - b.bound.rect.x);
|
|
1610
1635
|
const elements = [];
|
|
1611
1636
|
const entries = [];
|
|
@@ -1615,12 +1640,12 @@ async function captureBrowserSnapshot(page, options) {
|
|
|
1615
1640
|
let treeLength = 0;
|
|
1616
1641
|
const treeLimit = Math.min(5e4, Math.max(4e3, options.textLimit * 2));
|
|
1617
1642
|
for (const { node, role, bound } of meaningful) {
|
|
1618
|
-
const name =
|
|
1643
|
+
const name = displayNameOf(node);
|
|
1619
1644
|
const description = clippedText(node.description?.value) || null;
|
|
1620
1645
|
const value = valueOf(node);
|
|
1621
1646
|
const states = propertyMap(node);
|
|
1622
|
-
const actionable = browserNodeActionable(role, states);
|
|
1623
|
-
const ref = actionable
|
|
1647
|
+
const actionable = browserNodeActionable(role, states) || bound.pointer;
|
|
1648
|
+
const ref = (actionable ? elements.length < options.elementLimit : !BROWSER_UNREFERENCED_ROLES.has(role)) ? String(node.backendDOMNodeId) : null;
|
|
1624
1649
|
const visibleRect = role === "RootWebArea" && node.sourceFrameId === page.id ? { x: 0, y: 0, width: viewport.width, height: viewport.height, centerX: viewport.width / 2, centerY: viewport.height / 2 } : intersectRects(bound.rect, bound.clip) ?? bound.rect;
|
|
1625
1650
|
const rect = roundBrowserRect(visibleRect);
|
|
1626
1651
|
const actions = [
|
|
@@ -1628,7 +1653,7 @@ async function captureBrowserSnapshot(page, options) {
|
|
|
1628
1653
|
...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
|
|
1629
1654
|
"scroll"
|
|
1630
1655
|
];
|
|
1631
|
-
if (ref) {
|
|
1656
|
+
if (ref && actionable) {
|
|
1632
1657
|
elements.push({
|
|
1633
1658
|
ref,
|
|
1634
1659
|
role,
|
|
@@ -1641,7 +1666,7 @@ async function captureBrowserSnapshot(page, options) {
|
|
|
1641
1666
|
frameId: node.sourceFrameId
|
|
1642
1667
|
});
|
|
1643
1668
|
}
|
|
1644
|
-
const stateText = Object.entries(states).filter(([key]) => key !== "
|
|
1669
|
+
const stateText = Object.entries(states).filter(([key, state]) => !BROWSER_HIDDEN_STATES.has(key) && (BROWSER_TOGGLE_STATES.has(key) || state !== false && state !== "false")).filter(([key]) => key !== "editable" || !["textbox", "searchbox"].includes(role)).map(([key, state]) => `${key}=${JSON.stringify(state)}`);
|
|
1645
1670
|
const url = typeof states.url === "string" ? clippedText(states.url, 300) : "";
|
|
1646
1671
|
const semantic = [role, name && JSON.stringify(name), value !== null && `value=${JSON.stringify(value)}`, description && `description=${JSON.stringify(description)}`, url && `url=${JSON.stringify(url)}`, ...stateText].filter(Boolean).join(" ");
|
|
1647
1672
|
let depth = 0;
|
|
@@ -1667,9 +1692,11 @@ async function captureBrowserSnapshot(page, options) {
|
|
|
1667
1692
|
}
|
|
1668
1693
|
}
|
|
1669
1694
|
const revision = createHash2("sha256").update(JSON.stringify([page.url, entries.map(({ key, semantic }) => [key, semantic])])).digest("hex").slice(0, 16);
|
|
1695
|
+
const mainFrame = isRecord(frameTree) && isRecord(frameTree.frame) ? frameTree.frame : {};
|
|
1696
|
+
const frameUrl = typeof mainFrame.url === "string" ? `${mainFrame.url}${typeof mainFrame.urlFragment === "string" ? mainFrame.urlFragment : ""}` : page.url;
|
|
1670
1697
|
const snapshot = {
|
|
1671
|
-
title:
|
|
1672
|
-
url:
|
|
1698
|
+
title: page.title.slice(0, 1e3),
|
|
1699
|
+
url: frameUrl.slice(0, 2e3),
|
|
1673
1700
|
documentId: browserDocumentId(frameTree),
|
|
1674
1701
|
revision,
|
|
1675
1702
|
viewport,
|
|
@@ -1679,7 +1706,7 @@ async function captureBrowserSnapshot(page, options) {
|
|
|
1679
1706
|
controls: elements,
|
|
1680
1707
|
nodeCount: entries.length,
|
|
1681
1708
|
elementCount: elements.length,
|
|
1682
|
-
truncated: entries.length < meaningful.length || elements.length < meaningful.filter(({ node, role }) => browserNodeActionable(role, propertyMap(node))).length
|
|
1709
|
+
truncated: entries.length < meaningful.length || elements.length < meaningful.filter(({ node, role, bound }) => browserNodeActionable(role, propertyMap(node)) || bound.pointer).length
|
|
1683
1710
|
};
|
|
1684
1711
|
return { snapshot, entries };
|
|
1685
1712
|
}
|
|
@@ -1702,10 +1729,11 @@ async function computerBrowserCommand(options = {}) {
|
|
|
1702
1729
|
for (const { target, page } of pages) {
|
|
1703
1730
|
if (options.snapshot && target.webSocketDebuggerUrl) {
|
|
1704
1731
|
try {
|
|
1705
|
-
const { snapshot } = await captureBrowserSnapshot(
|
|
1706
|
-
|
|
1732
|
+
const { snapshot } = await withChromeSession(target.webSocketDebuggerUrl, async (session) => await captureBrowserSnapshot(
|
|
1733
|
+
session,
|
|
1734
|
+
{ id: target.id, url: target.url ?? "", title: target.title ?? "" },
|
|
1707
1735
|
{ textLimit, elementLimit }
|
|
1708
|
-
);
|
|
1736
|
+
));
|
|
1709
1737
|
pageResults.push({ ...page, snapshot });
|
|
1710
1738
|
} catch (error) {
|
|
1711
1739
|
pageResults.push({ ...page, snapshotError: error instanceof Error ? error.message : "snapshot failed" });
|
|
@@ -1720,89 +1748,36 @@ async function computerBrowserCommand(options = {}) {
|
|
|
1720
1748
|
pages: pageResults
|
|
1721
1749
|
}, null, 2));
|
|
1722
1750
|
}
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
const value = 'value' in element
|
|
1744
|
-
? element instanceof HTMLInputElement && element.type === 'password'
|
|
1745
|
-
? String(element.value).length
|
|
1746
|
-
: String(element.value)
|
|
1747
|
-
: element.textContent;
|
|
1748
|
-
return [
|
|
1749
|
-
index,
|
|
1750
|
-
element === active,
|
|
1751
|
-
value,
|
|
1752
|
-
'checked' in element ? element.checked : null,
|
|
1753
|
-
'selectedIndex' in element ? element.selectedIndex : null,
|
|
1754
|
-
'disabled' in element ? element.disabled : null,
|
|
1755
|
-
element.getAttribute('aria-checked'),
|
|
1756
|
-
element.getAttribute('aria-expanded'),
|
|
1757
|
-
element.getAttribute('aria-selected'),
|
|
1758
|
-
].join('|');
|
|
1759
|
-
}).join('\\u001f');
|
|
1760
|
-
let controlRevision = 2166136261;
|
|
1761
|
-
for (let index = 0; index < controlState.length; index++) {
|
|
1762
|
-
controlRevision ^= controlState.charCodeAt(index);
|
|
1763
|
-
controlRevision = Math.imul(controlRevision, 16777619);
|
|
1764
|
-
}
|
|
1765
|
-
return {
|
|
1766
|
-
revision: globalThis[key].state.revision,
|
|
1767
|
-
controlRevision: (controlRevision >>> 0).toString(16),
|
|
1768
|
-
readyState: document.readyState,
|
|
1769
|
-
busy: !!document.querySelector('[aria-busy="true"]'),
|
|
1770
|
-
url: location.href.slice(0, 2000),
|
|
1771
|
-
scrollX: Math.round(scrollX),
|
|
1772
|
-
scrollY: Math.round(scrollY),
|
|
1773
|
-
innerWidth,
|
|
1774
|
-
innerHeight,
|
|
1775
|
-
devicePixelRatio,
|
|
1776
|
-
};
|
|
1777
|
-
})()`;
|
|
1778
|
-
}
|
|
1779
|
-
async function waitForBrowserStability(page, options) {
|
|
1751
|
+
var BROWSER_QUIET_JS = `(() => {
|
|
1752
|
+
const key = '__replicasComputerUseQuiet';
|
|
1753
|
+
if (!globalThis[key]) {
|
|
1754
|
+
const tracker = { lastChange: performance.now() };
|
|
1755
|
+
const touch = () => { tracker.lastChange = performance.now(); };
|
|
1756
|
+
new MutationObserver(touch).observe(document, { subtree: true, childList: true, attributes: true, characterData: true });
|
|
1757
|
+
for (const type of ['input', 'change', 'scroll']) addEventListener(type, touch, { capture: true, passive: true });
|
|
1758
|
+
globalThis[key] = tracker;
|
|
1759
|
+
}
|
|
1760
|
+
return {
|
|
1761
|
+
quietMs: Math.round(performance.now() - globalThis[key].lastChange),
|
|
1762
|
+
readyState: document.readyState,
|
|
1763
|
+
busy: !!document.querySelector('[aria-busy="true"]'),
|
|
1764
|
+
animating: document.getAnimations().some((animation) => animation.playState === 'running' && animation.effect?.getTiming().iterations !== Infinity),
|
|
1765
|
+
url: location.href.slice(0, 2000),
|
|
1766
|
+
title: document.title.slice(0, 1000),
|
|
1767
|
+
devicePixelRatio,
|
|
1768
|
+
};
|
|
1769
|
+
})()`;
|
|
1770
|
+
async function waitForBrowserQuiet(session, options) {
|
|
1780
1771
|
const startedAt = Date.now();
|
|
1781
|
-
let
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
state = await evaluateChromeTarget(page.webSocketDebuggerUrl, buildBrowserStabilityExpression());
|
|
1789
|
-
samples++;
|
|
1790
|
-
const signature = JSON.stringify(state);
|
|
1791
|
-
const now = Date.now();
|
|
1792
|
-
if (lastSignature === null || signature !== lastSignature) {
|
|
1793
|
-
if (lastSignature !== null) changes++;
|
|
1794
|
-
lastSignature = signature;
|
|
1795
|
-
lastChangeAt = now;
|
|
1796
|
-
}
|
|
1797
|
-
const ready = typeof state === "object" && state !== null && "readyState" in state && state.readyState !== "loading" && "busy" in state && state.busy !== true;
|
|
1798
|
-
if (samples > 1 && ready && now - lastChangeAt >= options.stableMs) {
|
|
1799
|
-
return { stable: true, elapsedMs: now - startedAt, samples, changes, state };
|
|
1800
|
-
}
|
|
1801
|
-
} catch {
|
|
1802
|
-
}
|
|
1772
|
+
let state;
|
|
1773
|
+
while (true) {
|
|
1774
|
+
const value = await evaluateChromeSession(session, BROWSER_QUIET_JS).catch(() => null);
|
|
1775
|
+
state = isRecord(value) ? value : {};
|
|
1776
|
+
const now = Date.now();
|
|
1777
|
+
const quiet = isRecord(value) && typeof value.quietMs === "number" && value.quietMs >= options.stableMs && value.readyState !== "loading" && value.busy !== true && value.animating !== true && now - (options.actedAt ?? 0) >= options.stableMs;
|
|
1778
|
+
if (quiet || now - startedAt >= options.timeoutMs) return { stable: quiet, elapsedMs: now - startedAt, state };
|
|
1803
1779
|
await sleep(options.pollMs);
|
|
1804
1780
|
}
|
|
1805
|
-
return { stable: false, elapsedMs: Date.now() - startedAt, samples, changes, state };
|
|
1806
1781
|
}
|
|
1807
1782
|
function browserStateCachePath(targetId) {
|
|
1808
1783
|
return `${STATE_DIR}/browser-state-${targetId.replace(/[^a-zA-Z0-9_-]/g, "_")}.json`;
|
|
@@ -1832,35 +1807,30 @@ function browserStateDiff(previous, current) {
|
|
|
1832
1807
|
}
|
|
1833
1808
|
return { tree: lines.join("\n") || "No accessibility changes.", added, changed, removed };
|
|
1834
1809
|
}
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
const
|
|
1838
|
-
const
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
const page = await selectChromePage(options);
|
|
1847
|
-
const stability = await waitForBrowserStability(page, { timeoutMs, stableMs, pollMs });
|
|
1848
|
-
const target = resolvePath(path);
|
|
1849
|
-
mkdirSync4(dirname4(target), { recursive: true });
|
|
1850
|
-
const [{ snapshot, entries }, screenshotResult] = await Promise.all([
|
|
1851
|
-
captureBrowserSnapshot(page, { textLimit, elementLimit }),
|
|
1852
|
-
sendChromeCommand(page.webSocketDebuggerUrl, "Page.captureScreenshot", {
|
|
1853
|
-
format: "png",
|
|
1854
|
-
fromSurface: true,
|
|
1855
|
-
captureBeyondViewport: false
|
|
1856
|
-
})
|
|
1810
|
+
var BROWSER_ACTION_OBSERVATION = { textLimit: 8e3, elementLimit: 120, timeoutMs: 2e3, stableMs: 300, pollMs: 50 };
|
|
1811
|
+
async function observeBrowserPage(session, page, options) {
|
|
1812
|
+
const settled = await waitForBrowserQuiet(session, options);
|
|
1813
|
+
const live = {
|
|
1814
|
+
id: page.id,
|
|
1815
|
+
url: typeof settled.state.url === "string" ? settled.state.url : page.url ?? "",
|
|
1816
|
+
title: typeof settled.state.title === "string" ? settled.state.title : page.title ?? ""
|
|
1817
|
+
};
|
|
1818
|
+
const [{ snapshot, entries }, capture] = await Promise.all([
|
|
1819
|
+
captureBrowserSnapshot(session, live, options),
|
|
1820
|
+
options.screenshotPath ? sendChromeSessionCommand(session, "Page.captureScreenshot", { format: isJpeg(options.screenshotPath) ? "jpeg" : "png", fromSurface: true, captureBeyondViewport: false }) : null
|
|
1857
1821
|
]);
|
|
1858
|
-
|
|
1859
|
-
if (
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1822
|
+
let screenshot = null;
|
|
1823
|
+
if (options.screenshotPath && capture) {
|
|
1824
|
+
if (typeof capture.data !== "string") fail("Chrome did not return screenshot data");
|
|
1825
|
+
mkdirSync4(dirname4(options.screenshotPath), { recursive: true });
|
|
1826
|
+
writeFileSync4(options.screenshotPath, Buffer.from(capture.data, "base64"));
|
|
1827
|
+
screenshot = {
|
|
1828
|
+
path: options.screenshotPath,
|
|
1829
|
+
...readImageDimensions(options.screenshotPath),
|
|
1830
|
+
devicePixelRatio: typeof settled.state.devicePixelRatio === "number" ? settled.state.devicePixelRatio : 1
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
const cachePath = browserStateCachePath(page.id ?? "unknown");
|
|
1864
1834
|
let mode = "full";
|
|
1865
1835
|
let tree = snapshot.tree;
|
|
1866
1836
|
let changes = { added: snapshot.nodeCount, changed: 0, removed: 0 };
|
|
@@ -1874,33 +1844,62 @@ async function computerBrowserStateCommand(path, options = {}) {
|
|
|
1874
1844
|
}
|
|
1875
1845
|
}
|
|
1876
1846
|
mkdirSync4(STATE_DIR, { recursive: true });
|
|
1877
|
-
writeFileSync4(cachePath, JSON.stringify({ url: snapshot.url, documentId: snapshot.documentId, entries }));
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1847
|
+
writeFileSync4(cachePath, JSON.stringify({ url: snapshot.url, documentId: snapshot.documentId, entries, textLimit: options.textLimit, elementLimit: options.elementLimit }));
|
|
1848
|
+
return {
|
|
1849
|
+
screenshot,
|
|
1850
|
+
stability: { stable: settled.stable, elapsedMs: settled.elapsedMs },
|
|
1851
|
+
state: {
|
|
1852
|
+
title: snapshot.title,
|
|
1853
|
+
url: snapshot.url,
|
|
1854
|
+
documentId: snapshot.documentId,
|
|
1855
|
+
revision: snapshot.revision,
|
|
1856
|
+
viewport: snapshot.viewport,
|
|
1857
|
+
tree,
|
|
1858
|
+
nodeCount: snapshot.nodeCount,
|
|
1859
|
+
elementCount: snapshot.elementCount,
|
|
1860
|
+
truncated: snapshot.truncated,
|
|
1861
|
+
mode,
|
|
1862
|
+
changes
|
|
1863
|
+
}
|
|
1892
1864
|
};
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1865
|
+
}
|
|
1866
|
+
async function observeAfterBrowserAction(page, actedAt) {
|
|
1867
|
+
const cachePath = browserStateCachePath(page.id ?? "unknown");
|
|
1868
|
+
const cached = existsSync4(cachePath) ? readBrowserStateCache(cachePath) : null;
|
|
1869
|
+
try {
|
|
1870
|
+
const { stability, state } = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await observeBrowserPage(session, page, {
|
|
1871
|
+
...BROWSER_ACTION_OBSERVATION,
|
|
1872
|
+
textLimit: cached?.textLimit ?? BROWSER_ACTION_OBSERVATION.textLimit,
|
|
1873
|
+
elementLimit: cached?.elementLimit ?? BROWSER_ACTION_OBSERVATION.elementLimit,
|
|
1874
|
+
actedAt
|
|
1875
|
+
}));
|
|
1876
|
+
return { stability, state };
|
|
1877
|
+
} catch (error) {
|
|
1878
|
+
return { error: `Action completed but the page could not be observed: ${error instanceof Error ? error.message : String(error)}` };
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
async function computerBrowserStateCommand(path, options = {}) {
|
|
1882
|
+
const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 8e3;
|
|
1883
|
+
const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 120;
|
|
1884
|
+
const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 5e3;
|
|
1885
|
+
const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 500;
|
|
1886
|
+
const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
|
|
1887
|
+
if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
|
|
1888
|
+
if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
|
|
1889
|
+
if (timeoutMs < 0) fail("--timeout must be >= 0");
|
|
1890
|
+
if (stableMs < 0) fail("--stable-ms must be >= 0");
|
|
1891
|
+
if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
|
|
1892
|
+
const page = await selectChromePage(options);
|
|
1893
|
+
const observation = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await observeBrowserPage(session, page, {
|
|
1894
|
+
full: options.full,
|
|
1895
|
+
textLimit,
|
|
1896
|
+
elementLimit,
|
|
1897
|
+
timeoutMs,
|
|
1898
|
+
stableMs,
|
|
1899
|
+
pollMs,
|
|
1900
|
+
screenshotPath: path === void 0 ? void 0 : resolvePath(path)
|
|
1901
|
+
}));
|
|
1902
|
+
console.log(JSON.stringify({ targetId: page.id ?? "unknown", ...observation }, null, 2));
|
|
1904
1903
|
}
|
|
1905
1904
|
var DESKTOP_POINT_JS = `
|
|
1906
1905
|
const desktopPoint = (rect) => {
|
|
@@ -1920,10 +1919,9 @@ function readBrowserRefCache(page) {
|
|
|
1920
1919
|
const targetId = page.id;
|
|
1921
1920
|
if (!targetId) fail("Chrome target has no stable ID. Capture fresh browser state and retry.");
|
|
1922
1921
|
const cachePath = browserStateCachePath(targetId);
|
|
1923
|
-
if (!existsSync4(cachePath)) fail(`No browser state is cached for target ${targetId}. Run replicas computer browser-state
|
|
1922
|
+
if (!existsSync4(cachePath)) fail(`No browser state is cached for target ${targetId}. Run replicas computer browser-state --target-id ${targetId} first.`);
|
|
1924
1923
|
const cached = readBrowserStateCache(cachePath);
|
|
1925
1924
|
if (!cached) fail(`Browser state for target ${targetId} is invalid. Capture fresh browser state and retry.`);
|
|
1926
|
-
if (cached.url !== (page.url ?? "").slice(0, 2e3)) fail("Cached browser state belongs to a different URL. Capture fresh browser state and retry.");
|
|
1927
1925
|
return { targetId, cached };
|
|
1928
1926
|
}
|
|
1929
1927
|
function browserViewportRect(layout) {
|
|
@@ -1938,11 +1936,10 @@ function browserViewportRect(layout) {
|
|
|
1938
1936
|
};
|
|
1939
1937
|
}
|
|
1940
1938
|
async function prepareBrowserElement(page, refValue) {
|
|
1941
|
-
const backendNodeId = parseBrowserRef(refValue);
|
|
1942
1939
|
const { cached } = readBrowserRefCache(page);
|
|
1943
1940
|
return await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
1944
1941
|
const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
|
|
1945
|
-
const rect = await prepareBrowserRefInSession(session,
|
|
1942
|
+
const { backendNodeId, rect } = await prepareBrowserRefInSession(session, parseBrowserRef(refValue), cached, browserViewportRect(layout));
|
|
1946
1943
|
const axNodes = (await sendChromeSessionCommand(session, "Accessibility.getPartialAXTree", {
|
|
1947
1944
|
backendNodeId,
|
|
1948
1945
|
fetchRelatives: false
|
|
@@ -1975,23 +1972,35 @@ ${DESKTOP_POINT_JS}
|
|
|
1975
1972
|
};
|
|
1976
1973
|
});
|
|
1977
1974
|
}
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1975
|
+
var browserRefIdentity = (semantic) => semantic.match(/^\S+ "(?:[^"\\]|\\.)+"/)?.[0] ?? null;
|
|
1976
|
+
async function findReplacedBrowserRef(session, semantic, cached) {
|
|
1977
|
+
const identity = browserRefIdentity(semantic);
|
|
1978
|
+
if (!identity || cached.entries.filter((entry) => browserRefIdentity(entry.semantic) === identity).length !== 1) return null;
|
|
1979
|
+
const { entries } = await captureBrowserSnapshot(session, { url: "", title: "" }, { textLimit: 5e4, elementLimit: 500 });
|
|
1980
|
+
const matches = entries.filter((entry) => entry.line.trimStart().startsWith("[ref=") && browserRefIdentity(entry.semantic) === identity);
|
|
1981
|
+
return matches.length === 1 ? Number(matches[0].key.split(":").pop()) : null;
|
|
1982
|
+
}
|
|
1983
|
+
async function prepareBrowserRefInSession(session, ref, cached, viewportRect) {
|
|
1984
|
+
const entry = cached.entries.find(({ line }) => line.trimStart().startsWith(`[ref=${ref}]`));
|
|
1985
|
+
if (!entry) fail(`Browser element ref ${ref} is not in the latest state. Capture fresh browser state and retry.`);
|
|
1982
1986
|
const frameTree = (await sendChromeSessionCommand(session, "Page.getFrameTree", {})).frameTree;
|
|
1983
1987
|
if (cached.documentId !== browserDocumentId(frameTree)) {
|
|
1984
|
-
fail(`Browser element ref ${
|
|
1985
|
-
}
|
|
1986
|
-
try {
|
|
1987
|
-
await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId });
|
|
1988
|
-
} catch (error) {
|
|
1989
|
-
fail(`Browser element ref ${backendNodeId} is stale or cannot be scrolled into view: ${error instanceof Error ? error.message : String(error)}`);
|
|
1988
|
+
fail(`Browser element ref ${ref} belongs to a previous document. Capture fresh browser state and retry.`);
|
|
1990
1989
|
}
|
|
1990
|
+
const backendNodeId = await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId: ref }).then(
|
|
1991
|
+
() => ref,
|
|
1992
|
+
async (error) => {
|
|
1993
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1994
|
+
const replacement = /detached|No node/i.test(message) ? await findReplacedBrowserRef(session, entry.semantic, cached) : null;
|
|
1995
|
+
if (replacement === null) fail(`Browser element ref ${ref} is stale or cannot be scrolled into view: ${message}`);
|
|
1996
|
+
await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId: replacement });
|
|
1997
|
+
return replacement;
|
|
1998
|
+
}
|
|
1999
|
+
);
|
|
1991
2000
|
const quads = (await sendChromeSessionCommand(session, "DOM.getContentQuads", { backendNodeId })).quads;
|
|
1992
2001
|
const candidates = browserContentQuadRects(quads, viewportRect);
|
|
1993
|
-
if (!candidates[0]) fail(`Browser element ref ${
|
|
1994
|
-
return roundBrowserRect(candidates[0]);
|
|
2002
|
+
if (!candidates[0]) fail(`Browser element ref ${ref} has no visible content quad. Fetch fresh browser state and retry.`);
|
|
2003
|
+
return { backendNodeId, rect: roundBrowserRect(candidates[0]) };
|
|
1995
2004
|
}
|
|
1996
2005
|
function buildBrowserClickExpression(query, options) {
|
|
1997
2006
|
return `(() => {
|
|
@@ -2094,6 +2103,62 @@ async function dispatchChromeClickInSession(session, x, y, button, clicks) {
|
|
|
2094
2103
|
});
|
|
2095
2104
|
}
|
|
2096
2105
|
}
|
|
2106
|
+
async function dragChromeInSession(session, from, to) {
|
|
2107
|
+
const intercepted = new Promise((resolve5) => session.on("Input.dragIntercepted", (params) => resolve5(params.data)));
|
|
2108
|
+
await sendChromeSessionCommand(session, "Input.setInterceptDrags", { enabled: true });
|
|
2109
|
+
let released = false;
|
|
2110
|
+
try {
|
|
2111
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseMoved", x: from.centerX, y: from.centerY });
|
|
2112
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mousePressed", x: from.centerX, y: from.centerY, button: "left", buttons: 1, clickCount: 1 });
|
|
2113
|
+
const steps = 10;
|
|
2114
|
+
for (let step = 1; step <= steps; step++) {
|
|
2115
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
|
|
2116
|
+
type: "mouseMoved",
|
|
2117
|
+
x: from.centerX + (to.centerX - from.centerX) * step / steps,
|
|
2118
|
+
y: from.centerY + (to.centerY - from.centerY) * step / steps,
|
|
2119
|
+
button: "left",
|
|
2120
|
+
buttons: 1
|
|
2121
|
+
});
|
|
2122
|
+
}
|
|
2123
|
+
const dragData = await Promise.race([intercepted, sleep(100).then(() => void 0)]);
|
|
2124
|
+
if (dragData) {
|
|
2125
|
+
for (const type of ["dragEnter", "dragOver", "drop"]) {
|
|
2126
|
+
await sendChromeSessionCommand(session, "Input.dispatchDragEvent", { type, x: to.centerX, y: to.centerY, data: dragData });
|
|
2127
|
+
}
|
|
2128
|
+
}
|
|
2129
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseReleased", x: to.centerX, y: to.centerY, button: "left", buttons: 0, clickCount: 1 });
|
|
2130
|
+
released = true;
|
|
2131
|
+
} finally {
|
|
2132
|
+
if (!released) {
|
|
2133
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseReleased", x: to.centerX, y: to.centerY, button: "left", buttons: 0, clickCount: 1 }).catch(() => void 0);
|
|
2134
|
+
}
|
|
2135
|
+
await sendChromeSessionCommand(session, "Input.setInterceptDrags", { enabled: false }).catch(() => void 0);
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
async function prepareBrowserDragInSession(session, refs, cached, viewportRect) {
|
|
2139
|
+
const to = await prepareBrowserRefInSession(session, refs.to, cached, viewportRect);
|
|
2140
|
+
const from = await prepareBrowserRefInSession(session, refs.from, cached, viewportRect);
|
|
2141
|
+
const [fromQuad, toQuad] = await Promise.all([from, to].map(async ({ backendNodeId }) => browserContentQuadRects((await sendChromeSessionCommand(session, "DOM.getContentQuads", { backendNodeId })).quads, viewportRect)[0]));
|
|
2142
|
+
if (!fromQuad || !toQuad) fail(`Browser element refs ${refs.from} and ${refs.to} are not visible together. Scroll so both are in view and retry.`);
|
|
2143
|
+
return { from: roundBrowserRect(fromQuad), to: roundBrowserRect(toQuad), refs: { from: from.backendNodeId, to: to.backendNodeId } };
|
|
2144
|
+
}
|
|
2145
|
+
async function computerBrowserDragCommand(options) {
|
|
2146
|
+
const page = await selectChromePage(options);
|
|
2147
|
+
const { cached } = readBrowserRefCache(page);
|
|
2148
|
+
const refs = { from: parseBrowserRef(options.ref), to: parseBrowserRef(options.toRef) };
|
|
2149
|
+
const rects = await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2150
|
+
const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
|
|
2151
|
+
const prepared = await prepareBrowserDragInSession(session, refs, cached, browserViewportRect(layout));
|
|
2152
|
+
await dragChromeInSession(session, prepared.from, prepared.to);
|
|
2153
|
+
return prepared;
|
|
2154
|
+
});
|
|
2155
|
+
logRecordingAction({ type: "drag" });
|
|
2156
|
+
console.log(JSON.stringify({
|
|
2157
|
+
targetId: page.id ?? null,
|
|
2158
|
+
result: { dragged: true, ref: String(rects.refs.from), toRef: String(rects.refs.to), from: rects.from, to: rects.to },
|
|
2159
|
+
observation: await observeAfterBrowserAction(page, Date.now())
|
|
2160
|
+
}, null, 2));
|
|
2161
|
+
}
|
|
2097
2162
|
async function replaceChromeText(page, text) {
|
|
2098
2163
|
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2099
2164
|
await replaceChromeTextInSession(session, text);
|
|
@@ -2129,8 +2194,14 @@ function browserFillInvalid(actual, expected, fieldType, semanticSelection = fal
|
|
|
2129
2194
|
if (fieldType === "color") return String(actual).toLowerCase() !== expected.toLowerCase();
|
|
2130
2195
|
return actual !== expected;
|
|
2131
2196
|
}
|
|
2132
|
-
async function
|
|
2197
|
+
async function fillBrowserRefInSession(session, ref, cached, viewportRect, value) {
|
|
2198
|
+
let { backendNodeId } = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
2133
2199
|
await sendChromeSessionCommand(session, "DOM.focus", { backendNodeId });
|
|
2200
|
+
const connected = await callFunctionOnChromeNodeInSession(session, backendNodeId, "function() { return this.isConnected; }").then(chromeCallValue, () => false);
|
|
2201
|
+
if (connected !== true) {
|
|
2202
|
+
({ backendNodeId } = await prepareBrowserRefInSession(session, ref, cached, viewportRect));
|
|
2203
|
+
await sendChromeSessionCommand(session, "DOM.focus", { backendNodeId });
|
|
2204
|
+
}
|
|
2134
2205
|
const directResult = await callFunctionOnChromeNodeInSession(
|
|
2135
2206
|
session,
|
|
2136
2207
|
backendNodeId,
|
|
@@ -2183,7 +2254,7 @@ async function fillChromeNodeInSession(session, backendNodeId, value) {
|
|
|
2183
2254
|
if (browserFillInvalid(actual.value, value, fieldType, semanticSelection, selectionMatched)) {
|
|
2184
2255
|
fail(`Chrome reported that ref ${backendNodeId} contains ${JSON.stringify(actual.value)} after filling, expected ${JSON.stringify(value)}.`);
|
|
2185
2256
|
}
|
|
2186
|
-
return { value: actual.value, valueLength: actual.valueLength, type: fieldType };
|
|
2257
|
+
return { backendNodeId, value: actual.value, valueLength: actual.valueLength, type: fieldType };
|
|
2187
2258
|
}
|
|
2188
2259
|
async function computerBrowserClickCommand(query, options = {}) {
|
|
2189
2260
|
const page = await selectChromePage(options);
|
|
@@ -2196,8 +2267,9 @@ async function computerBrowserClickCommand(query, options = {}) {
|
|
|
2196
2267
|
await dispatchChromeClick(page, target.rect.centerX, target.rect.centerY, button, clicks);
|
|
2197
2268
|
if (target.desktopPoint) logRecordingAction({ type: "click", ...target.desktopPoint });
|
|
2198
2269
|
console.log(JSON.stringify({
|
|
2199
|
-
|
|
2200
|
-
result: { clicked: true, ref: target.ref, button, clickCount: clicks, rect: target.rect, element: target.element }
|
|
2270
|
+
targetId: page.id ?? null,
|
|
2271
|
+
result: { clicked: true, ref: target.ref, button, clickCount: clicks, rect: target.rect, element: target.element },
|
|
2272
|
+
observation: await observeAfterBrowserAction(page, Date.now())
|
|
2201
2273
|
}, null, 2));
|
|
2202
2274
|
return;
|
|
2203
2275
|
}
|
|
@@ -2213,8 +2285,9 @@ async function computerBrowserClickCommand(query, options = {}) {
|
|
|
2213
2285
|
const point = browserActionPoint(result);
|
|
2214
2286
|
if (point) logRecordingAction({ type: "click", ...point });
|
|
2215
2287
|
console.log(JSON.stringify({
|
|
2216
|
-
|
|
2217
|
-
result
|
|
2288
|
+
targetId: page.id ?? null,
|
|
2289
|
+
result,
|
|
2290
|
+
observation: await observeAfterBrowserAction(page, Date.now())
|
|
2218
2291
|
}, null, 2));
|
|
2219
2292
|
}
|
|
2220
2293
|
function buildBrowserFillExpression(query, value, options) {
|
|
@@ -2336,13 +2409,18 @@ ${DESKTOP_POINT_JS}
|
|
|
2336
2409
|
async function computerBrowserFillCommand(query, value, options = {}) {
|
|
2337
2410
|
const page = await selectChromePage(options);
|
|
2338
2411
|
if (options.ref) {
|
|
2412
|
+
const ref = parseBrowserRef(options.ref);
|
|
2339
2413
|
const target = await prepareBrowserElement(page, options.ref);
|
|
2340
|
-
const
|
|
2341
|
-
const actual = await withChromeSession(page.webSocketDebuggerUrl, async (session) =>
|
|
2414
|
+
const { cached } = readBrowserRefCache(page);
|
|
2415
|
+
const actual = await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2416
|
+
const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
|
|
2417
|
+
return await fillBrowserRefInSession(session, ref, cached, browserViewportRect(layout), value);
|
|
2418
|
+
});
|
|
2342
2419
|
if (target.desktopPoint) logRecordingAction({ type: "type", ...target.desktopPoint });
|
|
2343
2420
|
console.log(JSON.stringify({
|
|
2344
|
-
|
|
2345
|
-
result: { filled: true, ref:
|
|
2421
|
+
targetId: page.id ?? null,
|
|
2422
|
+
result: { filled: true, ref: String(actual.backendNodeId), valueLength: actual.valueLength, rect: target.rect, element: target.element },
|
|
2423
|
+
observation: await observeAfterBrowserAction(page, Date.now())
|
|
2346
2424
|
}, null, 2));
|
|
2347
2425
|
return;
|
|
2348
2426
|
}
|
|
@@ -2375,8 +2453,9 @@ async function computerBrowserFillCommand(query, value, options = {}) {
|
|
|
2375
2453
|
}
|
|
2376
2454
|
if (point) logRecordingAction({ type: "type", ...point });
|
|
2377
2455
|
console.log(JSON.stringify({
|
|
2378
|
-
|
|
2379
|
-
result
|
|
2456
|
+
targetId: page.id ?? null,
|
|
2457
|
+
result,
|
|
2458
|
+
observation: await observeAfterBrowserAction(page, Date.now())
|
|
2380
2459
|
}, null, 2));
|
|
2381
2460
|
}
|
|
2382
2461
|
function buildBrowserWaitExpression(query, options) {
|
|
@@ -2438,7 +2517,34 @@ async function computerBrowserWaitCommand(query, options = {}) {
|
|
|
2438
2517
|
await sleep(pollMs);
|
|
2439
2518
|
}
|
|
2440
2519
|
console.log(JSON.stringify({ ok: false, elapsedMs: Date.now() - start, attempts, result: lastResult }, null, 2));
|
|
2441
|
-
|
|
2520
|
+
fail(`Timed out after ${timeoutMs}ms waiting for ${JSON.stringify(query)}`);
|
|
2521
|
+
}
|
|
2522
|
+
async function browserScrollPointInSession(session, viewportRect, deltaX, deltaY) {
|
|
2523
|
+
const point = await evaluateChromeSession(session, `(() => {
|
|
2524
|
+
const vertical = ${deltaY !== 0};
|
|
2525
|
+
const root = document.scrollingElement || document.documentElement;
|
|
2526
|
+
if (vertical ? root.scrollHeight > innerHeight : root.scrollWidth > innerWidth) return null;
|
|
2527
|
+
let best = null;
|
|
2528
|
+
let bestArea = 0;
|
|
2529
|
+
for (const el of document.querySelectorAll('*')) {
|
|
2530
|
+
const style = getComputedStyle(el);
|
|
2531
|
+
const scrollable = vertical
|
|
2532
|
+
? /(auto|scroll)/.test(style.overflowY) && el.scrollHeight > el.clientHeight
|
|
2533
|
+
: /(auto|scroll)/.test(style.overflowX) && el.scrollWidth > el.clientWidth;
|
|
2534
|
+
if (!scrollable) continue;
|
|
2535
|
+
const rect = el.getBoundingClientRect();
|
|
2536
|
+
const left = Math.max(rect.left, 0);
|
|
2537
|
+
const top = Math.max(rect.top, 0);
|
|
2538
|
+
const width = Math.min(rect.right, innerWidth) - left;
|
|
2539
|
+
const height = Math.min(rect.bottom, innerHeight) - top;
|
|
2540
|
+
if (width > 0 && height > 0 && width * height > bestArea) {
|
|
2541
|
+
bestArea = width * height;
|
|
2542
|
+
best = { x: left + width / 2, y: top + height / 2 };
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
return best;
|
|
2546
|
+
})()`);
|
|
2547
|
+
return isRecord(point) && typeof point.x === "number" && typeof point.y === "number" ? { x: point.x, y: point.y } : { x: viewportRect.centerX, y: viewportRect.centerY };
|
|
2442
2548
|
}
|
|
2443
2549
|
async function computerBrowserScrollCommand(direction, options = {}) {
|
|
2444
2550
|
const page = await selectChromePage(options);
|
|
@@ -2446,25 +2552,54 @@ async function computerBrowserScrollCommand(direction, options = {}) {
|
|
|
2446
2552
|
if (!["up", "down", "left", "right"].includes(normalized)) fail("direction must be one of up|down|left|right");
|
|
2447
2553
|
const amount = options.amount ? parseCoord(options.amount, "--amount") : 600;
|
|
2448
2554
|
if (amount <= 0 || amount > 1e4) fail("--amount must be between 1 and 10000 CSS pixels");
|
|
2449
|
-
const
|
|
2450
|
-
const viewport = chromeVisualViewport(layout);
|
|
2555
|
+
const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 1e4;
|
|
2451
2556
|
const target = options.ref ? await prepareBrowserElement(page, options.ref) : null;
|
|
2452
|
-
const x = target?.rect.centerX ?? viewport.width / 2;
|
|
2453
|
-
const y = target?.rect.centerY ?? viewport.height / 2;
|
|
2454
2557
|
const deltaX = normalized === "left" ? -amount : normalized === "right" ? amount : 0;
|
|
2455
2558
|
const deltaY = normalized === "up" ? -amount : normalized === "down" ? amount : 0;
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
x,
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2559
|
+
const startedAt = Date.now();
|
|
2560
|
+
const { point, scrolls, search } = await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2561
|
+
const point2 = target ? { x: target.rect.centerX, y: target.rect.centerY } : await browserScrollPointInSession(session, browserViewportRect(await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {})), deltaX, deltaY);
|
|
2562
|
+
const wheel = () => sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseWheel", ...point2, deltaX, deltaY });
|
|
2563
|
+
if (options.until === void 0) {
|
|
2564
|
+
await wheel();
|
|
2565
|
+
return { point: point2, scrolls: 1, search: null };
|
|
2566
|
+
}
|
|
2567
|
+
const expression = `(() => {
|
|
2568
|
+
const needle = ${JSON.stringify(options.until)}.replace(/\\s+/g, ' ').trim().toLowerCase();
|
|
2569
|
+
const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
|
|
2570
|
+
const has = (el) => normalize(el.innerText).toLowerCase().includes(needle);
|
|
2571
|
+
${VISIBLE_JS}
|
|
2572
|
+
const offsets = [scrollX, scrollY, document.getElementsByTagName('*').length];
|
|
2573
|
+
for (let el = document.elementFromPoint(${point2.x}, ${point2.y}); el; el = el.parentElement) offsets.push(el.scrollLeft, el.scrollTop);
|
|
2574
|
+
if (!document.body || !has(document.body)) return { match: null, signature: offsets.join(',') };
|
|
2575
|
+
let el = document.body;
|
|
2576
|
+
for (let child = el; child; child = Array.from(el.children).find((item) => visible(item) && has(item))) el = child;
|
|
2577
|
+
el.scrollIntoView({ block: 'center', inline: 'nearest' });
|
|
2578
|
+
return { match: normalize(el.innerText).slice(0, 300), signature: null };
|
|
2579
|
+
})()`;
|
|
2580
|
+
let scrolls2 = 0;
|
|
2581
|
+
let unchanged = 0;
|
|
2582
|
+
let previous = null;
|
|
2583
|
+
while (true) {
|
|
2584
|
+
const found = await evaluateChromeSession(session, expression);
|
|
2585
|
+
if (!isRecord(found)) fail("Chrome did not return a text search result");
|
|
2586
|
+
if (typeof found.match === "string") return { point: point2, scrolls: scrolls2, search: { found: true, match: found.match } };
|
|
2587
|
+
unchanged = found.signature === previous ? unchanged + 1 : 0;
|
|
2588
|
+
previous = found.signature;
|
|
2589
|
+
if (unchanged >= 2) return { point: point2, scrolls: scrolls2, search: { found: false, reason: "Reached the end of the scrollable content" } };
|
|
2590
|
+
if (Date.now() - startedAt >= timeoutMs) return { point: point2, scrolls: scrolls2, search: { found: false, reason: `Timed out after ${timeoutMs}ms` } };
|
|
2591
|
+
await wheel();
|
|
2592
|
+
scrolls2++;
|
|
2593
|
+
await waitForBrowserQuiet(session, { timeoutMs: 2e3, stableMs: 300, pollMs: 50, actedAt: Date.now() });
|
|
2594
|
+
}
|
|
2462
2595
|
});
|
|
2463
2596
|
logRecordingAction({ type: "scroll", ...target?.desktopPoint ?? {} });
|
|
2464
2597
|
console.log(JSON.stringify({
|
|
2465
|
-
|
|
2466
|
-
result: { scrolled: true, direction: normalized, amount, ref: target?.ref ?? null, x: Math.round(x), y: Math.round(y) }
|
|
2598
|
+
targetId: page.id ?? null,
|
|
2599
|
+
result: { scrolled: true, direction: normalized, amount, scrolls, ref: target?.ref ?? null, x: Math.round(point.x), y: Math.round(point.y), ...search && { until: options.until, ...search } },
|
|
2600
|
+
observation: await observeAfterBrowserAction(page, Date.now())
|
|
2467
2601
|
}, null, 2));
|
|
2602
|
+
if (search?.found === false) fail(`${search.reason} without finding ${JSON.stringify(options.until)}`);
|
|
2468
2603
|
}
|
|
2469
2604
|
function browserKeyDetails(combo) {
|
|
2470
2605
|
const tokens = combo.split("+").map((token) => token.trim()).filter(Boolean);
|
|
@@ -2521,7 +2656,7 @@ async function computerBrowserKeyCommand(combo, options = {}) {
|
|
|
2521
2656
|
await dispatchChromeKeyInSession(session, key);
|
|
2522
2657
|
});
|
|
2523
2658
|
logRecordingAction({ type: "key" });
|
|
2524
|
-
console.log(JSON.stringify({ targetId: page.id ?? null, result: { pressed: combo } }, null, 2));
|
|
2659
|
+
console.log(JSON.stringify({ targetId: page.id ?? null, result: { pressed: combo }, observation: await observeAfterBrowserAction(page, Date.now()) }, null, 2));
|
|
2525
2660
|
}
|
|
2526
2661
|
async function dispatchChromeKeyInSession(session, key) {
|
|
2527
2662
|
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
@@ -2542,7 +2677,7 @@ async function computerBrowserTypeCommand(text, options = {}) {
|
|
|
2542
2677
|
const page = await selectChromePage(options);
|
|
2543
2678
|
await sendChromeCommand(page.webSocketDebuggerUrl, "Input.insertText", { text });
|
|
2544
2679
|
logRecordingAction({ type: "type" });
|
|
2545
|
-
console.log(JSON.stringify({ targetId: page.id ?? null, result: { typed: true, length: text.length } }, null, 2));
|
|
2680
|
+
console.log(JSON.stringify({ targetId: page.id ?? null, result: { typed: true, length: text.length }, observation: await observeAfterBrowserAction(page, Date.now()) }, null, 2));
|
|
2546
2681
|
}
|
|
2547
2682
|
function parseBrowserBatchActions(value) {
|
|
2548
2683
|
let input;
|
|
@@ -2570,6 +2705,12 @@ function parseBrowserBatchActions(value) {
|
|
|
2570
2705
|
}
|
|
2571
2706
|
return { action: "fill", ref: String(item.ref), value: item.value };
|
|
2572
2707
|
}
|
|
2708
|
+
if (item.action === "drag") {
|
|
2709
|
+
if (typeof item.ref !== "string" && typeof item.ref !== "number" || typeof item.toRef !== "string" && typeof item.toRef !== "number") {
|
|
2710
|
+
fail(`Batch drag ${index} requires ref and toRef`);
|
|
2711
|
+
}
|
|
2712
|
+
return { action: "drag", ref: String(item.ref), toRef: String(item.toRef) };
|
|
2713
|
+
}
|
|
2573
2714
|
if (item.action === "key") {
|
|
2574
2715
|
if (typeof item.combo !== "string") fail(`Batch key ${index} requires combo`);
|
|
2575
2716
|
return { action: "key", combo: item.combo };
|
|
@@ -2621,16 +2762,20 @@ async function computerBrowserBatchCommand(actionsJson, options = {}) {
|
|
|
2621
2762
|
const actionStartedAt = Date.now();
|
|
2622
2763
|
if (action.action === "click") {
|
|
2623
2764
|
const ref = parseBrowserRef(action.ref);
|
|
2624
|
-
const rect = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
2765
|
+
const { backendNodeId, rect } = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
2625
2766
|
await dispatchChromeClickInSession(session, rect.centerX, rect.centerY, action.button, action.clicks);
|
|
2626
2767
|
logRecordingAction({ type: "click" });
|
|
2627
|
-
completed.push({ index, action: action.action, ref: String(
|
|
2768
|
+
completed.push({ index, action: action.action, ref: String(backendNodeId), elapsedMs: Date.now() - actionStartedAt });
|
|
2628
2769
|
} else if (action.action === "fill") {
|
|
2629
|
-
const
|
|
2630
|
-
await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
2631
|
-
const result = await fillChromeNodeInSession(session, ref, action.value);
|
|
2770
|
+
const result = await fillBrowserRefInSession(session, parseBrowserRef(action.ref), cached, viewportRect, action.value);
|
|
2632
2771
|
logRecordingAction({ type: "type" });
|
|
2633
|
-
completed.push({ index, action: action.action, ref: String(
|
|
2772
|
+
completed.push({ index, action: action.action, ref: String(result.backendNodeId), valueLength: result.valueLength, elapsedMs: Date.now() - actionStartedAt });
|
|
2773
|
+
} else if (action.action === "drag") {
|
|
2774
|
+
const refs = { from: parseBrowserRef(action.ref), to: parseBrowserRef(action.toRef) };
|
|
2775
|
+
const rects = await prepareBrowserDragInSession(session, refs, cached, viewportRect);
|
|
2776
|
+
await dragChromeInSession(session, rects.from, rects.to);
|
|
2777
|
+
logRecordingAction({ type: "drag" });
|
|
2778
|
+
completed.push({ index, action: action.action, ref: String(rects.refs.from), toRef: String(rects.refs.to), elapsedMs: Date.now() - actionStartedAt });
|
|
2634
2779
|
} else if (action.action === "key") {
|
|
2635
2780
|
await dispatchChromeKeyInSession(session, browserKeyDetails(action.combo));
|
|
2636
2781
|
logRecordingAction({ type: "key" });
|
|
@@ -2641,16 +2786,11 @@ async function computerBrowserBatchCommand(actionsJson, options = {}) {
|
|
|
2641
2786
|
completed.push({ index, action: action.action, length: action.text.length, elapsedMs: Date.now() - actionStartedAt });
|
|
2642
2787
|
} else if (action.action === "scroll") {
|
|
2643
2788
|
const ref = action.ref ? parseBrowserRef(action.ref) : null;
|
|
2644
|
-
const rect = ref ? await prepareBrowserRefInSession(session, ref, cached, viewportRect) : viewportRect;
|
|
2645
2789
|
const deltaX = action.direction === "left" ? -action.amount : action.direction === "right" ? action.amount : 0;
|
|
2646
2790
|
const deltaY = action.direction === "up" ? -action.amount : action.direction === "down" ? action.amount : 0;
|
|
2647
|
-
await
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
y: rect.centerY,
|
|
2651
|
-
deltaX,
|
|
2652
|
-
deltaY
|
|
2653
|
-
});
|
|
2791
|
+
const rect = ref ? (await prepareBrowserRefInSession(session, ref, cached, viewportRect)).rect : null;
|
|
2792
|
+
const point = rect ? { x: rect.centerX, y: rect.centerY } : await browserScrollPointInSession(session, viewportRect, deltaX, deltaY);
|
|
2793
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", { type: "mouseWheel", ...point, deltaX, deltaY });
|
|
2654
2794
|
logRecordingAction({ type: "scroll" });
|
|
2655
2795
|
completed.push({ index, action: action.action, direction: action.direction, amount: action.amount, elapsedMs: Date.now() - actionStartedAt });
|
|
2656
2796
|
} else {
|
|
@@ -2670,10 +2810,11 @@ async function computerBrowserBatchCommand(actionsJson, options = {}) {
|
|
|
2670
2810
|
});
|
|
2671
2811
|
} catch (error) {
|
|
2672
2812
|
const message = error instanceof Error ? error.message : String(error);
|
|
2673
|
-
|
|
2813
|
+
const observation = await observeAfterBrowserAction(page, Date.now());
|
|
2814
|
+
console.log(JSON.stringify({ ok: false, targetId, elapsedMs: Date.now() - startedAt, completed, failedIndex: completed.length, error: message, observation }, null, 2));
|
|
2674
2815
|
throw error;
|
|
2675
2816
|
}
|
|
2676
|
-
console.log(JSON.stringify({ ok: true, targetId, elapsedMs: Date.now() - startedAt, completed }, null, 2));
|
|
2817
|
+
console.log(JSON.stringify({ ok: true, targetId, elapsedMs: Date.now() - startedAt, completed, observation: await observeAfterBrowserAction(page, Date.now()) }, null, 2));
|
|
2677
2818
|
}
|
|
2678
2819
|
var APP_ALIASES = {
|
|
2679
2820
|
chrome: [CHROME_WRAPPER],
|
|
@@ -2730,8 +2871,8 @@ async function computerLaunchCommand(app, args) {
|
|
|
2730
2871
|
const candidates = expectsNewChromePage && existingChromePageIds.size > 0 ? pages.filter((page) => page.id && !existingChromePageIds.has(page.id)) : pages;
|
|
2731
2872
|
for (const page of candidates) {
|
|
2732
2873
|
if (!page.webSocketDebuggerUrl || !page.url || page.url === "about:blank") continue;
|
|
2733
|
-
const
|
|
2734
|
-
if (readyState === "interactive" || readyState === "complete") {
|
|
2874
|
+
const quiet = await evaluateChromeTarget(page.webSocketDebuggerUrl, BROWSER_QUIET_JS);
|
|
2875
|
+
if (isRecord(quiet) && (quiet.readyState === "interactive" || quiet.readyState === "complete")) {
|
|
2735
2876
|
readyPage = page;
|
|
2736
2877
|
break;
|
|
2737
2878
|
}
|
|
@@ -2764,7 +2905,7 @@ async function computerClickCommand(xStr, yStr, options) {
|
|
|
2764
2905
|
const x = parseScreenCoord(xStr, "x", dimensions.width);
|
|
2765
2906
|
const y = parseScreenCoord(yStr, "y", dimensions.height);
|
|
2766
2907
|
const button = options.button ?? "1";
|
|
2767
|
-
const args = ["mousemove",
|
|
2908
|
+
const args = ["mousemove", String(x), String(y)];
|
|
2768
2909
|
if (options.modifiers) {
|
|
2769
2910
|
for (const mod of options.modifiers.split("+")) {
|
|
2770
2911
|
args.push("keydown", mod);
|
|
@@ -2788,7 +2929,7 @@ async function computerMoveCommand(xStr, yStr) {
|
|
|
2788
2929
|
const dimensions = getDisplayDimensions();
|
|
2789
2930
|
const x = parseScreenCoord(xStr, "x", dimensions.width);
|
|
2790
2931
|
const y = parseScreenCoord(yStr, "y", dimensions.height);
|
|
2791
|
-
runDesktopInputCmd(["mousemove",
|
|
2932
|
+
runDesktopInputCmd(["mousemove", String(x), String(y)]);
|
|
2792
2933
|
logRecordingAction({ type: "move", x, y });
|
|
2793
2934
|
console.log(`moved to (${x},${y})`);
|
|
2794
2935
|
}
|
|
@@ -2819,12 +2960,7 @@ async function computerScrollCommand(direction, options) {
|
|
|
2819
2960
|
x: parseScreenCoord(options.x, "--x", dimensions.width),
|
|
2820
2961
|
y: parseScreenCoord(options.y, "--y", dimensions.height)
|
|
2821
2962
|
};
|
|
2822
|
-
args.push(
|
|
2823
|
-
"mousemove",
|
|
2824
|
-
"--sync",
|
|
2825
|
-
String(hoverPosition.x),
|
|
2826
|
-
String(hoverPosition.y)
|
|
2827
|
-
);
|
|
2963
|
+
args.push("mousemove", String(hoverPosition.x), String(hoverPosition.y));
|
|
2828
2964
|
}
|
|
2829
2965
|
args.push("click", "--repeat", String(amount), "--delay", "30", button);
|
|
2830
2966
|
runDesktopInputCmd(args);
|
|
@@ -2842,7 +2978,6 @@ async function computerDragCommand(fx, fy, tx, ty) {
|
|
|
2842
2978
|
const steps = clamp(Math.ceil(distance / 70), 8, 24);
|
|
2843
2979
|
const args = [
|
|
2844
2980
|
"mousemove",
|
|
2845
|
-
"--sync",
|
|
2846
2981
|
String(fromX),
|
|
2847
2982
|
String(fromY),
|
|
2848
2983
|
"mousedown",
|
|
@@ -2851,8 +2986,10 @@ async function computerDragCommand(fx, fy, tx, ty) {
|
|
|
2851
2986
|
for (let i = 1; i <= steps; i++) {
|
|
2852
2987
|
const x = Math.round(fromX + (toX - fromX) * i / steps);
|
|
2853
2988
|
const y = Math.round(fromY + (toY - fromY) * i / steps);
|
|
2854
|
-
args.push("mousemove",
|
|
2989
|
+
args.push("mousemove", String(x), String(y), "sleep", "0.018");
|
|
2855
2990
|
}
|
|
2991
|
+
const nudgeX = toX > 0 ? toX - 1 : toX + 1;
|
|
2992
|
+
for (const x of [nudgeX, toX, nudgeX, toX]) args.push("mousemove", String(x), String(toY), "sleep", "0.018");
|
|
2856
2993
|
args.push("mouseup", "1");
|
|
2857
2994
|
runDesktopInputCmd(args);
|
|
2858
2995
|
logRecordingAction({ type: "drag", x: fromX, y: fromY, toX, toY });
|
|
@@ -2932,7 +3069,7 @@ async function computerStatusCommand() {
|
|
|
2932
3069
|
}
|
|
2933
3070
|
|
|
2934
3071
|
// src/version.ts
|
|
2935
|
-
var COMPUTER_USE_VERSION = "0.3.
|
|
3072
|
+
var COMPUTER_USE_VERSION = "0.3.23";
|
|
2936
3073
|
var COMPUTER_USE_PROTOCOL_VERSION = 1;
|
|
2937
3074
|
|
|
2938
3075
|
// src/program.ts
|
|
@@ -2947,9 +3084,9 @@ function createProgram() {
|
|
|
2947
3084
|
program.command("info").description("Print the live noVNC viewer URL").action(() => computerInfoCommand());
|
|
2948
3085
|
program.command("status").description("Show desktop services and the active preview URL").action(() => computerStatusCommand());
|
|
2949
3086
|
program.command("screenshot <path>").description("Capture the current desktop to a PNG file").option("--raw", "Save a 1:1 desktop capture").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").action((path, options) => computerScreenshotCommand(path, options));
|
|
2950
|
-
program.command("observe <path>").description("Wait for the desktop to settle and capture screen context").option("--raw", "Save a 1:1 desktop capture without a grid").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").option("--timeout <ms>", "Maximum stability wait. Default 3000.").option("--stable-ms <ms>", "Required unchanged time. Default 600.").option("--poll-ms <ms>", "Polling interval. Default
|
|
3087
|
+
program.command("observe <path>").description("Wait for the desktop to settle and capture screen context").option("--raw", "Save a 1:1 desktop capture without a grid (default)").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").option("--timeout <ms>", "Maximum stability wait. Default 3000.").option("--stable-ms <ms>", "Required unchanged time. Default 600.").option("--poll-ms <ms>", "Polling interval. Default 100.").action((path, options) => computerObserveCommand(path, options));
|
|
2951
3088
|
program.command("browser").description("Print JSON for controllable Chrome pages").option("--snapshot", "Include visible text and controls").option("--limit <chars>", "Maximum body text characters. Default 4000.").option("--element-limit <n>", "Maximum controls. Default 80.").option("--target-id <id>", "Only include this Chrome target").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((options) => computerBrowserCommand(options));
|
|
2952
|
-
program.command("browser-state
|
|
3089
|
+
program.command("browser-state [path]").description("Capture Chrome accessibility state, and a viewport screenshot when a path is given").option("--full", "Return the full tree instead of a diff").option("--limit <chars>", "Maximum visible text characters. Default 8000.").option("--element-limit <n>", "Maximum actionable elements. Default 120.").option("--timeout <ms>", "Maximum stability wait. Default 5000.").option("--stable-ms <ms>", "Required unchanged time. Default 500.").option("--poll-ms <ms>", "Polling interval. Default 100.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((path, options) => computerBrowserStateCommand(path, options));
|
|
2953
3090
|
program.command("browser-batch <actions>").description("Run an ordered, fail-fast JSON array of browser actions").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((actions, options) => computerBrowserBatchCommand(actions, options));
|
|
2954
3091
|
program.command("browser-click [text]").description("Click a Chrome element by ref or visible text").option("--ref <id>", "Backend DOM node ref from browser-state").option("--exact", "Require an exact text match").option("--index <n>", "Select the nth match. Default 0.").option("--button <button>", "left, middle, or right").option("--double", "Double-click").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserClickCommand(text, options));
|
|
2955
3092
|
program.command("browser-fill [field] [value]").description("Fill a Chrome field by ref or visible text").option("--ref <id>", "Backend DOM node ref from browser-state").option("--exact", "Require an exact field match").option("--index <n>", "Select the nth match. Default 0.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((field, value, options) => {
|
|
@@ -2958,7 +3095,8 @@ function createProgram() {
|
|
|
2958
3095
|
if (resolvedValue === void 0) throw new Error("Provide <field> <value>, or --ref <id> <value>");
|
|
2959
3096
|
return computerBrowserFillCommand(resolvedField, resolvedValue, options);
|
|
2960
3097
|
});
|
|
2961
|
-
program.command("browser-
|
|
3098
|
+
program.command("browser-drag").description("Drag a Chrome element onto another by ref").requiredOption("--ref <id>", "Element to drag, from the latest browser state").requiredOption("--to-ref <id>", "Element to drop it on").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((options) => computerBrowserDragCommand(options));
|
|
3099
|
+
program.command("browser-scroll <direction>").description("Send trusted wheel input to Chrome").option("--ref <id>", "Scroll from a backend DOM node ref").option("--amount <px>", "Scroll distance. Default 600.").option("--until <text>", "Keep scrolling until this text is on the page, then center it").option("--timeout <ms>", "Give up on --until after this long. Default 10000.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((direction, options) => computerBrowserScrollCommand(direction, options));
|
|
2962
3100
|
program.command("browser-key <combo>").description("Press a key chord in Chrome").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((combo, options) => computerBrowserKeyCommand(combo, options));
|
|
2963
3101
|
program.command("browser-type <text>").description("Type literal text in Chrome").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserTypeCommand(text, options));
|
|
2964
3102
|
program.command("browser-wait <text>").description("Wait for Chrome title, URL, text, or controls").option("--mode <mode>", "any, text, title, url, or control").option("--exact", "Require an exact match").option("--timeout <ms>", "Maximum wait. Default 10000.").option("--poll-ms <ms>", "Polling interval. Default 250.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserWaitCommand(text, options));
|