replicas-cli 0.2.355 → 0.2.357
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.mjs +1353 -164
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -7878,7 +7878,7 @@ var SANDBOX_PATHS = {
|
|
|
7878
7878
|
HOME_DIR: "/home/user",
|
|
7879
7879
|
WORKSPACES_DIR: "/home/user/workspaces",
|
|
7880
7880
|
REPLICAS_DIR: "/home/user/.replicas",
|
|
7881
|
-
|
|
7881
|
+
REPLICAS_CANVAS_DIR: "/home/user/.replicas/canvas",
|
|
7882
7882
|
REPOS_PREPARED_MARKER: "/home/user/.replicas/repos-prepared-for-engine-init",
|
|
7883
7883
|
REPLICAS_RUNTIME_ENV_FILE: "/home/user/.replicas/runtime-env.sh"
|
|
7884
7884
|
};
|
|
@@ -8010,13 +8010,12 @@ replicas computer info
|
|
|
8010
8010
|
# 2) Launch a browser on the workspace display.
|
|
8011
8011
|
replicas computer launch chrome https://news.ycombinator.com
|
|
8012
8012
|
|
|
8013
|
-
# 3)
|
|
8014
|
-
replicas computer
|
|
8015
|
-
replicas computer browser --snapshot
|
|
8013
|
+
# 3) Capture the settled viewport and native accessibility tree together.
|
|
8014
|
+
replicas computer browser-state /tmp/state.png --full
|
|
8016
8015
|
|
|
8017
|
-
# 4)
|
|
8018
|
-
replicas computer
|
|
8019
|
-
replicas computer
|
|
8016
|
+
# 4) Batch actions that use refs from the same state, then capture the semantic diff.
|
|
8017
|
+
replicas computer browser-batch '[{"action":"fill","ref":17,"value":"hello"},{"action":"click","ref":42}]'
|
|
8018
|
+
replicas computer browser-state /tmp/after-click.png
|
|
8020
8019
|
|
|
8021
8020
|
# 5) (Optional) Record a screencap to share back.
|
|
8022
8021
|
replicas computer record start /tmp/demo.mp4 --fps 60
|
|
@@ -8045,21 +8044,33 @@ Waits briefly for the screen to stop changing, saves a 1:1 screenshot, and print
|
|
|
8045
8044
|
|
|
8046
8045
|
Use this instead of hand-written \`sleep && screenshot\` loops after clicks, navigation, typing, or page loads. By default it saves a 100px coordinate-grid screenshot and waits up to 3s for 600ms of visual stability. Pass \`--raw\` if you need an unannotated 1:1 screenshot.
|
|
8047
8046
|
|
|
8048
|
-
### \`replicas computer browser\`
|
|
8049
|
-
|
|
8047
|
+
### \`replicas computer browser-state <path> [--full] [--timeout MS] [--stable-ms MS]\`
|
|
8048
|
+
Waits for the selected Chrome page to settle, saves its viewport screenshot, and prints a compact native Chrome accessibility tree in the same coordinate space. Actionable nodes have DOM-backed refs such as \`[ref=42] button "Save"\`; the tree includes accessible roles, names, values, states, visible bounds, and iframe content.
|
|
8049
|
+
|
|
8050
|
+
The first call returns a full tree. Later calls for the same target and document return semantic changes; navigation or reload forces a new full tree. Pass \`--full\` to reset the baseline. Refs are valid for the current document.
|
|
8051
|
+
|
|
8052
|
+
### \`replicas computer browser [--snapshot]\`
|
|
8053
|
+
Lists Chrome tabs. \`--snapshot\` returns the compatibility accessibility snapshot. Use \`browser-state\` for the primary action/observe loop.
|
|
8054
|
+
|
|
8055
|
+
### \`replicas computer browser-batch '<actions-json>'\`
|
|
8056
|
+
Runs up to 100 ordered browser actions through one Chrome session and stops at the first failure. Supported actions are \`click\`, \`fill\`, \`key\`, \`type\`, \`scroll\`, and \`wait\`. Prefer this when several actions use refs from the same \`browser-state\`; it avoids repeated process, tab-selection, and connection setup. Do not carry refs across navigation\u2014capture fresh state and start a new batch.
|
|
8057
|
+
|
|
8058
|
+
### \`replicas computer browser-click --ref ID [--button left|middle|right] [--double]\`
|
|
8059
|
+
Scrolls the referenced node into view and sends trusted Chrome mouse input at its current visible bounds. Prefer refs from \`browser-state\`. \`browser-click <text> [--exact] [--index N]\` remains a fallback when no fresh ref is available.
|
|
8050
8060
|
|
|
8051
|
-
|
|
8061
|
+
### \`replicas computer browser-fill --ref ID <value>\`
|
|
8062
|
+
Focuses the referenced field, fills it with trusted Chrome input, and verifies the resulting value. Select controls use their semantic option value. \`browser-fill <field> <value> [--exact] [--index N]\` remains a text-matching fallback.
|
|
8052
8063
|
|
|
8053
|
-
### \`replicas computer browser-
|
|
8054
|
-
|
|
8064
|
+
### \`replicas computer browser-scroll <up|down|left|right> [--ref ID] [--amount PX]\`
|
|
8065
|
+
Sends trusted wheel input at a referenced element or the center of the viewport.
|
|
8055
8066
|
|
|
8056
|
-
### \`replicas computer browser-
|
|
8057
|
-
|
|
8067
|
+
### \`replicas computer browser-key <combo>\` / \`browser-type <text>\`
|
|
8068
|
+
Sends trusted keyboard input to the focused element in the selected Chrome target. Prefer \`browser-fill\` when setting a known field value.
|
|
8058
8069
|
|
|
8059
8070
|
### \`replicas computer browser-wait <text> [--mode any|text|title|url|control] [--exact] [--timeout MS]\`
|
|
8060
8071
|
Waits until the active Chrome page matches text in the title, URL, body text, or visible controls. Use this after \`browser-click\` / \`browser-fill\` when you need web app state to settle without screenshot polling.
|
|
8061
8072
|
|
|
8062
|
-
For
|
|
8073
|
+
For browser state and actions, pass \`--target-id <id>\`, \`--title <text>\`, \`--url <text>\`, or \`--page <n>\` when multiple Chrome tabs are open. Prefer \`--target-id\` because titles and URLs can change after an action. The matched tab is brought to the foreground before each action, so driving multiple tabs or recording the screen switches which tab is visible.
|
|
8063
8074
|
|
|
8064
8075
|
### \`replicas computer click <x> <y> [--button N] [--double] [--modifiers ctrl+shift]\`
|
|
8065
8076
|
Move to (x, y) and click. Coordinates can be absolute pixels or percentages such as \`50%\` \`50%\`. Default is left-click (button 1); pass \`--button 3\` for right-click. \`--modifiers\` holds keys during the click (e.g. ctrl-click a link to open in a new tab).
|
|
@@ -8092,26 +8103,29 @@ Spawns an app on the workspace display. Built-in aliases:
|
|
|
8092
8103
|
Anything else gets \`exec\`'d verbatim, so \`replicas computer launch xeyes\` works if xeyes is installed.
|
|
8093
8104
|
When opening a known page, prefer passing the URL directly: \`replicas computer launch chrome http://localhost:3000/\`.
|
|
8094
8105
|
|
|
8106
|
+
Chrome launches do not return until the requested page is controllable. The output includes its page ID; carry that ID with \`--target-id\` into \`browser-state\` and browser actions so unrelated tabs cannot steal task context.
|
|
8107
|
+
|
|
8095
8108
|
### \`replicas computer record start <path> [--fps N]\`
|
|
8096
|
-
Starts an ffmpeg screen recorder. Output is post-processed when stopped: action windows stay at normal speed, idle gaps accelerate, click moments get eased camera zoom, and a synthetic cursor animates between logged mouse positions. The raw capture is fragmented MP4 while active (still playable if the workspace dies mid-record). Default 60fps; drop to 30 if the workspace is CPU-constrained.
|
|
8109
|
+
Starts an ffmpeg screen recorder and returns only after ffmpeg has written the first bytes. Output is post-processed when stopped: action windows stay at normal speed, idle gaps accelerate, click moments get eased camera zoom, and a synthetic cursor animates between logged mouse positions. The raw capture is fragmented MP4 while active (still playable if the workspace dies mid-record). Default 60fps; drop to 30 if the workspace is CPU-constrained.
|
|
8097
8110
|
|
|
8098
8111
|
Only one recording at a time. Re-running \`start\` while one is active fails - call \`stop\` first.
|
|
8099
8112
|
|
|
8100
8113
|
### \`replicas computer record stop\`
|
|
8101
|
-
SIGINTs ffmpeg,
|
|
8114
|
+
SIGINTs ffmpeg, verifies that the process has finalized the MP4, then prints the output path. A finalization timeout preserves the recording state so \`stop\` can be retried safely. Upload it with \`replicas media upload <path>\` to share it.
|
|
8102
8115
|
|
|
8103
8116
|
## Patterns
|
|
8104
8117
|
|
|
8105
|
-
###
|
|
8106
|
-
|
|
8118
|
+
### Browser action / state loop
|
|
8119
|
+
For Chrome, use the native accessibility state before interpreting pixels or guessing coordinates:
|
|
8107
8120
|
|
|
8108
8121
|
\`\`\`bash
|
|
8109
|
-
replicas computer
|
|
8110
|
-
replicas computer
|
|
8111
|
-
|
|
8122
|
+
replicas computer launch chrome https://example.com
|
|
8123
|
+
replicas computer browser-state /tmp/before.png --full --target-id <page-id>
|
|
8124
|
+
replicas computer browser-batch '[{"action":"fill","ref":17,"value":"hello"},{"action":"click","ref":23}]' --target-id <page-id>
|
|
8125
|
+
replicas computer browser-state /tmp/after.png --target-id <page-id>
|
|
8112
8126
|
\`\`\`
|
|
8113
8127
|
|
|
8114
|
-
|
|
8128
|
+
Read the screenshot and tree as one state. Batch ordered actions whose refs all come from that state, then inspect the next semantic diff. A batch is fail-fast and validates each ref against the cached document. Split at navigation or whenever a later action depends on newly rendered refs. Fall back to \`observe\` plus desktop coordinates for canvas content, browser chrome, non-Chrome apps, or controls missing from the accessibility tree.
|
|
8115
8129
|
|
|
8116
8130
|
### Typing into an address bar
|
|
8117
8131
|
\`\`\`bash
|
|
@@ -8120,10 +8134,9 @@ replicas computer observe /tmp/browser-open.png
|
|
|
8120
8134
|
replicas computer key ctrl+l
|
|
8121
8135
|
replicas computer type "https://example.com"
|
|
8122
8136
|
replicas computer key Return
|
|
8123
|
-
replicas computer
|
|
8124
|
-
replicas computer browser --
|
|
8125
|
-
replicas computer browser-
|
|
8126
|
-
replicas computer browser-click "More information" --title "Example"
|
|
8137
|
+
replicas computer browser-state /tmp/loaded.png --full
|
|
8138
|
+
replicas computer browser-fill --ref 17 "replicas"
|
|
8139
|
+
replicas computer browser-click --ref 23 --title "Example"
|
|
8127
8140
|
replicas computer browser-wait "Example Domain" --mode title --title "Example"
|
|
8128
8141
|
\`\`\`
|
|
8129
8142
|
|
|
@@ -9679,7 +9692,7 @@ var HOOK_EXEC_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
|
|
|
9679
9692
|
var REPLICAS_CONFIG_FILENAMES = ["replicas.json", "replicas.yaml", "replicas.yml"];
|
|
9680
9693
|
|
|
9681
9694
|
// ../shared/src/cli-version.ts
|
|
9682
|
-
var CLI_VERSION = "0.2.
|
|
9695
|
+
var CLI_VERSION = "0.2.357";
|
|
9683
9696
|
|
|
9684
9697
|
// ../shared/src/version.ts
|
|
9685
9698
|
function compareVersions(v1, v2) {
|
|
@@ -9734,8 +9747,6 @@ function workspaceConfigWithCapabilities(config2, capabilities = {}) {
|
|
|
9734
9747
|
function workspaceConfigWithPrFollowups(config2, prFollowups = config2?.capabilities?.pr_followups === true) {
|
|
9735
9748
|
return workspaceConfigWithCapabilities(config2, { pr_followups: prFollowups });
|
|
9736
9749
|
}
|
|
9737
|
-
var WORKSPACE_FILE_UPLOAD_MAX_SIZE_BYTES = 20 * 1024 * 1024;
|
|
9738
|
-
var WORKSPACE_FILE_CONTENT_MAX_SIZE_BYTES = 1 * 1024 * 1024;
|
|
9739
9750
|
var WORKSPACE_SIDEBAR_VIEWS = ["owned", "shared", "team", "automated"];
|
|
9740
9751
|
var WORKSPACE_STATUS_FILTERS = WORKSPACE_STATUSES.map((status) => status === "error" ? "failed" : status);
|
|
9741
9752
|
var WORKSPACE_SORT_CHOICES = [
|
|
@@ -15035,7 +15046,7 @@ var sleep = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
|
15035
15046
|
|
|
15036
15047
|
// src/commands/computer/recording.ts
|
|
15037
15048
|
import { spawn as spawn3 } from "child_process";
|
|
15038
|
-
import { appendFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, rmSync as rmSync2, writeFileSync as writeFileSync2 } from "fs";
|
|
15049
|
+
import { appendFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, rmSync as rmSync2, statSync, writeFileSync as writeFileSync2 } from "fs";
|
|
15039
15050
|
import { dirname as dirname2 } from "path";
|
|
15040
15051
|
|
|
15041
15052
|
// src/commands/computer/recording/render.ts
|
|
@@ -15516,6 +15527,18 @@ var RECORD_STARTED_AT_FILE = `${STATE_DIR}/recording-started-at.txt`;
|
|
|
15516
15527
|
var RECORD_FPS_FILE = `${STATE_DIR}/recording-fps.txt`;
|
|
15517
15528
|
var RECORD_DIMENSIONS_FILE = `${STATE_DIR}/recording-dimensions.json`;
|
|
15518
15529
|
var RECORD_ACTIONS_FILE = `${STATE_DIR}/recording-actions.jsonl`;
|
|
15530
|
+
var RECORD_STATE_FILES = [
|
|
15531
|
+
RECORD_PID_FILE,
|
|
15532
|
+
RECORD_PATH_FILE,
|
|
15533
|
+
RECORD_RAW_PATH_FILE,
|
|
15534
|
+
RECORD_STARTED_AT_FILE,
|
|
15535
|
+
RECORD_FPS_FILE,
|
|
15536
|
+
RECORD_DIMENSIONS_FILE,
|
|
15537
|
+
RECORD_ACTIONS_FILE
|
|
15538
|
+
];
|
|
15539
|
+
function clearRecordingState() {
|
|
15540
|
+
for (const file of RECORD_STATE_FILES) rmSync2(file, { force: true });
|
|
15541
|
+
}
|
|
15519
15542
|
function recordingStartedAt() {
|
|
15520
15543
|
if (!existsSync2(RECORD_STARTED_AT_FILE)) return null;
|
|
15521
15544
|
const startedAt = Number.parseInt(readFileSync2(RECORD_STARTED_AT_FILE, "utf8").trim(), 10);
|
|
@@ -15573,13 +15596,13 @@ async function computerRecordStartCommand(path6, options) {
|
|
|
15573
15596
|
if (existsSync2(RECORD_PID_FILE)) {
|
|
15574
15597
|
const pid = parseInt(readFileSync2(RECORD_PID_FILE, "utf8").trim(), 10);
|
|
15575
15598
|
if (Number.isFinite(pid)) {
|
|
15576
|
-
let
|
|
15599
|
+
let alive2 = false;
|
|
15577
15600
|
try {
|
|
15578
15601
|
process.kill(pid, 0);
|
|
15579
|
-
|
|
15602
|
+
alive2 = true;
|
|
15580
15603
|
} catch {
|
|
15581
15604
|
}
|
|
15582
|
-
if (
|
|
15605
|
+
if (alive2) fail(`recording already in progress (pid ${pid}). run \`replicas computer record stop\` first.`);
|
|
15583
15606
|
}
|
|
15584
15607
|
}
|
|
15585
15608
|
const target = resolvePath(path6);
|
|
@@ -15626,7 +15649,29 @@ async function computerRecordStartCommand(path6, options) {
|
|
|
15626
15649
|
writeFileSync2(RECORD_STARTED_AT_FILE, String(Date.now()));
|
|
15627
15650
|
writeFileSync2(RECORD_FPS_FILE, String(fps));
|
|
15628
15651
|
writeFileSync2(RECORD_DIMENSIONS_FILE, JSON.stringify({ width, height }));
|
|
15629
|
-
|
|
15652
|
+
const startedAt = Date.now();
|
|
15653
|
+
while (Date.now() - startedAt < 5e3) {
|
|
15654
|
+
try {
|
|
15655
|
+
process.kill(child.pid, 0);
|
|
15656
|
+
if (existsSync2(rawTarget) && statSync(rawTarget).size > 0) {
|
|
15657
|
+
console.log(`${target} (recording ready in ${Date.now() - startedAt}ms)`);
|
|
15658
|
+
return;
|
|
15659
|
+
}
|
|
15660
|
+
} catch {
|
|
15661
|
+
break;
|
|
15662
|
+
}
|
|
15663
|
+
await sleep(100);
|
|
15664
|
+
}
|
|
15665
|
+
let alive = false;
|
|
15666
|
+
try {
|
|
15667
|
+
process.kill(child.pid, 0);
|
|
15668
|
+
alive = true;
|
|
15669
|
+
} catch {
|
|
15670
|
+
}
|
|
15671
|
+
if (alive) fail("ffmpeg is running but did not produce recording output within 5 seconds; run `replicas computer record stop` to finalize or retry");
|
|
15672
|
+
clearRecordingState();
|
|
15673
|
+
rmSync2(rawTarget, { force: true });
|
|
15674
|
+
fail("ffmpeg exited before screen recording became ready");
|
|
15630
15675
|
}
|
|
15631
15676
|
async function computerRecordStopCommand() {
|
|
15632
15677
|
if (!existsSync2(RECORD_PID_FILE) && !existsSync2(RECORD_PATH_FILE)) fail("no recording in progress");
|
|
@@ -15637,15 +15682,17 @@ async function computerRecordStopCommand() {
|
|
|
15637
15682
|
process.kill(pid, "SIGINT");
|
|
15638
15683
|
} catch {
|
|
15639
15684
|
}
|
|
15640
|
-
|
|
15685
|
+
let alive = true;
|
|
15686
|
+
for (let i = 0; i < 150; i++) {
|
|
15641
15687
|
try {
|
|
15642
15688
|
process.kill(pid, 0);
|
|
15643
15689
|
} catch {
|
|
15690
|
+
alive = false;
|
|
15644
15691
|
break;
|
|
15645
15692
|
}
|
|
15646
|
-
await
|
|
15693
|
+
await sleep(200);
|
|
15647
15694
|
}
|
|
15648
|
-
|
|
15695
|
+
if (alive) fail(`ffmpeg did not finalize recording within 30 seconds (pid ${pid})`);
|
|
15649
15696
|
}
|
|
15650
15697
|
if (existsSync2(RECORD_PATH_FILE)) {
|
|
15651
15698
|
const target = readFileSync2(RECORD_PATH_FILE, "utf8").trim();
|
|
@@ -15658,13 +15705,8 @@ async function computerRecordStopCommand() {
|
|
|
15658
15705
|
rmSync2(rawPath, { force: true });
|
|
15659
15706
|
}
|
|
15660
15707
|
console.log(target);
|
|
15661
|
-
rmSync2(RECORD_PATH_FILE, { force: true });
|
|
15662
|
-
rmSync2(RECORD_RAW_PATH_FILE, { force: true });
|
|
15663
|
-
rmSync2(RECORD_STARTED_AT_FILE, { force: true });
|
|
15664
|
-
rmSync2(RECORD_FPS_FILE, { force: true });
|
|
15665
|
-
rmSync2(RECORD_DIMENSIONS_FILE, { force: true });
|
|
15666
|
-
rmSync2(RECORD_ACTIONS_FILE, { force: true });
|
|
15667
15708
|
}
|
|
15709
|
+
clearRecordingState();
|
|
15668
15710
|
}
|
|
15669
15711
|
|
|
15670
15712
|
// src/commands/computer/index.ts
|
|
@@ -16003,8 +16045,8 @@ async function getChromePages() {
|
|
|
16003
16045
|
async function selectChromePage(options = {}) {
|
|
16004
16046
|
const pages = await getChromePages();
|
|
16005
16047
|
let matches = pages;
|
|
16006
|
-
if (options.
|
|
16007
|
-
matches = matches.filter((page2) => page2.id === options.
|
|
16048
|
+
if (options.targetId) {
|
|
16049
|
+
matches = matches.filter((page2) => page2.id === options.targetId);
|
|
16008
16050
|
}
|
|
16009
16051
|
if (options.title) {
|
|
16010
16052
|
const needle = options.title.toLowerCase();
|
|
@@ -16023,121 +16065,528 @@ async function selectChromePage(options = {}) {
|
|
|
16023
16065
|
`No matching debuggable Chrome page found (${matches.length} match${matches.length === 1 ? "" : "es"}, index ${index}). Use \`replicas computer browser\` to list pages.`
|
|
16024
16066
|
);
|
|
16025
16067
|
}
|
|
16068
|
+
await sendChromeCommand(webSocketDebuggerUrl, "Page.bringToFront", {});
|
|
16026
16069
|
return { ...page, webSocketDebuggerUrl };
|
|
16027
16070
|
}
|
|
16028
|
-
|
|
16029
|
-
return `(() => {
|
|
16030
|
-
const clip = (value, limit) => String(value || '').replace(/\\s+/g, ' ').trim().slice(0, limit);
|
|
16031
|
-
const escape = (value) => globalThis.CSS && CSS.escape ? CSS.escape(value) : String(value).replace(/["\\\\]/g, '\\\\$&');
|
|
16071
|
+
var VISIBLE_JS = `
|
|
16032
16072
|
const visible = (el) => {
|
|
16033
16073
|
const rect = el.getBoundingClientRect();
|
|
16034
16074
|
const style = getComputedStyle(el);
|
|
16035
16075
|
return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
|
|
16036
|
-
}
|
|
16037
|
-
|
|
16038
|
-
|
|
16039
|
-
const id = el.getAttribute('id');
|
|
16040
|
-
if (id) values.push(...Array.from(document.querySelectorAll('label[for="' + escape(id) + '"]')).map((label) => label.innerText));
|
|
16041
|
-
const wrappingLabel = el.closest('label');
|
|
16042
|
-
if (wrappingLabel) values.push(wrappingLabel.innerText);
|
|
16043
|
-
values.push(el.innerText, el.value, el.getAttribute('aria-label'), el.getAttribute('title'), el.getAttribute('placeholder'), el.href, el.id, el.name, el.tagName);
|
|
16044
|
-
return values.map((value) => clip(value, 160)).filter(Boolean);
|
|
16045
|
-
};
|
|
16046
|
-
const name = (el) => labels(el)[0] || clip(el.tagName, 160);
|
|
16047
|
-
const selector = 'a,button,input,textarea,select,[role="button"],[role="link"],[role="textbox"],[contenteditable="true"]';
|
|
16048
|
-
const controls = Array.from(document.querySelectorAll(selector))
|
|
16049
|
-
.filter(visible)
|
|
16050
|
-
.slice(0, ${elementLimit})
|
|
16051
|
-
.map((el) => {
|
|
16052
|
-
const rect = el.getBoundingClientRect();
|
|
16053
|
-
return {
|
|
16054
|
-
tag: el.tagName.toLowerCase(),
|
|
16055
|
-
role: el.getAttribute('role') || null,
|
|
16056
|
-
type: el.getAttribute('type') || null,
|
|
16057
|
-
text: name(el),
|
|
16058
|
-
href: el.href || null,
|
|
16059
|
-
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
|
|
16060
|
-
rect: {
|
|
16061
|
-
x: Math.round(rect.x),
|
|
16062
|
-
y: Math.round(rect.y),
|
|
16063
|
-
width: Math.round(rect.width),
|
|
16064
|
-
height: Math.round(rect.height),
|
|
16065
|
-
centerX: Math.round(rect.x + rect.width / 2),
|
|
16066
|
-
centerY: Math.round(rect.y + rect.height / 2),
|
|
16067
|
-
},
|
|
16068
|
-
};
|
|
16069
|
-
});
|
|
16070
|
-
return {
|
|
16071
|
-
title: document.title,
|
|
16072
|
-
url: location.href,
|
|
16073
|
-
text: clip(document.body ? document.body.innerText : '', ${textLimit}),
|
|
16074
|
-
controls,
|
|
16075
|
-
};
|
|
16076
|
-
})()`;
|
|
16076
|
+
};`;
|
|
16077
|
+
async function sendChromeSessionCommand(session, method, params) {
|
|
16078
|
+
return chromeResult(await session.send({ method, params }), method);
|
|
16077
16079
|
}
|
|
16078
|
-
async function
|
|
16080
|
+
async function withChromeSession(webSocketDebuggerUrl, callback) {
|
|
16079
16081
|
return await new Promise((resolve2, reject) => {
|
|
16080
16082
|
const ws = new WebSocket(webSocketDebuggerUrl);
|
|
16083
|
+
const pending = /* @__PURE__ */ new Map();
|
|
16084
|
+
let nextId = 1;
|
|
16081
16085
|
let settled = false;
|
|
16082
|
-
const
|
|
16086
|
+
const failSession = (error) => {
|
|
16083
16087
|
if (settled) return;
|
|
16084
16088
|
settled = true;
|
|
16085
|
-
|
|
16089
|
+
for (const request of pending.values()) {
|
|
16090
|
+
clearTimeout(request.timeout);
|
|
16091
|
+
request.reject(error);
|
|
16092
|
+
}
|
|
16093
|
+
pending.clear();
|
|
16086
16094
|
ws.close();
|
|
16087
|
-
|
|
16095
|
+
reject(error);
|
|
16088
16096
|
};
|
|
16089
|
-
const
|
|
16090
|
-
|
|
16091
|
-
|
|
16092
|
-
|
|
16093
|
-
|
|
16094
|
-
|
|
16095
|
-
|
|
16096
|
-
|
|
16097
|
-
|
|
16098
|
-
|
|
16099
|
-
|
|
16100
|
-
|
|
16101
|
-
|
|
16097
|
+
const finishSession = (value) => {
|
|
16098
|
+
if (settled) return;
|
|
16099
|
+
settled = true;
|
|
16100
|
+
for (const request of pending.values()) clearTimeout(request.timeout);
|
|
16101
|
+
pending.clear();
|
|
16102
|
+
ws.close();
|
|
16103
|
+
resolve2(value);
|
|
16104
|
+
};
|
|
16105
|
+
const session = {
|
|
16106
|
+
send: ({ method, params }) => new Promise((resolveCommand, rejectCommand) => {
|
|
16107
|
+
const id = nextId++;
|
|
16108
|
+
const timeout = setTimeout(() => {
|
|
16109
|
+
pending.delete(id);
|
|
16110
|
+
rejectCommand(new Error(`Chrome DevTools ${method} timed out`));
|
|
16111
|
+
}, 1e4);
|
|
16112
|
+
pending.set(id, { method, resolve: resolveCommand, reject: rejectCommand, timeout });
|
|
16113
|
+
ws.send(JSON.stringify({ id, method, params }));
|
|
16114
|
+
})
|
|
16115
|
+
};
|
|
16116
|
+
ws.addEventListener("open", async () => {
|
|
16117
|
+
try {
|
|
16118
|
+
finishSession(await callback(session));
|
|
16119
|
+
} catch (error) {
|
|
16120
|
+
failSession(error instanceof Error ? error : new Error(String(error)));
|
|
16121
|
+
}
|
|
16102
16122
|
});
|
|
16103
16123
|
ws.addEventListener("message", (event) => {
|
|
16104
16124
|
const data = typeof event.data === "string" ? event.data : Buffer.isBuffer(event.data) ? event.data.toString("utf8") : Buffer.from(event.data).toString("utf8");
|
|
16105
|
-
let
|
|
16125
|
+
let message;
|
|
16106
16126
|
try {
|
|
16107
|
-
|
|
16127
|
+
message = JSON.parse(data);
|
|
16108
16128
|
} catch (error) {
|
|
16109
|
-
|
|
16110
|
-
finish(() => reject(new Error(`Chrome DevTools returned invalid WebSocket JSON: ${reason}`)));
|
|
16111
|
-
return;
|
|
16112
|
-
}
|
|
16113
|
-
if (typeof parsed !== "object" || parsed === null) {
|
|
16114
|
-
finish(() => reject(new Error("Chrome DevTools returned an invalid WebSocket message")));
|
|
16115
|
-
return;
|
|
16116
|
-
}
|
|
16117
|
-
const message = parsed;
|
|
16118
|
-
if (message.id !== 1) return;
|
|
16119
|
-
if (message.error) {
|
|
16120
|
-
finish(() => reject(new Error(message.error?.message || "Chrome DevTools evaluation failed")));
|
|
16121
|
-
return;
|
|
16122
|
-
}
|
|
16123
|
-
if (message.result?.exceptionDetails) {
|
|
16124
|
-
finish(() => reject(new Error(message.result.exceptionDetails.text || "Chrome DevTools evaluation threw")));
|
|
16129
|
+
failSession(new Error(`Chrome DevTools returned invalid WebSocket JSON: ${error instanceof Error ? error.message : String(error)}`));
|
|
16125
16130
|
return;
|
|
16126
16131
|
}
|
|
16127
|
-
|
|
16132
|
+
if (!isRecord(message) || typeof message.id !== "number") return;
|
|
16133
|
+
const request = pending.get(message.id);
|
|
16134
|
+
if (!request) return;
|
|
16135
|
+
clearTimeout(request.timeout);
|
|
16136
|
+
pending.delete(message.id);
|
|
16137
|
+
const detail = isRecord(message.error) && typeof message.error.message === "string" ? message.error.message : void 0;
|
|
16138
|
+
request.resolve(message.error ? { error: { message: detail } } : { result: isRecord(message.result) ? message.result : {} });
|
|
16128
16139
|
});
|
|
16129
|
-
ws.addEventListener("error", () =>
|
|
16130
|
-
|
|
16140
|
+
ws.addEventListener("error", () => failSession(new Error("Chrome DevTools WebSocket failed")));
|
|
16141
|
+
ws.addEventListener("close", () => {
|
|
16142
|
+
if (!settled) failSession(new Error("Chrome DevTools WebSocket closed unexpectedly"));
|
|
16131
16143
|
});
|
|
16132
16144
|
});
|
|
16133
16145
|
}
|
|
16146
|
+
async function sendChromeCommands(webSocketDebuggerUrl, commands) {
|
|
16147
|
+
return await withChromeSession(webSocketDebuggerUrl, async (session) => await Promise.all(commands.map((command) => session.send(command))));
|
|
16148
|
+
}
|
|
16149
|
+
async function sendChromeCommand(webSocketDebuggerUrl, method, params) {
|
|
16150
|
+
const response = (await sendChromeCommands(webSocketDebuggerUrl, [{ method, params }]))[0];
|
|
16151
|
+
if (response.error) throw new Error(response.error.message || `Chrome DevTools ${method} failed`);
|
|
16152
|
+
return response.result ?? {};
|
|
16153
|
+
}
|
|
16154
|
+
async function callFunctionOnChromeNodeInSession(session, backendNodeId, functionDeclaration, args = []) {
|
|
16155
|
+
const resolved = await sendChromeSessionCommand(session, "DOM.resolveNode", { backendNodeId });
|
|
16156
|
+
const objectId = isRecord(resolved.object) ? resolved.object.objectId : void 0;
|
|
16157
|
+
if (typeof objectId !== "string") throw new Error(`Could not resolve browser element ref ${backendNodeId}`);
|
|
16158
|
+
return await sendChromeSessionCommand(session, "Runtime.callFunctionOn", {
|
|
16159
|
+
objectId,
|
|
16160
|
+
functionDeclaration,
|
|
16161
|
+
arguments: args.map((value) => ({ value })),
|
|
16162
|
+
returnByValue: true
|
|
16163
|
+
});
|
|
16164
|
+
}
|
|
16165
|
+
function chromeCallValue(response) {
|
|
16166
|
+
return isRecord(response.result) ? response.result.value : void 0;
|
|
16167
|
+
}
|
|
16168
|
+
async function evaluateChromeTarget(webSocketDebuggerUrl, expression) {
|
|
16169
|
+
return await withChromeSession(webSocketDebuggerUrl, async (session) => await evaluateChromeSession(session, expression));
|
|
16170
|
+
}
|
|
16171
|
+
async function evaluateChromeSession(session, expression) {
|
|
16172
|
+
const response = await sendChromeSessionCommand(session, "Runtime.evaluate", {
|
|
16173
|
+
expression,
|
|
16174
|
+
awaitPromise: true,
|
|
16175
|
+
returnByValue: true
|
|
16176
|
+
});
|
|
16177
|
+
const exception = response.exceptionDetails;
|
|
16178
|
+
if (exception) {
|
|
16179
|
+
const text = typeof exception === "object" && exception !== null && "text" in exception && typeof exception.text === "string" ? exception.text : "";
|
|
16180
|
+
throw new Error(text || "Chrome DevTools evaluation threw");
|
|
16181
|
+
}
|
|
16182
|
+
const evaluated = response.result;
|
|
16183
|
+
return typeof evaluated === "object" && evaluated !== null && "value" in evaluated ? evaluated.value ?? null : null;
|
|
16184
|
+
}
|
|
16185
|
+
function isRawAXNode(value) {
|
|
16186
|
+
return isRecord(value);
|
|
16187
|
+
}
|
|
16188
|
+
function isDOMSnapshotDocument(value) {
|
|
16189
|
+
return isRecord(value);
|
|
16190
|
+
}
|
|
16191
|
+
function chromeVisualViewport(layout) {
|
|
16192
|
+
const viewport = isRecord(layout.cssVisualViewport) ? layout.cssVisualViewport : {};
|
|
16193
|
+
return {
|
|
16194
|
+
width: typeof viewport.clientWidth === "number" ? viewport.clientWidth : 0,
|
|
16195
|
+
height: typeof viewport.clientHeight === "number" ? viewport.clientHeight : 0
|
|
16196
|
+
};
|
|
16197
|
+
}
|
|
16198
|
+
function roundBrowserRect(rect) {
|
|
16199
|
+
return {
|
|
16200
|
+
x: Math.round(rect.x),
|
|
16201
|
+
y: Math.round(rect.y),
|
|
16202
|
+
width: Math.round(rect.width),
|
|
16203
|
+
height: Math.round(rect.height),
|
|
16204
|
+
centerX: Math.round(rect.centerX),
|
|
16205
|
+
centerY: Math.round(rect.centerY)
|
|
16206
|
+
};
|
|
16207
|
+
}
|
|
16208
|
+
function browserStateProperties(node) {
|
|
16209
|
+
const properties = [];
|
|
16210
|
+
for (const { name, value } of node.properties ?? []) {
|
|
16211
|
+
const state = value?.value;
|
|
16212
|
+
if (!name || !BROWSER_STATE_PROPERTIES.has(name) || typeof state !== "string" && typeof state !== "number" && typeof state !== "boolean") continue;
|
|
16213
|
+
properties.push([name, name === "url" ? clippedText(state, 300) : state]);
|
|
16214
|
+
}
|
|
16215
|
+
return Object.fromEntries(properties);
|
|
16216
|
+
}
|
|
16217
|
+
function readBrowserStateCache(path6) {
|
|
16218
|
+
let value;
|
|
16219
|
+
try {
|
|
16220
|
+
value = JSON.parse(readFileSync3(path6, "utf8"));
|
|
16221
|
+
} catch {
|
|
16222
|
+
return null;
|
|
16223
|
+
}
|
|
16224
|
+
if (!isRecord(value) || !Array.isArray(value.entries)) return null;
|
|
16225
|
+
const entries = [];
|
|
16226
|
+
for (const entry of value.entries) {
|
|
16227
|
+
if (!isRecord(entry) || typeof entry.key !== "string" || typeof entry.semantic !== "string" || typeof entry.line !== "string") return null;
|
|
16228
|
+
entries.push({ key: entry.key, semantic: entry.semantic, line: entry.line });
|
|
16229
|
+
}
|
|
16230
|
+
return {
|
|
16231
|
+
url: typeof value.url === "string" ? value.url : void 0,
|
|
16232
|
+
documentId: typeof value.documentId === "string" || value.documentId === null ? value.documentId : void 0,
|
|
16233
|
+
entries
|
|
16234
|
+
};
|
|
16235
|
+
}
|
|
16236
|
+
function chromeResult(response, method) {
|
|
16237
|
+
if (response.error) throw new Error(response.error.message || `Chrome DevTools ${method} failed`);
|
|
16238
|
+
return response.result ?? {};
|
|
16239
|
+
}
|
|
16240
|
+
function collectFrameIds(frameTree) {
|
|
16241
|
+
const ids = [];
|
|
16242
|
+
const visit = (value) => {
|
|
16243
|
+
if (!isRecord(value)) return;
|
|
16244
|
+
const frame = isRecord(value.frame) ? value.frame : {};
|
|
16245
|
+
if (typeof frame.id === "string") ids.push(frame.id);
|
|
16246
|
+
if (Array.isArray(value.childFrames)) value.childFrames.forEach(visit);
|
|
16247
|
+
};
|
|
16248
|
+
visit(frameTree);
|
|
16249
|
+
return ids;
|
|
16250
|
+
}
|
|
16251
|
+
function browserDocumentId(frameTree) {
|
|
16252
|
+
if (!isRecord(frameTree) || !isRecord(frameTree.frame)) return null;
|
|
16253
|
+
const loaderId = frameTree.frame.loaderId;
|
|
16254
|
+
return typeof loaderId === "string" ? loaderId : null;
|
|
16255
|
+
}
|
|
16256
|
+
function intersectRects(a, b) {
|
|
16257
|
+
const x = Math.max(a.x, b.x);
|
|
16258
|
+
const y = Math.max(a.y, b.y);
|
|
16259
|
+
const right = Math.min(a.x + a.width, b.x + b.width);
|
|
16260
|
+
const bottom = Math.min(a.y + a.height, b.y + b.height);
|
|
16261
|
+
if (right <= x || bottom <= y) return null;
|
|
16262
|
+
return {
|
|
16263
|
+
x,
|
|
16264
|
+
y,
|
|
16265
|
+
width: right - x,
|
|
16266
|
+
height: bottom - y,
|
|
16267
|
+
centerX: x + (right - x) / 2,
|
|
16268
|
+
centerY: y + (bottom - y) / 2
|
|
16269
|
+
};
|
|
16270
|
+
}
|
|
16271
|
+
function browserContentQuadRects(quads, viewportRect) {
|
|
16272
|
+
if (!Array.isArray(quads)) return [];
|
|
16273
|
+
return quads.filter((quad) => Array.isArray(quad) && quad.length >= 8 && quad.every(Number.isFinite)).map((quad) => {
|
|
16274
|
+
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
|
16275
|
+
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
|
16276
|
+
const x = Math.min(...xs);
|
|
16277
|
+
const y = Math.min(...ys);
|
|
16278
|
+
const width = Math.max(...xs) - x;
|
|
16279
|
+
const height = Math.max(...ys) - y;
|
|
16280
|
+
return { x, y, width, height, centerX: x + width / 2, centerY: y + height / 2 };
|
|
16281
|
+
}).map((rect) => intersectRects(rect, viewportRect)).filter((rect) => !!rect).sort((a, b) => b.width * b.height - a.width * a.height);
|
|
16282
|
+
}
|
|
16283
|
+
function buildBrowserBounds(result, viewport, targetId) {
|
|
16284
|
+
const strings = Array.isArray(result.strings) ? result.strings : [];
|
|
16285
|
+
const documents = Array.isArray(result.documents) ? result.documents.filter(isDOMSnapshotDocument) : [];
|
|
16286
|
+
const localRects = documents.map((document) => {
|
|
16287
|
+
const rects = /* @__PURE__ */ new Map();
|
|
16288
|
+
const nodeIndexes = document.layout?.nodeIndex ?? [];
|
|
16289
|
+
const bounds = document.layout?.bounds ?? [];
|
|
16290
|
+
const scrollX = document.scrollOffsetX ?? 0;
|
|
16291
|
+
const scrollY = document.scrollOffsetY ?? 0;
|
|
16292
|
+
nodeIndexes.forEach((nodeIndex, index) => {
|
|
16293
|
+
const bound = bounds[index];
|
|
16294
|
+
if (!Array.isArray(bound) || bound.length < 4 || !bound.every(Number.isFinite)) return;
|
|
16295
|
+
const [rawX, rawY, width, height] = bound;
|
|
16296
|
+
rects.set(nodeIndex, {
|
|
16297
|
+
x: rawX - scrollX,
|
|
16298
|
+
y: rawY - scrollY,
|
|
16299
|
+
width,
|
|
16300
|
+
height,
|
|
16301
|
+
centerX: rawX - scrollX + width / 2,
|
|
16302
|
+
centerY: rawY - scrollY + height / 2
|
|
16303
|
+
});
|
|
16304
|
+
});
|
|
16305
|
+
return rects;
|
|
16306
|
+
});
|
|
16307
|
+
const parents = /* @__PURE__ */ new Map();
|
|
16308
|
+
documents.forEach((document, documentIndex) => {
|
|
16309
|
+
const sparse = document.nodes?.contentDocumentIndex;
|
|
16310
|
+
sparse?.index?.forEach((nodeIndex, index) => {
|
|
16311
|
+
const childDocumentIndex = sparse.value?.[index];
|
|
16312
|
+
if (typeof childDocumentIndex === "number") parents.set(childDocumentIndex, { documentIndex, nodeIndex });
|
|
16313
|
+
});
|
|
16314
|
+
});
|
|
16315
|
+
const targetDocumentIndex = documents.findIndex(
|
|
16316
|
+
(document) => typeof document.frameId === "number" && strings[document.frameId] === targetId
|
|
16317
|
+
);
|
|
16318
|
+
const rootIndex = targetDocumentIndex >= 0 ? targetDocumentIndex : documents.findIndex((_, index) => !parents.has(index));
|
|
16319
|
+
const viewportRect = {
|
|
16320
|
+
x: 0,
|
|
16321
|
+
y: 0,
|
|
16322
|
+
width: viewport.width,
|
|
16323
|
+
height: viewport.height,
|
|
16324
|
+
centerX: viewport.width / 2,
|
|
16325
|
+
centerY: viewport.height / 2
|
|
16326
|
+
};
|
|
16327
|
+
const origins = /* @__PURE__ */ new Map();
|
|
16328
|
+
origins.set(rootIndex < 0 ? 0 : rootIndex, { x: 0, y: 0, clip: viewportRect });
|
|
16329
|
+
for (let pass = 0; pass < documents.length; pass++) {
|
|
16330
|
+
let changed = false;
|
|
16331
|
+
for (const [childIndex, parent] of parents) {
|
|
16332
|
+
if (origins.has(childIndex)) continue;
|
|
16333
|
+
const parentOrigin = origins.get(parent.documentIndex);
|
|
16334
|
+
const frameRect = localRects[parent.documentIndex]?.get(parent.nodeIndex);
|
|
16335
|
+
if (!parentOrigin || !frameRect) continue;
|
|
16336
|
+
const globalFrameRect = {
|
|
16337
|
+
...frameRect,
|
|
16338
|
+
x: parentOrigin.x + frameRect.x,
|
|
16339
|
+
y: parentOrigin.y + frameRect.y,
|
|
16340
|
+
centerX: parentOrigin.x + frameRect.centerX,
|
|
16341
|
+
centerY: parentOrigin.y + frameRect.centerY
|
|
16342
|
+
};
|
|
16343
|
+
const childDocument = documents[childIndex];
|
|
16344
|
+
const insetX = Math.max(0, (globalFrameRect.width - (childDocument?.contentWidth ?? globalFrameRect.width)) / 2);
|
|
16345
|
+
const insetY = Math.max(0, (globalFrameRect.height - (childDocument?.contentHeight ?? globalFrameRect.height)) / 2);
|
|
16346
|
+
const contentRect = {
|
|
16347
|
+
x: globalFrameRect.x + insetX,
|
|
16348
|
+
y: globalFrameRect.y + insetY,
|
|
16349
|
+
width: Math.max(0, globalFrameRect.width - insetX * 2),
|
|
16350
|
+
height: Math.max(0, globalFrameRect.height - insetY * 2),
|
|
16351
|
+
centerX: globalFrameRect.centerX,
|
|
16352
|
+
centerY: globalFrameRect.centerY
|
|
16353
|
+
};
|
|
16354
|
+
const clip = intersectRects(parentOrigin.clip, contentRect);
|
|
16355
|
+
if (!clip) continue;
|
|
16356
|
+
origins.set(childIndex, { x: contentRect.x, y: contentRect.y, clip });
|
|
16357
|
+
changed = true;
|
|
16358
|
+
}
|
|
16359
|
+
if (!changed) break;
|
|
16360
|
+
}
|
|
16361
|
+
const byBackendNodeId = /* @__PURE__ */ new Map();
|
|
16362
|
+
documents.forEach((document, documentIndex) => {
|
|
16363
|
+
const origin = origins.get(documentIndex);
|
|
16364
|
+
if (!origin) return;
|
|
16365
|
+
const backendNodeIds = document.nodes?.backendNodeId ?? [];
|
|
16366
|
+
for (const [nodeIndex, local] of localRects[documentIndex] ?? []) {
|
|
16367
|
+
const backendNodeId = backendNodeIds[nodeIndex];
|
|
16368
|
+
if (!Number.isInteger(backendNodeId)) continue;
|
|
16369
|
+
const rect = {
|
|
16370
|
+
...local,
|
|
16371
|
+
x: origin.x + local.x,
|
|
16372
|
+
y: origin.y + local.y,
|
|
16373
|
+
centerX: origin.x + local.centerX,
|
|
16374
|
+
centerY: origin.y + local.centerY
|
|
16375
|
+
};
|
|
16376
|
+
if (intersectRects(rect, origin.clip)) byBackendNodeId.set(backendNodeId, { rect, clip: origin.clip });
|
|
16377
|
+
}
|
|
16378
|
+
});
|
|
16379
|
+
return byBackendNodeId;
|
|
16380
|
+
}
|
|
16381
|
+
var BROWSER_INTERACTIVE_ROLES = /* @__PURE__ */ new Set([
|
|
16382
|
+
"button",
|
|
16383
|
+
"checkbox",
|
|
16384
|
+
"combobox",
|
|
16385
|
+
"link",
|
|
16386
|
+
"listbox",
|
|
16387
|
+
"menuitem",
|
|
16388
|
+
"menuitemcheckbox",
|
|
16389
|
+
"menuitemradio",
|
|
16390
|
+
"option",
|
|
16391
|
+
"radio",
|
|
16392
|
+
"scrollbar",
|
|
16393
|
+
"searchbox",
|
|
16394
|
+
"slider",
|
|
16395
|
+
"spinbutton",
|
|
16396
|
+
"switch",
|
|
16397
|
+
"tab",
|
|
16398
|
+
"textbox",
|
|
16399
|
+
"treeitem"
|
|
16400
|
+
]);
|
|
16401
|
+
var BROWSER_STRUCTURAL_ROLES = /* @__PURE__ */ new Set([
|
|
16402
|
+
"alert",
|
|
16403
|
+
"cell",
|
|
16404
|
+
"columnheader",
|
|
16405
|
+
"dialog",
|
|
16406
|
+
"figure",
|
|
16407
|
+
"gridcell",
|
|
16408
|
+
"heading",
|
|
16409
|
+
"image",
|
|
16410
|
+
"listitem",
|
|
16411
|
+
"main",
|
|
16412
|
+
"paragraph",
|
|
16413
|
+
"region",
|
|
16414
|
+
"row",
|
|
16415
|
+
"rowheader",
|
|
16416
|
+
"status",
|
|
16417
|
+
"StaticText"
|
|
16418
|
+
]);
|
|
16419
|
+
var BROWSER_STATE_PROPERTIES = /* @__PURE__ */ new Set([
|
|
16420
|
+
"autocomplete",
|
|
16421
|
+
"checked",
|
|
16422
|
+
"disabled",
|
|
16423
|
+
"editable",
|
|
16424
|
+
"expanded",
|
|
16425
|
+
"focusable",
|
|
16426
|
+
"focused",
|
|
16427
|
+
"haspopup",
|
|
16428
|
+
"invalid",
|
|
16429
|
+
"level",
|
|
16430
|
+
"multiselectable",
|
|
16431
|
+
"orientation",
|
|
16432
|
+
"pressed",
|
|
16433
|
+
"readonly",
|
|
16434
|
+
"protected",
|
|
16435
|
+
"required",
|
|
16436
|
+
"selected",
|
|
16437
|
+
"url"
|
|
16438
|
+
]);
|
|
16439
|
+
function clippedText(value, limit = 240) {
|
|
16440
|
+
return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
|
|
16441
|
+
}
|
|
16442
|
+
function browserTargetOutput(page) {
|
|
16443
|
+
return {
|
|
16444
|
+
targetId: page.id ?? null,
|
|
16445
|
+
title: (page.title ?? "").slice(0, 1e3),
|
|
16446
|
+
url: (page.url ?? "").slice(0, 2e3)
|
|
16447
|
+
};
|
|
16448
|
+
}
|
|
16449
|
+
function browserNodeActionable(role, states) {
|
|
16450
|
+
return BROWSER_INTERACTIVE_ROLES.has(role) || states.editable === true || states.focusable === true && role !== "RootWebArea";
|
|
16451
|
+
}
|
|
16452
|
+
async function captureBrowserSnapshot(page, options) {
|
|
16453
|
+
const initialCommands = [
|
|
16454
|
+
{ method: "Page.getFrameTree", params: {} },
|
|
16455
|
+
{ method: "Page.getLayoutMetrics", params: {} },
|
|
16456
|
+
{ method: "DOMSnapshot.captureSnapshot", params: { computedStyles: [], includeDOMRects: true, includePaintOrder: true } }
|
|
16457
|
+
];
|
|
16458
|
+
const initial = await sendChromeCommands(page.webSocketDebuggerUrl, initialCommands);
|
|
16459
|
+
const frameTree = chromeResult(initial[0], initialCommands[0].method).frameTree;
|
|
16460
|
+
const layout = chromeResult(initial[1], initialCommands[1].method);
|
|
16461
|
+
const domSnapshot = chromeResult(initial[2], initialCommands[2].method);
|
|
16462
|
+
const viewport = chromeVisualViewport(layout);
|
|
16463
|
+
const frameIds = collectFrameIds(frameTree);
|
|
16464
|
+
if (frameIds.length === 0 && page.id) frameIds.push(page.id);
|
|
16465
|
+
const axCommands = frameIds.map((frameId) => ({
|
|
16466
|
+
method: "Accessibility.getFullAXTree",
|
|
16467
|
+
params: { frameId }
|
|
16468
|
+
}));
|
|
16469
|
+
const axResults = await sendChromeCommands(page.webSocketDebuggerUrl, axCommands);
|
|
16470
|
+
const nodes = [];
|
|
16471
|
+
axResults.forEach((response, index) => {
|
|
16472
|
+
if (response.error) return;
|
|
16473
|
+
const responseNodes = response.result?.nodes;
|
|
16474
|
+
if (!Array.isArray(responseNodes)) return;
|
|
16475
|
+
for (const node of responseNodes) {
|
|
16476
|
+
if (!isRawAXNode(node)) continue;
|
|
16477
|
+
nodes.push({ ...node, sourceFrameId: frameIds[index] });
|
|
16478
|
+
}
|
|
16479
|
+
});
|
|
16480
|
+
const bounds = buildBrowserBounds(domSnapshot, viewport, page.id);
|
|
16481
|
+
const nodesByKey = new Map(nodes.map((node) => [`${node.sourceFrameId}:${node.nodeId}`, node]));
|
|
16482
|
+
const propertyMap = browserStateProperties;
|
|
16483
|
+
const roleOf = (node) => clippedText(node.role?.value, 80);
|
|
16484
|
+
const nameOf = (node) => clippedText(node.name?.value);
|
|
16485
|
+
const valueOf = (node) => {
|
|
16486
|
+
if (propertyMap(node).protected === true) return null;
|
|
16487
|
+
const value = node.value?.value;
|
|
16488
|
+
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
16489
|
+
const text = clippedText(value);
|
|
16490
|
+
if (/^[•●*]+$/.test(text)) return "[protected]";
|
|
16491
|
+
return text || null;
|
|
16492
|
+
};
|
|
16493
|
+
const meaningful = nodes.filter((node) => !node.ignored && Number.isInteger(node.backendDOMNodeId)).map((node) => ({
|
|
16494
|
+
node,
|
|
16495
|
+
role: roleOf(node),
|
|
16496
|
+
bound: typeof node.backendDOMNodeId === "number" ? bounds.get(node.backendDOMNodeId) : void 0
|
|
16497
|
+
})).filter(({ node, role, bound }) => {
|
|
16498
|
+
if (!bound || bound.rect.width <= 0 || bound.rect.height <= 0) return false;
|
|
16499
|
+
if (role === "InlineTextBox" || role === "none" || role === "generic") return false;
|
|
16500
|
+
if (BROWSER_INTERACTIVE_ROLES.has(role) || BROWSER_STRUCTURAL_ROLES.has(role)) return true;
|
|
16501
|
+
return !!nameOf(node) || valueOf(node) !== null || propertyMap(node).focusable === true;
|
|
16502
|
+
}).filter(({ node, role }) => {
|
|
16503
|
+
if (role !== "StaticText") return true;
|
|
16504
|
+
const parent = node.parentId ? nodesByKey.get(`${node.sourceFrameId}:${node.parentId}`) : void 0;
|
|
16505
|
+
return !parent || nameOf(parent) !== nameOf(node);
|
|
16506
|
+
}).sort((a, b) => a.bound.rect.y - b.bound.rect.y || a.bound.rect.x - b.bound.rect.x);
|
|
16507
|
+
const elements = [];
|
|
16508
|
+
const entries = [];
|
|
16509
|
+
const meaningfulKeys = new Set(meaningful.map(({ node }) => `${node.sourceFrameId}:${node.nodeId}`));
|
|
16510
|
+
const visibleText = [];
|
|
16511
|
+
let visibleTextLength = 0;
|
|
16512
|
+
let treeLength = 0;
|
|
16513
|
+
const treeLimit = Math.min(5e4, Math.max(4e3, options.textLimit * 2));
|
|
16514
|
+
for (const { node, role, bound } of meaningful) {
|
|
16515
|
+
const name = nameOf(node);
|
|
16516
|
+
const description = clippedText(node.description?.value) || null;
|
|
16517
|
+
const value = valueOf(node);
|
|
16518
|
+
const states = propertyMap(node);
|
|
16519
|
+
const actionable = browserNodeActionable(role, states);
|
|
16520
|
+
const ref = actionable && elements.length < options.elementLimit ? String(node.backendDOMNodeId) : null;
|
|
16521
|
+
const visibleRect = role === "RootWebArea" && node.sourceFrameId === page.id ? { x: 0, y: 0, width: viewport.width, height: viewport.height, centerX: viewport.width / 2, centerY: viewport.height / 2 } : intersectRects(bound.rect, bound.clip) ?? bound.rect;
|
|
16522
|
+
const rect = roundBrowserRect(visibleRect);
|
|
16523
|
+
const actions = [
|
|
16524
|
+
...actionable ? ["click"] : [],
|
|
16525
|
+
...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
|
|
16526
|
+
"scroll"
|
|
16527
|
+
];
|
|
16528
|
+
if (ref) {
|
|
16529
|
+
elements.push({
|
|
16530
|
+
ref,
|
|
16531
|
+
role,
|
|
16532
|
+
name: name || null,
|
|
16533
|
+
description,
|
|
16534
|
+
value,
|
|
16535
|
+
states,
|
|
16536
|
+
actions,
|
|
16537
|
+
rect,
|
|
16538
|
+
frameId: node.sourceFrameId
|
|
16539
|
+
});
|
|
16540
|
+
}
|
|
16541
|
+
const stateText = Object.entries(states).filter(([key]) => key !== "url" && key !== "focusable").map(([key, state]) => `${key}=${JSON.stringify(state)}`);
|
|
16542
|
+
const url = typeof states.url === "string" ? clippedText(states.url, 300) : "";
|
|
16543
|
+
const semantic = [role, name && JSON.stringify(name), value !== null && `value=${JSON.stringify(value)}`, description && `description=${JSON.stringify(description)}`, url && `url=${JSON.stringify(url)}`, ...stateText].filter(Boolean).join(" ");
|
|
16544
|
+
let depth = 0;
|
|
16545
|
+
let parentId = node.parentId;
|
|
16546
|
+
const visited = /* @__PURE__ */ new Set();
|
|
16547
|
+
while (parentId && depth < 8) {
|
|
16548
|
+
const parentKey = `${node.sourceFrameId}:${parentId}`;
|
|
16549
|
+
if (visited.has(parentKey)) break;
|
|
16550
|
+
visited.add(parentKey);
|
|
16551
|
+
if (meaningfulKeys.has(parentKey)) depth++;
|
|
16552
|
+
parentId = nodesByKey.get(parentKey)?.parentId;
|
|
16553
|
+
}
|
|
16554
|
+
const line = `${" ".repeat(depth)}${ref ? `[ref=${ref}]` : "-"} ${semantic} (${rect.x},${rect.y} ${rect.width}x${rect.height})`;
|
|
16555
|
+
if (treeLength + line.length + 1 <= treeLimit) {
|
|
16556
|
+
const key = Number.isInteger(node.backendDOMNodeId) ? `dom:${node.sourceFrameId}:${node.backendDOMNodeId}` : `ax:${node.sourceFrameId}:${node.nodeId}`;
|
|
16557
|
+
entries.push({ key, semantic, line });
|
|
16558
|
+
treeLength += line.length + 1;
|
|
16559
|
+
}
|
|
16560
|
+
if (name && visibleTextLength < options.textLimit && (role === "StaticText" || role === "heading" || BROWSER_INTERACTIVE_ROLES.has(role))) {
|
|
16561
|
+
const remaining = options.textLimit - visibleTextLength;
|
|
16562
|
+
visibleText.push(name.slice(0, remaining));
|
|
16563
|
+
visibleTextLength += Math.min(name.length, remaining) + 1;
|
|
16564
|
+
}
|
|
16565
|
+
}
|
|
16566
|
+
const revision = createHash2("sha256").update(JSON.stringify([page.url, entries.map(({ key, semantic }) => [key, semantic])])).digest("hex").slice(0, 16);
|
|
16567
|
+
const snapshot = {
|
|
16568
|
+
title: (page.title ?? "").slice(0, 1e3),
|
|
16569
|
+
url: (page.url ?? "").slice(0, 2e3),
|
|
16570
|
+
documentId: browserDocumentId(frameTree),
|
|
16571
|
+
revision,
|
|
16572
|
+
viewport,
|
|
16573
|
+
text: visibleText.join(" ").slice(0, options.textLimit),
|
|
16574
|
+
tree: entries.map(({ line }) => line).join("\n"),
|
|
16575
|
+
elements,
|
|
16576
|
+
controls: elements,
|
|
16577
|
+
nodeCount: entries.length,
|
|
16578
|
+
elementCount: elements.length,
|
|
16579
|
+
truncated: entries.length < meaningful.length || elements.length < meaningful.filter(({ node, role }) => browserNodeActionable(role, propertyMap(node))).length
|
|
16580
|
+
};
|
|
16581
|
+
return { snapshot, entries };
|
|
16582
|
+
}
|
|
16134
16583
|
async function computerBrowserCommand(options = {}) {
|
|
16135
16584
|
const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 4e3;
|
|
16136
16585
|
const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 80;
|
|
16137
16586
|
if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
|
|
16138
16587
|
if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
|
|
16139
|
-
const
|
|
16140
|
-
const pages =
|
|
16588
|
+
const targets = options.targetId || options.page || options.title || options.url ? [await selectChromePage(options)] : await getChromePages();
|
|
16589
|
+
const pages = targets.map((target) => ({
|
|
16141
16590
|
target,
|
|
16142
16591
|
page: {
|
|
16143
16592
|
id: target.id ?? null,
|
|
@@ -16150,7 +16599,11 @@ async function computerBrowserCommand(options = {}) {
|
|
|
16150
16599
|
for (const { target, page } of pages) {
|
|
16151
16600
|
if (options.snapshot && target.webSocketDebuggerUrl) {
|
|
16152
16601
|
try {
|
|
16153
|
-
|
|
16602
|
+
const { snapshot } = await captureBrowserSnapshot(
|
|
16603
|
+
{ ...target, webSocketDebuggerUrl: target.webSocketDebuggerUrl },
|
|
16604
|
+
{ textLimit, elementLimit }
|
|
16605
|
+
);
|
|
16606
|
+
pageResults.push({ ...page, snapshot });
|
|
16154
16607
|
} catch (error) {
|
|
16155
16608
|
pageResults.push({ ...page, snapshotError: error instanceof Error ? error.message : "snapshot failed" });
|
|
16156
16609
|
}
|
|
@@ -16164,6 +16617,188 @@ async function computerBrowserCommand(options = {}) {
|
|
|
16164
16617
|
pages: pageResults
|
|
16165
16618
|
}, null, 2));
|
|
16166
16619
|
}
|
|
16620
|
+
function buildBrowserStabilityExpression() {
|
|
16621
|
+
return `(() => {
|
|
16622
|
+
const key = '__replicasComputerUseObserver';
|
|
16623
|
+
if (!globalThis[key]) {
|
|
16624
|
+
const state = { revision: 0 };
|
|
16625
|
+
const observer = new MutationObserver(() => state.revision++);
|
|
16626
|
+
globalThis[key] = { state, observer, observing: false };
|
|
16627
|
+
}
|
|
16628
|
+
if (document.documentElement && !globalThis[key].observing) {
|
|
16629
|
+
globalThis[key].observer.observe(document.documentElement, {
|
|
16630
|
+
subtree: true,
|
|
16631
|
+
childList: true,
|
|
16632
|
+
attributes: true,
|
|
16633
|
+
characterData: true,
|
|
16634
|
+
});
|
|
16635
|
+
globalThis[key].observing = true;
|
|
16636
|
+
}
|
|
16637
|
+
const controls = Array.from(document.querySelectorAll('input,textarea,select,[contenteditable="true"],[role="checkbox"],[role="combobox"],[role="slider"],[role="switch"],[role="textbox"]')).slice(0, 200);
|
|
16638
|
+
const active = document.activeElement;
|
|
16639
|
+
const controlState = controls.map((element, index) => {
|
|
16640
|
+
const value = 'value' in element
|
|
16641
|
+
? element instanceof HTMLInputElement && element.type === 'password'
|
|
16642
|
+
? String(element.value).length
|
|
16643
|
+
: String(element.value)
|
|
16644
|
+
: element.textContent;
|
|
16645
|
+
return [
|
|
16646
|
+
index,
|
|
16647
|
+
element === active,
|
|
16648
|
+
value,
|
|
16649
|
+
'checked' in element ? element.checked : null,
|
|
16650
|
+
'selectedIndex' in element ? element.selectedIndex : null,
|
|
16651
|
+
'disabled' in element ? element.disabled : null,
|
|
16652
|
+
element.getAttribute('aria-checked'),
|
|
16653
|
+
element.getAttribute('aria-expanded'),
|
|
16654
|
+
element.getAttribute('aria-selected'),
|
|
16655
|
+
].join('|');
|
|
16656
|
+
}).join('\\u001f');
|
|
16657
|
+
let controlRevision = 2166136261;
|
|
16658
|
+
for (let index = 0; index < controlState.length; index++) {
|
|
16659
|
+
controlRevision ^= controlState.charCodeAt(index);
|
|
16660
|
+
controlRevision = Math.imul(controlRevision, 16777619);
|
|
16661
|
+
}
|
|
16662
|
+
return {
|
|
16663
|
+
revision: globalThis[key].state.revision,
|
|
16664
|
+
controlRevision: (controlRevision >>> 0).toString(16),
|
|
16665
|
+
readyState: document.readyState,
|
|
16666
|
+
busy: !!document.querySelector('[aria-busy="true"]'),
|
|
16667
|
+
url: location.href.slice(0, 2000),
|
|
16668
|
+
scrollX: Math.round(scrollX),
|
|
16669
|
+
scrollY: Math.round(scrollY),
|
|
16670
|
+
innerWidth,
|
|
16671
|
+
innerHeight,
|
|
16672
|
+
devicePixelRatio,
|
|
16673
|
+
};
|
|
16674
|
+
})()`;
|
|
16675
|
+
}
|
|
16676
|
+
async function waitForBrowserStability(page, options) {
|
|
16677
|
+
const startedAt = Date.now();
|
|
16678
|
+
let lastSignature = null;
|
|
16679
|
+
let lastChangeAt = startedAt;
|
|
16680
|
+
let samples = 0;
|
|
16681
|
+
let changes = 0;
|
|
16682
|
+
let state = null;
|
|
16683
|
+
while (Date.now() - startedAt <= options.timeoutMs) {
|
|
16684
|
+
try {
|
|
16685
|
+
state = await evaluateChromeTarget(page.webSocketDebuggerUrl, buildBrowserStabilityExpression());
|
|
16686
|
+
samples++;
|
|
16687
|
+
const signature = JSON.stringify(state);
|
|
16688
|
+
const now = Date.now();
|
|
16689
|
+
if (lastSignature === null || signature !== lastSignature) {
|
|
16690
|
+
if (lastSignature !== null) changes++;
|
|
16691
|
+
lastSignature = signature;
|
|
16692
|
+
lastChangeAt = now;
|
|
16693
|
+
}
|
|
16694
|
+
const ready = typeof state === "object" && state !== null && "readyState" in state && state.readyState !== "loading" && "busy" in state && state.busy !== true;
|
|
16695
|
+
if (samples > 1 && ready && now - lastChangeAt >= options.stableMs) {
|
|
16696
|
+
return { stable: true, elapsedMs: now - startedAt, samples, changes, state };
|
|
16697
|
+
}
|
|
16698
|
+
} catch {
|
|
16699
|
+
}
|
|
16700
|
+
await sleep(options.pollMs);
|
|
16701
|
+
}
|
|
16702
|
+
return { stable: false, elapsedMs: Date.now() - startedAt, samples, changes, state };
|
|
16703
|
+
}
|
|
16704
|
+
function browserStateCachePath(targetId) {
|
|
16705
|
+
return `${STATE_DIR}/browser-state-${targetId.replace(/[^a-zA-Z0-9_-]/g, "_")}.json`;
|
|
16706
|
+
}
|
|
16707
|
+
function browserStateDiff(previous, current) {
|
|
16708
|
+
const before = new Map(previous.map((entry) => [entry.key, entry]));
|
|
16709
|
+
const after = new Map(current.map((entry) => [entry.key, entry]));
|
|
16710
|
+
const lines = [];
|
|
16711
|
+
let added = 0;
|
|
16712
|
+
let changed = 0;
|
|
16713
|
+
let removed = 0;
|
|
16714
|
+
for (const entry of current) {
|
|
16715
|
+
const old = before.get(entry.key);
|
|
16716
|
+
if (!old) {
|
|
16717
|
+
added++;
|
|
16718
|
+
lines.push(`+ ${entry.line}`);
|
|
16719
|
+
} else if (old.semantic !== entry.semantic) {
|
|
16720
|
+
changed++;
|
|
16721
|
+
lines.push(`~ ${old.line}
|
|
16722
|
+
${entry.line}`);
|
|
16723
|
+
}
|
|
16724
|
+
}
|
|
16725
|
+
for (const entry of previous) {
|
|
16726
|
+
if (after.has(entry.key)) continue;
|
|
16727
|
+
removed++;
|
|
16728
|
+
lines.push(`- ${entry.line}`);
|
|
16729
|
+
}
|
|
16730
|
+
return { tree: lines.join("\n") || "No accessibility changes.", added, changed, removed };
|
|
16731
|
+
}
|
|
16732
|
+
async function computerBrowserStateCommand(path6, options = {}) {
|
|
16733
|
+
const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 8e3;
|
|
16734
|
+
const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 120;
|
|
16735
|
+
const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 5e3;
|
|
16736
|
+
const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 500;
|
|
16737
|
+
const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
|
|
16738
|
+
if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
|
|
16739
|
+
if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
|
|
16740
|
+
if (timeoutMs < 0) fail("--timeout must be >= 0");
|
|
16741
|
+
if (stableMs < 0) fail("--stable-ms must be >= 0");
|
|
16742
|
+
if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
|
|
16743
|
+
const page = await selectChromePage(options);
|
|
16744
|
+
const stability = await waitForBrowserStability(page, { timeoutMs, stableMs, pollMs });
|
|
16745
|
+
const target = resolvePath(path6);
|
|
16746
|
+
mkdirSync3(dirname3(target), { recursive: true });
|
|
16747
|
+
const [{ snapshot, entries }, screenshotResult] = await Promise.all([
|
|
16748
|
+
captureBrowserSnapshot(page, { textLimit, elementLimit }),
|
|
16749
|
+
sendChromeCommand(page.webSocketDebuggerUrl, "Page.captureScreenshot", {
|
|
16750
|
+
format: "png",
|
|
16751
|
+
fromSurface: true,
|
|
16752
|
+
captureBeyondViewport: false
|
|
16753
|
+
})
|
|
16754
|
+
]);
|
|
16755
|
+
const data = screenshotResult.data;
|
|
16756
|
+
if (typeof data !== "string") fail("Chrome did not return screenshot data");
|
|
16757
|
+
writeFileSync3(target, Buffer.from(data, "base64"));
|
|
16758
|
+
const screenshot = readPngDimensions(target);
|
|
16759
|
+
const targetId = page.id ?? "unknown";
|
|
16760
|
+
const cachePath = browserStateCachePath(targetId);
|
|
16761
|
+
let mode = "full";
|
|
16762
|
+
let tree = snapshot.tree;
|
|
16763
|
+
let changes = { added: snapshot.nodeCount, changed: 0, removed: 0 };
|
|
16764
|
+
if (!options.full && existsSync3(cachePath)) {
|
|
16765
|
+
const cached = readBrowserStateCache(cachePath);
|
|
16766
|
+
if (cached?.url === snapshot.url && cached.documentId === snapshot.documentId) {
|
|
16767
|
+
mode = "diff";
|
|
16768
|
+
const diff = browserStateDiff(cached.entries, entries);
|
|
16769
|
+
tree = diff.tree;
|
|
16770
|
+
changes = { added: diff.added, changed: diff.changed, removed: diff.removed };
|
|
16771
|
+
}
|
|
16772
|
+
}
|
|
16773
|
+
mkdirSync3(STATE_DIR, { recursive: true });
|
|
16774
|
+
writeFileSync3(cachePath, JSON.stringify({ url: snapshot.url, documentId: snapshot.documentId, entries }));
|
|
16775
|
+
const stableState = isRecord(stability.state) ? stability.state : {};
|
|
16776
|
+
const state = {
|
|
16777
|
+
title: snapshot.title,
|
|
16778
|
+
url: snapshot.url,
|
|
16779
|
+
documentId: snapshot.documentId,
|
|
16780
|
+
revision: snapshot.revision,
|
|
16781
|
+
viewport: snapshot.viewport,
|
|
16782
|
+
text: snapshot.text,
|
|
16783
|
+
tree,
|
|
16784
|
+
nodeCount: snapshot.nodeCount,
|
|
16785
|
+
elementCount: snapshot.elementCount,
|
|
16786
|
+
truncated: snapshot.truncated,
|
|
16787
|
+
mode,
|
|
16788
|
+
changes
|
|
16789
|
+
};
|
|
16790
|
+
console.log(JSON.stringify({
|
|
16791
|
+
targetId,
|
|
16792
|
+
screenshot: {
|
|
16793
|
+
path: target,
|
|
16794
|
+
width: screenshot.width,
|
|
16795
|
+
height: screenshot.height,
|
|
16796
|
+
devicePixelRatio: typeof stableState.devicePixelRatio === "number" ? stableState.devicePixelRatio : 1
|
|
16797
|
+
},
|
|
16798
|
+
stability,
|
|
16799
|
+
state
|
|
16800
|
+
}, null, 2));
|
|
16801
|
+
}
|
|
16167
16802
|
var DESKTOP_POINT_JS = `
|
|
16168
16803
|
const desktopPoint = (rect) => {
|
|
16169
16804
|
const borderX = Math.max(0, (window.outerWidth - window.innerWidth) / 2);
|
|
@@ -16173,17 +16808,95 @@ var DESKTOP_POINT_JS = `
|
|
|
16173
16808
|
y: Math.round(window.screenY + topChrome + rect.y + rect.height / 2),
|
|
16174
16809
|
};
|
|
16175
16810
|
};`;
|
|
16811
|
+
function parseBrowserRef(value) {
|
|
16812
|
+
const ref = parseCoord(value.replace(/^ref=/, ""), "--ref");
|
|
16813
|
+
if (ref <= 0) fail("--ref must be a positive backend DOM node ID from the latest browser state");
|
|
16814
|
+
return ref;
|
|
16815
|
+
}
|
|
16816
|
+
function readBrowserRefCache(page) {
|
|
16817
|
+
const targetId = page.id;
|
|
16818
|
+
if (!targetId) fail("Chrome target has no stable ID. Capture fresh browser state and retry.");
|
|
16819
|
+
const cachePath = browserStateCachePath(targetId);
|
|
16820
|
+
if (!existsSync3(cachePath)) fail(`No browser state is cached for target ${targetId}. Run replicas computer browser-state <path> --target-id ${targetId} first.`);
|
|
16821
|
+
const cached = readBrowserStateCache(cachePath);
|
|
16822
|
+
if (!cached) fail(`Browser state for target ${targetId} is invalid. Capture fresh browser state and retry.`);
|
|
16823
|
+
if (cached.url !== (page.url ?? "").slice(0, 2e3)) fail("Cached browser state belongs to a different URL. Capture fresh browser state and retry.");
|
|
16824
|
+
return { targetId, cached };
|
|
16825
|
+
}
|
|
16826
|
+
function browserViewportRect(layout) {
|
|
16827
|
+
const viewport = chromeVisualViewport(layout);
|
|
16828
|
+
return {
|
|
16829
|
+
x: 0,
|
|
16830
|
+
y: 0,
|
|
16831
|
+
width: viewport.width,
|
|
16832
|
+
height: viewport.height,
|
|
16833
|
+
centerX: viewport.width / 2,
|
|
16834
|
+
centerY: viewport.height / 2
|
|
16835
|
+
};
|
|
16836
|
+
}
|
|
16837
|
+
async function prepareBrowserElement(page, refValue) {
|
|
16838
|
+
const backendNodeId = parseBrowserRef(refValue);
|
|
16839
|
+
const { cached } = readBrowserRefCache(page);
|
|
16840
|
+
return await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
16841
|
+
const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
|
|
16842
|
+
const rect = await prepareBrowserRefInSession(session, backendNodeId, cached, browserViewportRect(layout));
|
|
16843
|
+
const axNodes = (await sendChromeSessionCommand(session, "Accessibility.getPartialAXTree", {
|
|
16844
|
+
backendNodeId,
|
|
16845
|
+
fetchRelatives: false
|
|
16846
|
+
})).nodes;
|
|
16847
|
+
const rawNode = Array.isArray(axNodes) && isRawAXNode(axNodes[0]) ? axNodes[0] : null;
|
|
16848
|
+
const states = rawNode ? browserStateProperties(rawNode) : {};
|
|
16849
|
+
const role = clippedText(rawNode?.role?.value, 80);
|
|
16850
|
+
const element = rawNode ? {
|
|
16851
|
+
ref: String(backendNodeId),
|
|
16852
|
+
role,
|
|
16853
|
+
name: clippedText(rawNode.name?.value) || null,
|
|
16854
|
+
description: clippedText(rawNode.description?.value) || null,
|
|
16855
|
+
value: typeof rawNode.value?.value === "string" ? clippedText(rawNode.value.value) : typeof rawNode.value?.value === "number" || typeof rawNode.value?.value === "boolean" ? rawNode.value.value : null,
|
|
16856
|
+
states,
|
|
16857
|
+
actions: [
|
|
16858
|
+
"click",
|
|
16859
|
+
...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
|
|
16860
|
+
"scroll"
|
|
16861
|
+
]
|
|
16862
|
+
} : null;
|
|
16863
|
+
const desktopPoint = await evaluateChromeSession(session, `(() => {
|
|
16864
|
+
${DESKTOP_POINT_JS}
|
|
16865
|
+
return desktopPoint({ x: ${rect.centerX}, y: ${rect.centerY}, width: 0, height: 0 });
|
|
16866
|
+
})()`);
|
|
16867
|
+
return {
|
|
16868
|
+
ref: String(backendNodeId),
|
|
16869
|
+
rect,
|
|
16870
|
+
element,
|
|
16871
|
+
desktopPoint: typeof desktopPoint === "object" && desktopPoint !== null && "x" in desktopPoint && typeof desktopPoint.x === "number" && "y" in desktopPoint && typeof desktopPoint.y === "number" ? { x: desktopPoint.x, y: desktopPoint.y } : null
|
|
16872
|
+
};
|
|
16873
|
+
});
|
|
16874
|
+
}
|
|
16875
|
+
async function prepareBrowserRefInSession(session, backendNodeId, cached, viewportRect) {
|
|
16876
|
+
if (!cached.entries.some(({ line }) => line.trimStart().startsWith(`[ref=${backendNodeId}]`))) {
|
|
16877
|
+
fail(`Browser element ref ${backendNodeId} is not in the latest state. Capture fresh browser state and retry.`);
|
|
16878
|
+
}
|
|
16879
|
+
const frameTree = (await sendChromeSessionCommand(session, "Page.getFrameTree", {})).frameTree;
|
|
16880
|
+
if (cached.documentId !== browserDocumentId(frameTree)) {
|
|
16881
|
+
fail(`Browser element ref ${backendNodeId} belongs to a previous document. Capture fresh browser state and retry.`);
|
|
16882
|
+
}
|
|
16883
|
+
try {
|
|
16884
|
+
await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId });
|
|
16885
|
+
} catch (error) {
|
|
16886
|
+
fail(`Browser element ref ${backendNodeId} is stale or cannot be scrolled into view: ${error instanceof Error ? error.message : String(error)}`);
|
|
16887
|
+
}
|
|
16888
|
+
const quads = (await sendChromeSessionCommand(session, "DOM.getContentQuads", { backendNodeId })).quads;
|
|
16889
|
+
const candidates = browserContentQuadRects(quads, viewportRect);
|
|
16890
|
+
if (!candidates[0]) fail(`Browser element ref ${backendNodeId} has no visible content quad. Fetch fresh browser state and retry.`);
|
|
16891
|
+
return roundBrowserRect(candidates[0]);
|
|
16892
|
+
}
|
|
16176
16893
|
function buildBrowserClickExpression(query, options) {
|
|
16177
16894
|
return `(() => {
|
|
16178
16895
|
const query = ${JSON.stringify(query)};
|
|
16179
16896
|
const exact = ${JSON.stringify(options.exact)};
|
|
16180
16897
|
const index = ${options.index};
|
|
16181
16898
|
const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
|
|
16182
|
-
|
|
16183
|
-
const rect = el.getBoundingClientRect();
|
|
16184
|
-
const style = getComputedStyle(el);
|
|
16185
|
-
return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
|
|
16186
|
-
};
|
|
16899
|
+
${VISIBLE_JS}
|
|
16187
16900
|
const label = (el) => normalize(el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('placeholder') || el.href || el.id || el.name || el.tagName);
|
|
16188
16901
|
${DESKTOP_POINT_JS}
|
|
16189
16902
|
const matches = (text) => {
|
|
@@ -16208,8 +16921,7 @@ ${DESKTOP_POINT_JS}
|
|
|
16208
16921
|
const { el, text } = candidate;
|
|
16209
16922
|
el.scrollIntoView({ block: 'center', inline: 'center' });
|
|
16210
16923
|
const rect = el.getBoundingClientRect();
|
|
16211
|
-
if (typeof el.focus === 'function') el.focus();
|
|
16212
|
-
el.click();
|
|
16924
|
+
if (typeof el.focus === 'function') el.focus({ preventScroll: true });
|
|
16213
16925
|
return {
|
|
16214
16926
|
clicked: true,
|
|
16215
16927
|
query,
|
|
@@ -16232,6 +16944,15 @@ ${DESKTOP_POINT_JS}
|
|
|
16232
16944
|
};
|
|
16233
16945
|
})()`;
|
|
16234
16946
|
}
|
|
16947
|
+
function browserActionRectCenter(result) {
|
|
16948
|
+
if (typeof result !== "object" || result === null || !("rect" in result)) return null;
|
|
16949
|
+
const rect = result.rect;
|
|
16950
|
+
if (typeof rect !== "object" || rect === null) return null;
|
|
16951
|
+
const x = "centerX" in rect ? rect.centerX : void 0;
|
|
16952
|
+
const y = "centerY" in rect ? rect.centerY : void 0;
|
|
16953
|
+
if (typeof x !== "number" || typeof y !== "number" || !Number.isFinite(x) || !Number.isFinite(y)) return null;
|
|
16954
|
+
return { x, y };
|
|
16955
|
+
}
|
|
16235
16956
|
function browserActionPoint(result) {
|
|
16236
16957
|
if (typeof result !== "object" || result === null) return null;
|
|
16237
16958
|
const action = "result" in result ? result.result : result;
|
|
@@ -16247,19 +16968,149 @@ function browserActionPoint(result) {
|
|
|
16247
16968
|
y: clamp(Math.round(y), 0, dimensions.height)
|
|
16248
16969
|
};
|
|
16249
16970
|
}
|
|
16971
|
+
async function dispatchChromeClick(page, x, y, button, clicks) {
|
|
16972
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
16973
|
+
await dispatchChromeClickInSession(session, x, y, button, clicks);
|
|
16974
|
+
});
|
|
16975
|
+
}
|
|
16976
|
+
async function dispatchChromeClickInSession(session, x, y, button, clicks) {
|
|
16977
|
+
for (let clickCount = 1; clickCount <= clicks; clickCount++) {
|
|
16978
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
|
|
16979
|
+
type: "mousePressed",
|
|
16980
|
+
x,
|
|
16981
|
+
y,
|
|
16982
|
+
button,
|
|
16983
|
+
clickCount
|
|
16984
|
+
});
|
|
16985
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
|
|
16986
|
+
type: "mouseReleased",
|
|
16987
|
+
x,
|
|
16988
|
+
y,
|
|
16989
|
+
button,
|
|
16990
|
+
clickCount
|
|
16991
|
+
});
|
|
16992
|
+
}
|
|
16993
|
+
}
|
|
16994
|
+
async function replaceChromeText(page, text) {
|
|
16995
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
16996
|
+
await replaceChromeTextInSession(session, text);
|
|
16997
|
+
});
|
|
16998
|
+
}
|
|
16999
|
+
async function replaceChromeTextInSession(session, text) {
|
|
17000
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
17001
|
+
type: "rawKeyDown",
|
|
17002
|
+
key: "a",
|
|
17003
|
+
code: "KeyA",
|
|
17004
|
+
windowsVirtualKeyCode: 65,
|
|
17005
|
+
nativeVirtualKeyCode: 65,
|
|
17006
|
+
modifiers: 2
|
|
17007
|
+
});
|
|
17008
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
17009
|
+
type: "keyUp",
|
|
17010
|
+
key: "a",
|
|
17011
|
+
code: "KeyA",
|
|
17012
|
+
windowsVirtualKeyCode: 65,
|
|
17013
|
+
nativeVirtualKeyCode: 65,
|
|
17014
|
+
modifiers: 2
|
|
17015
|
+
});
|
|
17016
|
+
await sendChromeSessionCommand(session, "Input.insertText", { text });
|
|
17017
|
+
}
|
|
17018
|
+
var BROWSER_DIRECT_VALUE_INPUT_TYPES = /* @__PURE__ */ new Set(["number", "range", "date", "datetime-local", "month", "week", "time", "color"]);
|
|
17019
|
+
function browserFillInvalid(actual, expected, fieldType, semanticSelection = false, selectionMatched = false) {
|
|
17020
|
+
if (semanticSelection) return !selectionMatched;
|
|
17021
|
+
if (["number", "range"].includes(fieldType)) {
|
|
17022
|
+
const actualNumber = typeof actual === "number" ? actual : Number(actual);
|
|
17023
|
+
const expectedNumber = Number(expected);
|
|
17024
|
+
return !Number.isFinite(actualNumber) || !Number.isFinite(expectedNumber) || actualNumber !== expectedNumber;
|
|
17025
|
+
}
|
|
17026
|
+
if (fieldType === "color") return String(actual).toLowerCase() !== expected.toLowerCase();
|
|
17027
|
+
return actual !== expected;
|
|
17028
|
+
}
|
|
17029
|
+
async function fillChromeNodeInSession(session, backendNodeId, value) {
|
|
17030
|
+
await sendChromeSessionCommand(session, "DOM.focus", { backendNodeId });
|
|
17031
|
+
const directResult = await callFunctionOnChromeNodeInSession(
|
|
17032
|
+
session,
|
|
17033
|
+
backendNodeId,
|
|
17034
|
+
`function(value) {
|
|
17035
|
+
const emit = () => {
|
|
17036
|
+
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
17037
|
+
this.dispatchEvent(new Event('change', { bubbles: true }));
|
|
17038
|
+
};
|
|
17039
|
+
if (this instanceof HTMLSelectElement) {
|
|
17040
|
+
const option = Array.from(this.options).find((item) => item.value === value || item.text.trim() === value);
|
|
17041
|
+
if (option) this.value = option.value;
|
|
17042
|
+
else this.value = value;
|
|
17043
|
+
emit();
|
|
17044
|
+
return { handled: true, semanticSelection: true, matched: !!option, value: this.value };
|
|
17045
|
+
}
|
|
17046
|
+
if (this instanceof HTMLInputElement && ${JSON.stringify([...BROWSER_DIRECT_VALUE_INPUT_TYPES])}.includes(this.type.toLowerCase())) {
|
|
17047
|
+
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
|
|
17048
|
+
if (!setter) return { handled: false };
|
|
17049
|
+
setter.call(this, value);
|
|
17050
|
+
emit();
|
|
17051
|
+
return { handled: true, semanticSelection: false, matched: true, value: this.value };
|
|
17052
|
+
}
|
|
17053
|
+
return { handled: false };
|
|
17054
|
+
}`,
|
|
17055
|
+
[value]
|
|
17056
|
+
);
|
|
17057
|
+
const directValue = chromeCallValue(directResult);
|
|
17058
|
+
const directHandled = isRecord(directValue) && directValue.handled === true;
|
|
17059
|
+
if (!directHandled) await replaceChromeTextInSession(session, value);
|
|
17060
|
+
const verification = await callFunctionOnChromeNodeInSession(
|
|
17061
|
+
session,
|
|
17062
|
+
backendNodeId,
|
|
17063
|
+
`function() {
|
|
17064
|
+
const value = this instanceof HTMLSelectElement || 'value' in this ? this.value : this.textContent;
|
|
17065
|
+
const type = this instanceof HTMLSelectElement
|
|
17066
|
+
? 'select'
|
|
17067
|
+
: this instanceof HTMLInputElement
|
|
17068
|
+
? this.type.toLowerCase()
|
|
17069
|
+
: this.isContentEditable
|
|
17070
|
+
? 'contenteditable'
|
|
17071
|
+
: 'text';
|
|
17072
|
+
return { value, valueLength: String(value ?? '').length, type };
|
|
17073
|
+
}`
|
|
17074
|
+
);
|
|
17075
|
+
const actual = chromeCallValue(verification);
|
|
17076
|
+
if (!isRecord(actual)) fail(`Chrome did not return a value for browser element ref ${backendNodeId}.`);
|
|
17077
|
+
const fieldType = typeof actual.type === "string" ? actual.type : "";
|
|
17078
|
+
const semanticSelection = isRecord(directValue) && directValue.semanticSelection === true;
|
|
17079
|
+
const selectionMatched = isRecord(directValue) && directValue.matched === true;
|
|
17080
|
+
if (browserFillInvalid(actual.value, value, fieldType, semanticSelection, selectionMatched)) {
|
|
17081
|
+
fail(`Chrome reported that ref ${backendNodeId} contains ${JSON.stringify(actual.value)} after filling, expected ${JSON.stringify(value)}.`);
|
|
17082
|
+
}
|
|
17083
|
+
return { value: actual.value, valueLength: actual.valueLength, type: fieldType };
|
|
17084
|
+
}
|
|
16250
17085
|
async function computerBrowserClickCommand(query, options = {}) {
|
|
16251
17086
|
const page = await selectChromePage(options);
|
|
17087
|
+
if (options.ref) {
|
|
17088
|
+
const target = await prepareBrowserElement(page, options.ref);
|
|
17089
|
+
const buttonNames = { "1": "left", "2": "middle", "3": "right", left: "left", middle: "middle", right: "right" };
|
|
17090
|
+
const button = buttonNames[(options.button ?? "left").toLowerCase()];
|
|
17091
|
+
if (!button) fail("--button must be one of left|middle|right|1|2|3");
|
|
17092
|
+
const clicks = options.double ? 2 : 1;
|
|
17093
|
+
await dispatchChromeClick(page, target.rect.centerX, target.rect.centerY, button, clicks);
|
|
17094
|
+
if (target.desktopPoint) logRecordingAction({ type: "click", ...target.desktopPoint });
|
|
17095
|
+
console.log(JSON.stringify({
|
|
17096
|
+
...browserTargetOutput(page),
|
|
17097
|
+
result: { clicked: true, ref: target.ref, button, clickCount: clicks, rect: target.rect, element: target.element }
|
|
17098
|
+
}, null, 2));
|
|
17099
|
+
return;
|
|
17100
|
+
}
|
|
17101
|
+
if (!query) fail("Provide control text or --ref <id> from the latest browser state");
|
|
16252
17102
|
const index = options.index ? parseCoord(options.index, "--index") : 0;
|
|
16253
17103
|
if (index < 0) fail("--index must be >= 0");
|
|
16254
17104
|
const result = await evaluateChromeTarget(
|
|
16255
17105
|
page.webSocketDebuggerUrl,
|
|
16256
17106
|
buildBrowserClickExpression(query, { exact: !!options.exact, index })
|
|
16257
17107
|
);
|
|
17108
|
+
const center = browserActionRectCenter(result);
|
|
17109
|
+
if (center) await dispatchChromeClick(page, center.x, center.y, "left", 1);
|
|
16258
17110
|
const point = browserActionPoint(result);
|
|
16259
17111
|
if (point) logRecordingAction({ type: "click", ...point });
|
|
16260
17112
|
console.log(JSON.stringify({
|
|
16261
|
-
|
|
16262
|
-
url: page.url ?? "",
|
|
17113
|
+
...browserTargetOutput(page),
|
|
16263
17114
|
result
|
|
16264
17115
|
}, null, 2));
|
|
16265
17116
|
}
|
|
@@ -16271,11 +17122,7 @@ function buildBrowserFillExpression(query, value, options) {
|
|
|
16271
17122
|
const index = ${options.index};
|
|
16272
17123
|
const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
|
|
16273
17124
|
const escape = (text) => globalThis.CSS && CSS.escape ? CSS.escape(text) : String(text).replace(/["\\\\]/g, '\\\\$&');
|
|
16274
|
-
|
|
16275
|
-
const rect = el.getBoundingClientRect();
|
|
16276
|
-
const style = getComputedStyle(el);
|
|
16277
|
-
return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
|
|
16278
|
-
};
|
|
17125
|
+
${VISIBLE_JS}
|
|
16279
17126
|
const matches = (text) => {
|
|
16280
17127
|
const haystack = text.toLowerCase();
|
|
16281
17128
|
const needle = query.toLowerCase();
|
|
@@ -16320,18 +17167,42 @@ ${DESKTOP_POINT_JS}
|
|
|
16320
17167
|
}
|
|
16321
17168
|
const { el, text } = candidate;
|
|
16322
17169
|
el.scrollIntoView({ block: 'center', inline: 'center' });
|
|
16323
|
-
if (typeof el.focus === 'function') el.focus();
|
|
17170
|
+
if (typeof el.focus === 'function') el.focus({ preventScroll: true });
|
|
17171
|
+
const fieldType = el.tagName.toLowerCase() === 'select' ? 'select' : String(el.getAttribute('type') || 'text').toLowerCase();
|
|
17172
|
+
let nativeInputRequired = false;
|
|
17173
|
+
let semanticSelection = false;
|
|
17174
|
+
let selectionMatched = false;
|
|
16324
17175
|
if (el.tagName.toLowerCase() === 'select') {
|
|
16325
17176
|
const option = Array.from(el.options).find((item) => item.value === value || normalize(item.text) === value);
|
|
16326
17177
|
if (option) el.value = option.value;
|
|
16327
17178
|
else el.value = value;
|
|
17179
|
+
semanticSelection = true;
|
|
17180
|
+
selectionMatched = !!option;
|
|
17181
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
17182
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
17183
|
+
} else if (el instanceof HTMLInputElement && ${JSON.stringify([...BROWSER_DIRECT_VALUE_INPUT_TYPES])}.includes(fieldType)) {
|
|
17184
|
+
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
|
|
17185
|
+
if (setter) {
|
|
17186
|
+
setter.call(el, value);
|
|
17187
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
17188
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
17189
|
+
} else {
|
|
17190
|
+
nativeInputRequired = true;
|
|
17191
|
+
}
|
|
16328
17192
|
} else if (el.isContentEditable) {
|
|
16329
|
-
|
|
17193
|
+
const selection = getSelection();
|
|
17194
|
+
const range = document.createRange();
|
|
17195
|
+
range.selectNodeContents(el);
|
|
17196
|
+
selection.removeAllRanges();
|
|
17197
|
+
selection.addRange(range);
|
|
17198
|
+
nativeInputRequired = true;
|
|
17199
|
+
} else if (typeof el.select === 'function') {
|
|
17200
|
+
el.select();
|
|
17201
|
+
nativeInputRequired = true;
|
|
16330
17202
|
} else {
|
|
16331
|
-
el.value
|
|
17203
|
+
el.setSelectionRange(0, String(el.value || '').length);
|
|
17204
|
+
nativeInputRequired = true;
|
|
16332
17205
|
}
|
|
16333
|
-
el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: value }));
|
|
16334
|
-
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
16335
17206
|
const rect = el.getBoundingClientRect();
|
|
16336
17207
|
return {
|
|
16337
17208
|
filled: true,
|
|
@@ -16340,9 +17211,13 @@ ${DESKTOP_POINT_JS}
|
|
|
16340
17211
|
index,
|
|
16341
17212
|
tag: el.tagName.toLowerCase(),
|
|
16342
17213
|
role: el.getAttribute('role') || null,
|
|
16343
|
-
type:
|
|
17214
|
+
type: fieldType,
|
|
16344
17215
|
text,
|
|
16345
|
-
|
|
17216
|
+
actualValue: el.isContentEditable ? el.textContent : 'value' in el ? el.value : null,
|
|
17217
|
+
valueLength: String(el.isContentEditable ? el.textContent : 'value' in el ? el.value : '').length,
|
|
17218
|
+
nativeInputRequired,
|
|
17219
|
+
semanticSelection,
|
|
17220
|
+
selectionMatched,
|
|
16346
17221
|
desktopPoint: desktopPoint(rect),
|
|
16347
17222
|
rect: {
|
|
16348
17223
|
x: Math.round(rect.x),
|
|
@@ -16357,6 +17232,18 @@ ${DESKTOP_POINT_JS}
|
|
|
16357
17232
|
}
|
|
16358
17233
|
async function computerBrowserFillCommand(query, value, options = {}) {
|
|
16359
17234
|
const page = await selectChromePage(options);
|
|
17235
|
+
if (options.ref) {
|
|
17236
|
+
const target = await prepareBrowserElement(page, options.ref);
|
|
17237
|
+
const backendNodeId = parseBrowserRef(options.ref);
|
|
17238
|
+
const actual = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await fillChromeNodeInSession(session, backendNodeId, value));
|
|
17239
|
+
if (target.desktopPoint) logRecordingAction({ type: "type", ...target.desktopPoint });
|
|
17240
|
+
console.log(JSON.stringify({
|
|
17241
|
+
...browserTargetOutput(page),
|
|
17242
|
+
result: { filled: true, ref: target.ref, valueLength: actual.valueLength, rect: target.rect, element: target.element }
|
|
17243
|
+
}, null, 2));
|
|
17244
|
+
return;
|
|
17245
|
+
}
|
|
17246
|
+
if (!query) fail("Provide field text or --ref <id> from the latest browser state");
|
|
16360
17247
|
const index = options.index ? parseCoord(options.index, "--index") : 0;
|
|
16361
17248
|
if (index < 0) fail("--index must be >= 0");
|
|
16362
17249
|
const result = await evaluateChromeTarget(
|
|
@@ -16364,10 +17251,28 @@ async function computerBrowserFillCommand(query, value, options = {}) {
|
|
|
16364
17251
|
buildBrowserFillExpression(query, value, { exact: !!options.exact, index })
|
|
16365
17252
|
);
|
|
16366
17253
|
const point = browserActionPoint(result);
|
|
17254
|
+
if (typeof result === "object" && result !== null && "nativeInputRequired" in result && result.nativeInputRequired === true) {
|
|
17255
|
+
await replaceChromeText(page, value);
|
|
17256
|
+
const actualValue = await evaluateChromeTarget(page.webSocketDebuggerUrl, `(() => {
|
|
17257
|
+
const el = document.activeElement;
|
|
17258
|
+
return el && el.isContentEditable ? el.textContent : el && 'value' in el ? el.value : null;
|
|
17259
|
+
})()`);
|
|
17260
|
+
const fieldType = "type" in result && typeof result.type === "string" ? result.type.toLowerCase() : "";
|
|
17261
|
+
if (browserFillInvalid(actualValue, value, fieldType)) {
|
|
17262
|
+
fail(`Chrome reported that ${JSON.stringify(query)} contains ${JSON.stringify(actualValue)} after filling, expected ${JSON.stringify(value)}.`);
|
|
17263
|
+
}
|
|
17264
|
+
if (isRecord(result)) {
|
|
17265
|
+
Object.assign(result, { actualValue, valueLength: String(actualValue ?? "").length });
|
|
17266
|
+
}
|
|
17267
|
+
} else if (isRecord(result) && result.filled === true) {
|
|
17268
|
+
const fieldType = typeof result.type === "string" ? result.type : "";
|
|
17269
|
+
if (browserFillInvalid(result.actualValue, value, fieldType, result.semanticSelection === true, result.selectionMatched === true)) {
|
|
17270
|
+
fail(`Chrome reported that ${JSON.stringify(query)} contains ${JSON.stringify(result.actualValue)} after filling, expected ${JSON.stringify(value)}.`);
|
|
17271
|
+
}
|
|
17272
|
+
}
|
|
16367
17273
|
if (point) logRecordingAction({ type: "type", ...point });
|
|
16368
17274
|
console.log(JSON.stringify({
|
|
16369
|
-
|
|
16370
|
-
url: page.url ?? "",
|
|
17275
|
+
...browserTargetOutput(page),
|
|
16371
17276
|
result
|
|
16372
17277
|
}, null, 2));
|
|
16373
17278
|
}
|
|
@@ -16377,11 +17282,7 @@ function buildBrowserWaitExpression(query, options) {
|
|
|
16377
17282
|
const mode = ${JSON.stringify(options.mode)};
|
|
16378
17283
|
const exact = ${JSON.stringify(options.exact)};
|
|
16379
17284
|
const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
|
|
16380
|
-
|
|
16381
|
-
const rect = el.getBoundingClientRect();
|
|
16382
|
-
const style = getComputedStyle(el);
|
|
16383
|
-
return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
|
|
16384
|
-
};
|
|
17285
|
+
${VISIBLE_JS}
|
|
16385
17286
|
const matches = (text) => {
|
|
16386
17287
|
const haystack = normalize(text).toLowerCase();
|
|
16387
17288
|
const needle = query.toLowerCase();
|
|
@@ -16436,6 +17337,241 @@ async function computerBrowserWaitCommand(query, options = {}) {
|
|
|
16436
17337
|
console.log(JSON.stringify({ ok: false, elapsedMs: Date.now() - start, attempts, result: lastResult }, null, 2));
|
|
16437
17338
|
process.exitCode = 1;
|
|
16438
17339
|
}
|
|
17340
|
+
async function computerBrowserScrollCommand(direction, options = {}) {
|
|
17341
|
+
const page = await selectChromePage(options);
|
|
17342
|
+
const normalized = direction.toLowerCase();
|
|
17343
|
+
if (!["up", "down", "left", "right"].includes(normalized)) fail("direction must be one of up|down|left|right");
|
|
17344
|
+
const amount = options.amount ? parseCoord(options.amount, "--amount") : 600;
|
|
17345
|
+
if (amount <= 0 || amount > 1e4) fail("--amount must be between 1 and 10000 CSS pixels");
|
|
17346
|
+
const layout = await sendChromeCommand(page.webSocketDebuggerUrl, "Page.getLayoutMetrics", {});
|
|
17347
|
+
const viewport = chromeVisualViewport(layout);
|
|
17348
|
+
const target = options.ref ? await prepareBrowserElement(page, options.ref) : null;
|
|
17349
|
+
const x = target?.rect.centerX ?? viewport.width / 2;
|
|
17350
|
+
const y = target?.rect.centerY ?? viewport.height / 2;
|
|
17351
|
+
const deltaX = normalized === "left" ? -amount : normalized === "right" ? amount : 0;
|
|
17352
|
+
const deltaY = normalized === "up" ? -amount : normalized === "down" ? amount : 0;
|
|
17353
|
+
await sendChromeCommand(page.webSocketDebuggerUrl, "Input.dispatchMouseEvent", {
|
|
17354
|
+
type: "mouseWheel",
|
|
17355
|
+
x,
|
|
17356
|
+
y,
|
|
17357
|
+
deltaX,
|
|
17358
|
+
deltaY
|
|
17359
|
+
});
|
|
17360
|
+
logRecordingAction({ type: "scroll", ...target?.desktopPoint ?? {} });
|
|
17361
|
+
console.log(JSON.stringify({
|
|
17362
|
+
...browserTargetOutput(page),
|
|
17363
|
+
result: { scrolled: true, direction: normalized, amount, ref: target?.ref ?? null, x: Math.round(x), y: Math.round(y) }
|
|
17364
|
+
}, null, 2));
|
|
17365
|
+
}
|
|
17366
|
+
function browserKeyDetails(combo) {
|
|
17367
|
+
const tokens = combo.split("+").map((token) => token.trim()).filter(Boolean);
|
|
17368
|
+
if (tokens.length === 0) fail("key combination cannot be empty");
|
|
17369
|
+
let modifiers = 0;
|
|
17370
|
+
for (const token of tokens.slice(0, -1)) {
|
|
17371
|
+
const normalized2 = token.toLowerCase();
|
|
17372
|
+
if (["alt", "option"].includes(normalized2)) modifiers |= 1;
|
|
17373
|
+
else if (["ctrl", "control"].includes(normalized2)) modifiers |= 2;
|
|
17374
|
+
else if (["meta", "cmd", "command", "super"].includes(normalized2)) modifiers |= 4;
|
|
17375
|
+
else if (normalized2 === "shift") modifiers |= 8;
|
|
17376
|
+
else fail(`Unsupported browser key modifier ${JSON.stringify(token)}`);
|
|
17377
|
+
}
|
|
17378
|
+
const requested = tokens[tokens.length - 1];
|
|
17379
|
+
const aliases = {
|
|
17380
|
+
enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 },
|
|
17381
|
+
return: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 },
|
|
17382
|
+
tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9 },
|
|
17383
|
+
escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
|
|
17384
|
+
esc: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
|
|
17385
|
+
backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 },
|
|
17386
|
+
delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 },
|
|
17387
|
+
space: { key: " ", code: "Space", windowsVirtualKeyCode: 32 },
|
|
17388
|
+
up: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
|
|
17389
|
+
arrowup: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
|
|
17390
|
+
down: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
|
|
17391
|
+
arrowdown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
|
|
17392
|
+
left: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
|
|
17393
|
+
arrowleft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
|
|
17394
|
+
right: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
|
|
17395
|
+
arrowright: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
|
|
17396
|
+
home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 },
|
|
17397
|
+
end: { key: "End", code: "End", windowsVirtualKeyCode: 35 },
|
|
17398
|
+
pageup: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 },
|
|
17399
|
+
pagedown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 }
|
|
17400
|
+
};
|
|
17401
|
+
const normalized = requested.toLowerCase();
|
|
17402
|
+
const alias = aliases[normalized];
|
|
17403
|
+
if (alias) return { ...alias, modifiers, ...alias.key === " " && modifiers === 0 ? { text: " " } : {} };
|
|
17404
|
+
if (/^[a-z]$/i.test(requested)) {
|
|
17405
|
+
const upper = requested.toUpperCase();
|
|
17406
|
+
const key = modifiers & 8 ? upper : requested.toLowerCase();
|
|
17407
|
+
return { key, code: `Key${upper}`, windowsVirtualKeyCode: upper.charCodeAt(0), modifiers, ...modifiers === 0 ? { text: key } : {} };
|
|
17408
|
+
}
|
|
17409
|
+
if (/^[0-9]$/.test(requested)) {
|
|
17410
|
+
return { key: requested, code: `Digit${requested}`, windowsVirtualKeyCode: requested.charCodeAt(0), modifiers, ...modifiers === 0 ? { text: requested } : {} };
|
|
17411
|
+
}
|
|
17412
|
+
fail(`Unsupported browser key ${JSON.stringify(requested)}`);
|
|
17413
|
+
}
|
|
17414
|
+
async function computerBrowserKeyCommand(combo, options = {}) {
|
|
17415
|
+
const page = await selectChromePage(options);
|
|
17416
|
+
const key = browserKeyDetails(combo);
|
|
17417
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
17418
|
+
await dispatchChromeKeyInSession(session, key);
|
|
17419
|
+
});
|
|
17420
|
+
logRecordingAction({ type: "key" });
|
|
17421
|
+
console.log(JSON.stringify({ targetId: page.id ?? null, result: { pressed: combo } }, null, 2));
|
|
17422
|
+
}
|
|
17423
|
+
async function dispatchChromeKeyInSession(session, key) {
|
|
17424
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
17425
|
+
type: "keyDown",
|
|
17426
|
+
...key,
|
|
17427
|
+
nativeVirtualKeyCode: key.windowsVirtualKeyCode
|
|
17428
|
+
});
|
|
17429
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
17430
|
+
type: "keyUp",
|
|
17431
|
+
key: key.key,
|
|
17432
|
+
code: key.code,
|
|
17433
|
+
modifiers: key.modifiers,
|
|
17434
|
+
windowsVirtualKeyCode: key.windowsVirtualKeyCode,
|
|
17435
|
+
nativeVirtualKeyCode: key.windowsVirtualKeyCode
|
|
17436
|
+
});
|
|
17437
|
+
}
|
|
17438
|
+
async function computerBrowserTypeCommand(text, options = {}) {
|
|
17439
|
+
const page = await selectChromePage(options);
|
|
17440
|
+
await sendChromeCommand(page.webSocketDebuggerUrl, "Input.insertText", { text });
|
|
17441
|
+
logRecordingAction({ type: "type" });
|
|
17442
|
+
console.log(JSON.stringify({ targetId: page.id ?? null, result: { typed: true, length: text.length } }, null, 2));
|
|
17443
|
+
}
|
|
17444
|
+
function parseBrowserBatchActions(value) {
|
|
17445
|
+
let input;
|
|
17446
|
+
try {
|
|
17447
|
+
input = JSON.parse(value);
|
|
17448
|
+
} catch (error) {
|
|
17449
|
+
fail(`Batch actions must be valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
17450
|
+
}
|
|
17451
|
+
if (!Array.isArray(input) || input.length === 0 || input.length > 100) {
|
|
17452
|
+
fail("Batch actions must be a JSON array containing 1 to 100 actions");
|
|
17453
|
+
}
|
|
17454
|
+
return input.map((item, index) => {
|
|
17455
|
+
if (!isRecord(item) || typeof item.action !== "string") fail(`Batch action ${index} must be an object with an action`);
|
|
17456
|
+
if (item.action === "click") {
|
|
17457
|
+
if (typeof item.ref !== "string" && typeof item.ref !== "number") fail(`Batch click ${index} requires ref`);
|
|
17458
|
+
const buttons = { "1": "left", "2": "middle", "3": "right", left: "left", middle: "middle", right: "right" };
|
|
17459
|
+
const requestedButton = item.button === void 0 ? "left" : String(item.button).toLowerCase();
|
|
17460
|
+
const button = buttons[requestedButton];
|
|
17461
|
+
if (!button) fail(`Batch click ${index} button must be left, middle, or right`);
|
|
17462
|
+
return { action: "click", ref: String(item.ref), button, clicks: item.double === true ? 2 : 1 };
|
|
17463
|
+
}
|
|
17464
|
+
if (item.action === "fill") {
|
|
17465
|
+
if (typeof item.ref !== "string" && typeof item.ref !== "number" || typeof item.value !== "string") {
|
|
17466
|
+
fail(`Batch fill ${index} requires ref and string value`);
|
|
17467
|
+
}
|
|
17468
|
+
return { action: "fill", ref: String(item.ref), value: item.value };
|
|
17469
|
+
}
|
|
17470
|
+
if (item.action === "key") {
|
|
17471
|
+
if (typeof item.combo !== "string") fail(`Batch key ${index} requires combo`);
|
|
17472
|
+
return { action: "key", combo: item.combo };
|
|
17473
|
+
}
|
|
17474
|
+
if (item.action === "type") {
|
|
17475
|
+
if (typeof item.text !== "string") fail(`Batch type ${index} requires text`);
|
|
17476
|
+
return { action: "type", text: item.text };
|
|
17477
|
+
}
|
|
17478
|
+
if (item.action === "scroll") {
|
|
17479
|
+
const directions = { up: "up", down: "down", left: "left", right: "right" };
|
|
17480
|
+
const direction = typeof item.direction === "string" ? directions[item.direction] : void 0;
|
|
17481
|
+
if (!direction) {
|
|
17482
|
+
fail(`Batch scroll ${index} direction must be up, down, left, or right`);
|
|
17483
|
+
}
|
|
17484
|
+
const amount = item.amount === void 0 ? 600 : Number(item.amount);
|
|
17485
|
+
if (!Number.isFinite(amount) || amount <= 0 || amount > 1e4) fail(`Batch scroll ${index} amount must be between 1 and 10000`);
|
|
17486
|
+
if (item.ref !== void 0 && typeof item.ref !== "string" && typeof item.ref !== "number") fail(`Batch scroll ${index} ref is invalid`);
|
|
17487
|
+
return {
|
|
17488
|
+
action: "scroll",
|
|
17489
|
+
direction,
|
|
17490
|
+
amount,
|
|
17491
|
+
...item.ref === void 0 ? {} : { ref: String(item.ref) }
|
|
17492
|
+
};
|
|
17493
|
+
}
|
|
17494
|
+
if (item.action === "wait") {
|
|
17495
|
+
if (typeof item.text !== "string") fail(`Batch wait ${index} requires text`);
|
|
17496
|
+
const mode = item.mode === void 0 ? "any" : String(item.mode);
|
|
17497
|
+
if (!["any", "text", "title", "url", "control"].includes(mode)) fail(`Batch wait ${index} mode is invalid`);
|
|
17498
|
+
const timeoutMs = item.timeoutMs === void 0 ? 1e4 : Number(item.timeoutMs);
|
|
17499
|
+
const pollMs = item.pollMs === void 0 ? 250 : Number(item.pollMs);
|
|
17500
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) fail(`Batch wait ${index} timeoutMs must be >= 0`);
|
|
17501
|
+
if (!Number.isFinite(pollMs) || pollMs < 50 || pollMs > 2e3) fail(`Batch wait ${index} pollMs must be between 50 and 2000`);
|
|
17502
|
+
return { action: "wait", text: item.text, mode, exact: item.exact === true, timeoutMs, pollMs };
|
|
17503
|
+
}
|
|
17504
|
+
fail(`Batch action ${index} has unsupported action ${JSON.stringify(item.action)}`);
|
|
17505
|
+
});
|
|
17506
|
+
}
|
|
17507
|
+
async function computerBrowserBatchCommand(actionsJson, options = {}) {
|
|
17508
|
+
const actions = parseBrowserBatchActions(actionsJson);
|
|
17509
|
+
const page = await selectChromePage(options);
|
|
17510
|
+
const { targetId, cached } = readBrowserRefCache(page);
|
|
17511
|
+
const startedAt = Date.now();
|
|
17512
|
+
const completed = [];
|
|
17513
|
+
try {
|
|
17514
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
17515
|
+
const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
|
|
17516
|
+
const viewportRect = browserViewportRect(layout);
|
|
17517
|
+
for (const [index, action] of actions.entries()) {
|
|
17518
|
+
const actionStartedAt = Date.now();
|
|
17519
|
+
if (action.action === "click") {
|
|
17520
|
+
const ref = parseBrowserRef(action.ref);
|
|
17521
|
+
const rect = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
17522
|
+
await dispatchChromeClickInSession(session, rect.centerX, rect.centerY, action.button, action.clicks);
|
|
17523
|
+
logRecordingAction({ type: "click" });
|
|
17524
|
+
completed.push({ index, action: action.action, ref: String(ref), elapsedMs: Date.now() - actionStartedAt });
|
|
17525
|
+
} else if (action.action === "fill") {
|
|
17526
|
+
const ref = parseBrowserRef(action.ref);
|
|
17527
|
+
await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
17528
|
+
const result = await fillChromeNodeInSession(session, ref, action.value);
|
|
17529
|
+
logRecordingAction({ type: "type" });
|
|
17530
|
+
completed.push({ index, action: action.action, ref: String(ref), valueLength: result.valueLength, elapsedMs: Date.now() - actionStartedAt });
|
|
17531
|
+
} else if (action.action === "key") {
|
|
17532
|
+
await dispatchChromeKeyInSession(session, browserKeyDetails(action.combo));
|
|
17533
|
+
logRecordingAction({ type: "key" });
|
|
17534
|
+
completed.push({ index, action: action.action, elapsedMs: Date.now() - actionStartedAt });
|
|
17535
|
+
} else if (action.action === "type") {
|
|
17536
|
+
await sendChromeSessionCommand(session, "Input.insertText", { text: action.text });
|
|
17537
|
+
logRecordingAction({ type: "type" });
|
|
17538
|
+
completed.push({ index, action: action.action, length: action.text.length, elapsedMs: Date.now() - actionStartedAt });
|
|
17539
|
+
} else if (action.action === "scroll") {
|
|
17540
|
+
const ref = action.ref ? parseBrowserRef(action.ref) : null;
|
|
17541
|
+
const rect = ref ? await prepareBrowserRefInSession(session, ref, cached, viewportRect) : viewportRect;
|
|
17542
|
+
const deltaX = action.direction === "left" ? -action.amount : action.direction === "right" ? action.amount : 0;
|
|
17543
|
+
const deltaY = action.direction === "up" ? -action.amount : action.direction === "down" ? action.amount : 0;
|
|
17544
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
|
|
17545
|
+
type: "mouseWheel",
|
|
17546
|
+
x: rect.centerX,
|
|
17547
|
+
y: rect.centerY,
|
|
17548
|
+
deltaX,
|
|
17549
|
+
deltaY
|
|
17550
|
+
});
|
|
17551
|
+
logRecordingAction({ type: "scroll" });
|
|
17552
|
+
completed.push({ index, action: action.action, direction: action.direction, amount: action.amount, elapsedMs: Date.now() - actionStartedAt });
|
|
17553
|
+
} else {
|
|
17554
|
+
const expression = buildBrowserWaitExpression(action.text, { mode: action.mode, exact: action.exact });
|
|
17555
|
+
let attempts = 0;
|
|
17556
|
+
let result = null;
|
|
17557
|
+
while (Date.now() - actionStartedAt <= action.timeoutMs) {
|
|
17558
|
+
attempts++;
|
|
17559
|
+
result = await evaluateChromeSession(session, expression);
|
|
17560
|
+
if (isRecord(result) && result.matched === true) break;
|
|
17561
|
+
await sleep(action.pollMs);
|
|
17562
|
+
}
|
|
17563
|
+
if (!isRecord(result) || result.matched !== true) fail(`Batch wait ${index} timed out after ${Date.now() - actionStartedAt}ms`);
|
|
17564
|
+
completed.push({ index, action: action.action, attempts, elapsedMs: Date.now() - actionStartedAt });
|
|
17565
|
+
}
|
|
17566
|
+
}
|
|
17567
|
+
});
|
|
17568
|
+
} catch (error) {
|
|
17569
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17570
|
+
console.log(JSON.stringify({ ok: false, targetId, elapsedMs: Date.now() - startedAt, completed, failedIndex: completed.length, error: message }, null, 2));
|
|
17571
|
+
throw error;
|
|
17572
|
+
}
|
|
17573
|
+
console.log(JSON.stringify({ ok: true, targetId, elapsedMs: Date.now() - startedAt, completed }, null, 2));
|
|
17574
|
+
}
|
|
16439
17575
|
async function computerClickCommand(xStr, yStr, options) {
|
|
16440
17576
|
const dimensions = getDisplayDimensions();
|
|
16441
17577
|
const x = parseScreenCoord(xStr, "x", dimensions.width);
|
|
@@ -16558,12 +17694,55 @@ async function computerLaunchCommand(app, args) {
|
|
|
16558
17694
|
const baseArgs = app === "chrome" && !existsSync3(CHROME_WRAPPER) ? LEGACY_CHROME_ALIAS : APP_ALIASES[app] ?? [app];
|
|
16559
17695
|
const bin = baseArgs[0];
|
|
16560
17696
|
const fullArgs = [...baseArgs.slice(1), ...args];
|
|
17697
|
+
const expectsNewChromePage = args.some((arg) => !arg.startsWith("-"));
|
|
17698
|
+
const existingChromePageIds = /* @__PURE__ */ new Set();
|
|
17699
|
+
if (app === "chrome") {
|
|
17700
|
+
try {
|
|
17701
|
+
for (const page of await getChromePages()) {
|
|
17702
|
+
if (page.id) existingChromePageIds.add(page.id);
|
|
17703
|
+
}
|
|
17704
|
+
} catch {
|
|
17705
|
+
}
|
|
17706
|
+
}
|
|
16561
17707
|
const child = spawn4(bin, fullArgs, {
|
|
16562
17708
|
env: withDisplay(),
|
|
16563
17709
|
detached: true,
|
|
16564
17710
|
stdio: "ignore"
|
|
16565
17711
|
});
|
|
16566
17712
|
child.unref();
|
|
17713
|
+
if (app === "chrome") {
|
|
17714
|
+
const startedAt = Date.now();
|
|
17715
|
+
let pageCount = 0;
|
|
17716
|
+
let readyPage = null;
|
|
17717
|
+
while (Date.now() - startedAt < 15e3) {
|
|
17718
|
+
let pages = [];
|
|
17719
|
+
try {
|
|
17720
|
+
pages = await getChromePages();
|
|
17721
|
+
} catch {
|
|
17722
|
+
}
|
|
17723
|
+
pageCount = pages.length;
|
|
17724
|
+
const candidates = expectsNewChromePage && existingChromePageIds.size > 0 ? pages.filter((page) => page.id && !existingChromePageIds.has(page.id)) : pages;
|
|
17725
|
+
for (const page of candidates) {
|
|
17726
|
+
if (!page.webSocketDebuggerUrl || !page.url || page.url === "about:blank") continue;
|
|
17727
|
+
const readyState = await evaluateChromeTarget(page.webSocketDebuggerUrl, "document.readyState");
|
|
17728
|
+
if (readyState === "interactive" || readyState === "complete") {
|
|
17729
|
+
readyPage = page;
|
|
17730
|
+
break;
|
|
17731
|
+
}
|
|
17732
|
+
}
|
|
17733
|
+
if (readyPage) break;
|
|
17734
|
+
try {
|
|
17735
|
+
if (child.pid) process.kill(child.pid, 0);
|
|
17736
|
+
} catch {
|
|
17737
|
+
fail(`Chrome exited before its desktop control channel became ready.`);
|
|
17738
|
+
}
|
|
17739
|
+
await sleep(100);
|
|
17740
|
+
}
|
|
17741
|
+
if (!readyPage?.webSocketDebuggerUrl) fail(`Chrome launched but its requested page was not controllable after 15 seconds.`);
|
|
17742
|
+
await sendChromeCommand(readyPage.webSocketDebuggerUrl, "Page.bringToFront", {});
|
|
17743
|
+
console.log(`launched ${bin} (pid ${child.pid}, ready in ${Date.now() - startedAt}ms, page ${readyPage.id}, ${pageCount} open)`);
|
|
17744
|
+
return;
|
|
17745
|
+
}
|
|
16567
17746
|
console.log(`launched ${bin} (pid ${child.pid})`);
|
|
16568
17747
|
}
|
|
16569
17748
|
|
|
@@ -20990,10 +22169,20 @@ if (isAgentMode()) {
|
|
|
20990
22169
|
computer.command("status").description("Show which desktop services are running and the active preview URL").action(wrap(() => computerStatusCommand()));
|
|
20991
22170
|
computer.command("screenshot <path>").description("Capture the current desktop to a PNG file. Use --raw or --grid for agent click planning; omit both for a branded shareable image.").option("--raw", "Save a 1:1 desktop capture with no branding, padding, or rounded corners").option("--grid [px]", "Overlay a coordinate grid on a 1:1 desktop capture. Default 100px.").action(wrap((path6, options) => computerScreenshotCommand(path6, options)));
|
|
20992
22171
|
computer.command("observe <path>").description("Wait briefly for the desktop to settle, save a 1:1 screenshot, and print JSON screen context for agents.").option("--raw", "Save a 1:1 desktop capture with no coordinate grid").option("--grid [px]", "Overlay a coordinate grid on the 1:1 desktop capture. Default 100px.").option("--timeout <ms>", "Maximum time to wait for visual stability. Default 3000.").option("--stable-ms <ms>", "Required unchanged time before the screen is considered stable. Default 600.").option("--poll-ms <ms>", "Screenshot polling interval while waiting. Default 200.").action(wrap((path6, options) => computerObserveCommand(path6, options)));
|
|
20993
|
-
computer.command("browser").description("Print JSON for Chrome tabs launched through Replicas, including page titles and URLs.").option("--snapshot", "Include visible page text and interactive controls with bounding boxes").option("--limit <chars>", "Maximum body text characters per page snapshot. Default 4000.").option("--element-limit <n>", "Maximum controls per page snapshot. Default 80.").action(wrap((options) => computerBrowserCommand(options)));
|
|
20994
|
-
computer.command("browser-
|
|
20995
|
-
computer.command("browser-
|
|
20996
|
-
computer.command("browser-
|
|
22172
|
+
computer.command("browser").description("Print JSON for Chrome tabs launched through Replicas, including page titles and URLs.").option("--snapshot", "Include visible page text and interactive controls with bounding boxes").option("--limit <chars>", "Maximum body text characters per page snapshot. Default 4000.").option("--element-limit <n>", "Maximum controls per page snapshot. Default 80.").option("--target-id <id>", "Only include the Chrome page with this target id").option("--page <n>", "When multiple pages match, include the nth page. Default 0.").option("--title <text>", "Only include pages whose title contains text").option("--url <text>", "Only include pages whose URL contains text").action(wrap((options) => computerBrowserCommand(options)));
|
|
22173
|
+
computer.command("browser-state <path>").description("Wait for a Chrome page to settle, capture its viewport, and print a bounded accessibility state with actionable refs.").option("--full", "Return the full accessibility tree instead of a diff from the previous state").option("--limit <chars>", "Maximum visible text characters. Default 8000.").option("--element-limit <n>", "Maximum actionable elements. Default 120.").option("--timeout <ms>", "Maximum time to wait for state stability. Default 5000.").option("--stable-ms <ms>", "Required unchanged time before capture. Default 500.").option("--poll-ms <ms>", "State polling interval. Default 100.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((path6, options) => computerBrowserStateCommand(path6, options)));
|
|
22174
|
+
computer.command("browser-batch <actions>").description("Run an ordered, fail-fast JSON array of browser actions through one Chrome session.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((actions, options) => computerBrowserBatchCommand(actions, options)));
|
|
22175
|
+
computer.command("browser-click [text]").description("Click a Chrome element by ref from browser-state, or fall back to visible control text.").option("--ref <id>", "Click the backend DOM node ref from the latest browser state").option("--exact", "Require an exact text match instead of substring matching").option("--index <n>", "When multiple controls match, click the nth match. Default 0.").option("--button <button>", "Mouse button for ref clicks: left, middle, or right. Default left.").option("--double", "Double-click a ref target").option("--target-id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserClickCommand(text, options)));
|
|
22176
|
+
computer.command("browser-fill [field] [value]").description("Fill a Chrome field by ref from browser-state, or fall back to visible field text.").option("--ref <id>", "Fill the backend DOM node ref from the latest browser state").option("--exact", "Require an exact field match instead of substring matching").option("--index <n>", "When multiple fields match, fill the nth match. Default 0.").option("--target-id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((field, value, options) => {
|
|
22177
|
+
const resolvedValue = value ?? (options.ref ? field : void 0);
|
|
22178
|
+
const resolvedField = value === void 0 && options.ref ? void 0 : field;
|
|
22179
|
+
if (resolvedValue === void 0) throw new Error("Provide <field> <value>, or --ref <id> <value>");
|
|
22180
|
+
return computerBrowserFillCommand(resolvedField, resolvedValue, options);
|
|
22181
|
+
}));
|
|
22182
|
+
computer.command("browser-scroll <direction>").description("Send trusted wheel input to a Chrome element ref or the center of the target page.").option("--ref <id>", "Scroll from the backend DOM node ref from the latest browser state").option("--amount <px>", "Scroll distance in CSS pixels. Default 600.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((direction, options) => computerBrowserScrollCommand(direction, options)));
|
|
22183
|
+
computer.command("browser-key <combo>").description("Press a key or modifier chord in the target Chrome page with trusted browser input.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((combo, options) => computerBrowserKeyCommand(combo, options)));
|
|
22184
|
+
computer.command("browser-type <text>").description("Type literal text into the focused field in the target Chrome page with trusted browser input.").option("--target-id <id>", "Target the Chrome page with this stable target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserTypeCommand(text, options)));
|
|
22185
|
+
computer.command("browser-wait <text>").description("Wait until Chrome page title, URL, body text, or controls match <text>.").option("--mode <mode>", "Where to match: any, text, title, url, or control. Default any.").option("--exact", "Require an exact match instead of substring matching").option("--timeout <ms>", "Maximum time to wait. Default 10000.").option("--poll-ms <ms>", "Polling interval. Default 250.").option("--target-id <id>", "Only target the Chrome page with this target id").option("--page <n>", "When multiple pages match, target the nth page. Default 0.").option("--title <text>", "Only target pages whose title contains text").option("--url <text>", "Only target pages whose URL contains text").action(wrap((text, options) => computerBrowserWaitCommand(text, options)));
|
|
20997
22186
|
computer.command("click <x> <y>").description("Move to (x, y) and click. Coordinates are pixels or percentages like 50% 50%.").option("-b, --button <n>", "Mouse button (1=left, 2=middle, 3=right). Default 1.").option("--double", "Double-click instead of single-click").option("--modifiers <mods>", "Hold modifier keys during the click, e.g. ctrl or ctrl+shift").action(wrap((x, y, options) => computerClickCommand(x, y, options)));
|
|
20998
22187
|
computer.command("move <x> <y>").description("Move the mouse to (x, y) without clicking. Coordinates are pixels or percentages like 50% 50%.").action(wrap((x, y) => computerMoveCommand(x, y)));
|
|
20999
22188
|
computer.command("type <text>").description("Type a literal string into the focused field. Use `key` for key combos like ctrl+l.").option("--delay <ms>", "Per-character delay in ms (default 12 \u2248 80 wpm)").action(wrap((text, options) => computerTypeCommand(text, options)));
|