replicas-computer 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/assets/desktop/brand-wallpaper.svg +54 -0
- package/assets/desktop/geist-mono.woff2 +0 -0
- package/assets/desktop/gtk-settings.ini +11 -0
- package/assets/desktop/novnc-index.html +282 -0
- package/assets/desktop/openbox-rc.xml +79 -0
- package/assets/desktop/openbox-theme/close.xbm +7 -0
- package/assets/desktop/openbox-theme/close_hover.xbm +7 -0
- package/assets/desktop/openbox-theme/close_pressed.xbm +7 -0
- package/assets/desktop/openbox-theme/iconify.xbm +7 -0
- package/assets/desktop/openbox-theme/iconify_hover.xbm +7 -0
- package/assets/desktop/openbox-theme/iconify_pressed.xbm +7 -0
- package/assets/desktop/openbox-theme/max.xbm +7 -0
- package/assets/desktop/openbox-theme/max_hover.xbm +7 -0
- package/assets/desktop/openbox-theme/max_pressed.xbm +7 -0
- package/assets/desktop/openbox-theme/max_toggled.xbm +7 -0
- package/assets/desktop/openbox-theme/max_toggled_hover.xbm +7 -0
- package/assets/desktop/openbox-theme/max_toggled_pressed.xbm +7 -0
- package/assets/desktop/openbox-theme/themerc +66 -0
- package/assets/desktop/picom.conf +44 -0
- package/assets/desktop/replicas-chrome +52 -0
- package/assets/desktop/replicas-chrome.desktop +21 -0
- package/assets/desktop/start-services.sh +383 -0
- package/assets/desktop/tint2rc +91 -0
- package/dist/index.js +3287 -0
- package/package.json +42 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,3287 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/client.ts
|
|
4
|
+
import { createConnection as createConnection2 } from "net";
|
|
5
|
+
|
|
6
|
+
// src/daemon.ts
|
|
7
|
+
import { spawnSync as spawnSync4 } from "child_process";
|
|
8
|
+
import { chmodSync, existsSync as existsSync4, mkdirSync as mkdirSync4, rmSync as rmSync4 } from "fs";
|
|
9
|
+
import { createConnection, createServer } from "net";
|
|
10
|
+
import { dirname as dirname4 } from "path";
|
|
11
|
+
|
|
12
|
+
// src/commands/desktop.ts
|
|
13
|
+
import { spawnSync } from "child_process";
|
|
14
|
+
import { existsSync, mkdirSync, readFileSync } from "fs";
|
|
15
|
+
import { dirname, isAbsolute, resolve } from "path";
|
|
16
|
+
var DESKTOP_NOVNC_PORT = 6080;
|
|
17
|
+
var DESKTOP_VIEWER_WIDTH = 1920;
|
|
18
|
+
var DESKTOP_VIEWER_HEIGHT = 1080;
|
|
19
|
+
var STATE_DIR = process.env.REPLICAS_DESKTOP_STATE_DIR || "/tmp/replicas-computer";
|
|
20
|
+
var DEFAULT_DISPLAY = process.env.REPLICAS_DESKTOP_DISPLAY || ":99";
|
|
21
|
+
var NOVNC_PORT = process.env.REPLICAS_DESKTOP_NOVNC_PORT ? parseInt(process.env.REPLICAS_DESKTOP_NOVNC_PORT, 10) : DESKTOP_NOVNC_PORT;
|
|
22
|
+
var SERVICES_SCRIPT = process.env.REPLICAS_COMPUTER_SERVICES_SCRIPT || "/usr/local/bin/replicas-start-desktop-services";
|
|
23
|
+
var INPUT_LOCK_FILE = process.env.REPLICAS_DESKTOP_INPUT_LOCK_FILE || `${STATE_DIR}/input.lock`;
|
|
24
|
+
var CHROME_WRAPPER = "/usr/local/bin/replicas-chrome";
|
|
25
|
+
var INFO_WAIT_TIMEOUT_MS = 1e4;
|
|
26
|
+
var INFO_WAIT_INTERVAL_MS = 500;
|
|
27
|
+
var lastDesktopHealthAt = 0;
|
|
28
|
+
var CHROME_DEBUG_PORT = (() => {
|
|
29
|
+
const value = process.env.REPLICAS_DESKTOP_CHROME_DEBUG_PORT;
|
|
30
|
+
if (!value) return 9222;
|
|
31
|
+
const port = parseInt(value, 10);
|
|
32
|
+
if (!Number.isInteger(port) || port <= 0 || port > 65535) {
|
|
33
|
+
fail(`REPLICAS_DESKTOP_CHROME_DEBUG_PORT must be a valid TCP port, got ${JSON.stringify(value)}`);
|
|
34
|
+
}
|
|
35
|
+
return port;
|
|
36
|
+
})();
|
|
37
|
+
function fail(msg) {
|
|
38
|
+
throw new Error(msg);
|
|
39
|
+
}
|
|
40
|
+
function getDesktopBridgeStatus() {
|
|
41
|
+
const r = spawnSync("bash", [SERVICES_SCRIPT, "--status-json"], { stdio: "pipe" });
|
|
42
|
+
if (r.status !== 0) return null;
|
|
43
|
+
try {
|
|
44
|
+
return JSON.parse(r.stdout.toString());
|
|
45
|
+
} catch {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
function desktopStackHealthy() {
|
|
50
|
+
const pids = {};
|
|
51
|
+
for (const name of ["openbox", "tint2", "x11vnc", "novnc"]) {
|
|
52
|
+
try {
|
|
53
|
+
const pid = Number.parseInt(readFileSync(`${STATE_DIR}/${name}.pid`, "utf8").trim(), 10);
|
|
54
|
+
if (!Number.isFinite(pid)) return false;
|
|
55
|
+
process.kill(pid, 0);
|
|
56
|
+
pids[name] = pid;
|
|
57
|
+
} catch {
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (spawnSync("xdpyinfo", [], { env: withDisplay(), stdio: "ignore" }).status !== 0) return false;
|
|
62
|
+
const wm = spawnSync("xprop", ["-root", "_NET_SUPPORTING_WM_CHECK"], { env: withDisplay(), stdio: "pipe" });
|
|
63
|
+
if (wm.status !== 0 || !wm.stdout?.toString().includes("window id")) return false;
|
|
64
|
+
const bridge = getDesktopBridgeStatus();
|
|
65
|
+
return !!bridge?.x11vnc.listenerPids?.includes(pids.x11vnc) && !!bridge.websockify.listenerPids?.includes(pids.novnc);
|
|
66
|
+
}
|
|
67
|
+
function ensureServicesRunning() {
|
|
68
|
+
if (!existsSync(SERVICES_SCRIPT)) {
|
|
69
|
+
fail(
|
|
70
|
+
`Desktop services script missing at ${SERVICES_SCRIPT}. The workspace image is out of date \u2014 Xvfb / openbox / x11vnc / websockify must be installed and \`replicas-computer setup\` must install the desktop runtime.`
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
if (Date.now() - lastDesktopHealthAt < 1e3) return;
|
|
74
|
+
if (desktopStackHealthy()) {
|
|
75
|
+
lastDesktopHealthAt = Date.now();
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const r = spawnSync("bash", [SERVICES_SCRIPT], { stdio: "pipe" });
|
|
79
|
+
if (r.status !== 0) {
|
|
80
|
+
fail(`Failed to start desktop services: ${r.stderr?.toString() || "unknown error"}`);
|
|
81
|
+
}
|
|
82
|
+
lastDesktopHealthAt = Date.now();
|
|
83
|
+
}
|
|
84
|
+
function withDisplay(env = process.env) {
|
|
85
|
+
return { ...env, DISPLAY: DEFAULT_DISPLAY };
|
|
86
|
+
}
|
|
87
|
+
function runDisplayCmd(bin, args) {
|
|
88
|
+
ensureServicesRunning();
|
|
89
|
+
const r = spawnSync(bin, args, { env: withDisplay(), stdio: "pipe" });
|
|
90
|
+
if (r.status !== 0) {
|
|
91
|
+
fail(`${bin} ${args.join(" ")} failed: ${r.stderr?.toString().trim() || `exit ${r.status}`}`);
|
|
92
|
+
}
|
|
93
|
+
return r.stdout?.toString() ?? "";
|
|
94
|
+
}
|
|
95
|
+
function runDesktopInputCmd(args) {
|
|
96
|
+
mkdirSync(dirname(INPUT_LOCK_FILE), { recursive: true });
|
|
97
|
+
return runDisplayCmd("flock", [
|
|
98
|
+
"--exclusive",
|
|
99
|
+
"--wait",
|
|
100
|
+
process.env.REPLICAS_DESKTOP_INPUT_LOCK_WAIT_SECONDS || "60",
|
|
101
|
+
INPUT_LOCK_FILE,
|
|
102
|
+
"xdotool",
|
|
103
|
+
...args
|
|
104
|
+
]);
|
|
105
|
+
}
|
|
106
|
+
function tryDisplayCmd(bin, args) {
|
|
107
|
+
ensureServicesRunning();
|
|
108
|
+
const r = spawnSync(bin, args, { env: withDisplay(), stdio: "pipe" });
|
|
109
|
+
if (r.status !== 0) return null;
|
|
110
|
+
const out = r.stdout?.toString().trim();
|
|
111
|
+
return out || null;
|
|
112
|
+
}
|
|
113
|
+
function parseCoord(value, label) {
|
|
114
|
+
if (!/^-?\d+$/.test(value)) fail(`${label} must be an integer (got "${value}")`);
|
|
115
|
+
const n = Number.parseInt(value, 10);
|
|
116
|
+
if (!Number.isFinite(n)) fail(`${label} must be an integer (got "${value}")`);
|
|
117
|
+
return n;
|
|
118
|
+
}
|
|
119
|
+
function getDisplayDimensions() {
|
|
120
|
+
const out = runDisplayCmd("xdpyinfo", []);
|
|
121
|
+
const match = out.match(/dimensions:\s+(\d+)x(\d+)\s+pixels/);
|
|
122
|
+
if (!match) fail("could not read display dimensions from xdpyinfo");
|
|
123
|
+
return { width: Number.parseInt(match[1], 10), height: Number.parseInt(match[2], 10) };
|
|
124
|
+
}
|
|
125
|
+
function getMouseLocation() {
|
|
126
|
+
const out = tryDisplayCmd("xdotool", ["getmouselocation", "--shell"]);
|
|
127
|
+
if (!out) return null;
|
|
128
|
+
const values = Object.fromEntries(
|
|
129
|
+
out.split("\n").map((line) => {
|
|
130
|
+
const [key, value] = line.split("=");
|
|
131
|
+
return [key.toLowerCase(), Number.parseInt(value, 10)];
|
|
132
|
+
})
|
|
133
|
+
);
|
|
134
|
+
if (!Number.isFinite(values.x) || !Number.isFinite(values.y)) return null;
|
|
135
|
+
return {
|
|
136
|
+
x: values.x,
|
|
137
|
+
y: values.y,
|
|
138
|
+
screen: Number.isFinite(values.screen) ? values.screen : 0,
|
|
139
|
+
window: Number.isFinite(values.window) ? values.window : 0
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
function parseScreenCoord(value, label, size) {
|
|
143
|
+
if (value.endsWith("%")) {
|
|
144
|
+
const rawPct = value.slice(0, -1);
|
|
145
|
+
if (!/^(?:\d+(?:\.\d+)?|\.\d+)$/.test(rawPct)) fail(`${label} percent must be a number between 0% and 100% (got "${value}")`);
|
|
146
|
+
const pct = Number.parseFloat(rawPct);
|
|
147
|
+
if (!Number.isFinite(pct) || pct < 0 || pct > 100) fail(`${label} percent must be between 0% and 100% (got "${value}")`);
|
|
148
|
+
return Math.round(pct / 100 * (size - 1));
|
|
149
|
+
}
|
|
150
|
+
const n = parseCoord(value, label);
|
|
151
|
+
if (n < 0 || n >= size) fail(`${label} must be between 0 and ${size - 1} pixels, or 0%-100% (got "${value}")`);
|
|
152
|
+
return n;
|
|
153
|
+
}
|
|
154
|
+
function resolvePath(p) {
|
|
155
|
+
return isAbsolute(p) ? p : resolve(process.cwd(), p);
|
|
156
|
+
}
|
|
157
|
+
function configuredDesktopDimensions() {
|
|
158
|
+
const resolution = process.env.REPLICAS_DESKTOP_RESOLUTION?.match(/^(\d+)x(\d+)$/);
|
|
159
|
+
const width = parseInt(resolution?.[1] || process.env.REPLICAS_DESKTOP_WIDTH || String(DESKTOP_VIEWER_WIDTH), 10);
|
|
160
|
+
const height = parseInt(resolution?.[2] || process.env.REPLICAS_DESKTOP_HEIGHT || String(DESKTOP_VIEWER_HEIGHT), 10);
|
|
161
|
+
if (!Number.isFinite(width) || width <= 0) fail(`REPLICAS_DESKTOP_WIDTH must be a positive integer`);
|
|
162
|
+
if (!Number.isFinite(height) || height <= 0) fail(`REPLICAS_DESKTOP_HEIGHT must be a positive integer`);
|
|
163
|
+
return { width, height };
|
|
164
|
+
}
|
|
165
|
+
function clamp(n, min, max) {
|
|
166
|
+
return Math.min(max, Math.max(min, n));
|
|
167
|
+
}
|
|
168
|
+
var sleep = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
169
|
+
|
|
170
|
+
// src/program.ts
|
|
171
|
+
import { format } from "util";
|
|
172
|
+
import { Command, CommanderError } from "commander";
|
|
173
|
+
|
|
174
|
+
// src/commands/index.ts
|
|
175
|
+
import { spawn as spawn2, spawnSync as spawnSync3 } from "child_process";
|
|
176
|
+
import { createHash } from "crypto";
|
|
177
|
+
import { closeSync, copyFileSync as copyFileSync2, existsSync as existsSync3, mkdirSync as mkdirSync3, openSync, readFileSync as readFileSync3, readSync, rmSync as rmSync3, writeFileSync as writeFileSync3 } from "fs";
|
|
178
|
+
import { homedir } from "os";
|
|
179
|
+
import { dirname as dirname3, join } from "path";
|
|
180
|
+
import chalk from "chalk";
|
|
181
|
+
|
|
182
|
+
// src/commands/recording.ts
|
|
183
|
+
import { spawn } from "child_process";
|
|
184
|
+
import { appendFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2, rmSync as rmSync2, statSync, writeFileSync as writeFileSync2 } from "fs";
|
|
185
|
+
import { dirname as dirname2 } from "path";
|
|
186
|
+
|
|
187
|
+
// src/commands/recording/render.ts
|
|
188
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
189
|
+
import { copyFileSync, rmSync, writeFileSync } from "fs";
|
|
190
|
+
|
|
191
|
+
// src/commands/recording/config.ts
|
|
192
|
+
var cameraMotion = {
|
|
193
|
+
clickLeadSeconds: 1.2,
|
|
194
|
+
clickTailSeconds: 0.5,
|
|
195
|
+
actionLeadSeconds: 0.9,
|
|
196
|
+
clickZoom: 1.24,
|
|
197
|
+
closeClickOutputGapSeconds: 6,
|
|
198
|
+
clickMoveSeconds: 1.2,
|
|
199
|
+
returnSeconds: 1,
|
|
200
|
+
minSegmentSeconds: 0.05
|
|
201
|
+
};
|
|
202
|
+
var cursorMotion = {
|
|
203
|
+
minMoveSeconds: 0.38,
|
|
204
|
+
maxMoveSeconds: 1.7,
|
|
205
|
+
speedPxPerSecond: 700,
|
|
206
|
+
defaultLeadSeconds: 0.16,
|
|
207
|
+
clickShrinkSeconds: 0.08,
|
|
208
|
+
clickGrowSeconds: 0.14,
|
|
209
|
+
clickDwellGapStartSeconds: 0.75,
|
|
210
|
+
clickDwellFullGapSeconds: 3.2,
|
|
211
|
+
clickMinDwellSeconds: 0.12,
|
|
212
|
+
clickMaxDwellSeconds: 1.15,
|
|
213
|
+
maxDwellGapShare: 0.45,
|
|
214
|
+
dragLeadSeconds: 0.55,
|
|
215
|
+
maxKeyframes: 120
|
|
216
|
+
};
|
|
217
|
+
var cursorStyle = {
|
|
218
|
+
size: 30,
|
|
219
|
+
clickSize: 24
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
// src/commands/recording/ffmpeg-expressions.ts
|
|
223
|
+
function expressionNumber(n) {
|
|
224
|
+
return Number.isInteger(n) ? String(n) : n.toFixed(3);
|
|
225
|
+
}
|
|
226
|
+
function mixNumber(from, to, progress) {
|
|
227
|
+
return from + (to - from) * progress;
|
|
228
|
+
}
|
|
229
|
+
function easeInOutSmootherNumber(progress) {
|
|
230
|
+
const p = clamp(progress, 0, 1);
|
|
231
|
+
return p * p * p * (p * (p * 6 - 15) + 10);
|
|
232
|
+
}
|
|
233
|
+
function easeInOutSmootherExpression(progress) {
|
|
234
|
+
const p = `(${progress})`;
|
|
235
|
+
return `${p}*${p}*${p}*(${p}*(${p}*6-15)+10)`;
|
|
236
|
+
}
|
|
237
|
+
function progressOverTime(timeExpression, seconds) {
|
|
238
|
+
return easeInOutSmootherExpression(`min(max((${timeExpression})/${seconds.toFixed(3)}\\,0)\\,1)`);
|
|
239
|
+
}
|
|
240
|
+
function progressBetween(start, duration) {
|
|
241
|
+
return easeInOutSmootherExpression(`min(max((t-${start.toFixed(3)})/${duration.toFixed(3)}\\,0)\\,1)`);
|
|
242
|
+
}
|
|
243
|
+
function mixExpression(from, to, progress) {
|
|
244
|
+
return `${expressionNumber(from)}+(${expressionNumber(to - from)})*${progress}`;
|
|
245
|
+
}
|
|
246
|
+
function cropOrigin(point, zoom, size) {
|
|
247
|
+
return clamp(point * zoom - size / 2, 0, size * zoom - size);
|
|
248
|
+
}
|
|
249
|
+
function easedCropOrigin(fromPoint, toPoint, fromZoom, toZoom, size, progress) {
|
|
250
|
+
return mixExpression(
|
|
251
|
+
cropOrigin(fromPoint, fromZoom, size),
|
|
252
|
+
cropOrigin(toPoint, toZoom, size),
|
|
253
|
+
progress
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
function betweenExpression(start, end) {
|
|
257
|
+
return `between(t\\,${start.toFixed(3)}\\,${end.toFixed(3)})`;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// src/commands/recording/timeline.ts
|
|
261
|
+
function centerOf(size) {
|
|
262
|
+
return { x: size.width / 2, y: size.height / 2 };
|
|
263
|
+
}
|
|
264
|
+
function actionPoint(action, size) {
|
|
265
|
+
const x = action.toX ?? action.x ?? size.width / 2;
|
|
266
|
+
const y = action.toY ?? action.y ?? size.height / 2;
|
|
267
|
+
return { x: clamp(x, 0, size.width), y: clamp(y, 0, size.height) };
|
|
268
|
+
}
|
|
269
|
+
function idleSpeed(duration) {
|
|
270
|
+
if (duration < 1.2) return 1;
|
|
271
|
+
return clamp(1 + Math.log1p(duration - 0.8) * 1.8, 1.5, 5);
|
|
272
|
+
}
|
|
273
|
+
function renderedGapSeconds(duration) {
|
|
274
|
+
return duration / idleSpeed(duration);
|
|
275
|
+
}
|
|
276
|
+
function timelineActions(actions) {
|
|
277
|
+
return actions.filter((action) => action.type !== "key" && action.type !== "move");
|
|
278
|
+
}
|
|
279
|
+
function actionWindow(action, duration) {
|
|
280
|
+
const shouldZoom = action.type === "click";
|
|
281
|
+
const at = clamp(action.atMs / 1e3, 0, duration);
|
|
282
|
+
const start = clamp(at - (shouldZoom ? cameraMotion.clickLeadSeconds : cameraMotion.actionLeadSeconds), 0, duration);
|
|
283
|
+
const tail = shouldZoom ? cameraMotion.clickTailSeconds : action.type === "type" ? 1.5 : 0.9;
|
|
284
|
+
return { start, end: clamp(at + tail, start, duration), shouldZoom };
|
|
285
|
+
}
|
|
286
|
+
function nextClickStart(actions, index, duration) {
|
|
287
|
+
const nextClick = actions.slice(index + 1).find((candidate) => candidate.type === "click");
|
|
288
|
+
return nextClick ? clamp(nextClick.atMs / 1e3 - cameraMotion.clickLeadSeconds, 0, duration) : null;
|
|
289
|
+
}
|
|
290
|
+
function shouldPreserveZoom(nextStart, cursor, lastZoom) {
|
|
291
|
+
return lastZoom > 1 && nextStart !== null && nextStart > cursor && renderedGapSeconds(nextStart - cursor) <= cameraMotion.closeClickOutputGapSeconds;
|
|
292
|
+
}
|
|
293
|
+
function fullFrameSegment(duration, size) {
|
|
294
|
+
const center = centerOf(size);
|
|
295
|
+
return {
|
|
296
|
+
start: 0,
|
|
297
|
+
end: duration,
|
|
298
|
+
speed: 1,
|
|
299
|
+
motionSeconds: cameraMotion.returnSeconds,
|
|
300
|
+
fromZoom: 1,
|
|
301
|
+
zoom: 1,
|
|
302
|
+
fromX: center.x,
|
|
303
|
+
fromY: center.y,
|
|
304
|
+
...center
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
function buildRecordingSegments(actions, duration, size) {
|
|
308
|
+
const actionsForTimeline = timelineActions(actions);
|
|
309
|
+
if (actionsForTimeline.length === 0) return [fullFrameSegment(duration, size)];
|
|
310
|
+
const center = centerOf(size);
|
|
311
|
+
const segments = [];
|
|
312
|
+
let cursor = 0;
|
|
313
|
+
let lastPoint = center;
|
|
314
|
+
let lastZoom = 1;
|
|
315
|
+
for (let i = 0; i < actionsForTimeline.length; i++) {
|
|
316
|
+
const action = actionsForTimeline[i];
|
|
317
|
+
const { start, end, shouldZoom } = actionWindow(action, duration);
|
|
318
|
+
const preserveZoom = !shouldZoom && shouldPreserveZoom(nextClickStart(actionsForTimeline, i, duration), cursor, lastZoom);
|
|
319
|
+
const point = shouldZoom ? actionPoint(action, size) : preserveZoom ? lastPoint : center;
|
|
320
|
+
const zoom = shouldZoom ? cameraMotion.clickZoom : preserveZoom ? lastZoom : 1;
|
|
321
|
+
if (start > cursor + cameraMotion.minSegmentSeconds) {
|
|
322
|
+
const gap = start - cursor;
|
|
323
|
+
const speed = idleSpeed(gap);
|
|
324
|
+
const bridgeZoom = shouldZoom && lastZoom > 1 && renderedGapSeconds(gap) <= cameraMotion.closeClickOutputGapSeconds;
|
|
325
|
+
segments.push({
|
|
326
|
+
start: cursor,
|
|
327
|
+
end: start,
|
|
328
|
+
speed,
|
|
329
|
+
motionSeconds: bridgeZoom ? Math.min(cameraMotion.clickMoveSeconds, Math.max(0.18, gap / speed)) : cameraMotion.returnSeconds,
|
|
330
|
+
fromZoom: lastZoom,
|
|
331
|
+
zoom: bridgeZoom ? cameraMotion.clickZoom : zoom,
|
|
332
|
+
fromX: lastPoint.x,
|
|
333
|
+
fromY: lastPoint.y,
|
|
334
|
+
...point
|
|
335
|
+
});
|
|
336
|
+
lastZoom = bridgeZoom ? cameraMotion.clickZoom : zoom;
|
|
337
|
+
lastPoint = point;
|
|
338
|
+
}
|
|
339
|
+
const actionStart = Math.max(cursor, start);
|
|
340
|
+
if (end > actionStart + cameraMotion.minSegmentSeconds) {
|
|
341
|
+
segments.push({
|
|
342
|
+
start: actionStart,
|
|
343
|
+
end,
|
|
344
|
+
speed: 1,
|
|
345
|
+
motionSeconds: shouldZoom ? cameraMotion.clickMoveSeconds : cameraMotion.returnSeconds,
|
|
346
|
+
fromZoom: lastZoom,
|
|
347
|
+
zoom,
|
|
348
|
+
fromX: lastPoint.x,
|
|
349
|
+
fromY: lastPoint.y,
|
|
350
|
+
...point
|
|
351
|
+
});
|
|
352
|
+
cursor = end;
|
|
353
|
+
lastZoom = zoom;
|
|
354
|
+
}
|
|
355
|
+
lastPoint = point;
|
|
356
|
+
}
|
|
357
|
+
if (duration > cursor + cameraMotion.minSegmentSeconds) {
|
|
358
|
+
segments.push({
|
|
359
|
+
start: cursor,
|
|
360
|
+
end: duration,
|
|
361
|
+
speed: idleSpeed(duration - cursor),
|
|
362
|
+
motionSeconds: cameraMotion.returnSeconds,
|
|
363
|
+
fromZoom: lastZoom,
|
|
364
|
+
zoom: 1,
|
|
365
|
+
fromX: lastPoint.x,
|
|
366
|
+
fromY: lastPoint.y,
|
|
367
|
+
...center
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
return segments.filter((segment) => segment.end > segment.start + cameraMotion.minSegmentSeconds);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// src/commands/recording/cursor.ts
|
|
374
|
+
var cursorHotspotRatio = { x: 8 / 32, y: 4 / 32 };
|
|
375
|
+
function cursorAssets(stamp) {
|
|
376
|
+
return {
|
|
377
|
+
path: `/tmp/replicas-cursor-${stamp}.svg`,
|
|
378
|
+
size: cursorStyle.size
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
function clickDwellSeconds(gapSeconds) {
|
|
382
|
+
if (gapSeconds <= cursorMotion.clickDwellGapStartSeconds) return cursorMotion.defaultLeadSeconds;
|
|
383
|
+
const progress = (gapSeconds - cursorMotion.clickDwellGapStartSeconds) / (cursorMotion.clickDwellFullGapSeconds - cursorMotion.clickDwellGapStartSeconds);
|
|
384
|
+
const dwell = mixNumber(
|
|
385
|
+
cursorMotion.clickMinDwellSeconds,
|
|
386
|
+
cursorMotion.clickMaxDwellSeconds,
|
|
387
|
+
easeInOutSmootherNumber(progress)
|
|
388
|
+
);
|
|
389
|
+
return Math.min(dwell, gapSeconds * cursorMotion.maxDwellGapShare);
|
|
390
|
+
}
|
|
391
|
+
function arrivalLeadSeconds(action, gapSeconds) {
|
|
392
|
+
return action.type === "click" ? clickDwellSeconds(gapSeconds) : cursorMotion.defaultLeadSeconds;
|
|
393
|
+
}
|
|
394
|
+
function cursorKeyframe(action, at, previousAt, size) {
|
|
395
|
+
if (typeof action.x !== "number" || typeof action.y !== "number") return null;
|
|
396
|
+
const point = actionPoint(action, size);
|
|
397
|
+
const lead = arrivalLeadSeconds(action, at - previousAt);
|
|
398
|
+
return { ...point, at, arriveAt: clamp(at - lead, previousAt, at) };
|
|
399
|
+
}
|
|
400
|
+
function dedupeKeyframes(points) {
|
|
401
|
+
return points.sort((a, b) => a.at - b.at).filter(
|
|
402
|
+
(point, index, all) => index === 0 || point.at - all[index - 1].at > 0.08 || Math.hypot(point.x - all[index - 1].x, point.y - all[index - 1].y) > 24
|
|
403
|
+
);
|
|
404
|
+
}
|
|
405
|
+
function downsampleKeyframes(points, actions, size) {
|
|
406
|
+
if (points.length <= cursorMotion.maxKeyframes) return points;
|
|
407
|
+
const anchors = actions.filter((action) => action.type !== "move").map((action) => actionPoint(action, size));
|
|
408
|
+
const result = points.filter(
|
|
409
|
+
(point, index) => index === 0 || index === points.length - 1 || anchors.some((anchor) => Math.hypot(anchor.x - point.x, anchor.y - point.y) < 1)
|
|
410
|
+
);
|
|
411
|
+
const step = points.length / Math.max(1, cursorMotion.maxKeyframes - result.length);
|
|
412
|
+
for (let i = 0; result.length < cursorMotion.maxKeyframes && i < points.length; i += step) {
|
|
413
|
+
const point = points[Math.floor(i)];
|
|
414
|
+
if (!result.includes(point)) result.push(point);
|
|
415
|
+
}
|
|
416
|
+
return result.sort((a, b) => a.at - b.at);
|
|
417
|
+
}
|
|
418
|
+
function cursorKeyframes(actions, duration, size) {
|
|
419
|
+
const center = { x: size.width / 2, y: size.height / 2 };
|
|
420
|
+
const points = [{ at: 0, arriveAt: 0, ...center }];
|
|
421
|
+
for (const action of actions) {
|
|
422
|
+
const at = clamp(action.atMs / 1e3, 0, duration);
|
|
423
|
+
const previousAt = points[points.length - 1].at;
|
|
424
|
+
if (action.type === "drag" && typeof action.x === "number" && typeof action.y === "number") {
|
|
425
|
+
const startAt = clamp(at - cursorMotion.dragLeadSeconds, 0, duration);
|
|
426
|
+
points.push({
|
|
427
|
+
at: startAt,
|
|
428
|
+
arriveAt: startAt,
|
|
429
|
+
x: clamp(action.x, 0, size.width),
|
|
430
|
+
y: clamp(action.y, 0, size.height)
|
|
431
|
+
});
|
|
432
|
+
if (typeof action.toX === "number" && typeof action.toY === "number") {
|
|
433
|
+
points.push({
|
|
434
|
+
at,
|
|
435
|
+
arriveAt: at,
|
|
436
|
+
x: clamp(action.toX, 0, size.width),
|
|
437
|
+
y: clamp(action.toY, 0, size.height)
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
const point = cursorKeyframe(action, at, previousAt, size);
|
|
443
|
+
if (point) points.push(point);
|
|
444
|
+
}
|
|
445
|
+
return downsampleKeyframes(dedupeKeyframes(points), actions, size);
|
|
446
|
+
}
|
|
447
|
+
function cursorAxisExpression(points, axis) {
|
|
448
|
+
if (points.length === 0) return "0";
|
|
449
|
+
let expression = expressionNumber(points[points.length - 1][axis]);
|
|
450
|
+
for (let i = points.length - 2; i >= 0; i--) {
|
|
451
|
+
const from = points[i];
|
|
452
|
+
const to = points[i + 1];
|
|
453
|
+
const arriveAt = clamp(to.arriveAt, from.at, to.at);
|
|
454
|
+
const moveWindow = Math.max(1e-3, arriveAt - from.at);
|
|
455
|
+
const distance = Math.hypot(to.x - from.x, to.y - from.y);
|
|
456
|
+
const moveSeconds = Math.min(
|
|
457
|
+
moveWindow,
|
|
458
|
+
Math.max(cursorMotion.minMoveSeconds, Math.min(cursorMotion.maxMoveSeconds, distance / cursorMotion.speedPxPerSecond))
|
|
459
|
+
);
|
|
460
|
+
const moveStart = Math.max(from.at, arriveAt - moveSeconds);
|
|
461
|
+
const duration = Math.max(1e-3, arriveAt - moveStart);
|
|
462
|
+
const interpolated = mixExpression(from[axis], to[axis], progressBetween(moveStart, duration));
|
|
463
|
+
expression = `if(lte(t\\,${moveStart.toFixed(3)})\\,${expressionNumber(from[axis])}\\,if(lte(t\\,${to.at.toFixed(3)})\\,${interpolated}\\,${expression}))`;
|
|
464
|
+
}
|
|
465
|
+
return expression;
|
|
466
|
+
}
|
|
467
|
+
function cursorSizeExpression(actions, normalSize) {
|
|
468
|
+
let expression = expressionNumber(normalSize);
|
|
469
|
+
const clicks = actions.filter((action) => action.type === "click").sort((a, b) => b.atMs - a.atMs);
|
|
470
|
+
for (const action of clicks) {
|
|
471
|
+
const at = action.atMs / 1e3;
|
|
472
|
+
const shrinkStart = Math.max(0, at - cursorMotion.clickShrinkSeconds);
|
|
473
|
+
const shrink = mixExpression(
|
|
474
|
+
normalSize,
|
|
475
|
+
cursorStyle.clickSize,
|
|
476
|
+
progressBetween(shrinkStart, Math.max(1e-3, at - shrinkStart))
|
|
477
|
+
);
|
|
478
|
+
const grow = mixExpression(cursorStyle.clickSize, normalSize, progressBetween(at, cursorMotion.clickGrowSeconds));
|
|
479
|
+
expression = `if(${betweenExpression(shrinkStart, at)}\\,${shrink}\\,if(${betweenExpression(at, at + cursorMotion.clickGrowSeconds)}\\,${grow}\\,${expression}))`;
|
|
480
|
+
}
|
|
481
|
+
return expression;
|
|
482
|
+
}
|
|
483
|
+
function cursorOverlayPositionExpression(cursor, size, hotspotRatio, overlaySize) {
|
|
484
|
+
return `max(0\\,min(${expressionNumber(size)}-${overlaySize}\\,(${cursor})-${hotspotRatio.toFixed(6)}*${overlaySize}))`;
|
|
485
|
+
}
|
|
486
|
+
function cursorSvg(size) {
|
|
487
|
+
return `<svg width="${size}" height="${size}" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg"><path d="M7.7 3.4c-1.45 0-2.62 1.17-2.62 2.62v19.7c0 2.46 3.02 3.65 4.7 1.85l5.06-5.44a5.42 5.42 0 0 1 3.96-1.73h5.82c2.48 0 3.63-3.08 1.76-4.7L9.42 4.04A2.62 2.62 0 0 0 7.7 3.4Z" fill="white" fill-opacity="0.98" stroke="black" stroke-opacity="0.88" stroke-width="2.9" stroke-linejoin="round"/></svg>`;
|
|
488
|
+
}
|
|
489
|
+
function buildCursorOverlayFilter(actions, duration, size, assets, projectCursor = null, inputLabel = "[screen]", outputLabel = "[v]") {
|
|
490
|
+
const points = cursorKeyframes(actions, duration, size);
|
|
491
|
+
const rawCursorX = cursorAxisExpression(points, "x");
|
|
492
|
+
const rawCursorY = cursorAxisExpression(points, "y");
|
|
493
|
+
const cursorX = projectCursor ? projectCursor.x(rawCursorX) : rawCursorX;
|
|
494
|
+
const cursorY = projectCursor ? projectCursor.y(rawCursorY) : rawCursorY;
|
|
495
|
+
const cursorSize = cursorSizeExpression(actions, assets.size);
|
|
496
|
+
const cursorOverlayX = cursorOverlayPositionExpression(cursorX, size.width, cursorHotspotRatio.x, "overlay_w");
|
|
497
|
+
const cursorOverlayY = cursorOverlayPositionExpression(cursorY, size.height, cursorHotspotRatio.y, "overlay_h");
|
|
498
|
+
return `[1:v]scale=w='${cursorSize}':h='${cursorSize}':eval=frame[cursor];${inputLabel}[cursor]overlay=x='${cursorOverlayX}':y='${cursorOverlayY}':eof_action=repeat:shortest=1:eval=frame${outputLabel}`;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// src/commands/recording/render.ts
|
|
502
|
+
function videoDuration(path) {
|
|
503
|
+
const r = spawnSync2("ffprobe", [
|
|
504
|
+
"-v",
|
|
505
|
+
"error",
|
|
506
|
+
"-show_entries",
|
|
507
|
+
"format=duration",
|
|
508
|
+
"-of",
|
|
509
|
+
"default=noprint_wrappers=1:nokey=1",
|
|
510
|
+
path
|
|
511
|
+
], { stdio: "pipe" });
|
|
512
|
+
if (r.status !== 0) fail(`ffprobe failed: ${r.stderr?.toString().trim() || `exit ${r.status}`}`);
|
|
513
|
+
const duration = Number.parseFloat(r.stdout?.toString().trim() ?? "");
|
|
514
|
+
if (!Number.isFinite(duration) || duration <= 0) fail(`could not read recording duration for ${path}`);
|
|
515
|
+
return duration;
|
|
516
|
+
}
|
|
517
|
+
function renderedSegmentSpans(segments) {
|
|
518
|
+
let outputStart = 0;
|
|
519
|
+
return segments.map((segment) => {
|
|
520
|
+
const outputEnd = outputStart + (segment.end - segment.start) / segment.speed;
|
|
521
|
+
const span = {
|
|
522
|
+
rawStart: segment.start,
|
|
523
|
+
rawEnd: segment.end,
|
|
524
|
+
outputStart,
|
|
525
|
+
outputEnd,
|
|
526
|
+
speed: segment.speed
|
|
527
|
+
};
|
|
528
|
+
outputStart = outputEnd;
|
|
529
|
+
return span;
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
function outputTimeForRawTime(spans, rawTime) {
|
|
533
|
+
const span = spans.find((candidate) => rawTime >= candidate.rawStart && rawTime <= candidate.rawEnd);
|
|
534
|
+
if (span) return span.outputStart + (rawTime - span.rawStart) / span.speed;
|
|
535
|
+
if (spans.length === 0) return rawTime;
|
|
536
|
+
if (rawTime <= spans[0].rawStart) return spans[0].outputStart;
|
|
537
|
+
const lastSpan = spans[spans.length - 1];
|
|
538
|
+
if (rawTime >= lastSpan.rawEnd) return lastSpan.outputEnd;
|
|
539
|
+
return spans.find((candidate) => rawTime < candidate.rawStart)?.outputStart ?? lastSpan.outputEnd;
|
|
540
|
+
}
|
|
541
|
+
function actionsOnRenderedTimeline(actions, spans) {
|
|
542
|
+
return actions.map((action) => ({
|
|
543
|
+
...action,
|
|
544
|
+
atMs: outputTimeForRawTime(spans, action.atMs / 1e3) * 1e3
|
|
545
|
+
}));
|
|
546
|
+
}
|
|
547
|
+
function segmentMotionSeconds(segment) {
|
|
548
|
+
return Math.min(segment.motionSeconds, Math.max(0.12, (segment.end - segment.start) / segment.speed));
|
|
549
|
+
}
|
|
550
|
+
function scaledFrameClampExpression(value, zoom, outputSize) {
|
|
551
|
+
return `max(0\\,min(${outputSize}*(${zoom})-${outputSize}\\,${value}))`;
|
|
552
|
+
}
|
|
553
|
+
function cameraTransformExpressions(segment, size, timeExpression) {
|
|
554
|
+
const progress = progressOverTime(timeExpression, segmentMotionSeconds(segment));
|
|
555
|
+
const zoom = mixExpression(segment.fromZoom, segment.zoom, progress);
|
|
556
|
+
return {
|
|
557
|
+
zoom,
|
|
558
|
+
cropX: scaledFrameClampExpression(
|
|
559
|
+
easedCropOrigin(segment.fromX, segment.x, segment.fromZoom, segment.zoom, size.width, progress),
|
|
560
|
+
zoom,
|
|
561
|
+
size.width
|
|
562
|
+
),
|
|
563
|
+
cropY: scaledFrameClampExpression(
|
|
564
|
+
easedCropOrigin(segment.fromY, segment.y, segment.fromZoom, segment.zoom, size.height, progress),
|
|
565
|
+
zoom,
|
|
566
|
+
size.height
|
|
567
|
+
)
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
function projectedCursorExpression(rawCursor, axis, segments, spans, size) {
|
|
571
|
+
let expression = rawCursor;
|
|
572
|
+
for (let i = segments.length - 1; i >= 0; i--) {
|
|
573
|
+
const span = spans[i];
|
|
574
|
+
const camera = cameraTransformExpressions(segments[i], size, `t-${span.outputStart.toFixed(3)}`);
|
|
575
|
+
const crop = axis === "x" ? camera.cropX : camera.cropY;
|
|
576
|
+
const projected = `(${rawCursor})*(${camera.zoom})-(${crop})`;
|
|
577
|
+
expression = `if(${betweenExpression(span.outputStart, span.outputEnd)}\\,${projected}\\,${expression})`;
|
|
578
|
+
}
|
|
579
|
+
return expression;
|
|
580
|
+
}
|
|
581
|
+
function renderRecording(rawPath, target, actions, fps, size) {
|
|
582
|
+
if (actions.length === 0) {
|
|
583
|
+
copyFileSync(rawPath, target);
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
const duration = videoDuration(rawPath);
|
|
587
|
+
const segments = buildRecordingSegments(actions, duration, size);
|
|
588
|
+
if (segments.length === 0) {
|
|
589
|
+
copyFileSync(rawPath, target);
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
const stamp = `${process.pid}-${Date.now()}`;
|
|
593
|
+
const cursor = cursorAssets(stamp);
|
|
594
|
+
writeFileSync(cursor.path, cursorSvg(cursor.size));
|
|
595
|
+
const spans = renderedSegmentSpans(segments);
|
|
596
|
+
const renderedDuration = spans.length ? spans[spans.length - 1].outputEnd : duration;
|
|
597
|
+
const renderedActions = actionsOnRenderedTimeline(actions, spans);
|
|
598
|
+
const cursorFilter = buildCursorOverlayFilter(
|
|
599
|
+
renderedActions,
|
|
600
|
+
renderedDuration,
|
|
601
|
+
size,
|
|
602
|
+
cursor,
|
|
603
|
+
{
|
|
604
|
+
x: (cursorX) => projectedCursorExpression(cursorX, "x", segments, spans, size),
|
|
605
|
+
y: (cursorY) => projectedCursorExpression(cursorY, "y", segments, spans, size)
|
|
606
|
+
}
|
|
607
|
+
);
|
|
608
|
+
const screenLabels = segments.map((_, index) => `[screen${index}]`).join("");
|
|
609
|
+
const screenSplitFilter = `[0:v]split=${segments.length}${screenLabels}`;
|
|
610
|
+
const filters = segments.map((segment, index) => {
|
|
611
|
+
const { zoom, cropX, cropY } = cameraTransformExpressions(segment, size, "t");
|
|
612
|
+
return `[screen${index}]trim=start=${segment.start.toFixed(3)}:end=${segment.end.toFixed(3)},setpts=(PTS-STARTPTS)/${segment.speed.toFixed(3)},scale=w=${size.width}*(${zoom}):h=${size.height}*(${zoom}):eval=frame,crop=${size.width}:${size.height}:x=${cropX}:y=${cropY},setsar=1[v${index}]`;
|
|
613
|
+
});
|
|
614
|
+
const concatInputs = segments.map((_, index) => `[v${index}]`).join("");
|
|
615
|
+
const filter = `${screenSplitFilter};${filters.join(";")};${concatInputs}concat=n=${segments.length}:v=1:a=0[screen];${cursorFilter}`;
|
|
616
|
+
const filterPath = `/tmp/replicas-recording-filter-${stamp}.ffgraph`;
|
|
617
|
+
writeFileSync(filterPath, filter);
|
|
618
|
+
try {
|
|
619
|
+
const r = spawnSync2("ffmpeg", [
|
|
620
|
+
"-y",
|
|
621
|
+
"-hide_banner",
|
|
622
|
+
"-loglevel",
|
|
623
|
+
"warning",
|
|
624
|
+
"-i",
|
|
625
|
+
rawPath,
|
|
626
|
+
"-loop",
|
|
627
|
+
"1",
|
|
628
|
+
"-i",
|
|
629
|
+
cursor.path,
|
|
630
|
+
"-filter_complex_script",
|
|
631
|
+
filterPath,
|
|
632
|
+
"-map",
|
|
633
|
+
"[v]",
|
|
634
|
+
"-r",
|
|
635
|
+
String(fps),
|
|
636
|
+
"-c:v",
|
|
637
|
+
"libx264",
|
|
638
|
+
"-preset",
|
|
639
|
+
"veryfast",
|
|
640
|
+
"-crf",
|
|
641
|
+
"23",
|
|
642
|
+
"-pix_fmt",
|
|
643
|
+
"yuv420p",
|
|
644
|
+
"-movflags",
|
|
645
|
+
"+faststart",
|
|
646
|
+
target
|
|
647
|
+
], { stdio: "pipe", maxBuffer: 20 * 1024 * 1024 });
|
|
648
|
+
if (r.status !== 0) {
|
|
649
|
+
fail(`recording post-processing failed: ${r.error?.message || r.stderr?.toString().trim() || `exit ${r.status}`}`);
|
|
650
|
+
}
|
|
651
|
+
} finally {
|
|
652
|
+
rmSync(filterPath, { force: true });
|
|
653
|
+
rmSync(cursor.path, { force: true });
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
// src/commands/recording.ts
|
|
658
|
+
var RECORD_PID_FILE = `${STATE_DIR}/ffmpeg.pid`;
|
|
659
|
+
var RECORD_PATH_FILE = `${STATE_DIR}/recording-path.txt`;
|
|
660
|
+
var RECORD_RAW_PATH_FILE = `${STATE_DIR}/recording-raw-path.txt`;
|
|
661
|
+
var RECORD_STARTED_AT_FILE = `${STATE_DIR}/recording-started-at.txt`;
|
|
662
|
+
var RECORD_FPS_FILE = `${STATE_DIR}/recording-fps.txt`;
|
|
663
|
+
var RECORD_DIMENSIONS_FILE = `${STATE_DIR}/recording-dimensions.json`;
|
|
664
|
+
var RECORD_ACTIONS_FILE = `${STATE_DIR}/recording-actions.jsonl`;
|
|
665
|
+
var RECORD_STATE_FILES = [
|
|
666
|
+
RECORD_PID_FILE,
|
|
667
|
+
RECORD_PATH_FILE,
|
|
668
|
+
RECORD_RAW_PATH_FILE,
|
|
669
|
+
RECORD_STARTED_AT_FILE,
|
|
670
|
+
RECORD_FPS_FILE,
|
|
671
|
+
RECORD_DIMENSIONS_FILE,
|
|
672
|
+
RECORD_ACTIONS_FILE
|
|
673
|
+
];
|
|
674
|
+
function clearRecordingState() {
|
|
675
|
+
for (const file of RECORD_STATE_FILES) rmSync2(file, { force: true });
|
|
676
|
+
}
|
|
677
|
+
function recordingStartedAt() {
|
|
678
|
+
if (!existsSync2(RECORD_STARTED_AT_FILE)) return null;
|
|
679
|
+
const startedAt = Number.parseInt(readFileSync2(RECORD_STARTED_AT_FILE, "utf8").trim(), 10);
|
|
680
|
+
return Number.isFinite(startedAt) ? startedAt : null;
|
|
681
|
+
}
|
|
682
|
+
function logRecordingAction(action) {
|
|
683
|
+
const startedAt = recordingStartedAt();
|
|
684
|
+
if (!startedAt) return;
|
|
685
|
+
const atMs = Date.now() - startedAt;
|
|
686
|
+
appendFileSync(RECORD_ACTIONS_FILE, `${JSON.stringify({ ...action, atMs })}
|
|
687
|
+
`);
|
|
688
|
+
}
|
|
689
|
+
function readRecordingDimensions() {
|
|
690
|
+
if (!existsSync2(RECORD_DIMENSIONS_FILE)) return configuredDesktopDimensions();
|
|
691
|
+
try {
|
|
692
|
+
const dimensions = JSON.parse(readFileSync2(RECORD_DIMENSIONS_FILE, "utf8"));
|
|
693
|
+
const width = dimensions?.width;
|
|
694
|
+
const height = dimensions?.height;
|
|
695
|
+
if (typeof width === "number" && Number.isFinite(width) && width > 0 && typeof height === "number" && Number.isFinite(height) && height > 0) {
|
|
696
|
+
return { width, height };
|
|
697
|
+
}
|
|
698
|
+
} catch {
|
|
699
|
+
}
|
|
700
|
+
fail("invalid recording dimensions state");
|
|
701
|
+
}
|
|
702
|
+
function isRecordingActionType(value) {
|
|
703
|
+
return value === "click" || value === "drag" || value === "key" || value === "move" || value === "scroll" || value === "type";
|
|
704
|
+
}
|
|
705
|
+
function isOptionalNumber(value) {
|
|
706
|
+
return value === void 0 || typeof value === "number";
|
|
707
|
+
}
|
|
708
|
+
function readRecordingActions() {
|
|
709
|
+
if (!existsSync2(RECORD_ACTIONS_FILE)) return [];
|
|
710
|
+
return readFileSync2(RECORD_ACTIONS_FILE, "utf8").split("\n").filter(Boolean).flatMap((line) => {
|
|
711
|
+
try {
|
|
712
|
+
const value = JSON.parse(line);
|
|
713
|
+
if (typeof value !== "object" || value === null) return [];
|
|
714
|
+
const type = "type" in value ? value.type : void 0;
|
|
715
|
+
const atMs = "atMs" in value ? value.atMs : void 0;
|
|
716
|
+
const x = "x" in value ? value.x : void 0;
|
|
717
|
+
const y = "y" in value ? value.y : void 0;
|
|
718
|
+
const toX = "toX" in value ? value.toX : void 0;
|
|
719
|
+
const toY = "toY" in value ? value.toY : void 0;
|
|
720
|
+
if (!isRecordingActionType(type) || typeof atMs !== "number" || !Number.isFinite(atMs) || !isOptionalNumber(x) || !isOptionalNumber(y) || !isOptionalNumber(toX) || !isOptionalNumber(toY)) {
|
|
721
|
+
return [];
|
|
722
|
+
}
|
|
723
|
+
return [{ type, atMs, x, y, toX, toY }];
|
|
724
|
+
} catch {
|
|
725
|
+
return [];
|
|
726
|
+
}
|
|
727
|
+
}).sort((a, b) => a.atMs - b.atMs);
|
|
728
|
+
}
|
|
729
|
+
async function computerRecordStartCommand(path, options) {
|
|
730
|
+
ensureServicesRunning();
|
|
731
|
+
if (existsSync2(RECORD_PID_FILE)) {
|
|
732
|
+
const pid = parseInt(readFileSync2(RECORD_PID_FILE, "utf8").trim(), 10);
|
|
733
|
+
if (Number.isFinite(pid)) {
|
|
734
|
+
let alive2 = false;
|
|
735
|
+
try {
|
|
736
|
+
process.kill(pid, 0);
|
|
737
|
+
alive2 = true;
|
|
738
|
+
} catch {
|
|
739
|
+
}
|
|
740
|
+
if (alive2) fail(`recording already in progress (pid ${pid}). run \`replicas computer record stop\` first.`);
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
const target = resolvePath(path);
|
|
744
|
+
mkdirSync2(dirname2(target), { recursive: true });
|
|
745
|
+
const fps = options.fps ? parseCoord(options.fps, "--fps") : 60;
|
|
746
|
+
const { width, height } = configuredDesktopDimensions();
|
|
747
|
+
mkdirSync2(STATE_DIR, { recursive: true });
|
|
748
|
+
const rawTarget = `${target}.raw-${Date.now()}.mp4`;
|
|
749
|
+
rmSync2(RECORD_ACTIONS_FILE, { force: true });
|
|
750
|
+
const child = spawn("ffmpeg", [
|
|
751
|
+
"-y",
|
|
752
|
+
"-hide_banner",
|
|
753
|
+
"-loglevel",
|
|
754
|
+
"warning",
|
|
755
|
+
"-f",
|
|
756
|
+
"x11grab",
|
|
757
|
+
"-framerate",
|
|
758
|
+
String(fps),
|
|
759
|
+
"-video_size",
|
|
760
|
+
`${width}x${height}`,
|
|
761
|
+
"-draw_mouse",
|
|
762
|
+
"0",
|
|
763
|
+
"-i",
|
|
764
|
+
DEFAULT_DISPLAY,
|
|
765
|
+
"-c:v",
|
|
766
|
+
"libx264",
|
|
767
|
+
"-preset",
|
|
768
|
+
"ultrafast",
|
|
769
|
+
"-tune",
|
|
770
|
+
"zerolatency",
|
|
771
|
+
"-crf",
|
|
772
|
+
"23",
|
|
773
|
+
"-pix_fmt",
|
|
774
|
+
"yuv420p",
|
|
775
|
+
"-movflags",
|
|
776
|
+
"+faststart+frag_keyframe+empty_moov",
|
|
777
|
+
rawTarget
|
|
778
|
+
], { detached: true, stdio: "ignore" });
|
|
779
|
+
child.unref();
|
|
780
|
+
if (!child.pid) fail("failed to launch ffmpeg");
|
|
781
|
+
writeFileSync2(RECORD_PID_FILE, String(child.pid));
|
|
782
|
+
writeFileSync2(RECORD_PATH_FILE, target);
|
|
783
|
+
writeFileSync2(RECORD_RAW_PATH_FILE, rawTarget);
|
|
784
|
+
writeFileSync2(RECORD_STARTED_AT_FILE, String(Date.now()));
|
|
785
|
+
writeFileSync2(RECORD_FPS_FILE, String(fps));
|
|
786
|
+
writeFileSync2(RECORD_DIMENSIONS_FILE, JSON.stringify({ width, height }));
|
|
787
|
+
const startedAt = Date.now();
|
|
788
|
+
while (Date.now() - startedAt < 5e3) {
|
|
789
|
+
try {
|
|
790
|
+
process.kill(child.pid, 0);
|
|
791
|
+
if (existsSync2(rawTarget) && statSync(rawTarget).size > 0) {
|
|
792
|
+
console.log(`${target} (recording ready in ${Date.now() - startedAt}ms)`);
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
} catch {
|
|
796
|
+
break;
|
|
797
|
+
}
|
|
798
|
+
await sleep(100);
|
|
799
|
+
}
|
|
800
|
+
let alive = false;
|
|
801
|
+
try {
|
|
802
|
+
process.kill(child.pid, 0);
|
|
803
|
+
alive = true;
|
|
804
|
+
} catch {
|
|
805
|
+
}
|
|
806
|
+
if (alive) fail("ffmpeg is running but did not produce recording output within 5 seconds; run `replicas computer record stop` to finalize or retry");
|
|
807
|
+
clearRecordingState();
|
|
808
|
+
rmSync2(rawTarget, { force: true });
|
|
809
|
+
fail("ffmpeg exited before screen recording became ready");
|
|
810
|
+
}
|
|
811
|
+
async function computerRecordStopCommand() {
|
|
812
|
+
if (!existsSync2(RECORD_PID_FILE) && !existsSync2(RECORD_PATH_FILE)) fail("no recording in progress");
|
|
813
|
+
if (existsSync2(RECORD_PID_FILE)) {
|
|
814
|
+
const pid = parseInt(readFileSync2(RECORD_PID_FILE, "utf8").trim(), 10);
|
|
815
|
+
if (!Number.isFinite(pid)) fail("invalid recording pidfile");
|
|
816
|
+
try {
|
|
817
|
+
process.kill(pid, "SIGINT");
|
|
818
|
+
} catch {
|
|
819
|
+
}
|
|
820
|
+
let alive = true;
|
|
821
|
+
for (let i = 0; i < 150; i++) {
|
|
822
|
+
try {
|
|
823
|
+
process.kill(pid, 0);
|
|
824
|
+
} catch {
|
|
825
|
+
alive = false;
|
|
826
|
+
break;
|
|
827
|
+
}
|
|
828
|
+
await sleep(200);
|
|
829
|
+
}
|
|
830
|
+
if (alive) fail(`ffmpeg did not finalize recording within 30 seconds (pid ${pid})`);
|
|
831
|
+
}
|
|
832
|
+
if (existsSync2(RECORD_PATH_FILE)) {
|
|
833
|
+
const target = readFileSync2(RECORD_PATH_FILE, "utf8").trim();
|
|
834
|
+
const rawPath = existsSync2(RECORD_RAW_PATH_FILE) ? readFileSync2(RECORD_RAW_PATH_FILE, "utf8").trim() : target;
|
|
835
|
+
const fps = existsSync2(RECORD_FPS_FILE) ? parseInt(readFileSync2(RECORD_FPS_FILE, "utf8").trim(), 10) : 60;
|
|
836
|
+
const size = readRecordingDimensions();
|
|
837
|
+
const actions = readRecordingActions();
|
|
838
|
+
if (rawPath !== target) {
|
|
839
|
+
renderRecording(rawPath, target, actions, Number.isFinite(fps) ? fps : 60, size);
|
|
840
|
+
rmSync2(rawPath, { force: true });
|
|
841
|
+
}
|
|
842
|
+
console.log(target);
|
|
843
|
+
}
|
|
844
|
+
clearRecordingState();
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// src/commands/index.ts
|
|
848
|
+
function isRecord(value) {
|
|
849
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
850
|
+
}
|
|
851
|
+
function getDesktopViewerUrl(publicUrl) {
|
|
852
|
+
const url = new URL(`${publicUrl.replace(/\/$/, "")}/`);
|
|
853
|
+
url.searchParams.set("v", "13");
|
|
854
|
+
return url.toString();
|
|
855
|
+
}
|
|
856
|
+
function bridgeStatus(details, includeBacklog = false, rootsOnly = false) {
|
|
857
|
+
const pids = rootsOnly ? details.rootPids ?? [] : details.listenerPids ?? [];
|
|
858
|
+
const listenerCount = rootsOnly ? details.rootListeners ?? pids.length : details.listeners ?? pids.length;
|
|
859
|
+
const listenerLabel = rootsOnly ? "root listener" : "listener";
|
|
860
|
+
const parts = [`${listenerCount} ${listenerLabel}${listenerCount === 1 ? "" : "s"} on ${details.port}`];
|
|
861
|
+
if (includeBacklog) parts.push(`backlog ${details.backlog ?? 0}`);
|
|
862
|
+
if (pids.length > 0) parts.push(`pid${pids.length === 1 ? "" : "s"} ${pids.join(",")}`);
|
|
863
|
+
return ` (${parts.join(", ")})`;
|
|
864
|
+
}
|
|
865
|
+
async function lookupDesktopViewerUrl() {
|
|
866
|
+
try {
|
|
867
|
+
const value = JSON.parse(readFileSync3(join(homedir(), ".replicas", "preview-ports.json"), "utf8"));
|
|
868
|
+
if (!isRecord(value) || !Array.isArray(value.previews)) return null;
|
|
869
|
+
const preview = value.previews.find(
|
|
870
|
+
(item) => isRecord(item) && item.port === NOVNC_PORT && typeof item.publicUrl === "string"
|
|
871
|
+
);
|
|
872
|
+
return isRecord(preview) && typeof preview.publicUrl === "string" ? getDesktopViewerUrl(preview.publicUrl) : null;
|
|
873
|
+
} catch {
|
|
874
|
+
return null;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
async function waitForDesktopViewerUrl(timeoutMs) {
|
|
878
|
+
const deadline = Date.now() + timeoutMs;
|
|
879
|
+
while (Date.now() < deadline) {
|
|
880
|
+
const url = await lookupDesktopViewerUrl();
|
|
881
|
+
if (url) return url;
|
|
882
|
+
await new Promise((resolve3) => setTimeout(resolve3, INFO_WAIT_INTERVAL_MS));
|
|
883
|
+
}
|
|
884
|
+
return await lookupDesktopViewerUrl();
|
|
885
|
+
}
|
|
886
|
+
async function computerInfoCommand() {
|
|
887
|
+
const viewerUrl = await waitForDesktopViewerUrl(INFO_WAIT_TIMEOUT_MS);
|
|
888
|
+
if (!viewerUrl) {
|
|
889
|
+
fail(
|
|
890
|
+
`Desktop preview for port ${NOVNC_PORT} is not registered. The engine registers it at startup \u2014 check /tmp/replicas-desktop-stack.log and engine logs for errors.`
|
|
891
|
+
);
|
|
892
|
+
}
|
|
893
|
+
console.log(viewerUrl);
|
|
894
|
+
console.error(chalk.dim(`Share this URL with the user to let them watch the desktop live.`));
|
|
895
|
+
}
|
|
896
|
+
async function computerStatusCommand() {
|
|
897
|
+
ensureServicesRunning();
|
|
898
|
+
const bridge = getDesktopBridgeStatus();
|
|
899
|
+
const procs = ["Xvfb", "openbox", "tint2", "x11vnc", "websockify"];
|
|
900
|
+
for (const p of procs) {
|
|
901
|
+
const r = spawnSync3("pgrep", ["-af", p], { stdio: "pipe" });
|
|
902
|
+
const running = r.status === 0 && !!r.stdout?.toString().trim();
|
|
903
|
+
const suffix = p === "x11vnc" && bridge ? bridgeStatus(bridge.x11vnc, true) : p === "websockify" && bridge ? bridgeStatus(bridge.websockify, false, true) : "";
|
|
904
|
+
console.log(` ${running ? chalk.green("\u25CF") : chalk.red("\u25CB")} ${p}${suffix}`);
|
|
905
|
+
}
|
|
906
|
+
const viewerUrl = await lookupDesktopViewerUrl();
|
|
907
|
+
if (viewerUrl) {
|
|
908
|
+
console.log(` ${chalk.cyan("preview")}: ${viewerUrl}`);
|
|
909
|
+
} else {
|
|
910
|
+
console.log(` ${chalk.dim("preview: not yet registered (engine registers it at startup)")}`);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
var PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
|
|
914
|
+
function readPngDimensions(filePath) {
|
|
915
|
+
const fd = openSync(filePath, "r");
|
|
916
|
+
try {
|
|
917
|
+
const buf = Buffer.alloc(24);
|
|
918
|
+
const bytesRead = readSync(fd, buf, 0, 24, 0);
|
|
919
|
+
if (bytesRead < 24 || !buf.subarray(0, 8).equals(PNG_SIGNATURE)) {
|
|
920
|
+
fail(`${filePath} is not a valid PNG (read ${bytesRead}/24 bytes)`);
|
|
921
|
+
}
|
|
922
|
+
return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };
|
|
923
|
+
} finally {
|
|
924
|
+
closeSync(fd);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
function brandSvgPath() {
|
|
928
|
+
const dir = process.env.REPLICAS_DESKTOP_TEMPLATES || "/usr/local/share/replicas/desktop";
|
|
929
|
+
return `${dir}/brand-wallpaper.svg`;
|
|
930
|
+
}
|
|
931
|
+
function loadBrandSvg(canvasW, canvasH) {
|
|
932
|
+
const path = brandSvgPath();
|
|
933
|
+
if (!existsSync3(path)) {
|
|
934
|
+
fail(
|
|
935
|
+
`Brand wallpaper SVG missing at ${path}. The workspace image is out of date \u2014 \`replicas-computer setup\` must install brand-wallpaper.svg at $REPLICAS_DESKTOP_TEMPLATES.`
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
return readFileSync3(path, "utf8").replace(/(<svg[^>]*\s)width="\d+"/, `$1width="${canvasW}"`).replace(/(<svg[^>]*\s)height="\d+"/, `$1height="${canvasH}"`);
|
|
939
|
+
}
|
|
940
|
+
var BRAND_PAD_FRACTION = 0.06;
|
|
941
|
+
var SCREENSHOT_CORNER_FRACTION = 0.022;
|
|
942
|
+
var SHADOW_SIGMA_FRACTION = 0.022;
|
|
943
|
+
var SHADOW_OFFSET_Y_FRACTION = 0.013;
|
|
944
|
+
var SHADOW_ALPHA = 0.6;
|
|
945
|
+
var SCREENSHOT_MASK_TEMPLATE = `<svg width="__W__" height="__H__" xmlns="http://www.w3.org/2000/svg"><rect width="__W__" height="__H__" rx="__R__" ry="__R__" fill="white"/></svg>`;
|
|
946
|
+
var SHADOW_MASK_TEMPLATE = `<svg width="__SW__" height="__SH__" xmlns="http://www.w3.org/2000/svg"><rect x="__M__" y="__M__" width="__W__" height="__H__" rx="__R__" ry="__R__" fill="white"/></svg>`;
|
|
947
|
+
function parseGridSize(value) {
|
|
948
|
+
if (value === void 0 || value === false) return null;
|
|
949
|
+
const size = value === true ? 100 : parseCoord(value, "--grid");
|
|
950
|
+
if (size < 25 || size > 500) fail("--grid must be between 25 and 500 pixels");
|
|
951
|
+
return size;
|
|
952
|
+
}
|
|
953
|
+
function buildGridSvg(width, height, size) {
|
|
954
|
+
const lines = [];
|
|
955
|
+
const labels = [];
|
|
956
|
+
for (let x = 0; x <= width; x += size) {
|
|
957
|
+
const strokeWidth = x % (size * 5) === 0 ? 2 : 1;
|
|
958
|
+
lines.push(`<line x1="${x}" y1="0" x2="${x}" y2="${height}" stroke="#00e5ff" stroke-opacity="0.42" stroke-width="${strokeWidth}"/>`);
|
|
959
|
+
if (x > 0 && x < width) labels.push(`<text x="${x + 4}" y="18">${x}</text>`);
|
|
960
|
+
}
|
|
961
|
+
for (let y = 0; y <= height; y += size) {
|
|
962
|
+
const strokeWidth = y % (size * 5) === 0 ? 2 : 1;
|
|
963
|
+
lines.push(`<line x1="0" y1="${y}" x2="${width}" y2="${y}" stroke="#00e5ff" stroke-opacity="0.42" stroke-width="${strokeWidth}"/>`);
|
|
964
|
+
if (y > 0 && y < height) labels.push(`<text x="4" y="${y - 4}">${y}</text>`);
|
|
965
|
+
}
|
|
966
|
+
return `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">
|
|
967
|
+
<style>text{font:16px monospace;fill:white;paint-order:stroke;stroke:black;stroke-width:4px;stroke-linejoin:round}</style>
|
|
968
|
+
${lines.join("\n")}
|
|
969
|
+
${labels.join("\n")}
|
|
970
|
+
</svg>`;
|
|
971
|
+
}
|
|
972
|
+
function overlayGrid(rawPath, target, width, height, gridSize, gridPath) {
|
|
973
|
+
writeFileSync3(gridPath, buildGridSvg(width, height, gridSize));
|
|
974
|
+
const r = spawnSync3(
|
|
975
|
+
"ffmpeg",
|
|
976
|
+
[
|
|
977
|
+
"-y",
|
|
978
|
+
"-hide_banner",
|
|
979
|
+
"-loglevel",
|
|
980
|
+
"error",
|
|
981
|
+
"-i",
|
|
982
|
+
rawPath,
|
|
983
|
+
"-i",
|
|
984
|
+
gridPath,
|
|
985
|
+
"-filter_complex",
|
|
986
|
+
"[0:v][1:v]overlay=0:0:format=auto",
|
|
987
|
+
"-frames:v",
|
|
988
|
+
"1",
|
|
989
|
+
"-update",
|
|
990
|
+
"1",
|
|
991
|
+
target
|
|
992
|
+
],
|
|
993
|
+
{ stdio: "pipe" }
|
|
994
|
+
);
|
|
995
|
+
if (r.status !== 0) {
|
|
996
|
+
fail(`ffmpeg grid overlay failed: ${r.stderr?.toString().trim() || `exit ${r.status}`}`);
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
async function computerScreenshotCommand(path, options = {}) {
|
|
1000
|
+
const target = resolvePath(path);
|
|
1001
|
+
mkdirSync3(dirname3(target), { recursive: true });
|
|
1002
|
+
const stamp = `${process.pid}-${Date.now()}`;
|
|
1003
|
+
const rawPath = `/tmp/replicas-screenshot-${stamp}.raw.png`;
|
|
1004
|
+
const svgPath = `/tmp/replicas-screenshot-${stamp}.brand.svg`;
|
|
1005
|
+
const maskPath = `/tmp/replicas-screenshot-${stamp}.mask.svg`;
|
|
1006
|
+
const shadowPath = `/tmp/replicas-screenshot-${stamp}.shadow.svg`;
|
|
1007
|
+
const gridPath = `/tmp/replicas-screenshot-${stamp}.grid.svg`;
|
|
1008
|
+
try {
|
|
1009
|
+
runDisplayCmd("scrot", ["-o", rawPath]);
|
|
1010
|
+
const { width, height } = readPngDimensions(rawPath);
|
|
1011
|
+
const gridSize = parseGridSize(options.grid);
|
|
1012
|
+
if (options.raw && gridSize === null) {
|
|
1013
|
+
copyFileSync2(rawPath, target);
|
|
1014
|
+
console.log(`${target} (${width}x${height}, raw 1:1 desktop pixels)`);
|
|
1015
|
+
return;
|
|
1016
|
+
}
|
|
1017
|
+
if (gridSize !== null) {
|
|
1018
|
+
overlayGrid(rawPath, target, width, height, gridSize, gridPath);
|
|
1019
|
+
console.log(`${target} (${width}x${height}, raw 1:1 desktop pixels, ${gridSize}px grid)`);
|
|
1020
|
+
return;
|
|
1021
|
+
}
|
|
1022
|
+
const padX = Math.round(width * BRAND_PAD_FRACTION);
|
|
1023
|
+
const padY = Math.round(height * BRAND_PAD_FRACTION);
|
|
1024
|
+
const canvasW = width + padX * 2;
|
|
1025
|
+
const canvasH = height + padY * 2;
|
|
1026
|
+
const minDim = Math.min(width, height);
|
|
1027
|
+
const cornerR = Math.round(minDim * SCREENSHOT_CORNER_FRACTION);
|
|
1028
|
+
const shadowSigma = Math.max(12, Math.round(minDim * SHADOW_SIGMA_FRACTION));
|
|
1029
|
+
const shadowOffsetY = Math.round(minDim * SHADOW_OFFSET_Y_FRACTION);
|
|
1030
|
+
const shadowMargin = shadowSigma * 3;
|
|
1031
|
+
const shadowW = width + shadowMargin * 2;
|
|
1032
|
+
const shadowH = height + shadowMargin * 2;
|
|
1033
|
+
writeFileSync3(svgPath, loadBrandSvg(canvasW, canvasH));
|
|
1034
|
+
writeFileSync3(
|
|
1035
|
+
maskPath,
|
|
1036
|
+
SCREENSHOT_MASK_TEMPLATE.replace(/__W__/g, String(width)).replace(/__H__/g, String(height)).replace(/__R__/g, String(cornerR))
|
|
1037
|
+
);
|
|
1038
|
+
writeFileSync3(
|
|
1039
|
+
shadowPath,
|
|
1040
|
+
SHADOW_MASK_TEMPLATE.replace(/__SW__/g, String(shadowW)).replace(/__SH__/g, String(shadowH)).replace(/__M__/g, String(shadowMargin)).replace(/__W__/g, String(width)).replace(/__H__/g, String(height)).replace(/__R__/g, String(cornerR))
|
|
1041
|
+
);
|
|
1042
|
+
const r = spawnSync3(
|
|
1043
|
+
"ffmpeg",
|
|
1044
|
+
[
|
|
1045
|
+
"-y",
|
|
1046
|
+
"-hide_banner",
|
|
1047
|
+
"-loglevel",
|
|
1048
|
+
"error",
|
|
1049
|
+
"-i",
|
|
1050
|
+
svgPath,
|
|
1051
|
+
"-i",
|
|
1052
|
+
rawPath,
|
|
1053
|
+
"-i",
|
|
1054
|
+
maskPath,
|
|
1055
|
+
"-i",
|
|
1056
|
+
shadowPath,
|
|
1057
|
+
"-filter_complex",
|
|
1058
|
+
`[2:v]format=rgba,alphaextract[mask];[3:v]format=rgba,colorchannelmixer=rr=0:gg=0:bb=0:aa=${SHADOW_ALPHA},gblur=sigma=${shadowSigma}[shadow];[1:v]format=rgba[scr];[scr][mask]alphamerge[rounded];[0:v]format=rgba[bg];[bg][shadow]overlay=${padX - shadowMargin}:${padY - shadowMargin + shadowOffsetY}:format=auto[bg_shadow];[bg_shadow][rounded]overlay=${padX}:${padY}:format=auto`,
|
|
1059
|
+
"-frames:v",
|
|
1060
|
+
"1",
|
|
1061
|
+
"-update",
|
|
1062
|
+
"1",
|
|
1063
|
+
target
|
|
1064
|
+
],
|
|
1065
|
+
{ stdio: "pipe" }
|
|
1066
|
+
);
|
|
1067
|
+
if (r.status !== 0) {
|
|
1068
|
+
fail(`ffmpeg branding failed: ${r.stderr?.toString().trim() || `exit ${r.status}`}`);
|
|
1069
|
+
}
|
|
1070
|
+
} finally {
|
|
1071
|
+
rmSync3(rawPath, { force: true });
|
|
1072
|
+
rmSync3(svgPath, { force: true });
|
|
1073
|
+
rmSync3(maskPath, { force: true });
|
|
1074
|
+
rmSync3(shadowPath, { force: true });
|
|
1075
|
+
rmSync3(gridPath, { force: true });
|
|
1076
|
+
}
|
|
1077
|
+
console.log(target);
|
|
1078
|
+
}
|
|
1079
|
+
function hashFile(path) {
|
|
1080
|
+
return createHash("sha256").update(readFileSync3(path)).digest("hex");
|
|
1081
|
+
}
|
|
1082
|
+
async function captureStableRawScreenshot(target, options) {
|
|
1083
|
+
const start = Date.now();
|
|
1084
|
+
const framePath = `${target}.frame.png`;
|
|
1085
|
+
let lastHash = null;
|
|
1086
|
+
let lastChangeAt = start;
|
|
1087
|
+
let frames = 0;
|
|
1088
|
+
let changes = 0;
|
|
1089
|
+
let width = 0;
|
|
1090
|
+
let height = 0;
|
|
1091
|
+
try {
|
|
1092
|
+
while (Date.now() - start <= options.timeoutMs) {
|
|
1093
|
+
runDisplayCmd("scrot", ["-o", framePath]);
|
|
1094
|
+
frames++;
|
|
1095
|
+
const dimensions = readPngDimensions(framePath);
|
|
1096
|
+
width = dimensions.width;
|
|
1097
|
+
height = dimensions.height;
|
|
1098
|
+
const hash = hashFile(framePath);
|
|
1099
|
+
const now = Date.now();
|
|
1100
|
+
if (lastHash === null) {
|
|
1101
|
+
lastChangeAt = now;
|
|
1102
|
+
} else if (hash !== lastHash) {
|
|
1103
|
+
changes++;
|
|
1104
|
+
lastChangeAt = now;
|
|
1105
|
+
}
|
|
1106
|
+
lastHash = hash;
|
|
1107
|
+
copyFileSync2(framePath, target);
|
|
1108
|
+
if (frames > 1 && now - lastChangeAt >= options.stableMs) {
|
|
1109
|
+
return { width, height, stable: true, elapsedMs: now - start, frames, changes };
|
|
1110
|
+
}
|
|
1111
|
+
await sleep(options.pollMs);
|
|
1112
|
+
}
|
|
1113
|
+
return { width, height, stable: false, elapsedMs: Date.now() - start, frames, changes };
|
|
1114
|
+
} finally {
|
|
1115
|
+
rmSync3(framePath, { force: true });
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
function getActiveWindowTitle() {
|
|
1119
|
+
return tryDisplayCmd("xdotool", ["getactivewindow", "getwindowname"]);
|
|
1120
|
+
}
|
|
1121
|
+
function getVisibleWindowTitles() {
|
|
1122
|
+
const out = tryDisplayCmd("xdotool", ["search", "--onlyvisible", "--name", ".", "getwindowname", "%@"]);
|
|
1123
|
+
return out ? out.split("\n").filter(Boolean).slice(0, 20) : [];
|
|
1124
|
+
}
|
|
1125
|
+
function recordingMousePosition() {
|
|
1126
|
+
const mouse = getMouseLocation();
|
|
1127
|
+
return mouse ? { x: mouse.x, y: mouse.y } : null;
|
|
1128
|
+
}
|
|
1129
|
+
async function computerObserveCommand(path, options = {}) {
|
|
1130
|
+
const target = resolvePath(path);
|
|
1131
|
+
mkdirSync3(dirname3(target), { recursive: true });
|
|
1132
|
+
const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 3e3;
|
|
1133
|
+
const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 600;
|
|
1134
|
+
const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 200;
|
|
1135
|
+
if (timeoutMs < 0) fail("--timeout must be >= 0");
|
|
1136
|
+
if (stableMs < 0) fail("--stable-ms must be >= 0");
|
|
1137
|
+
if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
|
|
1138
|
+
const stamp = `${process.pid}-${Date.now()}`;
|
|
1139
|
+
const rawPath = `/tmp/replicas-observe-${stamp}.raw.png`;
|
|
1140
|
+
const gridPath = `/tmp/replicas-observe-${stamp}.grid.svg`;
|
|
1141
|
+
try {
|
|
1142
|
+
ensureServicesRunning();
|
|
1143
|
+
const capture = await captureStableRawScreenshot(rawPath, { timeoutMs, stableMs, pollMs });
|
|
1144
|
+
const gridSize = options.raw ? null : parseGridSize(options.grid ?? true);
|
|
1145
|
+
if (gridSize === null) {
|
|
1146
|
+
copyFileSync2(rawPath, target);
|
|
1147
|
+
} else {
|
|
1148
|
+
overlayGrid(rawPath, target, capture.width, capture.height, gridSize, gridPath);
|
|
1149
|
+
}
|
|
1150
|
+
console.log(JSON.stringify({
|
|
1151
|
+
screenshot: target,
|
|
1152
|
+
width: capture.width,
|
|
1153
|
+
height: capture.height,
|
|
1154
|
+
stable: capture.stable,
|
|
1155
|
+
elapsedMs: capture.elapsedMs,
|
|
1156
|
+
frames: capture.frames,
|
|
1157
|
+
changes: capture.changes,
|
|
1158
|
+
mouse: getMouseLocation(),
|
|
1159
|
+
activeWindow: getActiveWindowTitle(),
|
|
1160
|
+
visibleWindows: getVisibleWindowTitles(),
|
|
1161
|
+
mode: gridSize === null ? "raw" : "grid",
|
|
1162
|
+
gridSize
|
|
1163
|
+
}, null, 2));
|
|
1164
|
+
} finally {
|
|
1165
|
+
rmSync3(rawPath, { force: true });
|
|
1166
|
+
rmSync3(gridPath, { force: true });
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
async function fetchChromeJson(path) {
|
|
1170
|
+
let res;
|
|
1171
|
+
try {
|
|
1172
|
+
res = await fetch(`http://127.0.0.1:${CHROME_DEBUG_PORT}${path}`);
|
|
1173
|
+
} catch {
|
|
1174
|
+
fail(
|
|
1175
|
+
`Chrome DevTools is not available on localhost:${CHROME_DEBUG_PORT}. Launch Chrome with \`replicas computer launch chrome <url>\` and try again.`
|
|
1176
|
+
);
|
|
1177
|
+
}
|
|
1178
|
+
if (!res.ok) fail(`Chrome DevTools returned HTTP ${res.status}`);
|
|
1179
|
+
try {
|
|
1180
|
+
return await res.json();
|
|
1181
|
+
} catch (error) {
|
|
1182
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1183
|
+
fail(`Chrome DevTools returned invalid JSON from ${path}: ${reason}`);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
async function getChromePages() {
|
|
1187
|
+
const targets = await fetchChromeJson("/json/list");
|
|
1188
|
+
if (!Array.isArray(targets)) fail("Chrome DevTools returned an unexpected target list");
|
|
1189
|
+
return targets.filter((target) => typeof target === "object" && target !== null).filter((target) => target.type === "page");
|
|
1190
|
+
}
|
|
1191
|
+
async function selectChromePage(options = {}) {
|
|
1192
|
+
const pages = await getChromePages();
|
|
1193
|
+
let matches = pages;
|
|
1194
|
+
if (options.targetId) {
|
|
1195
|
+
matches = matches.filter((page2) => page2.id === options.targetId);
|
|
1196
|
+
}
|
|
1197
|
+
if (options.title) {
|
|
1198
|
+
const needle = options.title.toLowerCase();
|
|
1199
|
+
matches = matches.filter((page2) => (page2.title ?? "").toLowerCase().includes(needle));
|
|
1200
|
+
}
|
|
1201
|
+
if (options.url) {
|
|
1202
|
+
const needle = options.url.toLowerCase();
|
|
1203
|
+
matches = matches.filter((page2) => (page2.url ?? "").toLowerCase().includes(needle));
|
|
1204
|
+
}
|
|
1205
|
+
const index = options.page ? parseCoord(options.page, "--page") : 0;
|
|
1206
|
+
if (index < 0) fail("--page must be >= 0");
|
|
1207
|
+
const page = matches[index];
|
|
1208
|
+
const webSocketDebuggerUrl = page?.webSocketDebuggerUrl;
|
|
1209
|
+
if (!webSocketDebuggerUrl) {
|
|
1210
|
+
fail(
|
|
1211
|
+
`No matching debuggable Chrome page found (${matches.length} match${matches.length === 1 ? "" : "es"}, index ${index}). Use \`replicas computer browser\` to list pages.`
|
|
1212
|
+
);
|
|
1213
|
+
}
|
|
1214
|
+
await sendChromeCommand(webSocketDebuggerUrl, "Page.bringToFront", {});
|
|
1215
|
+
return { ...page, webSocketDebuggerUrl };
|
|
1216
|
+
}
|
|
1217
|
+
var VISIBLE_JS = `
|
|
1218
|
+
const visible = (el) => {
|
|
1219
|
+
const rect = el.getBoundingClientRect();
|
|
1220
|
+
const style = getComputedStyle(el);
|
|
1221
|
+
return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
|
|
1222
|
+
};`;
|
|
1223
|
+
async function sendChromeSessionCommand(session, method, params) {
|
|
1224
|
+
return chromeResult(await session.send({ method, params }), method);
|
|
1225
|
+
}
|
|
1226
|
+
async function withChromeSession(webSocketDebuggerUrl, callback) {
|
|
1227
|
+
return await new Promise((resolve3, reject) => {
|
|
1228
|
+
const ws = new WebSocket(webSocketDebuggerUrl);
|
|
1229
|
+
const pending = /* @__PURE__ */ new Map();
|
|
1230
|
+
let nextId = 1;
|
|
1231
|
+
let settled = false;
|
|
1232
|
+
const failSession = (error) => {
|
|
1233
|
+
if (settled) return;
|
|
1234
|
+
settled = true;
|
|
1235
|
+
for (const request of pending.values()) {
|
|
1236
|
+
clearTimeout(request.timeout);
|
|
1237
|
+
request.reject(error);
|
|
1238
|
+
}
|
|
1239
|
+
pending.clear();
|
|
1240
|
+
ws.close();
|
|
1241
|
+
reject(error);
|
|
1242
|
+
};
|
|
1243
|
+
const finishSession = (value) => {
|
|
1244
|
+
if (settled) return;
|
|
1245
|
+
settled = true;
|
|
1246
|
+
for (const request of pending.values()) clearTimeout(request.timeout);
|
|
1247
|
+
pending.clear();
|
|
1248
|
+
ws.close();
|
|
1249
|
+
resolve3(value);
|
|
1250
|
+
};
|
|
1251
|
+
const session = {
|
|
1252
|
+
send: ({ method, params }) => new Promise((resolveCommand, rejectCommand) => {
|
|
1253
|
+
const id = nextId++;
|
|
1254
|
+
const timeout = setTimeout(() => {
|
|
1255
|
+
pending.delete(id);
|
|
1256
|
+
rejectCommand(new Error(`Chrome DevTools ${method} timed out`));
|
|
1257
|
+
}, 1e4);
|
|
1258
|
+
pending.set(id, { method, resolve: resolveCommand, reject: rejectCommand, timeout });
|
|
1259
|
+
ws.send(JSON.stringify({ id, method, params }));
|
|
1260
|
+
})
|
|
1261
|
+
};
|
|
1262
|
+
ws.addEventListener("open", async () => {
|
|
1263
|
+
try {
|
|
1264
|
+
finishSession(await callback(session));
|
|
1265
|
+
} catch (error) {
|
|
1266
|
+
failSession(error instanceof Error ? error : new Error(String(error)));
|
|
1267
|
+
}
|
|
1268
|
+
});
|
|
1269
|
+
ws.addEventListener("message", (event) => {
|
|
1270
|
+
const data = typeof event.data === "string" ? event.data : Buffer.isBuffer(event.data) ? event.data.toString("utf8") : Buffer.from(event.data).toString("utf8");
|
|
1271
|
+
let message;
|
|
1272
|
+
try {
|
|
1273
|
+
message = JSON.parse(data);
|
|
1274
|
+
} catch (error) {
|
|
1275
|
+
failSession(new Error(`Chrome DevTools returned invalid WebSocket JSON: ${error instanceof Error ? error.message : String(error)}`));
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
if (!isRecord(message) || typeof message.id !== "number") return;
|
|
1279
|
+
const request = pending.get(message.id);
|
|
1280
|
+
if (!request) return;
|
|
1281
|
+
clearTimeout(request.timeout);
|
|
1282
|
+
pending.delete(message.id);
|
|
1283
|
+
const detail = isRecord(message.error) && typeof message.error.message === "string" ? message.error.message : void 0;
|
|
1284
|
+
request.resolve(message.error ? { error: { message: detail } } : { result: isRecord(message.result) ? message.result : {} });
|
|
1285
|
+
});
|
|
1286
|
+
ws.addEventListener("error", () => failSession(new Error("Chrome DevTools WebSocket failed")));
|
|
1287
|
+
ws.addEventListener("close", () => {
|
|
1288
|
+
if (!settled) failSession(new Error("Chrome DevTools WebSocket closed unexpectedly"));
|
|
1289
|
+
});
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
async function sendChromeCommands(webSocketDebuggerUrl, commands) {
|
|
1293
|
+
return await withChromeSession(webSocketDebuggerUrl, async (session) => await Promise.all(commands.map((command) => session.send(command))));
|
|
1294
|
+
}
|
|
1295
|
+
async function sendChromeCommand(webSocketDebuggerUrl, method, params) {
|
|
1296
|
+
const response = (await sendChromeCommands(webSocketDebuggerUrl, [{ method, params }]))[0];
|
|
1297
|
+
if (response.error) throw new Error(response.error.message || `Chrome DevTools ${method} failed`);
|
|
1298
|
+
return response.result ?? {};
|
|
1299
|
+
}
|
|
1300
|
+
async function callFunctionOnChromeNodeInSession(session, backendNodeId, functionDeclaration, args = []) {
|
|
1301
|
+
const resolved = await sendChromeSessionCommand(session, "DOM.resolveNode", { backendNodeId });
|
|
1302
|
+
const objectId = isRecord(resolved.object) ? resolved.object.objectId : void 0;
|
|
1303
|
+
if (typeof objectId !== "string") throw new Error(`Could not resolve browser element ref ${backendNodeId}`);
|
|
1304
|
+
return await sendChromeSessionCommand(session, "Runtime.callFunctionOn", {
|
|
1305
|
+
objectId,
|
|
1306
|
+
functionDeclaration,
|
|
1307
|
+
arguments: args.map((value) => ({ value })),
|
|
1308
|
+
returnByValue: true
|
|
1309
|
+
});
|
|
1310
|
+
}
|
|
1311
|
+
function chromeCallValue(response) {
|
|
1312
|
+
return isRecord(response.result) ? response.result.value : void 0;
|
|
1313
|
+
}
|
|
1314
|
+
async function evaluateChromeTarget(webSocketDebuggerUrl, expression) {
|
|
1315
|
+
return await withChromeSession(webSocketDebuggerUrl, async (session) => await evaluateChromeSession(session, expression));
|
|
1316
|
+
}
|
|
1317
|
+
async function evaluateChromeSession(session, expression) {
|
|
1318
|
+
const response = await sendChromeSessionCommand(session, "Runtime.evaluate", {
|
|
1319
|
+
expression,
|
|
1320
|
+
awaitPromise: true,
|
|
1321
|
+
returnByValue: true
|
|
1322
|
+
});
|
|
1323
|
+
const exception = response.exceptionDetails;
|
|
1324
|
+
if (exception) {
|
|
1325
|
+
const text = typeof exception === "object" && exception !== null && "text" in exception && typeof exception.text === "string" ? exception.text : "";
|
|
1326
|
+
throw new Error(text || "Chrome DevTools evaluation threw");
|
|
1327
|
+
}
|
|
1328
|
+
const evaluated = response.result;
|
|
1329
|
+
return typeof evaluated === "object" && evaluated !== null && "value" in evaluated ? evaluated.value ?? null : null;
|
|
1330
|
+
}
|
|
1331
|
+
function isRawAXNode(value) {
|
|
1332
|
+
return isRecord(value);
|
|
1333
|
+
}
|
|
1334
|
+
function isDOMSnapshotDocument(value) {
|
|
1335
|
+
return isRecord(value);
|
|
1336
|
+
}
|
|
1337
|
+
function chromeVisualViewport(layout) {
|
|
1338
|
+
const viewport = isRecord(layout.cssVisualViewport) ? layout.cssVisualViewport : {};
|
|
1339
|
+
return {
|
|
1340
|
+
width: typeof viewport.clientWidth === "number" ? viewport.clientWidth : 0,
|
|
1341
|
+
height: typeof viewport.clientHeight === "number" ? viewport.clientHeight : 0
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
function roundBrowserRect(rect) {
|
|
1345
|
+
return {
|
|
1346
|
+
x: Math.round(rect.x),
|
|
1347
|
+
y: Math.round(rect.y),
|
|
1348
|
+
width: Math.round(rect.width),
|
|
1349
|
+
height: Math.round(rect.height),
|
|
1350
|
+
centerX: Math.round(rect.centerX),
|
|
1351
|
+
centerY: Math.round(rect.centerY)
|
|
1352
|
+
};
|
|
1353
|
+
}
|
|
1354
|
+
function browserStateProperties(node) {
|
|
1355
|
+
const properties = [];
|
|
1356
|
+
for (const { name, value } of node.properties ?? []) {
|
|
1357
|
+
const state = value?.value;
|
|
1358
|
+
if (!name || !BROWSER_STATE_PROPERTIES.has(name) || typeof state !== "string" && typeof state !== "number" && typeof state !== "boolean") continue;
|
|
1359
|
+
properties.push([name, name === "url" ? clippedText(state, 300) : state]);
|
|
1360
|
+
}
|
|
1361
|
+
return Object.fromEntries(properties);
|
|
1362
|
+
}
|
|
1363
|
+
function readBrowserStateCache(path) {
|
|
1364
|
+
let value;
|
|
1365
|
+
try {
|
|
1366
|
+
value = JSON.parse(readFileSync3(path, "utf8"));
|
|
1367
|
+
} catch {
|
|
1368
|
+
return null;
|
|
1369
|
+
}
|
|
1370
|
+
if (!isRecord(value) || !Array.isArray(value.entries)) return null;
|
|
1371
|
+
const entries = [];
|
|
1372
|
+
for (const entry of value.entries) {
|
|
1373
|
+
if (!isRecord(entry) || typeof entry.key !== "string" || typeof entry.semantic !== "string" || typeof entry.line !== "string") return null;
|
|
1374
|
+
entries.push({ key: entry.key, semantic: entry.semantic, line: entry.line });
|
|
1375
|
+
}
|
|
1376
|
+
return {
|
|
1377
|
+
url: typeof value.url === "string" ? value.url : void 0,
|
|
1378
|
+
documentId: typeof value.documentId === "string" || value.documentId === null ? value.documentId : void 0,
|
|
1379
|
+
entries
|
|
1380
|
+
};
|
|
1381
|
+
}
|
|
1382
|
+
function chromeResult(response, method) {
|
|
1383
|
+
if (response.error) throw new Error(response.error.message || `Chrome DevTools ${method} failed`);
|
|
1384
|
+
return response.result ?? {};
|
|
1385
|
+
}
|
|
1386
|
+
function collectFrameIds(frameTree) {
|
|
1387
|
+
const ids = [];
|
|
1388
|
+
const visit = (value) => {
|
|
1389
|
+
if (!isRecord(value)) return;
|
|
1390
|
+
const frame = isRecord(value.frame) ? value.frame : {};
|
|
1391
|
+
if (typeof frame.id === "string") ids.push(frame.id);
|
|
1392
|
+
if (Array.isArray(value.childFrames)) value.childFrames.forEach(visit);
|
|
1393
|
+
};
|
|
1394
|
+
visit(frameTree);
|
|
1395
|
+
return ids;
|
|
1396
|
+
}
|
|
1397
|
+
function browserDocumentId(frameTree) {
|
|
1398
|
+
if (!isRecord(frameTree) || !isRecord(frameTree.frame)) return null;
|
|
1399
|
+
const loaderId = frameTree.frame.loaderId;
|
|
1400
|
+
return typeof loaderId === "string" ? loaderId : null;
|
|
1401
|
+
}
|
|
1402
|
+
function intersectRects(a, b) {
|
|
1403
|
+
const x = Math.max(a.x, b.x);
|
|
1404
|
+
const y = Math.max(a.y, b.y);
|
|
1405
|
+
const right = Math.min(a.x + a.width, b.x + b.width);
|
|
1406
|
+
const bottom = Math.min(a.y + a.height, b.y + b.height);
|
|
1407
|
+
if (right <= x || bottom <= y) return null;
|
|
1408
|
+
return {
|
|
1409
|
+
x,
|
|
1410
|
+
y,
|
|
1411
|
+
width: right - x,
|
|
1412
|
+
height: bottom - y,
|
|
1413
|
+
centerX: x + (right - x) / 2,
|
|
1414
|
+
centerY: y + (bottom - y) / 2
|
|
1415
|
+
};
|
|
1416
|
+
}
|
|
1417
|
+
function browserContentQuadRects(quads, viewportRect) {
|
|
1418
|
+
if (!Array.isArray(quads)) return [];
|
|
1419
|
+
return quads.filter((quad) => Array.isArray(quad) && quad.length >= 8 && quad.every(Number.isFinite)).map((quad) => {
|
|
1420
|
+
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
|
1421
|
+
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
|
1422
|
+
const x = Math.min(...xs);
|
|
1423
|
+
const y = Math.min(...ys);
|
|
1424
|
+
const width = Math.max(...xs) - x;
|
|
1425
|
+
const height = Math.max(...ys) - y;
|
|
1426
|
+
return { x, y, width, height, centerX: x + width / 2, centerY: y + height / 2 };
|
|
1427
|
+
}).map((rect) => intersectRects(rect, viewportRect)).filter((rect) => !!rect).sort((a, b) => b.width * b.height - a.width * a.height);
|
|
1428
|
+
}
|
|
1429
|
+
function buildBrowserBounds(result, viewport, targetId) {
|
|
1430
|
+
const strings = Array.isArray(result.strings) ? result.strings : [];
|
|
1431
|
+
const documents = Array.isArray(result.documents) ? result.documents.filter(isDOMSnapshotDocument) : [];
|
|
1432
|
+
const localRects = documents.map((document) => {
|
|
1433
|
+
const rects = /* @__PURE__ */ new Map();
|
|
1434
|
+
const nodeIndexes = document.layout?.nodeIndex ?? [];
|
|
1435
|
+
const bounds = document.layout?.bounds ?? [];
|
|
1436
|
+
const scrollX = document.scrollOffsetX ?? 0;
|
|
1437
|
+
const scrollY = document.scrollOffsetY ?? 0;
|
|
1438
|
+
nodeIndexes.forEach((nodeIndex, index) => {
|
|
1439
|
+
const bound = bounds[index];
|
|
1440
|
+
if (!Array.isArray(bound) || bound.length < 4 || !bound.every(Number.isFinite)) return;
|
|
1441
|
+
const [rawX, rawY, width, height] = bound;
|
|
1442
|
+
rects.set(nodeIndex, {
|
|
1443
|
+
x: rawX - scrollX,
|
|
1444
|
+
y: rawY - scrollY,
|
|
1445
|
+
width,
|
|
1446
|
+
height,
|
|
1447
|
+
centerX: rawX - scrollX + width / 2,
|
|
1448
|
+
centerY: rawY - scrollY + height / 2
|
|
1449
|
+
});
|
|
1450
|
+
});
|
|
1451
|
+
return rects;
|
|
1452
|
+
});
|
|
1453
|
+
const parents = /* @__PURE__ */ new Map();
|
|
1454
|
+
documents.forEach((document, documentIndex) => {
|
|
1455
|
+
const sparse = document.nodes?.contentDocumentIndex;
|
|
1456
|
+
sparse?.index?.forEach((nodeIndex, index) => {
|
|
1457
|
+
const childDocumentIndex = sparse.value?.[index];
|
|
1458
|
+
if (typeof childDocumentIndex === "number") parents.set(childDocumentIndex, { documentIndex, nodeIndex });
|
|
1459
|
+
});
|
|
1460
|
+
});
|
|
1461
|
+
const targetDocumentIndex = documents.findIndex(
|
|
1462
|
+
(document) => typeof document.frameId === "number" && strings[document.frameId] === targetId
|
|
1463
|
+
);
|
|
1464
|
+
const rootIndex = targetDocumentIndex >= 0 ? targetDocumentIndex : documents.findIndex((_, index) => !parents.has(index));
|
|
1465
|
+
const viewportRect = {
|
|
1466
|
+
x: 0,
|
|
1467
|
+
y: 0,
|
|
1468
|
+
width: viewport.width,
|
|
1469
|
+
height: viewport.height,
|
|
1470
|
+
centerX: viewport.width / 2,
|
|
1471
|
+
centerY: viewport.height / 2
|
|
1472
|
+
};
|
|
1473
|
+
const origins = /* @__PURE__ */ new Map();
|
|
1474
|
+
origins.set(rootIndex < 0 ? 0 : rootIndex, { x: 0, y: 0, clip: viewportRect });
|
|
1475
|
+
for (let pass = 0; pass < documents.length; pass++) {
|
|
1476
|
+
let changed = false;
|
|
1477
|
+
for (const [childIndex, parent] of parents) {
|
|
1478
|
+
if (origins.has(childIndex)) continue;
|
|
1479
|
+
const parentOrigin = origins.get(parent.documentIndex);
|
|
1480
|
+
const frameRect = localRects[parent.documentIndex]?.get(parent.nodeIndex);
|
|
1481
|
+
if (!parentOrigin || !frameRect) continue;
|
|
1482
|
+
const globalFrameRect = {
|
|
1483
|
+
...frameRect,
|
|
1484
|
+
x: parentOrigin.x + frameRect.x,
|
|
1485
|
+
y: parentOrigin.y + frameRect.y,
|
|
1486
|
+
centerX: parentOrigin.x + frameRect.centerX,
|
|
1487
|
+
centerY: parentOrigin.y + frameRect.centerY
|
|
1488
|
+
};
|
|
1489
|
+
const childDocument = documents[childIndex];
|
|
1490
|
+
const insetX = Math.max(0, (globalFrameRect.width - (childDocument?.contentWidth ?? globalFrameRect.width)) / 2);
|
|
1491
|
+
const insetY = Math.max(0, (globalFrameRect.height - (childDocument?.contentHeight ?? globalFrameRect.height)) / 2);
|
|
1492
|
+
const contentRect = {
|
|
1493
|
+
x: globalFrameRect.x + insetX,
|
|
1494
|
+
y: globalFrameRect.y + insetY,
|
|
1495
|
+
width: Math.max(0, globalFrameRect.width - insetX * 2),
|
|
1496
|
+
height: Math.max(0, globalFrameRect.height - insetY * 2),
|
|
1497
|
+
centerX: globalFrameRect.centerX,
|
|
1498
|
+
centerY: globalFrameRect.centerY
|
|
1499
|
+
};
|
|
1500
|
+
const clip = intersectRects(parentOrigin.clip, contentRect);
|
|
1501
|
+
if (!clip) continue;
|
|
1502
|
+
origins.set(childIndex, { x: contentRect.x, y: contentRect.y, clip });
|
|
1503
|
+
changed = true;
|
|
1504
|
+
}
|
|
1505
|
+
if (!changed) break;
|
|
1506
|
+
}
|
|
1507
|
+
const byBackendNodeId = /* @__PURE__ */ new Map();
|
|
1508
|
+
documents.forEach((document, documentIndex) => {
|
|
1509
|
+
const origin = origins.get(documentIndex);
|
|
1510
|
+
if (!origin) return;
|
|
1511
|
+
const backendNodeIds = document.nodes?.backendNodeId ?? [];
|
|
1512
|
+
for (const [nodeIndex, local] of localRects[documentIndex] ?? []) {
|
|
1513
|
+
const backendNodeId = backendNodeIds[nodeIndex];
|
|
1514
|
+
if (!Number.isInteger(backendNodeId)) continue;
|
|
1515
|
+
const rect = {
|
|
1516
|
+
...local,
|
|
1517
|
+
x: origin.x + local.x,
|
|
1518
|
+
y: origin.y + local.y,
|
|
1519
|
+
centerX: origin.x + local.centerX,
|
|
1520
|
+
centerY: origin.y + local.centerY
|
|
1521
|
+
};
|
|
1522
|
+
if (intersectRects(rect, origin.clip)) byBackendNodeId.set(backendNodeId, { rect, clip: origin.clip });
|
|
1523
|
+
}
|
|
1524
|
+
});
|
|
1525
|
+
return byBackendNodeId;
|
|
1526
|
+
}
|
|
1527
|
+
var BROWSER_INTERACTIVE_ROLES = /* @__PURE__ */ new Set([
|
|
1528
|
+
"button",
|
|
1529
|
+
"checkbox",
|
|
1530
|
+
"combobox",
|
|
1531
|
+
"link",
|
|
1532
|
+
"listbox",
|
|
1533
|
+
"menuitem",
|
|
1534
|
+
"menuitemcheckbox",
|
|
1535
|
+
"menuitemradio",
|
|
1536
|
+
"option",
|
|
1537
|
+
"radio",
|
|
1538
|
+
"scrollbar",
|
|
1539
|
+
"searchbox",
|
|
1540
|
+
"slider",
|
|
1541
|
+
"spinbutton",
|
|
1542
|
+
"switch",
|
|
1543
|
+
"tab",
|
|
1544
|
+
"textbox",
|
|
1545
|
+
"treeitem"
|
|
1546
|
+
]);
|
|
1547
|
+
var BROWSER_STRUCTURAL_ROLES = /* @__PURE__ */ new Set([
|
|
1548
|
+
"alert",
|
|
1549
|
+
"cell",
|
|
1550
|
+
"columnheader",
|
|
1551
|
+
"dialog",
|
|
1552
|
+
"figure",
|
|
1553
|
+
"gridcell",
|
|
1554
|
+
"heading",
|
|
1555
|
+
"image",
|
|
1556
|
+
"listitem",
|
|
1557
|
+
"main",
|
|
1558
|
+
"paragraph",
|
|
1559
|
+
"region",
|
|
1560
|
+
"row",
|
|
1561
|
+
"rowheader",
|
|
1562
|
+
"status",
|
|
1563
|
+
"StaticText"
|
|
1564
|
+
]);
|
|
1565
|
+
var BROWSER_STATE_PROPERTIES = /* @__PURE__ */ new Set([
|
|
1566
|
+
"autocomplete",
|
|
1567
|
+
"checked",
|
|
1568
|
+
"disabled",
|
|
1569
|
+
"editable",
|
|
1570
|
+
"expanded",
|
|
1571
|
+
"focusable",
|
|
1572
|
+
"focused",
|
|
1573
|
+
"haspopup",
|
|
1574
|
+
"invalid",
|
|
1575
|
+
"level",
|
|
1576
|
+
"multiselectable",
|
|
1577
|
+
"orientation",
|
|
1578
|
+
"pressed",
|
|
1579
|
+
"readonly",
|
|
1580
|
+
"protected",
|
|
1581
|
+
"required",
|
|
1582
|
+
"selected",
|
|
1583
|
+
"url"
|
|
1584
|
+
]);
|
|
1585
|
+
function clippedText(value, limit = 240) {
|
|
1586
|
+
return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
|
|
1587
|
+
}
|
|
1588
|
+
function browserTargetOutput(page) {
|
|
1589
|
+
return {
|
|
1590
|
+
targetId: page.id ?? null,
|
|
1591
|
+
title: (page.title ?? "").slice(0, 1e3),
|
|
1592
|
+
url: (page.url ?? "").slice(0, 2e3)
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
function browserNodeActionable(role, states) {
|
|
1596
|
+
return BROWSER_INTERACTIVE_ROLES.has(role) || states.editable === true || states.focusable === true && role !== "RootWebArea";
|
|
1597
|
+
}
|
|
1598
|
+
async function captureBrowserSnapshot(page, options) {
|
|
1599
|
+
const initialCommands = [
|
|
1600
|
+
{ method: "Page.getFrameTree", params: {} },
|
|
1601
|
+
{ method: "Page.getLayoutMetrics", params: {} },
|
|
1602
|
+
{ method: "DOMSnapshot.captureSnapshot", params: { computedStyles: [], includeDOMRects: true, includePaintOrder: true } }
|
|
1603
|
+
];
|
|
1604
|
+
const initial = await sendChromeCommands(page.webSocketDebuggerUrl, initialCommands);
|
|
1605
|
+
const frameTree = chromeResult(initial[0], initialCommands[0].method).frameTree;
|
|
1606
|
+
const layout = chromeResult(initial[1], initialCommands[1].method);
|
|
1607
|
+
const domSnapshot = chromeResult(initial[2], initialCommands[2].method);
|
|
1608
|
+
const viewport = chromeVisualViewport(layout);
|
|
1609
|
+
const frameIds = collectFrameIds(frameTree);
|
|
1610
|
+
if (frameIds.length === 0 && page.id) frameIds.push(page.id);
|
|
1611
|
+
const axCommands = frameIds.map((frameId) => ({
|
|
1612
|
+
method: "Accessibility.getFullAXTree",
|
|
1613
|
+
params: { frameId }
|
|
1614
|
+
}));
|
|
1615
|
+
const axResults = await sendChromeCommands(page.webSocketDebuggerUrl, axCommands);
|
|
1616
|
+
const nodes = [];
|
|
1617
|
+
axResults.forEach((response, index) => {
|
|
1618
|
+
if (response.error) return;
|
|
1619
|
+
const responseNodes = response.result?.nodes;
|
|
1620
|
+
if (!Array.isArray(responseNodes)) return;
|
|
1621
|
+
for (const node of responseNodes) {
|
|
1622
|
+
if (!isRawAXNode(node)) continue;
|
|
1623
|
+
nodes.push({ ...node, sourceFrameId: frameIds[index] });
|
|
1624
|
+
}
|
|
1625
|
+
});
|
|
1626
|
+
const bounds = buildBrowserBounds(domSnapshot, viewport, page.id);
|
|
1627
|
+
const nodesByKey = new Map(nodes.map((node) => [`${node.sourceFrameId}:${node.nodeId}`, node]));
|
|
1628
|
+
const propertyMap = browserStateProperties;
|
|
1629
|
+
const roleOf = (node) => clippedText(node.role?.value, 80);
|
|
1630
|
+
const nameOf = (node) => clippedText(node.name?.value);
|
|
1631
|
+
const valueOf = (node) => {
|
|
1632
|
+
if (propertyMap(node).protected === true) return null;
|
|
1633
|
+
const value = node.value?.value;
|
|
1634
|
+
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
1635
|
+
const text = clippedText(value);
|
|
1636
|
+
if (/^[•●*]+$/.test(text)) return "[protected]";
|
|
1637
|
+
return text || null;
|
|
1638
|
+
};
|
|
1639
|
+
const meaningful = nodes.filter((node) => !node.ignored && Number.isInteger(node.backendDOMNodeId)).map((node) => ({
|
|
1640
|
+
node,
|
|
1641
|
+
role: roleOf(node),
|
|
1642
|
+
bound: typeof node.backendDOMNodeId === "number" ? bounds.get(node.backendDOMNodeId) : void 0
|
|
1643
|
+
})).filter(({ node, role, bound }) => {
|
|
1644
|
+
if (!bound || bound.rect.width <= 0 || bound.rect.height <= 0) return false;
|
|
1645
|
+
if (role === "InlineTextBox" || role === "none" || role === "generic") return false;
|
|
1646
|
+
if (BROWSER_INTERACTIVE_ROLES.has(role) || BROWSER_STRUCTURAL_ROLES.has(role)) return true;
|
|
1647
|
+
return !!nameOf(node) || valueOf(node) !== null || propertyMap(node).focusable === true;
|
|
1648
|
+
}).filter(({ node, role }) => {
|
|
1649
|
+
if (role !== "StaticText") return true;
|
|
1650
|
+
const parent = node.parentId ? nodesByKey.get(`${node.sourceFrameId}:${node.parentId}`) : void 0;
|
|
1651
|
+
return !parent || nameOf(parent) !== nameOf(node);
|
|
1652
|
+
}).sort((a, b) => a.bound.rect.y - b.bound.rect.y || a.bound.rect.x - b.bound.rect.x);
|
|
1653
|
+
const elements = [];
|
|
1654
|
+
const entries = [];
|
|
1655
|
+
const meaningfulKeys = new Set(meaningful.map(({ node }) => `${node.sourceFrameId}:${node.nodeId}`));
|
|
1656
|
+
const visibleText = [];
|
|
1657
|
+
let visibleTextLength = 0;
|
|
1658
|
+
let treeLength = 0;
|
|
1659
|
+
const treeLimit = Math.min(5e4, Math.max(4e3, options.textLimit * 2));
|
|
1660
|
+
for (const { node, role, bound } of meaningful) {
|
|
1661
|
+
const name = nameOf(node);
|
|
1662
|
+
const description = clippedText(node.description?.value) || null;
|
|
1663
|
+
const value = valueOf(node);
|
|
1664
|
+
const states = propertyMap(node);
|
|
1665
|
+
const actionable = browserNodeActionable(role, states);
|
|
1666
|
+
const ref = actionable && elements.length < options.elementLimit ? String(node.backendDOMNodeId) : null;
|
|
1667
|
+
const visibleRect = role === "RootWebArea" && node.sourceFrameId === page.id ? { x: 0, y: 0, width: viewport.width, height: viewport.height, centerX: viewport.width / 2, centerY: viewport.height / 2 } : intersectRects(bound.rect, bound.clip) ?? bound.rect;
|
|
1668
|
+
const rect = roundBrowserRect(visibleRect);
|
|
1669
|
+
const actions = [
|
|
1670
|
+
...actionable ? ["click"] : [],
|
|
1671
|
+
...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
|
|
1672
|
+
"scroll"
|
|
1673
|
+
];
|
|
1674
|
+
if (ref) {
|
|
1675
|
+
elements.push({
|
|
1676
|
+
ref,
|
|
1677
|
+
role,
|
|
1678
|
+
name: name || null,
|
|
1679
|
+
description,
|
|
1680
|
+
value,
|
|
1681
|
+
states,
|
|
1682
|
+
actions,
|
|
1683
|
+
rect,
|
|
1684
|
+
frameId: node.sourceFrameId
|
|
1685
|
+
});
|
|
1686
|
+
}
|
|
1687
|
+
const stateText = Object.entries(states).filter(([key]) => key !== "url" && key !== "focusable").map(([key, state]) => `${key}=${JSON.stringify(state)}`);
|
|
1688
|
+
const url = typeof states.url === "string" ? clippedText(states.url, 300) : "";
|
|
1689
|
+
const semantic = [role, name && JSON.stringify(name), value !== null && `value=${JSON.stringify(value)}`, description && `description=${JSON.stringify(description)}`, url && `url=${JSON.stringify(url)}`, ...stateText].filter(Boolean).join(" ");
|
|
1690
|
+
let depth = 0;
|
|
1691
|
+
let parentId = node.parentId;
|
|
1692
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1693
|
+
while (parentId && depth < 8) {
|
|
1694
|
+
const parentKey = `${node.sourceFrameId}:${parentId}`;
|
|
1695
|
+
if (visited.has(parentKey)) break;
|
|
1696
|
+
visited.add(parentKey);
|
|
1697
|
+
if (meaningfulKeys.has(parentKey)) depth++;
|
|
1698
|
+
parentId = nodesByKey.get(parentKey)?.parentId;
|
|
1699
|
+
}
|
|
1700
|
+
const line = `${" ".repeat(depth)}${ref ? `[ref=${ref}]` : "-"} ${semantic} (${rect.x},${rect.y} ${rect.width}x${rect.height})`;
|
|
1701
|
+
if (treeLength + line.length + 1 <= treeLimit) {
|
|
1702
|
+
const key = Number.isInteger(node.backendDOMNodeId) ? `dom:${node.sourceFrameId}:${node.backendDOMNodeId}` : `ax:${node.sourceFrameId}:${node.nodeId}`;
|
|
1703
|
+
entries.push({ key, semantic, line });
|
|
1704
|
+
treeLength += line.length + 1;
|
|
1705
|
+
}
|
|
1706
|
+
if (name && visibleTextLength < options.textLimit && (role === "StaticText" || role === "heading" || BROWSER_INTERACTIVE_ROLES.has(role))) {
|
|
1707
|
+
const remaining = options.textLimit - visibleTextLength;
|
|
1708
|
+
visibleText.push(name.slice(0, remaining));
|
|
1709
|
+
visibleTextLength += Math.min(name.length, remaining) + 1;
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
const revision = createHash("sha256").update(JSON.stringify([page.url, entries.map(({ key, semantic }) => [key, semantic])])).digest("hex").slice(0, 16);
|
|
1713
|
+
const snapshot = {
|
|
1714
|
+
title: (page.title ?? "").slice(0, 1e3),
|
|
1715
|
+
url: (page.url ?? "").slice(0, 2e3),
|
|
1716
|
+
documentId: browserDocumentId(frameTree),
|
|
1717
|
+
revision,
|
|
1718
|
+
viewport,
|
|
1719
|
+
text: visibleText.join(" ").slice(0, options.textLimit),
|
|
1720
|
+
tree: entries.map(({ line }) => line).join("\n"),
|
|
1721
|
+
elements,
|
|
1722
|
+
controls: elements,
|
|
1723
|
+
nodeCount: entries.length,
|
|
1724
|
+
elementCount: elements.length,
|
|
1725
|
+
truncated: entries.length < meaningful.length || elements.length < meaningful.filter(({ node, role }) => browserNodeActionable(role, propertyMap(node))).length
|
|
1726
|
+
};
|
|
1727
|
+
return { snapshot, entries };
|
|
1728
|
+
}
|
|
1729
|
+
async function computerBrowserCommand(options = {}) {
|
|
1730
|
+
const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 4e3;
|
|
1731
|
+
const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 80;
|
|
1732
|
+
if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
|
|
1733
|
+
if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
|
|
1734
|
+
const targets = options.targetId || options.page || options.title || options.url ? [await selectChromePage(options)] : await getChromePages();
|
|
1735
|
+
const pages = targets.map((target) => ({
|
|
1736
|
+
target,
|
|
1737
|
+
page: {
|
|
1738
|
+
id: target.id ?? null,
|
|
1739
|
+
title: target.title ?? "",
|
|
1740
|
+
url: target.url ?? "",
|
|
1741
|
+
attached: !!target.attached
|
|
1742
|
+
}
|
|
1743
|
+
}));
|
|
1744
|
+
const pageResults = [];
|
|
1745
|
+
for (const { target, page } of pages) {
|
|
1746
|
+
if (options.snapshot && target.webSocketDebuggerUrl) {
|
|
1747
|
+
try {
|
|
1748
|
+
const { snapshot } = await captureBrowserSnapshot(
|
|
1749
|
+
{ ...target, webSocketDebuggerUrl: target.webSocketDebuggerUrl },
|
|
1750
|
+
{ textLimit, elementLimit }
|
|
1751
|
+
);
|
|
1752
|
+
pageResults.push({ ...page, snapshot });
|
|
1753
|
+
} catch (error) {
|
|
1754
|
+
pageResults.push({ ...page, snapshotError: error instanceof Error ? error.message : "snapshot failed" });
|
|
1755
|
+
}
|
|
1756
|
+
} else {
|
|
1757
|
+
pageResults.push(page);
|
|
1758
|
+
}
|
|
1759
|
+
}
|
|
1760
|
+
console.log(JSON.stringify({
|
|
1761
|
+
port: CHROME_DEBUG_PORT,
|
|
1762
|
+
pageCount: pageResults.length,
|
|
1763
|
+
pages: pageResults
|
|
1764
|
+
}, null, 2));
|
|
1765
|
+
}
|
|
1766
|
+
function buildBrowserStabilityExpression() {
|
|
1767
|
+
return `(() => {
|
|
1768
|
+
const key = '__replicasComputerUseObserver';
|
|
1769
|
+
if (!globalThis[key]) {
|
|
1770
|
+
const state = { revision: 0 };
|
|
1771
|
+
const observer = new MutationObserver(() => state.revision++);
|
|
1772
|
+
globalThis[key] = { state, observer, observing: false };
|
|
1773
|
+
}
|
|
1774
|
+
if (document.documentElement && !globalThis[key].observing) {
|
|
1775
|
+
globalThis[key].observer.observe(document.documentElement, {
|
|
1776
|
+
subtree: true,
|
|
1777
|
+
childList: true,
|
|
1778
|
+
attributes: true,
|
|
1779
|
+
characterData: true,
|
|
1780
|
+
});
|
|
1781
|
+
globalThis[key].observing = true;
|
|
1782
|
+
}
|
|
1783
|
+
const controls = Array.from(document.querySelectorAll('input,textarea,select,[contenteditable="true"],[role="checkbox"],[role="combobox"],[role="slider"],[role="switch"],[role="textbox"]')).slice(0, 200);
|
|
1784
|
+
const active = document.activeElement;
|
|
1785
|
+
const controlState = controls.map((element, index) => {
|
|
1786
|
+
const value = 'value' in element
|
|
1787
|
+
? element instanceof HTMLInputElement && element.type === 'password'
|
|
1788
|
+
? String(element.value).length
|
|
1789
|
+
: String(element.value)
|
|
1790
|
+
: element.textContent;
|
|
1791
|
+
return [
|
|
1792
|
+
index,
|
|
1793
|
+
element === active,
|
|
1794
|
+
value,
|
|
1795
|
+
'checked' in element ? element.checked : null,
|
|
1796
|
+
'selectedIndex' in element ? element.selectedIndex : null,
|
|
1797
|
+
'disabled' in element ? element.disabled : null,
|
|
1798
|
+
element.getAttribute('aria-checked'),
|
|
1799
|
+
element.getAttribute('aria-expanded'),
|
|
1800
|
+
element.getAttribute('aria-selected'),
|
|
1801
|
+
].join('|');
|
|
1802
|
+
}).join('\\u001f');
|
|
1803
|
+
let controlRevision = 2166136261;
|
|
1804
|
+
for (let index = 0; index < controlState.length; index++) {
|
|
1805
|
+
controlRevision ^= controlState.charCodeAt(index);
|
|
1806
|
+
controlRevision = Math.imul(controlRevision, 16777619);
|
|
1807
|
+
}
|
|
1808
|
+
return {
|
|
1809
|
+
revision: globalThis[key].state.revision,
|
|
1810
|
+
controlRevision: (controlRevision >>> 0).toString(16),
|
|
1811
|
+
readyState: document.readyState,
|
|
1812
|
+
busy: !!document.querySelector('[aria-busy="true"]'),
|
|
1813
|
+
url: location.href.slice(0, 2000),
|
|
1814
|
+
scrollX: Math.round(scrollX),
|
|
1815
|
+
scrollY: Math.round(scrollY),
|
|
1816
|
+
innerWidth,
|
|
1817
|
+
innerHeight,
|
|
1818
|
+
devicePixelRatio,
|
|
1819
|
+
};
|
|
1820
|
+
})()`;
|
|
1821
|
+
}
|
|
1822
|
+
async function waitForBrowserStability(page, options) {
|
|
1823
|
+
const startedAt = Date.now();
|
|
1824
|
+
let lastSignature = null;
|
|
1825
|
+
let lastChangeAt = startedAt;
|
|
1826
|
+
let samples = 0;
|
|
1827
|
+
let changes = 0;
|
|
1828
|
+
let state = null;
|
|
1829
|
+
while (Date.now() - startedAt <= options.timeoutMs) {
|
|
1830
|
+
try {
|
|
1831
|
+
state = await evaluateChromeTarget(page.webSocketDebuggerUrl, buildBrowserStabilityExpression());
|
|
1832
|
+
samples++;
|
|
1833
|
+
const signature = JSON.stringify(state);
|
|
1834
|
+
const now = Date.now();
|
|
1835
|
+
if (lastSignature === null || signature !== lastSignature) {
|
|
1836
|
+
if (lastSignature !== null) changes++;
|
|
1837
|
+
lastSignature = signature;
|
|
1838
|
+
lastChangeAt = now;
|
|
1839
|
+
}
|
|
1840
|
+
const ready = typeof state === "object" && state !== null && "readyState" in state && state.readyState !== "loading" && "busy" in state && state.busy !== true;
|
|
1841
|
+
if (samples > 1 && ready && now - lastChangeAt >= options.stableMs) {
|
|
1842
|
+
return { stable: true, elapsedMs: now - startedAt, samples, changes, state };
|
|
1843
|
+
}
|
|
1844
|
+
} catch {
|
|
1845
|
+
}
|
|
1846
|
+
await sleep(options.pollMs);
|
|
1847
|
+
}
|
|
1848
|
+
return { stable: false, elapsedMs: Date.now() - startedAt, samples, changes, state };
|
|
1849
|
+
}
|
|
1850
|
+
function browserStateCachePath(targetId) {
|
|
1851
|
+
return `${STATE_DIR}/browser-state-${targetId.replace(/[^a-zA-Z0-9_-]/g, "_")}.json`;
|
|
1852
|
+
}
|
|
1853
|
+
function browserStateDiff(previous, current) {
|
|
1854
|
+
const before = new Map(previous.map((entry) => [entry.key, entry]));
|
|
1855
|
+
const after = new Map(current.map((entry) => [entry.key, entry]));
|
|
1856
|
+
const lines = [];
|
|
1857
|
+
let added = 0;
|
|
1858
|
+
let changed = 0;
|
|
1859
|
+
let removed = 0;
|
|
1860
|
+
for (const entry of current) {
|
|
1861
|
+
const old = before.get(entry.key);
|
|
1862
|
+
if (!old) {
|
|
1863
|
+
added++;
|
|
1864
|
+
lines.push(`+ ${entry.line}`);
|
|
1865
|
+
} else if (old.semantic !== entry.semantic) {
|
|
1866
|
+
changed++;
|
|
1867
|
+
lines.push(`~ ${old.line}
|
|
1868
|
+
${entry.line}`);
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
for (const entry of previous) {
|
|
1872
|
+
if (after.has(entry.key)) continue;
|
|
1873
|
+
removed++;
|
|
1874
|
+
lines.push(`- ${entry.line}`);
|
|
1875
|
+
}
|
|
1876
|
+
return { tree: lines.join("\n") || "No accessibility changes.", added, changed, removed };
|
|
1877
|
+
}
|
|
1878
|
+
async function computerBrowserStateCommand(path, options = {}) {
|
|
1879
|
+
const textLimit = options.limit ? parseCoord(options.limit, "--limit") : 8e3;
|
|
1880
|
+
const elementLimit = options.elementLimit ? parseCoord(options.elementLimit, "--element-limit") : 120;
|
|
1881
|
+
const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 5e3;
|
|
1882
|
+
const stableMs = options.stableMs ? parseCoord(options.stableMs, "--stable-ms") : 500;
|
|
1883
|
+
const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 100;
|
|
1884
|
+
if (textLimit < 0 || textLimit > 5e4) fail("--limit must be between 0 and 50000");
|
|
1885
|
+
if (elementLimit < 0 || elementLimit > 500) fail("--element-limit must be between 0 and 500");
|
|
1886
|
+
if (timeoutMs < 0) fail("--timeout must be >= 0");
|
|
1887
|
+
if (stableMs < 0) fail("--stable-ms must be >= 0");
|
|
1888
|
+
if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
|
|
1889
|
+
const page = await selectChromePage(options);
|
|
1890
|
+
const stability = await waitForBrowserStability(page, { timeoutMs, stableMs, pollMs });
|
|
1891
|
+
const target = resolvePath(path);
|
|
1892
|
+
mkdirSync3(dirname3(target), { recursive: true });
|
|
1893
|
+
const [{ snapshot, entries }, screenshotResult] = await Promise.all([
|
|
1894
|
+
captureBrowserSnapshot(page, { textLimit, elementLimit }),
|
|
1895
|
+
sendChromeCommand(page.webSocketDebuggerUrl, "Page.captureScreenshot", {
|
|
1896
|
+
format: "png",
|
|
1897
|
+
fromSurface: true,
|
|
1898
|
+
captureBeyondViewport: false
|
|
1899
|
+
})
|
|
1900
|
+
]);
|
|
1901
|
+
const data = screenshotResult.data;
|
|
1902
|
+
if (typeof data !== "string") fail("Chrome did not return screenshot data");
|
|
1903
|
+
writeFileSync3(target, Buffer.from(data, "base64"));
|
|
1904
|
+
const screenshot = readPngDimensions(target);
|
|
1905
|
+
const targetId = page.id ?? "unknown";
|
|
1906
|
+
const cachePath = browserStateCachePath(targetId);
|
|
1907
|
+
let mode = "full";
|
|
1908
|
+
let tree = snapshot.tree;
|
|
1909
|
+
let changes = { added: snapshot.nodeCount, changed: 0, removed: 0 };
|
|
1910
|
+
if (!options.full && existsSync3(cachePath)) {
|
|
1911
|
+
const cached = readBrowserStateCache(cachePath);
|
|
1912
|
+
if (cached?.url === snapshot.url && cached.documentId === snapshot.documentId) {
|
|
1913
|
+
mode = "diff";
|
|
1914
|
+
const diff = browserStateDiff(cached.entries, entries);
|
|
1915
|
+
tree = diff.tree;
|
|
1916
|
+
changes = { added: diff.added, changed: diff.changed, removed: diff.removed };
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
mkdirSync3(STATE_DIR, { recursive: true });
|
|
1920
|
+
writeFileSync3(cachePath, JSON.stringify({ url: snapshot.url, documentId: snapshot.documentId, entries }));
|
|
1921
|
+
const stableState = isRecord(stability.state) ? stability.state : {};
|
|
1922
|
+
const state = {
|
|
1923
|
+
title: snapshot.title,
|
|
1924
|
+
url: snapshot.url,
|
|
1925
|
+
documentId: snapshot.documentId,
|
|
1926
|
+
revision: snapshot.revision,
|
|
1927
|
+
viewport: snapshot.viewport,
|
|
1928
|
+
text: snapshot.text,
|
|
1929
|
+
tree,
|
|
1930
|
+
nodeCount: snapshot.nodeCount,
|
|
1931
|
+
elementCount: snapshot.elementCount,
|
|
1932
|
+
truncated: snapshot.truncated,
|
|
1933
|
+
mode,
|
|
1934
|
+
changes
|
|
1935
|
+
};
|
|
1936
|
+
console.log(JSON.stringify({
|
|
1937
|
+
targetId,
|
|
1938
|
+
screenshot: {
|
|
1939
|
+
path: target,
|
|
1940
|
+
width: screenshot.width,
|
|
1941
|
+
height: screenshot.height,
|
|
1942
|
+
devicePixelRatio: typeof stableState.devicePixelRatio === "number" ? stableState.devicePixelRatio : 1
|
|
1943
|
+
},
|
|
1944
|
+
stability,
|
|
1945
|
+
state
|
|
1946
|
+
}, null, 2));
|
|
1947
|
+
}
|
|
1948
|
+
var DESKTOP_POINT_JS = `
|
|
1949
|
+
const desktopPoint = (rect) => {
|
|
1950
|
+
const borderX = Math.max(0, (window.outerWidth - window.innerWidth) / 2);
|
|
1951
|
+
const topChrome = Math.max(0, window.outerHeight - window.innerHeight - borderX);
|
|
1952
|
+
return {
|
|
1953
|
+
x: Math.round(window.screenX + borderX + rect.x + rect.width / 2),
|
|
1954
|
+
y: Math.round(window.screenY + topChrome + rect.y + rect.height / 2),
|
|
1955
|
+
};
|
|
1956
|
+
};`;
|
|
1957
|
+
function parseBrowserRef(value) {
|
|
1958
|
+
const ref = parseCoord(value.replace(/^ref=/, ""), "--ref");
|
|
1959
|
+
if (ref <= 0) fail("--ref must be a positive backend DOM node ID from the latest browser state");
|
|
1960
|
+
return ref;
|
|
1961
|
+
}
|
|
1962
|
+
function readBrowserRefCache(page) {
|
|
1963
|
+
const targetId = page.id;
|
|
1964
|
+
if (!targetId) fail("Chrome target has no stable ID. Capture fresh browser state and retry.");
|
|
1965
|
+
const cachePath = browserStateCachePath(targetId);
|
|
1966
|
+
if (!existsSync3(cachePath)) fail(`No browser state is cached for target ${targetId}. Run replicas computer browser-state <path> --target-id ${targetId} first.`);
|
|
1967
|
+
const cached = readBrowserStateCache(cachePath);
|
|
1968
|
+
if (!cached) fail(`Browser state for target ${targetId} is invalid. Capture fresh browser state and retry.`);
|
|
1969
|
+
if (cached.url !== (page.url ?? "").slice(0, 2e3)) fail("Cached browser state belongs to a different URL. Capture fresh browser state and retry.");
|
|
1970
|
+
return { targetId, cached };
|
|
1971
|
+
}
|
|
1972
|
+
function browserViewportRect(layout) {
|
|
1973
|
+
const viewport = chromeVisualViewport(layout);
|
|
1974
|
+
return {
|
|
1975
|
+
x: 0,
|
|
1976
|
+
y: 0,
|
|
1977
|
+
width: viewport.width,
|
|
1978
|
+
height: viewport.height,
|
|
1979
|
+
centerX: viewport.width / 2,
|
|
1980
|
+
centerY: viewport.height / 2
|
|
1981
|
+
};
|
|
1982
|
+
}
|
|
1983
|
+
async function prepareBrowserElement(page, refValue) {
|
|
1984
|
+
const backendNodeId = parseBrowserRef(refValue);
|
|
1985
|
+
const { cached } = readBrowserRefCache(page);
|
|
1986
|
+
return await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
1987
|
+
const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
|
|
1988
|
+
const rect = await prepareBrowserRefInSession(session, backendNodeId, cached, browserViewportRect(layout));
|
|
1989
|
+
const axNodes = (await sendChromeSessionCommand(session, "Accessibility.getPartialAXTree", {
|
|
1990
|
+
backendNodeId,
|
|
1991
|
+
fetchRelatives: false
|
|
1992
|
+
})).nodes;
|
|
1993
|
+
const rawNode = Array.isArray(axNodes) && isRawAXNode(axNodes[0]) ? axNodes[0] : null;
|
|
1994
|
+
const states = rawNode ? browserStateProperties(rawNode) : {};
|
|
1995
|
+
const role = clippedText(rawNode?.role?.value, 80);
|
|
1996
|
+
const element = rawNode ? {
|
|
1997
|
+
ref: String(backendNodeId),
|
|
1998
|
+
role,
|
|
1999
|
+
name: clippedText(rawNode.name?.value) || null,
|
|
2000
|
+
description: clippedText(rawNode.description?.value) || null,
|
|
2001
|
+
value: typeof rawNode.value?.value === "string" ? clippedText(rawNode.value.value) : typeof rawNode.value?.value === "number" || typeof rawNode.value?.value === "boolean" ? rawNode.value.value : null,
|
|
2002
|
+
states,
|
|
2003
|
+
actions: [
|
|
2004
|
+
"click",
|
|
2005
|
+
...["textbox", "searchbox", "combobox", "spinbutton", "slider"].includes(role) || states.editable === true ? ["set_value"] : [],
|
|
2006
|
+
"scroll"
|
|
2007
|
+
]
|
|
2008
|
+
} : null;
|
|
2009
|
+
const desktopPoint = await evaluateChromeSession(session, `(() => {
|
|
2010
|
+
${DESKTOP_POINT_JS}
|
|
2011
|
+
return desktopPoint({ x: ${rect.centerX}, y: ${rect.centerY}, width: 0, height: 0 });
|
|
2012
|
+
})()`);
|
|
2013
|
+
return {
|
|
2014
|
+
ref: String(backendNodeId),
|
|
2015
|
+
rect,
|
|
2016
|
+
element,
|
|
2017
|
+
desktopPoint: typeof desktopPoint === "object" && desktopPoint !== null && "x" in desktopPoint && typeof desktopPoint.x === "number" && "y" in desktopPoint && typeof desktopPoint.y === "number" ? { x: desktopPoint.x, y: desktopPoint.y } : null
|
|
2018
|
+
};
|
|
2019
|
+
});
|
|
2020
|
+
}
|
|
2021
|
+
async function prepareBrowserRefInSession(session, backendNodeId, cached, viewportRect) {
|
|
2022
|
+
if (!cached.entries.some(({ line }) => line.trimStart().startsWith(`[ref=${backendNodeId}]`))) {
|
|
2023
|
+
fail(`Browser element ref ${backendNodeId} is not in the latest state. Capture fresh browser state and retry.`);
|
|
2024
|
+
}
|
|
2025
|
+
const frameTree = (await sendChromeSessionCommand(session, "Page.getFrameTree", {})).frameTree;
|
|
2026
|
+
if (cached.documentId !== browserDocumentId(frameTree)) {
|
|
2027
|
+
fail(`Browser element ref ${backendNodeId} belongs to a previous document. Capture fresh browser state and retry.`);
|
|
2028
|
+
}
|
|
2029
|
+
try {
|
|
2030
|
+
await sendChromeSessionCommand(session, "DOM.scrollIntoViewIfNeeded", { backendNodeId });
|
|
2031
|
+
} catch (error) {
|
|
2032
|
+
fail(`Browser element ref ${backendNodeId} is stale or cannot be scrolled into view: ${error instanceof Error ? error.message : String(error)}`);
|
|
2033
|
+
}
|
|
2034
|
+
const quads = (await sendChromeSessionCommand(session, "DOM.getContentQuads", { backendNodeId })).quads;
|
|
2035
|
+
const candidates = browserContentQuadRects(quads, viewportRect);
|
|
2036
|
+
if (!candidates[0]) fail(`Browser element ref ${backendNodeId} has no visible content quad. Fetch fresh browser state and retry.`);
|
|
2037
|
+
return roundBrowserRect(candidates[0]);
|
|
2038
|
+
}
|
|
2039
|
+
function buildBrowserClickExpression(query, options) {
|
|
2040
|
+
return `(() => {
|
|
2041
|
+
const query = ${JSON.stringify(query)};
|
|
2042
|
+
const exact = ${JSON.stringify(options.exact)};
|
|
2043
|
+
const index = ${options.index};
|
|
2044
|
+
const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
|
|
2045
|
+
${VISIBLE_JS}
|
|
2046
|
+
const label = (el) => normalize(el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('placeholder') || el.href || el.id || el.name || el.tagName);
|
|
2047
|
+
${DESKTOP_POINT_JS}
|
|
2048
|
+
const matches = (text) => {
|
|
2049
|
+
const haystack = text.toLowerCase();
|
|
2050
|
+
const needle = query.toLowerCase();
|
|
2051
|
+
return exact ? haystack === needle : haystack.includes(needle);
|
|
2052
|
+
};
|
|
2053
|
+
const selector = 'a,button,input,textarea,select,[role="button"],[role="link"],[role="textbox"],[contenteditable="true"]';
|
|
2054
|
+
const candidates = Array.from(document.querySelectorAll(selector))
|
|
2055
|
+
.filter(visible)
|
|
2056
|
+
.map((el) => ({ el, text: label(el) }))
|
|
2057
|
+
.filter((candidate) => candidate.text && matches(candidate.text));
|
|
2058
|
+
const candidate = candidates[index];
|
|
2059
|
+
if (!candidate) {
|
|
2060
|
+
return {
|
|
2061
|
+
clicked: false,
|
|
2062
|
+
query,
|
|
2063
|
+
matchCount: candidates.length,
|
|
2064
|
+
reason: candidates.length ? 'index out of range' : 'no visible matching control',
|
|
2065
|
+
};
|
|
2066
|
+
}
|
|
2067
|
+
const { el, text } = candidate;
|
|
2068
|
+
el.scrollIntoView({ block: 'center', inline: 'center' });
|
|
2069
|
+
const rect = el.getBoundingClientRect();
|
|
2070
|
+
if (typeof el.focus === 'function') el.focus({ preventScroll: true });
|
|
2071
|
+
return {
|
|
2072
|
+
clicked: true,
|
|
2073
|
+
query,
|
|
2074
|
+
matchCount: candidates.length,
|
|
2075
|
+
index,
|
|
2076
|
+
tag: el.tagName.toLowerCase(),
|
|
2077
|
+
role: el.getAttribute('role') || null,
|
|
2078
|
+
type: el.getAttribute('type') || null,
|
|
2079
|
+
text,
|
|
2080
|
+
href: el.href || null,
|
|
2081
|
+
desktopPoint: desktopPoint(rect),
|
|
2082
|
+
rect: {
|
|
2083
|
+
x: Math.round(rect.x),
|
|
2084
|
+
y: Math.round(rect.y),
|
|
2085
|
+
width: Math.round(rect.width),
|
|
2086
|
+
height: Math.round(rect.height),
|
|
2087
|
+
centerX: Math.round(rect.x + rect.width / 2),
|
|
2088
|
+
centerY: Math.round(rect.y + rect.height / 2),
|
|
2089
|
+
},
|
|
2090
|
+
};
|
|
2091
|
+
})()`;
|
|
2092
|
+
}
|
|
2093
|
+
function browserActionRectCenter(result) {
|
|
2094
|
+
if (typeof result !== "object" || result === null || !("rect" in result)) return null;
|
|
2095
|
+
const rect = result.rect;
|
|
2096
|
+
if (typeof rect !== "object" || rect === null) return null;
|
|
2097
|
+
const x = "centerX" in rect ? rect.centerX : void 0;
|
|
2098
|
+
const y = "centerY" in rect ? rect.centerY : void 0;
|
|
2099
|
+
if (typeof x !== "number" || typeof y !== "number" || !Number.isFinite(x) || !Number.isFinite(y)) return null;
|
|
2100
|
+
return { x, y };
|
|
2101
|
+
}
|
|
2102
|
+
function browserActionPoint(result) {
|
|
2103
|
+
if (typeof result !== "object" || result === null) return null;
|
|
2104
|
+
const action = "result" in result ? result.result : result;
|
|
2105
|
+
if (typeof action !== "object" || action === null || !("desktopPoint" in action)) return null;
|
|
2106
|
+
const point = action.desktopPoint;
|
|
2107
|
+
if (typeof point !== "object" || point === null) return null;
|
|
2108
|
+
const x = "x" in point ? point.x : void 0;
|
|
2109
|
+
const y = "y" in point ? point.y : void 0;
|
|
2110
|
+
if (typeof x !== "number" || typeof y !== "number" || !Number.isFinite(x) || !Number.isFinite(y)) return null;
|
|
2111
|
+
const dimensions = getDisplayDimensions();
|
|
2112
|
+
return {
|
|
2113
|
+
x: clamp(Math.round(x), 0, dimensions.width),
|
|
2114
|
+
y: clamp(Math.round(y), 0, dimensions.height)
|
|
2115
|
+
};
|
|
2116
|
+
}
|
|
2117
|
+
async function dispatchChromeClick(page, x, y, button, clicks) {
|
|
2118
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2119
|
+
await dispatchChromeClickInSession(session, x, y, button, clicks);
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
async function dispatchChromeClickInSession(session, x, y, button, clicks) {
|
|
2123
|
+
for (let clickCount = 1; clickCount <= clicks; clickCount++) {
|
|
2124
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
|
|
2125
|
+
type: "mousePressed",
|
|
2126
|
+
x,
|
|
2127
|
+
y,
|
|
2128
|
+
button,
|
|
2129
|
+
clickCount
|
|
2130
|
+
});
|
|
2131
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
|
|
2132
|
+
type: "mouseReleased",
|
|
2133
|
+
x,
|
|
2134
|
+
y,
|
|
2135
|
+
button,
|
|
2136
|
+
clickCount
|
|
2137
|
+
});
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2140
|
+
async function replaceChromeText(page, text) {
|
|
2141
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2142
|
+
await replaceChromeTextInSession(session, text);
|
|
2143
|
+
});
|
|
2144
|
+
}
|
|
2145
|
+
async function replaceChromeTextInSession(session, text) {
|
|
2146
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
2147
|
+
type: "rawKeyDown",
|
|
2148
|
+
key: "a",
|
|
2149
|
+
code: "KeyA",
|
|
2150
|
+
windowsVirtualKeyCode: 65,
|
|
2151
|
+
nativeVirtualKeyCode: 65,
|
|
2152
|
+
modifiers: 2
|
|
2153
|
+
});
|
|
2154
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
2155
|
+
type: "keyUp",
|
|
2156
|
+
key: "a",
|
|
2157
|
+
code: "KeyA",
|
|
2158
|
+
windowsVirtualKeyCode: 65,
|
|
2159
|
+
nativeVirtualKeyCode: 65,
|
|
2160
|
+
modifiers: 2
|
|
2161
|
+
});
|
|
2162
|
+
await sendChromeSessionCommand(session, "Input.insertText", { text });
|
|
2163
|
+
}
|
|
2164
|
+
var BROWSER_DIRECT_VALUE_INPUT_TYPES = /* @__PURE__ */ new Set(["number", "range", "date", "datetime-local", "month", "week", "time", "color"]);
|
|
2165
|
+
function browserFillInvalid(actual, expected, fieldType, semanticSelection = false, selectionMatched = false) {
|
|
2166
|
+
if (semanticSelection) return !selectionMatched;
|
|
2167
|
+
if (["number", "range"].includes(fieldType)) {
|
|
2168
|
+
const actualNumber = typeof actual === "number" ? actual : Number(actual);
|
|
2169
|
+
const expectedNumber = Number(expected);
|
|
2170
|
+
return !Number.isFinite(actualNumber) || !Number.isFinite(expectedNumber) || actualNumber !== expectedNumber;
|
|
2171
|
+
}
|
|
2172
|
+
if (fieldType === "color") return String(actual).toLowerCase() !== expected.toLowerCase();
|
|
2173
|
+
return actual !== expected;
|
|
2174
|
+
}
|
|
2175
|
+
async function fillChromeNodeInSession(session, backendNodeId, value) {
|
|
2176
|
+
await sendChromeSessionCommand(session, "DOM.focus", { backendNodeId });
|
|
2177
|
+
const directResult = await callFunctionOnChromeNodeInSession(
|
|
2178
|
+
session,
|
|
2179
|
+
backendNodeId,
|
|
2180
|
+
`function(value) {
|
|
2181
|
+
const emit = () => {
|
|
2182
|
+
this.dispatchEvent(new Event('input', { bubbles: true }));
|
|
2183
|
+
this.dispatchEvent(new Event('change', { bubbles: true }));
|
|
2184
|
+
};
|
|
2185
|
+
if (this instanceof HTMLSelectElement) {
|
|
2186
|
+
const option = Array.from(this.options).find((item) => item.value === value || item.text.trim() === value);
|
|
2187
|
+
if (option) this.value = option.value;
|
|
2188
|
+
else this.value = value;
|
|
2189
|
+
emit();
|
|
2190
|
+
return { handled: true, semanticSelection: true, matched: !!option, value: this.value };
|
|
2191
|
+
}
|
|
2192
|
+
if (this instanceof HTMLInputElement && ${JSON.stringify([...BROWSER_DIRECT_VALUE_INPUT_TYPES])}.includes(this.type.toLowerCase())) {
|
|
2193
|
+
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
|
|
2194
|
+
if (!setter) return { handled: false };
|
|
2195
|
+
setter.call(this, value);
|
|
2196
|
+
emit();
|
|
2197
|
+
return { handled: true, semanticSelection: false, matched: true, value: this.value };
|
|
2198
|
+
}
|
|
2199
|
+
return { handled: false };
|
|
2200
|
+
}`,
|
|
2201
|
+
[value]
|
|
2202
|
+
);
|
|
2203
|
+
const directValue = chromeCallValue(directResult);
|
|
2204
|
+
const directHandled = isRecord(directValue) && directValue.handled === true;
|
|
2205
|
+
if (!directHandled) await replaceChromeTextInSession(session, value);
|
|
2206
|
+
const verification = await callFunctionOnChromeNodeInSession(
|
|
2207
|
+
session,
|
|
2208
|
+
backendNodeId,
|
|
2209
|
+
`function() {
|
|
2210
|
+
const value = this instanceof HTMLSelectElement || 'value' in this ? this.value : this.textContent;
|
|
2211
|
+
const type = this instanceof HTMLSelectElement
|
|
2212
|
+
? 'select'
|
|
2213
|
+
: this instanceof HTMLInputElement
|
|
2214
|
+
? this.type.toLowerCase()
|
|
2215
|
+
: this.isContentEditable
|
|
2216
|
+
? 'contenteditable'
|
|
2217
|
+
: 'text';
|
|
2218
|
+
return { value, valueLength: String(value ?? '').length, type };
|
|
2219
|
+
}`
|
|
2220
|
+
);
|
|
2221
|
+
const actual = chromeCallValue(verification);
|
|
2222
|
+
if (!isRecord(actual)) fail(`Chrome did not return a value for browser element ref ${backendNodeId}.`);
|
|
2223
|
+
const fieldType = typeof actual.type === "string" ? actual.type : "";
|
|
2224
|
+
const semanticSelection = isRecord(directValue) && directValue.semanticSelection === true;
|
|
2225
|
+
const selectionMatched = isRecord(directValue) && directValue.matched === true;
|
|
2226
|
+
if (browserFillInvalid(actual.value, value, fieldType, semanticSelection, selectionMatched)) {
|
|
2227
|
+
fail(`Chrome reported that ref ${backendNodeId} contains ${JSON.stringify(actual.value)} after filling, expected ${JSON.stringify(value)}.`);
|
|
2228
|
+
}
|
|
2229
|
+
return { value: actual.value, valueLength: actual.valueLength, type: fieldType };
|
|
2230
|
+
}
|
|
2231
|
+
async function computerBrowserClickCommand(query, options = {}) {
|
|
2232
|
+
const page = await selectChromePage(options);
|
|
2233
|
+
if (options.ref) {
|
|
2234
|
+
const target = await prepareBrowserElement(page, options.ref);
|
|
2235
|
+
const buttonNames = { "1": "left", "2": "middle", "3": "right", left: "left", middle: "middle", right: "right" };
|
|
2236
|
+
const button = buttonNames[(options.button ?? "left").toLowerCase()];
|
|
2237
|
+
if (!button) fail("--button must be one of left|middle|right|1|2|3");
|
|
2238
|
+
const clicks = options.double ? 2 : 1;
|
|
2239
|
+
await dispatchChromeClick(page, target.rect.centerX, target.rect.centerY, button, clicks);
|
|
2240
|
+
if (target.desktopPoint) logRecordingAction({ type: "click", ...target.desktopPoint });
|
|
2241
|
+
console.log(JSON.stringify({
|
|
2242
|
+
...browserTargetOutput(page),
|
|
2243
|
+
result: { clicked: true, ref: target.ref, button, clickCount: clicks, rect: target.rect, element: target.element }
|
|
2244
|
+
}, null, 2));
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
2247
|
+
if (!query) fail("Provide control text or --ref <id> from the latest browser state");
|
|
2248
|
+
const index = options.index ? parseCoord(options.index, "--index") : 0;
|
|
2249
|
+
if (index < 0) fail("--index must be >= 0");
|
|
2250
|
+
const result = await evaluateChromeTarget(
|
|
2251
|
+
page.webSocketDebuggerUrl,
|
|
2252
|
+
buildBrowserClickExpression(query, { exact: !!options.exact, index })
|
|
2253
|
+
);
|
|
2254
|
+
const center = browserActionRectCenter(result);
|
|
2255
|
+
if (center) await dispatchChromeClick(page, center.x, center.y, "left", 1);
|
|
2256
|
+
const point = browserActionPoint(result);
|
|
2257
|
+
if (point) logRecordingAction({ type: "click", ...point });
|
|
2258
|
+
console.log(JSON.stringify({
|
|
2259
|
+
...browserTargetOutput(page),
|
|
2260
|
+
result
|
|
2261
|
+
}, null, 2));
|
|
2262
|
+
}
|
|
2263
|
+
function buildBrowserFillExpression(query, value, options) {
|
|
2264
|
+
return `(() => {
|
|
2265
|
+
const query = ${JSON.stringify(query)};
|
|
2266
|
+
const value = ${JSON.stringify(value)};
|
|
2267
|
+
const exact = ${JSON.stringify(options.exact)};
|
|
2268
|
+
const index = ${options.index};
|
|
2269
|
+
const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
|
|
2270
|
+
const escape = (text) => globalThis.CSS && CSS.escape ? CSS.escape(text) : String(text).replace(/["\\\\]/g, '\\\\$&');
|
|
2271
|
+
${VISIBLE_JS}
|
|
2272
|
+
const matches = (text) => {
|
|
2273
|
+
const haystack = text.toLowerCase();
|
|
2274
|
+
const needle = query.toLowerCase();
|
|
2275
|
+
return exact ? haystack === needle : haystack.includes(needle);
|
|
2276
|
+
};
|
|
2277
|
+
const labelParts = (el) => {
|
|
2278
|
+
const id = el.getAttribute('id');
|
|
2279
|
+
const labels = [];
|
|
2280
|
+
if (id) labels.push(...Array.from(document.querySelectorAll('label[for="' + escape(id) + '"]')).map((label) => label.innerText));
|
|
2281
|
+
const wrappingLabel = el.closest('label');
|
|
2282
|
+
if (wrappingLabel) labels.push(wrappingLabel.innerText);
|
|
2283
|
+
labels.push(
|
|
2284
|
+
el.getAttribute('aria-label'),
|
|
2285
|
+
el.getAttribute('title'),
|
|
2286
|
+
el.getAttribute('placeholder'),
|
|
2287
|
+
el.getAttribute('name'),
|
|
2288
|
+
el.getAttribute('id'),
|
|
2289
|
+
el.value,
|
|
2290
|
+
el.innerText,
|
|
2291
|
+
);
|
|
2292
|
+
return labels.map(normalize).filter(Boolean);
|
|
2293
|
+
};
|
|
2294
|
+
const labelText = (el) => labelParts(el).join(' ');
|
|
2295
|
+
${DESKTOP_POINT_JS}
|
|
2296
|
+
const fieldMatches = (el) => {
|
|
2297
|
+
const parts = labelParts(el);
|
|
2298
|
+
return exact ? parts.some(matches) : matches(parts.join(' '));
|
|
2299
|
+
};
|
|
2300
|
+
const selector = 'input:not([type="button"]):not([type="submit"]):not([type="reset"]):not([type="checkbox"]):not([type="radio"]),textarea,select,[role="textbox"],[contenteditable="true"]';
|
|
2301
|
+
const candidates = Array.from(document.querySelectorAll(selector))
|
|
2302
|
+
.filter(visible)
|
|
2303
|
+
.map((el) => ({ el, text: labelText(el) }))
|
|
2304
|
+
.filter((candidate) => candidate.text && fieldMatches(candidate.el));
|
|
2305
|
+
const candidate = candidates[index];
|
|
2306
|
+
if (!candidate) {
|
|
2307
|
+
return {
|
|
2308
|
+
filled: false,
|
|
2309
|
+
query,
|
|
2310
|
+
matchCount: candidates.length,
|
|
2311
|
+
reason: candidates.length ? 'index out of range' : 'no visible matching field',
|
|
2312
|
+
};
|
|
2313
|
+
}
|
|
2314
|
+
const { el, text } = candidate;
|
|
2315
|
+
el.scrollIntoView({ block: 'center', inline: 'center' });
|
|
2316
|
+
if (typeof el.focus === 'function') el.focus({ preventScroll: true });
|
|
2317
|
+
const fieldType = el.tagName.toLowerCase() === 'select' ? 'select' : String(el.getAttribute('type') || 'text').toLowerCase();
|
|
2318
|
+
let nativeInputRequired = false;
|
|
2319
|
+
let semanticSelection = false;
|
|
2320
|
+
let selectionMatched = false;
|
|
2321
|
+
if (el.tagName.toLowerCase() === 'select') {
|
|
2322
|
+
const option = Array.from(el.options).find((item) => item.value === value || normalize(item.text) === value);
|
|
2323
|
+
if (option) el.value = option.value;
|
|
2324
|
+
else el.value = value;
|
|
2325
|
+
semanticSelection = true;
|
|
2326
|
+
selectionMatched = !!option;
|
|
2327
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
2328
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
2329
|
+
} else if (el instanceof HTMLInputElement && ${JSON.stringify([...BROWSER_DIRECT_VALUE_INPUT_TYPES])}.includes(fieldType)) {
|
|
2330
|
+
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
|
|
2331
|
+
if (setter) {
|
|
2332
|
+
setter.call(el, value);
|
|
2333
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
2334
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
2335
|
+
} else {
|
|
2336
|
+
nativeInputRequired = true;
|
|
2337
|
+
}
|
|
2338
|
+
} else if (el.isContentEditable) {
|
|
2339
|
+
const selection = getSelection();
|
|
2340
|
+
const range = document.createRange();
|
|
2341
|
+
range.selectNodeContents(el);
|
|
2342
|
+
selection.removeAllRanges();
|
|
2343
|
+
selection.addRange(range);
|
|
2344
|
+
nativeInputRequired = true;
|
|
2345
|
+
} else if (typeof el.select === 'function') {
|
|
2346
|
+
el.select();
|
|
2347
|
+
nativeInputRequired = true;
|
|
2348
|
+
} else {
|
|
2349
|
+
el.setSelectionRange(0, String(el.value || '').length);
|
|
2350
|
+
nativeInputRequired = true;
|
|
2351
|
+
}
|
|
2352
|
+
const rect = el.getBoundingClientRect();
|
|
2353
|
+
return {
|
|
2354
|
+
filled: true,
|
|
2355
|
+
query,
|
|
2356
|
+
matchCount: candidates.length,
|
|
2357
|
+
index,
|
|
2358
|
+
tag: el.tagName.toLowerCase(),
|
|
2359
|
+
role: el.getAttribute('role') || null,
|
|
2360
|
+
type: fieldType,
|
|
2361
|
+
text,
|
|
2362
|
+
actualValue: el.isContentEditable ? el.textContent : 'value' in el ? el.value : null,
|
|
2363
|
+
valueLength: String(el.isContentEditable ? el.textContent : 'value' in el ? el.value : '').length,
|
|
2364
|
+
nativeInputRequired,
|
|
2365
|
+
semanticSelection,
|
|
2366
|
+
selectionMatched,
|
|
2367
|
+
desktopPoint: desktopPoint(rect),
|
|
2368
|
+
rect: {
|
|
2369
|
+
x: Math.round(rect.x),
|
|
2370
|
+
y: Math.round(rect.y),
|
|
2371
|
+
width: Math.round(rect.width),
|
|
2372
|
+
height: Math.round(rect.height),
|
|
2373
|
+
centerX: Math.round(rect.x + rect.width / 2),
|
|
2374
|
+
centerY: Math.round(rect.y + rect.height / 2),
|
|
2375
|
+
},
|
|
2376
|
+
};
|
|
2377
|
+
})()`;
|
|
2378
|
+
}
|
|
2379
|
+
async function computerBrowserFillCommand(query, value, options = {}) {
|
|
2380
|
+
const page = await selectChromePage(options);
|
|
2381
|
+
if (options.ref) {
|
|
2382
|
+
const target = await prepareBrowserElement(page, options.ref);
|
|
2383
|
+
const backendNodeId = parseBrowserRef(options.ref);
|
|
2384
|
+
const actual = await withChromeSession(page.webSocketDebuggerUrl, async (session) => await fillChromeNodeInSession(session, backendNodeId, value));
|
|
2385
|
+
if (target.desktopPoint) logRecordingAction({ type: "type", ...target.desktopPoint });
|
|
2386
|
+
console.log(JSON.stringify({
|
|
2387
|
+
...browserTargetOutput(page),
|
|
2388
|
+
result: { filled: true, ref: target.ref, valueLength: actual.valueLength, rect: target.rect, element: target.element }
|
|
2389
|
+
}, null, 2));
|
|
2390
|
+
return;
|
|
2391
|
+
}
|
|
2392
|
+
if (!query) fail("Provide field text or --ref <id> from the latest browser state");
|
|
2393
|
+
const index = options.index ? parseCoord(options.index, "--index") : 0;
|
|
2394
|
+
if (index < 0) fail("--index must be >= 0");
|
|
2395
|
+
const result = await evaluateChromeTarget(
|
|
2396
|
+
page.webSocketDebuggerUrl,
|
|
2397
|
+
buildBrowserFillExpression(query, value, { exact: !!options.exact, index })
|
|
2398
|
+
);
|
|
2399
|
+
const point = browserActionPoint(result);
|
|
2400
|
+
if (typeof result === "object" && result !== null && "nativeInputRequired" in result && result.nativeInputRequired === true) {
|
|
2401
|
+
await replaceChromeText(page, value);
|
|
2402
|
+
const actualValue = await evaluateChromeTarget(page.webSocketDebuggerUrl, `(() => {
|
|
2403
|
+
const el = document.activeElement;
|
|
2404
|
+
return el && el.isContentEditable ? el.textContent : el && 'value' in el ? el.value : null;
|
|
2405
|
+
})()`);
|
|
2406
|
+
const fieldType = "type" in result && typeof result.type === "string" ? result.type.toLowerCase() : "";
|
|
2407
|
+
if (browserFillInvalid(actualValue, value, fieldType)) {
|
|
2408
|
+
fail(`Chrome reported that ${JSON.stringify(query)} contains ${JSON.stringify(actualValue)} after filling, expected ${JSON.stringify(value)}.`);
|
|
2409
|
+
}
|
|
2410
|
+
if (isRecord(result)) {
|
|
2411
|
+
Object.assign(result, { actualValue, valueLength: String(actualValue ?? "").length });
|
|
2412
|
+
}
|
|
2413
|
+
} else if (isRecord(result) && result.filled === true) {
|
|
2414
|
+
const fieldType = typeof result.type === "string" ? result.type : "";
|
|
2415
|
+
if (browserFillInvalid(result.actualValue, value, fieldType, result.semanticSelection === true, result.selectionMatched === true)) {
|
|
2416
|
+
fail(`Chrome reported that ${JSON.stringify(query)} contains ${JSON.stringify(result.actualValue)} after filling, expected ${JSON.stringify(value)}.`);
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
if (point) logRecordingAction({ type: "type", ...point });
|
|
2420
|
+
console.log(JSON.stringify({
|
|
2421
|
+
...browserTargetOutput(page),
|
|
2422
|
+
result
|
|
2423
|
+
}, null, 2));
|
|
2424
|
+
}
|
|
2425
|
+
function buildBrowserWaitExpression(query, options) {
|
|
2426
|
+
return `(() => {
|
|
2427
|
+
const query = ${JSON.stringify(query)};
|
|
2428
|
+
const mode = ${JSON.stringify(options.mode)};
|
|
2429
|
+
const exact = ${JSON.stringify(options.exact)};
|
|
2430
|
+
const normalize = (text) => String(text || '').replace(/\\s+/g, ' ').trim();
|
|
2431
|
+
${VISIBLE_JS}
|
|
2432
|
+
const matches = (text) => {
|
|
2433
|
+
const haystack = normalize(text).toLowerCase();
|
|
2434
|
+
const needle = query.toLowerCase();
|
|
2435
|
+
return exact ? haystack === needle : haystack.includes(needle);
|
|
2436
|
+
};
|
|
2437
|
+
const controlText = (el) => normalize(el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('placeholder') || el.href || el.id || el.name || el.tagName);
|
|
2438
|
+
const controls = () => Array.from(document.querySelectorAll('a,button,input,textarea,select,[role="button"],[role="link"],[role="textbox"],[contenteditable="true"]'))
|
|
2439
|
+
.filter(visible)
|
|
2440
|
+
.map(controlText)
|
|
2441
|
+
.filter(Boolean);
|
|
2442
|
+
const values = {
|
|
2443
|
+
title: [document.title],
|
|
2444
|
+
url: [location.href],
|
|
2445
|
+
text: [document.body ? document.body.innerText : ''],
|
|
2446
|
+
control: controls(),
|
|
2447
|
+
any: [document.title, location.href, document.body ? document.body.innerText : '', ...controls()],
|
|
2448
|
+
}[mode];
|
|
2449
|
+
if (!values) return { matched: false, query, mode, reason: 'invalid mode' };
|
|
2450
|
+
const match = values.find(matches) || null;
|
|
2451
|
+
return {
|
|
2452
|
+
matched: !!match,
|
|
2453
|
+
query,
|
|
2454
|
+
mode,
|
|
2455
|
+
exact,
|
|
2456
|
+
match: match ? normalize(match).slice(0, 500) : null,
|
|
2457
|
+
title: document.title,
|
|
2458
|
+
url: location.href,
|
|
2459
|
+
};
|
|
2460
|
+
})()`;
|
|
2461
|
+
}
|
|
2462
|
+
async function computerBrowserWaitCommand(query, options = {}) {
|
|
2463
|
+
const mode = options.mode ?? "any";
|
|
2464
|
+
if (!["any", "text", "title", "url", "control"].includes(mode)) fail("--mode must be one of any|text|title|url|control");
|
|
2465
|
+
const timeoutMs = options.timeout ? parseCoord(options.timeout, "--timeout") : 1e4;
|
|
2466
|
+
const pollMs = options.pollMs ? parseCoord(options.pollMs, "--poll-ms") : 250;
|
|
2467
|
+
if (timeoutMs < 0) fail("--timeout must be >= 0");
|
|
2468
|
+
if (pollMs < 50 || pollMs > 2e3) fail("--poll-ms must be between 50 and 2000");
|
|
2469
|
+
const page = await selectChromePage(options);
|
|
2470
|
+
const start = Date.now();
|
|
2471
|
+
const expression = buildBrowserWaitExpression(query, { mode, exact: !!options.exact });
|
|
2472
|
+
let attempts = 0;
|
|
2473
|
+
let lastResult = null;
|
|
2474
|
+
while (Date.now() - start <= timeoutMs) {
|
|
2475
|
+
attempts++;
|
|
2476
|
+
lastResult = await evaluateChromeTarget(page.webSocketDebuggerUrl, expression);
|
|
2477
|
+
if (typeof lastResult === "object" && lastResult !== null && "matched" in lastResult && lastResult.matched) {
|
|
2478
|
+
console.log(JSON.stringify({ ok: true, elapsedMs: Date.now() - start, attempts, result: lastResult }, null, 2));
|
|
2479
|
+
return;
|
|
2480
|
+
}
|
|
2481
|
+
await sleep(pollMs);
|
|
2482
|
+
}
|
|
2483
|
+
console.log(JSON.stringify({ ok: false, elapsedMs: Date.now() - start, attempts, result: lastResult }, null, 2));
|
|
2484
|
+
process.exitCode = 1;
|
|
2485
|
+
}
|
|
2486
|
+
async function computerBrowserScrollCommand(direction, options = {}) {
|
|
2487
|
+
const page = await selectChromePage(options);
|
|
2488
|
+
const normalized = direction.toLowerCase();
|
|
2489
|
+
if (!["up", "down", "left", "right"].includes(normalized)) fail("direction must be one of up|down|left|right");
|
|
2490
|
+
const amount = options.amount ? parseCoord(options.amount, "--amount") : 600;
|
|
2491
|
+
if (amount <= 0 || amount > 1e4) fail("--amount must be between 1 and 10000 CSS pixels");
|
|
2492
|
+
const layout = await sendChromeCommand(page.webSocketDebuggerUrl, "Page.getLayoutMetrics", {});
|
|
2493
|
+
const viewport = chromeVisualViewport(layout);
|
|
2494
|
+
const target = options.ref ? await prepareBrowserElement(page, options.ref) : null;
|
|
2495
|
+
const x = target?.rect.centerX ?? viewport.width / 2;
|
|
2496
|
+
const y = target?.rect.centerY ?? viewport.height / 2;
|
|
2497
|
+
const deltaX = normalized === "left" ? -amount : normalized === "right" ? amount : 0;
|
|
2498
|
+
const deltaY = normalized === "up" ? -amount : normalized === "down" ? amount : 0;
|
|
2499
|
+
await sendChromeCommand(page.webSocketDebuggerUrl, "Input.dispatchMouseEvent", {
|
|
2500
|
+
type: "mouseWheel",
|
|
2501
|
+
x,
|
|
2502
|
+
y,
|
|
2503
|
+
deltaX,
|
|
2504
|
+
deltaY
|
|
2505
|
+
});
|
|
2506
|
+
logRecordingAction({ type: "scroll", ...target?.desktopPoint ?? {} });
|
|
2507
|
+
console.log(JSON.stringify({
|
|
2508
|
+
...browserTargetOutput(page),
|
|
2509
|
+
result: { scrolled: true, direction: normalized, amount, ref: target?.ref ?? null, x: Math.round(x), y: Math.round(y) }
|
|
2510
|
+
}, null, 2));
|
|
2511
|
+
}
|
|
2512
|
+
function browserKeyDetails(combo) {
|
|
2513
|
+
const tokens = combo.split("+").map((token) => token.trim()).filter(Boolean);
|
|
2514
|
+
if (tokens.length === 0) fail("key combination cannot be empty");
|
|
2515
|
+
let modifiers = 0;
|
|
2516
|
+
for (const token of tokens.slice(0, -1)) {
|
|
2517
|
+
const normalized2 = token.toLowerCase();
|
|
2518
|
+
if (["alt", "option"].includes(normalized2)) modifiers |= 1;
|
|
2519
|
+
else if (["ctrl", "control"].includes(normalized2)) modifiers |= 2;
|
|
2520
|
+
else if (["meta", "cmd", "command", "super"].includes(normalized2)) modifiers |= 4;
|
|
2521
|
+
else if (normalized2 === "shift") modifiers |= 8;
|
|
2522
|
+
else fail(`Unsupported browser key modifier ${JSON.stringify(token)}`);
|
|
2523
|
+
}
|
|
2524
|
+
const requested = tokens[tokens.length - 1];
|
|
2525
|
+
const aliases = {
|
|
2526
|
+
enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 },
|
|
2527
|
+
return: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 },
|
|
2528
|
+
tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9 },
|
|
2529
|
+
escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
|
|
2530
|
+
esc: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
|
|
2531
|
+
backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 },
|
|
2532
|
+
delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 },
|
|
2533
|
+
space: { key: " ", code: "Space", windowsVirtualKeyCode: 32 },
|
|
2534
|
+
up: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
|
|
2535
|
+
arrowup: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
|
|
2536
|
+
down: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
|
|
2537
|
+
arrowdown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
|
|
2538
|
+
left: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
|
|
2539
|
+
arrowleft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
|
|
2540
|
+
right: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
|
|
2541
|
+
arrowright: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
|
|
2542
|
+
home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 },
|
|
2543
|
+
end: { key: "End", code: "End", windowsVirtualKeyCode: 35 },
|
|
2544
|
+
pageup: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 },
|
|
2545
|
+
pagedown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 }
|
|
2546
|
+
};
|
|
2547
|
+
const normalized = requested.toLowerCase();
|
|
2548
|
+
const alias = aliases[normalized];
|
|
2549
|
+
if (alias) return { ...alias, modifiers, ...alias.key === " " && modifiers === 0 ? { text: " " } : {} };
|
|
2550
|
+
if (/^[a-z]$/i.test(requested)) {
|
|
2551
|
+
const upper = requested.toUpperCase();
|
|
2552
|
+
const key = modifiers & 8 ? upper : requested.toLowerCase();
|
|
2553
|
+
return { key, code: `Key${upper}`, windowsVirtualKeyCode: upper.charCodeAt(0), modifiers, ...modifiers === 0 ? { text: key } : {} };
|
|
2554
|
+
}
|
|
2555
|
+
if (/^[0-9]$/.test(requested)) {
|
|
2556
|
+
return { key: requested, code: `Digit${requested}`, windowsVirtualKeyCode: requested.charCodeAt(0), modifiers, ...modifiers === 0 ? { text: requested } : {} };
|
|
2557
|
+
}
|
|
2558
|
+
fail(`Unsupported browser key ${JSON.stringify(requested)}`);
|
|
2559
|
+
}
|
|
2560
|
+
async function computerBrowserKeyCommand(combo, options = {}) {
|
|
2561
|
+
const page = await selectChromePage(options);
|
|
2562
|
+
const key = browserKeyDetails(combo);
|
|
2563
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2564
|
+
await dispatchChromeKeyInSession(session, key);
|
|
2565
|
+
});
|
|
2566
|
+
logRecordingAction({ type: "key" });
|
|
2567
|
+
console.log(JSON.stringify({ targetId: page.id ?? null, result: { pressed: combo } }, null, 2));
|
|
2568
|
+
}
|
|
2569
|
+
async function dispatchChromeKeyInSession(session, key) {
|
|
2570
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
2571
|
+
type: "keyDown",
|
|
2572
|
+
...key,
|
|
2573
|
+
nativeVirtualKeyCode: key.windowsVirtualKeyCode
|
|
2574
|
+
});
|
|
2575
|
+
await sendChromeSessionCommand(session, "Input.dispatchKeyEvent", {
|
|
2576
|
+
type: "keyUp",
|
|
2577
|
+
key: key.key,
|
|
2578
|
+
code: key.code,
|
|
2579
|
+
modifiers: key.modifiers,
|
|
2580
|
+
windowsVirtualKeyCode: key.windowsVirtualKeyCode,
|
|
2581
|
+
nativeVirtualKeyCode: key.windowsVirtualKeyCode
|
|
2582
|
+
});
|
|
2583
|
+
}
|
|
2584
|
+
async function computerBrowserTypeCommand(text, options = {}) {
|
|
2585
|
+
const page = await selectChromePage(options);
|
|
2586
|
+
await sendChromeCommand(page.webSocketDebuggerUrl, "Input.insertText", { text });
|
|
2587
|
+
logRecordingAction({ type: "type" });
|
|
2588
|
+
console.log(JSON.stringify({ targetId: page.id ?? null, result: { typed: true, length: text.length } }, null, 2));
|
|
2589
|
+
}
|
|
2590
|
+
function parseBrowserBatchActions(value) {
|
|
2591
|
+
let input;
|
|
2592
|
+
try {
|
|
2593
|
+
input = JSON.parse(value);
|
|
2594
|
+
} catch (error) {
|
|
2595
|
+
fail(`Batch actions must be valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
2596
|
+
}
|
|
2597
|
+
if (!Array.isArray(input) || input.length === 0 || input.length > 100) {
|
|
2598
|
+
fail("Batch actions must be a JSON array containing 1 to 100 actions");
|
|
2599
|
+
}
|
|
2600
|
+
return input.map((item, index) => {
|
|
2601
|
+
if (!isRecord(item) || typeof item.action !== "string") fail(`Batch action ${index} must be an object with an action`);
|
|
2602
|
+
if (item.action === "click") {
|
|
2603
|
+
if (typeof item.ref !== "string" && typeof item.ref !== "number") fail(`Batch click ${index} requires ref`);
|
|
2604
|
+
const buttons = { "1": "left", "2": "middle", "3": "right", left: "left", middle: "middle", right: "right" };
|
|
2605
|
+
const requestedButton = item.button === void 0 ? "left" : String(item.button).toLowerCase();
|
|
2606
|
+
const button = buttons[requestedButton];
|
|
2607
|
+
if (!button) fail(`Batch click ${index} button must be left, middle, or right`);
|
|
2608
|
+
return { action: "click", ref: String(item.ref), button, clicks: item.double === true ? 2 : 1 };
|
|
2609
|
+
}
|
|
2610
|
+
if (item.action === "fill") {
|
|
2611
|
+
if (typeof item.ref !== "string" && typeof item.ref !== "number" || typeof item.value !== "string") {
|
|
2612
|
+
fail(`Batch fill ${index} requires ref and string value`);
|
|
2613
|
+
}
|
|
2614
|
+
return { action: "fill", ref: String(item.ref), value: item.value };
|
|
2615
|
+
}
|
|
2616
|
+
if (item.action === "key") {
|
|
2617
|
+
if (typeof item.combo !== "string") fail(`Batch key ${index} requires combo`);
|
|
2618
|
+
return { action: "key", combo: item.combo };
|
|
2619
|
+
}
|
|
2620
|
+
if (item.action === "type") {
|
|
2621
|
+
if (typeof item.text !== "string") fail(`Batch type ${index} requires text`);
|
|
2622
|
+
return { action: "type", text: item.text };
|
|
2623
|
+
}
|
|
2624
|
+
if (item.action === "scroll") {
|
|
2625
|
+
const directions = { up: "up", down: "down", left: "left", right: "right" };
|
|
2626
|
+
const direction = typeof item.direction === "string" ? directions[item.direction] : void 0;
|
|
2627
|
+
if (!direction) {
|
|
2628
|
+
fail(`Batch scroll ${index} direction must be up, down, left, or right`);
|
|
2629
|
+
}
|
|
2630
|
+
const amount = item.amount === void 0 ? 600 : Number(item.amount);
|
|
2631
|
+
if (!Number.isFinite(amount) || amount <= 0 || amount > 1e4) fail(`Batch scroll ${index} amount must be between 1 and 10000`);
|
|
2632
|
+
if (item.ref !== void 0 && typeof item.ref !== "string" && typeof item.ref !== "number") fail(`Batch scroll ${index} ref is invalid`);
|
|
2633
|
+
return {
|
|
2634
|
+
action: "scroll",
|
|
2635
|
+
direction,
|
|
2636
|
+
amount,
|
|
2637
|
+
...item.ref === void 0 ? {} : { ref: String(item.ref) }
|
|
2638
|
+
};
|
|
2639
|
+
}
|
|
2640
|
+
if (item.action === "wait") {
|
|
2641
|
+
if (typeof item.text !== "string") fail(`Batch wait ${index} requires text`);
|
|
2642
|
+
const mode = item.mode === void 0 ? "any" : String(item.mode);
|
|
2643
|
+
if (!["any", "text", "title", "url", "control"].includes(mode)) fail(`Batch wait ${index} mode is invalid`);
|
|
2644
|
+
const timeoutMs = item.timeoutMs === void 0 ? 1e4 : Number(item.timeoutMs);
|
|
2645
|
+
const pollMs = item.pollMs === void 0 ? 250 : Number(item.pollMs);
|
|
2646
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) fail(`Batch wait ${index} timeoutMs must be >= 0`);
|
|
2647
|
+
if (!Number.isFinite(pollMs) || pollMs < 50 || pollMs > 2e3) fail(`Batch wait ${index} pollMs must be between 50 and 2000`);
|
|
2648
|
+
return { action: "wait", text: item.text, mode, exact: item.exact === true, timeoutMs, pollMs };
|
|
2649
|
+
}
|
|
2650
|
+
fail(`Batch action ${index} has unsupported action ${JSON.stringify(item.action)}`);
|
|
2651
|
+
});
|
|
2652
|
+
}
|
|
2653
|
+
async function computerBrowserBatchCommand(actionsJson, options = {}) {
|
|
2654
|
+
const actions = parseBrowserBatchActions(actionsJson);
|
|
2655
|
+
const page = await selectChromePage(options);
|
|
2656
|
+
const { targetId, cached } = readBrowserRefCache(page);
|
|
2657
|
+
const startedAt = Date.now();
|
|
2658
|
+
const completed = [];
|
|
2659
|
+
try {
|
|
2660
|
+
await withChromeSession(page.webSocketDebuggerUrl, async (session) => {
|
|
2661
|
+
const layout = await sendChromeSessionCommand(session, "Page.getLayoutMetrics", {});
|
|
2662
|
+
const viewportRect = browserViewportRect(layout);
|
|
2663
|
+
for (const [index, action] of actions.entries()) {
|
|
2664
|
+
const actionStartedAt = Date.now();
|
|
2665
|
+
if (action.action === "click") {
|
|
2666
|
+
const ref = parseBrowserRef(action.ref);
|
|
2667
|
+
const rect = await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
2668
|
+
await dispatchChromeClickInSession(session, rect.centerX, rect.centerY, action.button, action.clicks);
|
|
2669
|
+
logRecordingAction({ type: "click" });
|
|
2670
|
+
completed.push({ index, action: action.action, ref: String(ref), elapsedMs: Date.now() - actionStartedAt });
|
|
2671
|
+
} else if (action.action === "fill") {
|
|
2672
|
+
const ref = parseBrowserRef(action.ref);
|
|
2673
|
+
await prepareBrowserRefInSession(session, ref, cached, viewportRect);
|
|
2674
|
+
const result = await fillChromeNodeInSession(session, ref, action.value);
|
|
2675
|
+
logRecordingAction({ type: "type" });
|
|
2676
|
+
completed.push({ index, action: action.action, ref: String(ref), valueLength: result.valueLength, elapsedMs: Date.now() - actionStartedAt });
|
|
2677
|
+
} else if (action.action === "key") {
|
|
2678
|
+
await dispatchChromeKeyInSession(session, browserKeyDetails(action.combo));
|
|
2679
|
+
logRecordingAction({ type: "key" });
|
|
2680
|
+
completed.push({ index, action: action.action, elapsedMs: Date.now() - actionStartedAt });
|
|
2681
|
+
} else if (action.action === "type") {
|
|
2682
|
+
await sendChromeSessionCommand(session, "Input.insertText", { text: action.text });
|
|
2683
|
+
logRecordingAction({ type: "type" });
|
|
2684
|
+
completed.push({ index, action: action.action, length: action.text.length, elapsedMs: Date.now() - actionStartedAt });
|
|
2685
|
+
} else if (action.action === "scroll") {
|
|
2686
|
+
const ref = action.ref ? parseBrowserRef(action.ref) : null;
|
|
2687
|
+
const rect = ref ? await prepareBrowserRefInSession(session, ref, cached, viewportRect) : viewportRect;
|
|
2688
|
+
const deltaX = action.direction === "left" ? -action.amount : action.direction === "right" ? action.amount : 0;
|
|
2689
|
+
const deltaY = action.direction === "up" ? -action.amount : action.direction === "down" ? action.amount : 0;
|
|
2690
|
+
await sendChromeSessionCommand(session, "Input.dispatchMouseEvent", {
|
|
2691
|
+
type: "mouseWheel",
|
|
2692
|
+
x: rect.centerX,
|
|
2693
|
+
y: rect.centerY,
|
|
2694
|
+
deltaX,
|
|
2695
|
+
deltaY
|
|
2696
|
+
});
|
|
2697
|
+
logRecordingAction({ type: "scroll" });
|
|
2698
|
+
completed.push({ index, action: action.action, direction: action.direction, amount: action.amount, elapsedMs: Date.now() - actionStartedAt });
|
|
2699
|
+
} else {
|
|
2700
|
+
const expression = buildBrowserWaitExpression(action.text, { mode: action.mode, exact: action.exact });
|
|
2701
|
+
let attempts = 0;
|
|
2702
|
+
let result = null;
|
|
2703
|
+
while (Date.now() - actionStartedAt <= action.timeoutMs) {
|
|
2704
|
+
attempts++;
|
|
2705
|
+
result = await evaluateChromeSession(session, expression);
|
|
2706
|
+
if (isRecord(result) && result.matched === true) break;
|
|
2707
|
+
await sleep(action.pollMs);
|
|
2708
|
+
}
|
|
2709
|
+
if (!isRecord(result) || result.matched !== true) fail(`Batch wait ${index} timed out after ${Date.now() - actionStartedAt}ms`);
|
|
2710
|
+
completed.push({ index, action: action.action, attempts, elapsedMs: Date.now() - actionStartedAt });
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
});
|
|
2714
|
+
} catch (error) {
|
|
2715
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2716
|
+
console.log(JSON.stringify({ ok: false, targetId, elapsedMs: Date.now() - startedAt, completed, failedIndex: completed.length, error: message }, null, 2));
|
|
2717
|
+
throw error;
|
|
2718
|
+
}
|
|
2719
|
+
console.log(JSON.stringify({ ok: true, targetId, elapsedMs: Date.now() - startedAt, completed }, null, 2));
|
|
2720
|
+
}
|
|
2721
|
+
async function computerClickCommand(xStr, yStr, options) {
|
|
2722
|
+
const dimensions = getDisplayDimensions();
|
|
2723
|
+
const x = parseScreenCoord(xStr, "x", dimensions.width);
|
|
2724
|
+
const y = parseScreenCoord(yStr, "y", dimensions.height);
|
|
2725
|
+
const button = options.button ?? "1";
|
|
2726
|
+
const args = ["mousemove", "--sync", String(x), String(y)];
|
|
2727
|
+
if (options.modifiers) {
|
|
2728
|
+
for (const mod of options.modifiers.split("+")) {
|
|
2729
|
+
args.push("keydown", mod);
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
if (options.double) {
|
|
2733
|
+
args.push("click", "--repeat", "2", "--delay", "50", button);
|
|
2734
|
+
} else {
|
|
2735
|
+
args.push("click", button);
|
|
2736
|
+
}
|
|
2737
|
+
if (options.modifiers) {
|
|
2738
|
+
for (const mod of options.modifiers.split("+").reverse()) {
|
|
2739
|
+
args.push("keyup", mod);
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
runDesktopInputCmd(args);
|
|
2743
|
+
logRecordingAction({ type: "click", x, y });
|
|
2744
|
+
console.log(`clicked ${button === "1" ? "left" : button === "2" ? "middle" : button === "3" ? "right" : `button ${button}`} at (${x},${y})${options.double ? " x2" : ""}`);
|
|
2745
|
+
}
|
|
2746
|
+
async function computerMoveCommand(xStr, yStr) {
|
|
2747
|
+
const dimensions = getDisplayDimensions();
|
|
2748
|
+
const x = parseScreenCoord(xStr, "x", dimensions.width);
|
|
2749
|
+
const y = parseScreenCoord(yStr, "y", dimensions.height);
|
|
2750
|
+
runDesktopInputCmd(["mousemove", "--sync", String(x), String(y)]);
|
|
2751
|
+
logRecordingAction({ type: "move", x, y });
|
|
2752
|
+
console.log(`moved to (${x},${y})`);
|
|
2753
|
+
}
|
|
2754
|
+
async function computerTypeCommand(text, options) {
|
|
2755
|
+
const delay = options.delay ? parseCoord(options.delay, "--delay") : 12;
|
|
2756
|
+
runDesktopInputCmd(["type", "--delay", String(delay), "--", text]);
|
|
2757
|
+
const position = recordingMousePosition();
|
|
2758
|
+
logRecordingAction({ type: "type", ...position ?? {} });
|
|
2759
|
+
console.log(`typed ${text.length} char${text.length === 1 ? "" : "s"}`);
|
|
2760
|
+
}
|
|
2761
|
+
async function computerKeyCommand(combo) {
|
|
2762
|
+
runDesktopInputCmd(["key", "--", combo]);
|
|
2763
|
+
const position = recordingMousePosition();
|
|
2764
|
+
logRecordingAction({ type: "key", ...position ?? {} });
|
|
2765
|
+
console.log(`pressed ${combo}`);
|
|
2766
|
+
}
|
|
2767
|
+
async function computerScrollCommand(direction, options) {
|
|
2768
|
+
const dir = direction.toLowerCase();
|
|
2769
|
+
const buttonMap = { up: "4", down: "5", left: "6", right: "7" };
|
|
2770
|
+
const button = buttonMap[dir];
|
|
2771
|
+
if (!button) fail(`direction must be one of up|down|left|right (got "${direction}")`);
|
|
2772
|
+
const amount = options.amount ? parseCoord(options.amount, "--amount") : 3;
|
|
2773
|
+
const args = [];
|
|
2774
|
+
let hoverPosition = null;
|
|
2775
|
+
if (options.x && options.y) {
|
|
2776
|
+
const dimensions = getDisplayDimensions();
|
|
2777
|
+
hoverPosition = {
|
|
2778
|
+
x: parseScreenCoord(options.x, "--x", dimensions.width),
|
|
2779
|
+
y: parseScreenCoord(options.y, "--y", dimensions.height)
|
|
2780
|
+
};
|
|
2781
|
+
args.push(
|
|
2782
|
+
"mousemove",
|
|
2783
|
+
"--sync",
|
|
2784
|
+
String(hoverPosition.x),
|
|
2785
|
+
String(hoverPosition.y)
|
|
2786
|
+
);
|
|
2787
|
+
}
|
|
2788
|
+
args.push("click", "--repeat", String(amount), "--delay", "30", button);
|
|
2789
|
+
runDesktopInputCmd(args);
|
|
2790
|
+
const position = hoverPosition ?? recordingMousePosition();
|
|
2791
|
+
logRecordingAction({ type: "scroll", ...position ?? {} });
|
|
2792
|
+
console.log(`scrolled ${dir} x${amount}`);
|
|
2793
|
+
}
|
|
2794
|
+
async function computerDragCommand(fx, fy, tx, ty) {
|
|
2795
|
+
const dimensions = getDisplayDimensions();
|
|
2796
|
+
const fromX = parseScreenCoord(fx, "fromX", dimensions.width);
|
|
2797
|
+
const fromY = parseScreenCoord(fy, "fromY", dimensions.height);
|
|
2798
|
+
const toX = parseScreenCoord(tx, "toX", dimensions.width);
|
|
2799
|
+
const toY = parseScreenCoord(ty, "toY", dimensions.height);
|
|
2800
|
+
const distance = Math.hypot(toX - fromX, toY - fromY);
|
|
2801
|
+
const steps = clamp(Math.ceil(distance / 70), 8, 24);
|
|
2802
|
+
const args = [
|
|
2803
|
+
"mousemove",
|
|
2804
|
+
"--sync",
|
|
2805
|
+
String(fromX),
|
|
2806
|
+
String(fromY),
|
|
2807
|
+
"mousedown",
|
|
2808
|
+
"1"
|
|
2809
|
+
];
|
|
2810
|
+
for (let i = 1; i <= steps; i++) {
|
|
2811
|
+
const x = Math.round(fromX + (toX - fromX) * i / steps);
|
|
2812
|
+
const y = Math.round(fromY + (toY - fromY) * i / steps);
|
|
2813
|
+
args.push("mousemove", "--sync", String(x), String(y), "sleep", "0.018");
|
|
2814
|
+
}
|
|
2815
|
+
args.push("mouseup", "1");
|
|
2816
|
+
runDesktopInputCmd(args);
|
|
2817
|
+
logRecordingAction({ type: "drag", x: fromX, y: fromY, toX, toY });
|
|
2818
|
+
console.log(`dragged (${fromX},${fromY}) -> (${toX},${toY})`);
|
|
2819
|
+
}
|
|
2820
|
+
var APP_ALIASES = {
|
|
2821
|
+
chrome: [CHROME_WRAPPER],
|
|
2822
|
+
chromium: ["chromium", "--no-first-run", "--no-default-browser-check"],
|
|
2823
|
+
firefox: ["firefox"],
|
|
2824
|
+
terminal: ["xfce4-terminal"],
|
|
2825
|
+
xterm: ["xterm"],
|
|
2826
|
+
notepad: ["mousepad"],
|
|
2827
|
+
editor: ["mousepad"],
|
|
2828
|
+
files: ["thunar"],
|
|
2829
|
+
filemanager: ["thunar"]
|
|
2830
|
+
};
|
|
2831
|
+
var LEGACY_CHROME_ALIAS = [
|
|
2832
|
+
"google-chrome",
|
|
2833
|
+
"--no-first-run",
|
|
2834
|
+
"--no-default-browser-check",
|
|
2835
|
+
"--start-maximized",
|
|
2836
|
+
"--user-data-dir=/tmp/replicas-computer/chrome-profile"
|
|
2837
|
+
];
|
|
2838
|
+
async function computerLaunchCommand(app, args) {
|
|
2839
|
+
ensureServicesRunning();
|
|
2840
|
+
const baseArgs = app === "chrome" && !existsSync3(CHROME_WRAPPER) ? LEGACY_CHROME_ALIAS : APP_ALIASES[app] ?? [app];
|
|
2841
|
+
const bin = baseArgs[0];
|
|
2842
|
+
const fullArgs = [...baseArgs.slice(1), ...args];
|
|
2843
|
+
const expectsNewChromePage = args.some((arg) => !arg.startsWith("-"));
|
|
2844
|
+
const existingChromePageIds = /* @__PURE__ */ new Set();
|
|
2845
|
+
if (app === "chrome") {
|
|
2846
|
+
try {
|
|
2847
|
+
for (const page of await getChromePages()) {
|
|
2848
|
+
if (page.id) existingChromePageIds.add(page.id);
|
|
2849
|
+
}
|
|
2850
|
+
} catch {
|
|
2851
|
+
}
|
|
2852
|
+
}
|
|
2853
|
+
const child = spawn2(bin, fullArgs, {
|
|
2854
|
+
env: withDisplay(),
|
|
2855
|
+
detached: true,
|
|
2856
|
+
stdio: "ignore"
|
|
2857
|
+
});
|
|
2858
|
+
child.unref();
|
|
2859
|
+
if (app === "chrome") {
|
|
2860
|
+
const startedAt = Date.now();
|
|
2861
|
+
let pageCount = 0;
|
|
2862
|
+
let readyPage = null;
|
|
2863
|
+
while (Date.now() - startedAt < 15e3) {
|
|
2864
|
+
let pages = [];
|
|
2865
|
+
try {
|
|
2866
|
+
pages = await getChromePages();
|
|
2867
|
+
} catch {
|
|
2868
|
+
}
|
|
2869
|
+
pageCount = pages.length;
|
|
2870
|
+
const candidates = expectsNewChromePage && existingChromePageIds.size > 0 ? pages.filter((page) => page.id && !existingChromePageIds.has(page.id)) : pages;
|
|
2871
|
+
for (const page of candidates) {
|
|
2872
|
+
if (!page.webSocketDebuggerUrl || !page.url || page.url === "about:blank") continue;
|
|
2873
|
+
const readyState = await evaluateChromeTarget(page.webSocketDebuggerUrl, "document.readyState");
|
|
2874
|
+
if (readyState === "interactive" || readyState === "complete") {
|
|
2875
|
+
readyPage = page;
|
|
2876
|
+
break;
|
|
2877
|
+
}
|
|
2878
|
+
}
|
|
2879
|
+
if (readyPage) break;
|
|
2880
|
+
try {
|
|
2881
|
+
if (child.pid) process.kill(child.pid, 0);
|
|
2882
|
+
} catch {
|
|
2883
|
+
fail(`Chrome exited before its desktop control channel became ready.`);
|
|
2884
|
+
}
|
|
2885
|
+
await sleep(100);
|
|
2886
|
+
}
|
|
2887
|
+
if (!readyPage?.webSocketDebuggerUrl) fail(`Chrome launched but its requested page was not controllable after 15 seconds.`);
|
|
2888
|
+
await sendChromeCommand(readyPage.webSocketDebuggerUrl, "Page.bringToFront", {});
|
|
2889
|
+
console.log(`launched ${bin} (pid ${child.pid}, ready in ${Date.now() - startedAt}ms, page ${readyPage.id}, ${pageCount} open)`);
|
|
2890
|
+
return;
|
|
2891
|
+
}
|
|
2892
|
+
console.log(`launched ${bin} (pid ${child.pid})`);
|
|
2893
|
+
}
|
|
2894
|
+
|
|
2895
|
+
// src/version.ts
|
|
2896
|
+
var COMPUTER_USE_VERSION = "0.1.0";
|
|
2897
|
+
var COMPUTER_USE_PROTOCOL_VERSION = 1;
|
|
2898
|
+
|
|
2899
|
+
// src/program.ts
|
|
2900
|
+
function createProgram() {
|
|
2901
|
+
const program = new Command().name("replicas-computer").description("Drive the Replicas workspace computer").version(COMPUTER_USE_VERSION).exitOverride();
|
|
2902
|
+
program.command("setup").description("Validate dependencies and install the computer-use runtime").option("--check", "Validate without installing files").action(() => {
|
|
2903
|
+
throw new Error("setup must run outside the computer-use daemon");
|
|
2904
|
+
});
|
|
2905
|
+
program.command("daemon").description("Run the computer-use daemon").action(() => {
|
|
2906
|
+
throw new Error("daemon must run outside the computer-use daemon");
|
|
2907
|
+
});
|
|
2908
|
+
program.command("info").description("Print the live noVNC viewer URL").action(() => computerInfoCommand());
|
|
2909
|
+
program.command("status").description("Show desktop services and the active preview URL").action(() => computerStatusCommand());
|
|
2910
|
+
program.command("screenshot <path>").description("Capture the current desktop to a PNG file").option("--raw", "Save a 1:1 desktop capture").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").action((path, options) => computerScreenshotCommand(path, options));
|
|
2911
|
+
program.command("observe <path>").description("Wait for the desktop to settle and capture screen context").option("--raw", "Save a 1:1 desktop capture without a grid").option("--grid [px]", "Overlay a coordinate grid. Default 100px.").option("--timeout <ms>", "Maximum stability wait. Default 3000.").option("--stable-ms <ms>", "Required unchanged time. Default 600.").option("--poll-ms <ms>", "Polling interval. Default 200.").action((path, options) => computerObserveCommand(path, options));
|
|
2912
|
+
program.command("browser").description("Print JSON for controllable Chrome pages").option("--snapshot", "Include visible text and controls").option("--limit <chars>", "Maximum body text characters. Default 4000.").option("--element-limit <n>", "Maximum controls. Default 80.").option("--target-id <id>", "Only include this Chrome target").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((options) => computerBrowserCommand(options));
|
|
2913
|
+
program.command("browser-state <path>").description("Capture a Chrome screenshot and accessibility state").option("--full", "Return the full tree instead of a diff").option("--limit <chars>", "Maximum visible text characters. Default 8000.").option("--element-limit <n>", "Maximum actionable elements. Default 120.").option("--timeout <ms>", "Maximum stability wait. Default 5000.").option("--stable-ms <ms>", "Required unchanged time. Default 500.").option("--poll-ms <ms>", "Polling interval. Default 100.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((path, options) => computerBrowserStateCommand(path, options));
|
|
2914
|
+
program.command("browser-batch <actions>").description("Run an ordered, fail-fast JSON array of browser actions").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((actions, options) => computerBrowserBatchCommand(actions, options));
|
|
2915
|
+
program.command("browser-click [text]").description("Click a Chrome element by ref or visible text").option("--ref <id>", "Backend DOM node ref from browser-state").option("--exact", "Require an exact text match").option("--index <n>", "Select the nth match. Default 0.").option("--button <button>", "left, middle, or right").option("--double", "Double-click").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserClickCommand(text, options));
|
|
2916
|
+
program.command("browser-fill [field] [value]").description("Fill a Chrome field by ref or visible text").option("--ref <id>", "Backend DOM node ref from browser-state").option("--exact", "Require an exact field match").option("--index <n>", "Select the nth match. Default 0.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((field, value, options) => {
|
|
2917
|
+
const resolvedValue = value ?? (options.ref ? field : void 0);
|
|
2918
|
+
const resolvedField = value === void 0 && options.ref ? void 0 : field;
|
|
2919
|
+
if (resolvedValue === void 0) throw new Error("Provide <field> <value>, or --ref <id> <value>");
|
|
2920
|
+
return computerBrowserFillCommand(resolvedField, resolvedValue, options);
|
|
2921
|
+
});
|
|
2922
|
+
program.command("browser-scroll <direction>").description("Send trusted wheel input to Chrome").option("--ref <id>", "Scroll from a backend DOM node ref").option("--amount <px>", "Scroll distance. Default 600.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((direction, options) => computerBrowserScrollCommand(direction, options));
|
|
2923
|
+
program.command("browser-key <combo>").description("Press a key chord in Chrome").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((combo, options) => computerBrowserKeyCommand(combo, options));
|
|
2924
|
+
program.command("browser-type <text>").description("Type literal text in Chrome").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserTypeCommand(text, options));
|
|
2925
|
+
program.command("browser-wait <text>").description("Wait for Chrome title, URL, text, or controls").option("--mode <mode>", "any, text, title, url, or control").option("--exact", "Require an exact match").option("--timeout <ms>", "Maximum wait. Default 10000.").option("--poll-ms <ms>", "Polling interval. Default 250.").option("--target-id <id>", "Target this Chrome page").option("--page <n>", "Select the nth matching page. Default 0.").option("--title <text>", "Filter by title").option("--url <text>", "Filter by URL").action((text, options) => computerBrowserWaitCommand(text, options));
|
|
2926
|
+
program.command("click <x> <y>").description("Move to a desktop coordinate and click").option("-b, --button <n>", "Mouse button. Default 1.").option("--double", "Double-click").option("--modifiers <mods>", "Modifier keys, such as ctrl+shift").action((x, y, options) => computerClickCommand(x, y, options));
|
|
2927
|
+
program.command("move <x> <y>").description("Move the desktop pointer").action((x, y) => computerMoveCommand(x, y));
|
|
2928
|
+
program.command("type <text>").description("Type literal desktop text").option("--delay <ms>", "Per-character delay").action((text, options) => computerTypeCommand(text, options));
|
|
2929
|
+
program.command("key <combo>").description("Press a desktop key chord").action((combo) => computerKeyCommand(combo));
|
|
2930
|
+
program.command("scroll <direction>").description("Scroll the desktop").option("--amount <n>", "Wheel ticks").option("--x <x>", "Pointer x").option("--y <y>", "Pointer y").action((direction, options) => computerScrollCommand(direction, options));
|
|
2931
|
+
program.command("drag <fromX> <fromY> <toX> <toY>").description("Drag between desktop coordinates").action((fromX, fromY, toX, toY) => computerDragCommand(fromX, fromY, toX, toY));
|
|
2932
|
+
program.command("launch <app> [args...]").description("Launch an application on the desktop").action((app, args) => computerLaunchCommand(app, args));
|
|
2933
|
+
const record = program.command("record").description("Record the workspace display");
|
|
2934
|
+
record.command("start <path>").description("Start a recording").option("--fps <n>", "Frame rate. Default 60.").action((path, options) => computerRecordStartCommand(path, options));
|
|
2935
|
+
record.command("stop").description("Stop and finalize the active recording").action(() => computerRecordStopCommand());
|
|
2936
|
+
return program;
|
|
2937
|
+
}
|
|
2938
|
+
async function runDirectCommand(argv, cwd = process.cwd()) {
|
|
2939
|
+
const stdout = [];
|
|
2940
|
+
const stderr = [];
|
|
2941
|
+
const previousCwd = process.cwd();
|
|
2942
|
+
const previousLog = console.log;
|
|
2943
|
+
const previousError = console.error;
|
|
2944
|
+
const program = createProgram().configureOutput({
|
|
2945
|
+
writeOut: (value) => stdout.push(value),
|
|
2946
|
+
writeErr: (value) => stderr.push(value)
|
|
2947
|
+
});
|
|
2948
|
+
console.log = (...values) => stdout.push(`${format(...values)}
|
|
2949
|
+
`);
|
|
2950
|
+
console.error = (...values) => stderr.push(`${format(...values)}
|
|
2951
|
+
`);
|
|
2952
|
+
try {
|
|
2953
|
+
process.chdir(cwd);
|
|
2954
|
+
await program.parseAsync(argv, { from: "user" });
|
|
2955
|
+
return { exitCode: 0, stdout: stdout.join(""), stderr: stderr.join("") };
|
|
2956
|
+
} catch (error) {
|
|
2957
|
+
if (error instanceof CommanderError && ["commander.helpDisplayed", "commander.version"].includes(error.code)) {
|
|
2958
|
+
return { exitCode: 0, stdout: stdout.join(""), stderr: stderr.join("") };
|
|
2959
|
+
}
|
|
2960
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2961
|
+
if (!stderr.join("").includes(message)) stderr.push(`Error: ${message}
|
|
2962
|
+
`);
|
|
2963
|
+
return { exitCode: error instanceof CommanderError ? error.exitCode : 1, stdout: stdout.join(""), stderr: stderr.join("") };
|
|
2964
|
+
} finally {
|
|
2965
|
+
process.chdir(previousCwd);
|
|
2966
|
+
console.log = previousLog;
|
|
2967
|
+
console.error = previousError;
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
|
|
2971
|
+
// src/daemon.ts
|
|
2972
|
+
var COMPUTER_USE_SOCKET_PATH = process.env.REPLICAS_COMPUTER_SOCKET || `${STATE_DIR}/control.sock`;
|
|
2973
|
+
var MAX_REQUEST_BYTES = 1024 * 1024;
|
|
2974
|
+
function isDaemonRequest(value) {
|
|
2975
|
+
return typeof value === "object" && value !== null && "protocolVersion" in value && value.protocolVersion === COMPUTER_USE_PROTOCOL_VERSION && "cwd" in value && typeof value.cwd === "string" && value.cwd.startsWith("/") && "argv" in value && Array.isArray(value.argv) && value.argv.length <= 100 && value.argv.every((arg) => typeof arg === "string" && arg.length <= 1e5);
|
|
2976
|
+
}
|
|
2977
|
+
function writeResult(socket, result) {
|
|
2978
|
+
socket.end(`${JSON.stringify({ protocolVersion: COMPUTER_USE_PROTOCOL_VERSION, ...result })}
|
|
2979
|
+
`);
|
|
2980
|
+
}
|
|
2981
|
+
async function hasSocketListener() {
|
|
2982
|
+
if (!existsSync4(COMPUTER_USE_SOCKET_PATH)) return false;
|
|
2983
|
+
return await new Promise((resolve3) => {
|
|
2984
|
+
const socket = createConnection(COMPUTER_USE_SOCKET_PATH);
|
|
2985
|
+
socket.setTimeout(500, () => socket.destroy());
|
|
2986
|
+
socket.once("connect", () => {
|
|
2987
|
+
socket.destroy();
|
|
2988
|
+
resolve3(true);
|
|
2989
|
+
});
|
|
2990
|
+
socket.once("error", () => resolve3(false));
|
|
2991
|
+
socket.once("close", () => resolve3(false));
|
|
2992
|
+
});
|
|
2993
|
+
}
|
|
2994
|
+
async function runComputerDaemon() {
|
|
2995
|
+
if (await hasSocketListener()) throw new Error(`Computer-use daemon is already listening on ${COMPUTER_USE_SOCKET_PATH}`);
|
|
2996
|
+
ensureServicesRunning();
|
|
2997
|
+
mkdirSync4(dirname4(COMPUTER_USE_SOCKET_PATH), { recursive: true, mode: 448 });
|
|
2998
|
+
rmSync4(COMPUTER_USE_SOCKET_PATH, { force: true });
|
|
2999
|
+
let queue = Promise.resolve();
|
|
3000
|
+
const server = createServer({ allowHalfOpen: true }, (socket) => {
|
|
3001
|
+
let input = "";
|
|
3002
|
+
let rejected = false;
|
|
3003
|
+
socket.setEncoding("utf8");
|
|
3004
|
+
socket.on("data", (chunk) => {
|
|
3005
|
+
if (rejected) return;
|
|
3006
|
+
input += chunk;
|
|
3007
|
+
if (Buffer.byteLength(input) > MAX_REQUEST_BYTES) {
|
|
3008
|
+
rejected = true;
|
|
3009
|
+
writeResult(socket, { exitCode: 2, stdout: "", stderr: "Computer-use request exceeds 1MB\n" });
|
|
3010
|
+
}
|
|
3011
|
+
});
|
|
3012
|
+
socket.on("end", () => {
|
|
3013
|
+
if (rejected) return;
|
|
3014
|
+
queue = queue.then(async () => {
|
|
3015
|
+
let request;
|
|
3016
|
+
try {
|
|
3017
|
+
request = JSON.parse(input);
|
|
3018
|
+
} catch {
|
|
3019
|
+
writeResult(socket, { exitCode: 2, stdout: "", stderr: "Invalid computer-use request JSON\n" });
|
|
3020
|
+
return;
|
|
3021
|
+
}
|
|
3022
|
+
if (!isDaemonRequest(request)) {
|
|
3023
|
+
writeResult(socket, { exitCode: 2, stdout: "", stderr: "Invalid or incompatible computer-use request\n" });
|
|
3024
|
+
return;
|
|
3025
|
+
}
|
|
3026
|
+
writeResult(socket, await runDirectCommand(request.argv, request.cwd));
|
|
3027
|
+
}).catch((error) => {
|
|
3028
|
+
writeResult(socket, { exitCode: 1, stdout: "", stderr: `${error instanceof Error ? error.message : String(error)}
|
|
3029
|
+
` });
|
|
3030
|
+
});
|
|
3031
|
+
});
|
|
3032
|
+
});
|
|
3033
|
+
await new Promise((resolve3, reject) => {
|
|
3034
|
+
server.once("error", reject);
|
|
3035
|
+
server.listen(COMPUTER_USE_SOCKET_PATH, () => {
|
|
3036
|
+
server.off("error", reject);
|
|
3037
|
+
chmodSync(COMPUTER_USE_SOCKET_PATH, 384);
|
|
3038
|
+
console.log(`replicas-computer ${COMPUTER_USE_VERSION} listening on ${COMPUTER_USE_SOCKET_PATH}`);
|
|
3039
|
+
resolve3();
|
|
3040
|
+
});
|
|
3041
|
+
});
|
|
3042
|
+
const healthTimer = setInterval(() => {
|
|
3043
|
+
queue = queue.then(() => ensureServicesRunning()).catch((error) => {
|
|
3044
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
3045
|
+
});
|
|
3046
|
+
}, 5e3);
|
|
3047
|
+
healthTimer.unref();
|
|
3048
|
+
await new Promise((resolve3) => {
|
|
3049
|
+
let closing = false;
|
|
3050
|
+
const close = () => {
|
|
3051
|
+
if (closing) return;
|
|
3052
|
+
closing = true;
|
|
3053
|
+
clearInterval(healthTimer);
|
|
3054
|
+
server.close(() => resolve3());
|
|
3055
|
+
};
|
|
3056
|
+
process.once("SIGTERM", close);
|
|
3057
|
+
process.once("SIGINT", close);
|
|
3058
|
+
});
|
|
3059
|
+
spawnSync4("bash", [SERVICES_SCRIPT, "--stop"], { stdio: "inherit" });
|
|
3060
|
+
rmSync4(COMPUTER_USE_SOCKET_PATH, { force: true });
|
|
3061
|
+
}
|
|
3062
|
+
|
|
3063
|
+
// src/client.ts
|
|
3064
|
+
var DAEMON_STARTUP_TIMEOUT_MS = 5e3;
|
|
3065
|
+
var ComputerDaemonUnavailableError = class extends Error {
|
|
3066
|
+
};
|
|
3067
|
+
function sendComputerRequest(argv) {
|
|
3068
|
+
return new Promise((resolve3, reject) => {
|
|
3069
|
+
const socket = createConnection2(COMPUTER_USE_SOCKET_PATH);
|
|
3070
|
+
let response = "";
|
|
3071
|
+
let settled = false;
|
|
3072
|
+
const finish = (error, result) => {
|
|
3073
|
+
if (settled) return;
|
|
3074
|
+
settled = true;
|
|
3075
|
+
if (error) reject(error);
|
|
3076
|
+
else if (result) resolve3(result);
|
|
3077
|
+
};
|
|
3078
|
+
socket.setEncoding("utf8");
|
|
3079
|
+
socket.setTimeout(10 * 60 * 1e3, () => socket.destroy(new Error("Computer-use command timed out")));
|
|
3080
|
+
socket.once("connect", () => socket.end(JSON.stringify({
|
|
3081
|
+
protocolVersion: COMPUTER_USE_PROTOCOL_VERSION,
|
|
3082
|
+
argv,
|
|
3083
|
+
cwd: process.cwd()
|
|
3084
|
+
})));
|
|
3085
|
+
socket.on("data", (chunk) => {
|
|
3086
|
+
response += chunk;
|
|
3087
|
+
});
|
|
3088
|
+
socket.once("error", (error) => finish(new ComputerDaemonUnavailableError(`Computer-use daemon is unavailable at ${COMPUTER_USE_SOCKET_PATH}: ${error.message}`)));
|
|
3089
|
+
socket.once("close", () => {
|
|
3090
|
+
if (!response) {
|
|
3091
|
+
finish(new ComputerDaemonUnavailableError("Computer-use daemon closed the connection without a response"));
|
|
3092
|
+
return;
|
|
3093
|
+
}
|
|
3094
|
+
try {
|
|
3095
|
+
const value = JSON.parse(response);
|
|
3096
|
+
if (typeof value !== "object" || value === null || !("protocolVersion" in value) || value.protocolVersion !== COMPUTER_USE_PROTOCOL_VERSION || !("exitCode" in value) || typeof value.exitCode !== "number" || !("stdout" in value) || typeof value.stdout !== "string" || !("stderr" in value) || typeof value.stderr !== "string") {
|
|
3097
|
+
finish(new Error("Computer-use daemon returned an incompatible response"));
|
|
3098
|
+
return;
|
|
3099
|
+
}
|
|
3100
|
+
finish(void 0, { exitCode: value.exitCode, stdout: value.stdout, stderr: value.stderr });
|
|
3101
|
+
} catch (error) {
|
|
3102
|
+
finish(new Error(`Computer-use daemon returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`));
|
|
3103
|
+
}
|
|
3104
|
+
});
|
|
3105
|
+
});
|
|
3106
|
+
}
|
|
3107
|
+
async function runComputerClient(argv) {
|
|
3108
|
+
const deadline = Date.now() + DAEMON_STARTUP_TIMEOUT_MS;
|
|
3109
|
+
while (true) {
|
|
3110
|
+
try {
|
|
3111
|
+
return await sendComputerRequest(argv);
|
|
3112
|
+
} catch (error) {
|
|
3113
|
+
if (!(error instanceof ComputerDaemonUnavailableError) || Date.now() >= deadline) throw error;
|
|
3114
|
+
await new Promise((resolve3) => setTimeout(resolve3, 100));
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
|
|
3119
|
+
// src/setup.ts
|
|
3120
|
+
import { spawnSync as spawnSync5 } from "child_process";
|
|
3121
|
+
import { accessSync, appendFileSync as appendFileSync2, chmodSync as chmodSync2, constants, cpSync, existsSync as existsSync5, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync4 } from "fs";
|
|
3122
|
+
import { dirname as dirname5, resolve as resolve2 } from "path";
|
|
3123
|
+
import { fileURLToPath } from "url";
|
|
3124
|
+
var dependencies = [
|
|
3125
|
+
["bash", "bash"],
|
|
3126
|
+
["Node.js", "node"],
|
|
3127
|
+
["Xvfb", "Xvfb"],
|
|
3128
|
+
["xdpyinfo", "xdpyinfo"],
|
|
3129
|
+
["xprop", "xprop"],
|
|
3130
|
+
["xdotool", "xdotool"],
|
|
3131
|
+
["scrot", "scrot"],
|
|
3132
|
+
["openbox", "openbox"],
|
|
3133
|
+
["tint2", "tint2"],
|
|
3134
|
+
["picom", "picom"],
|
|
3135
|
+
["ffmpeg", "ffmpeg"],
|
|
3136
|
+
["x11vnc", "x11vnc"],
|
|
3137
|
+
["websockify", "websockify"],
|
|
3138
|
+
["Google Chrome", "google-chrome-stable"],
|
|
3139
|
+
["flock", "flock"],
|
|
3140
|
+
["ss", "ss"],
|
|
3141
|
+
["pgrep", "pgrep"],
|
|
3142
|
+
["setsid", "setsid"],
|
|
3143
|
+
["sudo", "sudo"],
|
|
3144
|
+
["systemctl", "systemctl"],
|
|
3145
|
+
["hsetroot", "hsetroot"],
|
|
3146
|
+
["XFCE terminal", "xfce4-terminal"],
|
|
3147
|
+
["Mousepad", "mousepad"],
|
|
3148
|
+
["Thunar", "thunar"]
|
|
3149
|
+
];
|
|
3150
|
+
var serviceUnit = `[Unit]
|
|
3151
|
+
Description=Replicas computer-use runtime
|
|
3152
|
+
After=basic.target local-fs.target
|
|
3153
|
+
|
|
3154
|
+
[Service]
|
|
3155
|
+
Type=exec
|
|
3156
|
+
User=user
|
|
3157
|
+
Group=user
|
|
3158
|
+
WorkingDirectory=/home/user/workspaces
|
|
3159
|
+
Environment=HOME=/home/user
|
|
3160
|
+
Environment=USER=user
|
|
3161
|
+
ExecStart=/bin/bash -lc 'export NVM_DIR=/usr/local/nvm; if [ -s /usr/local/nvm/nvm.sh ]; then . /usr/local/nvm/nvm.sh; fi; if [ -f /home/user/.replicas/runtime-env.sh ]; then . /home/user/.replicas/runtime-env.sh; fi; exec replicas-computer daemon'
|
|
3162
|
+
Restart=on-failure
|
|
3163
|
+
RestartSec=2
|
|
3164
|
+
TimeoutStopSec=15
|
|
3165
|
+
StandardOutput=journal
|
|
3166
|
+
StandardError=inherit
|
|
3167
|
+
|
|
3168
|
+
[Install]
|
|
3169
|
+
WantedBy=multi-user.target
|
|
3170
|
+
`;
|
|
3171
|
+
function commandVersion(command) {
|
|
3172
|
+
const result = spawnSync5(command, ["--version"], { encoding: "utf8" });
|
|
3173
|
+
return `${result.stdout || result.stderr}`.trim().split("\n")[0]?.slice(0, 200) || "available";
|
|
3174
|
+
}
|
|
3175
|
+
function hasCommand(command) {
|
|
3176
|
+
return (process.env.PATH ?? "").split(":").some((directory) => {
|
|
3177
|
+
try {
|
|
3178
|
+
accessSync(resolve2(directory, command), constants.X_OK);
|
|
3179
|
+
return true;
|
|
3180
|
+
} catch {
|
|
3181
|
+
return false;
|
|
3182
|
+
}
|
|
3183
|
+
});
|
|
3184
|
+
}
|
|
3185
|
+
function setupComputer(options = {}) {
|
|
3186
|
+
const found = {};
|
|
3187
|
+
const missing = [];
|
|
3188
|
+
for (const [name, command] of dependencies) {
|
|
3189
|
+
if (hasCommand(command)) {
|
|
3190
|
+
found[command] = commandVersion(command);
|
|
3191
|
+
console.log(`\u2713 ${name}`);
|
|
3192
|
+
} else {
|
|
3193
|
+
missing.push(`${name} (${command})`);
|
|
3194
|
+
console.error(`\u2717 ${name} \u2014 ${command} is not on PATH`);
|
|
3195
|
+
}
|
|
3196
|
+
}
|
|
3197
|
+
if (found.ffmpeg) {
|
|
3198
|
+
const encoders = spawnSync5("ffmpeg", ["-hide_banner", "-encoders"], { encoding: "utf8" });
|
|
3199
|
+
if (!`${encoders.stdout}${encoders.stderr}`.includes("libx264")) {
|
|
3200
|
+
missing.push("ffmpeg encoder libx264");
|
|
3201
|
+
console.error("\u2717 ffmpeg encoder libx264 is unavailable");
|
|
3202
|
+
} else {
|
|
3203
|
+
console.log("\u2713 ffmpeg encoder libx264");
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
const noVncRoot = ["/usr/share/novnc", "/usr/share/novnc-common"].find(existsSync5);
|
|
3207
|
+
if (!noVncRoot) {
|
|
3208
|
+
missing.push("noVNC web root");
|
|
3209
|
+
console.error("\u2717 noVNC web root \u2014 /usr/share/novnc and /usr/share/novnc-common are missing");
|
|
3210
|
+
} else {
|
|
3211
|
+
console.log(`\u2713 noVNC web root (${noVncRoot})`);
|
|
3212
|
+
}
|
|
3213
|
+
if (spawnSync5("id", ["user"], { stdio: "ignore" }).status !== 0) {
|
|
3214
|
+
missing.push("workspace user account");
|
|
3215
|
+
console.error("\u2717 workspace user account \u2014 user does not exist");
|
|
3216
|
+
} else {
|
|
3217
|
+
console.log("\u2713 workspace user account");
|
|
3218
|
+
}
|
|
3219
|
+
if (missing.length > 0) throw new Error(`Missing ${missing.length} computer-use requirement${missing.length === 1 ? "" : "s"}: ${missing.join(", ")}`);
|
|
3220
|
+
if (!noVncRoot) throw new Error("noVNC web root is unavailable");
|
|
3221
|
+
if (options.checkOnly) return;
|
|
3222
|
+
if (process.getuid?.() !== 0) throw new Error("replicas-computer setup must run as root to install runtime files");
|
|
3223
|
+
const packageRoot = resolve2(dirname5(fileURLToPath(import.meta.url)), "..");
|
|
3224
|
+
const assets = resolve2(packageRoot, "assets", "desktop");
|
|
3225
|
+
if (!existsSync5(assets)) throw new Error(`Computer-use assets are missing from ${assets}`);
|
|
3226
|
+
mkdirSync5("/usr/local/share/replicas/desktop", { recursive: true });
|
|
3227
|
+
cpSync(assets, "/usr/local/share/replicas/desktop", { recursive: true, force: true });
|
|
3228
|
+
cpSync(resolve2(assets, "start-services.sh"), "/usr/local/bin/replicas-start-desktop-services", { force: true });
|
|
3229
|
+
cpSync(resolve2(assets, "replicas-chrome"), "/usr/local/bin/replicas-chrome", { force: true });
|
|
3230
|
+
cpSync(resolve2(assets, "replicas-chrome.desktop"), "/usr/share/applications/replicas-chrome.desktop", { force: true });
|
|
3231
|
+
chmodSync2("/usr/local/bin/replicas-start-desktop-services", 493);
|
|
3232
|
+
chmodSync2("/usr/local/bin/replicas-chrome", 493);
|
|
3233
|
+
cpSync(resolve2(assets, "novnc-index.html"), resolve2(noVncRoot, "index.html"), { force: true });
|
|
3234
|
+
cpSync(resolve2(assets, "geist-mono.woff2"), resolve2(noVncRoot, "geist-mono.woff2"), { force: true });
|
|
3235
|
+
const noVncStyles = resolve2(noVncRoot, "app", "styles", "base.css");
|
|
3236
|
+
const styleOverride = "\n#noVNC_status { display: none !important; }\n#noVNC_control_bar_anchor { display: none !important; }\n";
|
|
3237
|
+
if (existsSync5(noVncStyles) && !readFileSync4(noVncStyles, "utf8").includes("#noVNC_status { display: none !important; }")) {
|
|
3238
|
+
appendFileSync2(noVncStyles, styleOverride);
|
|
3239
|
+
}
|
|
3240
|
+
writeFileSync4("/etc/systemd/system/replicas-computer.service", serviceUnit, { mode: 420 });
|
|
3241
|
+
if (existsSync5("/run/systemd/system")) {
|
|
3242
|
+
const reload = spawnSync5("systemctl", ["daemon-reload"], { stdio: "pipe" });
|
|
3243
|
+
if (reload.status !== 0) throw new Error(`Failed to reload systemd: ${reload.stderr?.toString().trim() || `exit ${reload.status}`}`);
|
|
3244
|
+
}
|
|
3245
|
+
mkdirSync5("/usr/local/share/replicas-computer", { recursive: true });
|
|
3246
|
+
writeFileSync4("/usr/local/share/replicas-computer/install.json", `${JSON.stringify({
|
|
3247
|
+
version: COMPUTER_USE_VERSION,
|
|
3248
|
+
protocolVersion: COMPUTER_USE_PROTOCOL_VERSION,
|
|
3249
|
+
dependencies: found
|
|
3250
|
+
}, null, 2)}
|
|
3251
|
+
`);
|
|
3252
|
+
console.log(`replicas-computer ${COMPUTER_USE_VERSION} setup complete`);
|
|
3253
|
+
}
|
|
3254
|
+
|
|
3255
|
+
// src/index.ts
|
|
3256
|
+
async function main() {
|
|
3257
|
+
const argv = process.argv.slice(2);
|
|
3258
|
+
if (["setup", "daemon"].includes(argv[0]) && argv.slice(1).some((arg) => ["--help", "-h"].includes(arg))) {
|
|
3259
|
+
const result2 = await runDirectCommand(argv);
|
|
3260
|
+
if (result2.stdout) process.stdout.write(result2.stdout);
|
|
3261
|
+
if (result2.stderr) process.stderr.write(result2.stderr);
|
|
3262
|
+
process.exitCode = result2.exitCode;
|
|
3263
|
+
return;
|
|
3264
|
+
}
|
|
3265
|
+
if (argv[0] === "setup") {
|
|
3266
|
+
const unsupported = argv.slice(1).filter((arg) => arg !== "--check");
|
|
3267
|
+
if (unsupported.length > 0) throw new Error(`Unknown setup option: ${unsupported[0]}`);
|
|
3268
|
+
setupComputer({ checkOnly: argv.includes("--check") });
|
|
3269
|
+
return;
|
|
3270
|
+
}
|
|
3271
|
+
if (argv[0] === "daemon") {
|
|
3272
|
+
if (argv.length > 1) throw new Error(`Unknown daemon option: ${argv[1]}`);
|
|
3273
|
+
await runComputerDaemon();
|
|
3274
|
+
return;
|
|
3275
|
+
}
|
|
3276
|
+
const direct = argv[0] === "--direct";
|
|
3277
|
+
const commandArgv = direct ? argv.slice(1) : argv;
|
|
3278
|
+
const local = commandArgv.length === 0 || ["--help", "-h", "--version", "-V", "help"].includes(commandArgv[0]);
|
|
3279
|
+
const result = direct || local ? await runDirectCommand(commandArgv) : await runComputerClient(commandArgv);
|
|
3280
|
+
if (result.stdout) process.stdout.write(result.stdout);
|
|
3281
|
+
if (result.stderr) process.stderr.write(result.stderr);
|
|
3282
|
+
process.exitCode = result.exitCode;
|
|
3283
|
+
}
|
|
3284
|
+
main().catch((error) => {
|
|
3285
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
3286
|
+
process.exitCode = 1;
|
|
3287
|
+
});
|