usecomputer 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +338 -0
- package/build.zig +1 -0
- package/dist/bridge-contract.test.js +124 -63
- package/dist/bridge.d.ts.map +1 -1
- package/dist/bridge.js +241 -46
- package/dist/cli-parsing.test.js +34 -11
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +328 -22
- package/dist/coord-map.d.ts +14 -0
- package/dist/coord-map.d.ts.map +1 -0
- package/dist/coord-map.js +75 -0
- package/dist/coord-map.test.d.ts +2 -0
- package/dist/coord-map.test.d.ts.map +1 -0
- package/dist/coord-map.test.js +157 -0
- package/dist/darwin-arm64/usecomputer.node +0 -0
- package/dist/darwin-x64/usecomputer.node +0 -0
- package/dist/debug-point-image.d.ts +8 -0
- package/dist/debug-point-image.d.ts.map +1 -0
- package/dist/debug-point-image.js +43 -0
- package/dist/debug-point-image.test.d.ts +2 -0
- package/dist/debug-point-image.test.d.ts.map +1 -0
- package/dist/debug-point-image.test.js +44 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/lib.d.ts +26 -0
- package/dist/lib.d.ts.map +1 -0
- package/dist/lib.js +88 -0
- package/dist/native-click-smoke.test.js +69 -29
- package/dist/native-lib.d.ts +59 -1
- package/dist/native-lib.d.ts.map +1 -1
- package/dist/terminal-table.d.ts +10 -0
- package/dist/terminal-table.d.ts.map +1 -0
- package/dist/terminal-table.js +55 -0
- package/dist/terminal-table.test.d.ts +2 -0
- package/dist/terminal-table.test.d.ts.map +1 -0
- package/dist/terminal-table.test.js +41 -0
- package/dist/types.d.ts +45 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +16 -4
- package/src/bridge-contract.test.ts +140 -69
- package/src/bridge.ts +293 -53
- package/src/cli-parsing.test.ts +61 -0
- package/src/cli.ts +401 -25
- package/src/coord-map.test.ts +178 -0
- package/src/coord-map.ts +105 -0
- package/src/debug-point-image.test.ts +50 -0
- package/src/debug-point-image.ts +69 -0
- package/src/index.ts +3 -1
- package/src/lib.ts +125 -0
- package/src/native-click-smoke.test.ts +81 -63
- package/src/native-lib.ts +39 -1
- package/src/terminal-table.test.ts +44 -0
- package/src/terminal-table.ts +88 -0
- package/src/types.ts +50 -0
- package/zig/src/lib.zig +1280 -163
- package/zig/src/scroll.zig +213 -0
- package/zig/src/window.zig +123 -0
package/dist/cli.js
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
// usecomputer CLI entrypoint and command wiring for desktop automation actions.
|
|
2
2
|
import { goke } from 'goke';
|
|
3
|
+
import pc from 'picocolors';
|
|
3
4
|
import { z } from 'zod';
|
|
4
5
|
import dedent from 'string-dedent';
|
|
5
6
|
import { createRequire } from 'node:module';
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import pathModule from 'node:path';
|
|
6
9
|
import url from 'node:url';
|
|
7
10
|
import { createBridge } from './bridge.js';
|
|
11
|
+
import { getRegionFromCoordMap, mapPointFromCoordMap, mapPointToCoordMap, parseCoordMapOrThrow, } from './coord-map.js';
|
|
8
12
|
import { parseDirection, parseModifiers, parsePoint, parseRegion } from './command-parsers.js';
|
|
13
|
+
import { drawDebugPointOnImage } from './debug-point-image.js';
|
|
14
|
+
import { renderAlignedTable } from './terminal-table.js';
|
|
9
15
|
const require = createRequire(import.meta.url);
|
|
10
16
|
const packageJson = require('../package.json');
|
|
11
17
|
function printJson(value) {
|
|
@@ -14,6 +20,38 @@ function printJson(value) {
|
|
|
14
20
|
function printLine(value) {
|
|
15
21
|
process.stdout.write(`${value}\n`);
|
|
16
22
|
}
|
|
23
|
+
function readTextFromStdin() {
|
|
24
|
+
return fs.readFileSync(0, 'utf8');
|
|
25
|
+
}
|
|
26
|
+
function parsePositiveInteger({ value, option, }) {
|
|
27
|
+
if (typeof value !== 'number') {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
31
|
+
throw new Error(`Option ${option} must be a positive number`);
|
|
32
|
+
}
|
|
33
|
+
return Math.round(value);
|
|
34
|
+
}
|
|
35
|
+
function splitIntoChunks({ text, chunkSize, }) {
|
|
36
|
+
if (!chunkSize || text.length <= chunkSize) {
|
|
37
|
+
return [text];
|
|
38
|
+
}
|
|
39
|
+
const chunkCount = Math.ceil(text.length / chunkSize);
|
|
40
|
+
return Array.from({ length: chunkCount }, (_, index) => {
|
|
41
|
+
const start = index * chunkSize;
|
|
42
|
+
const end = start + chunkSize;
|
|
43
|
+
return text.slice(start, end);
|
|
44
|
+
}).filter((chunk) => {
|
|
45
|
+
return chunk.length > 0;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
function sleep({ ms, }) {
|
|
49
|
+
return new Promise((resolve) => {
|
|
50
|
+
setTimeout(() => {
|
|
51
|
+
resolve();
|
|
52
|
+
}, ms);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
17
55
|
function parsePointOrThrow(input) {
|
|
18
56
|
const parsed = parsePoint(input);
|
|
19
57
|
if (parsed instanceof Error) {
|
|
@@ -21,6 +59,21 @@ function parsePointOrThrow(input) {
|
|
|
21
59
|
}
|
|
22
60
|
return parsed;
|
|
23
61
|
}
|
|
62
|
+
function resolveOutputPath({ path }) {
|
|
63
|
+
if (!path) {
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
return path.startsWith('/')
|
|
67
|
+
? path
|
|
68
|
+
: `${process.cwd()}/${path}`;
|
|
69
|
+
}
|
|
70
|
+
function ensureParentDirectory({ filePath }) {
|
|
71
|
+
if (!filePath) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const parentDirectory = pathModule.dirname(filePath);
|
|
75
|
+
fs.mkdirSync(parentDirectory, { recursive: true });
|
|
76
|
+
}
|
|
24
77
|
function resolvePointInput({ x, y, target, command, }) {
|
|
25
78
|
if (typeof x === 'number' || typeof y === 'number') {
|
|
26
79
|
if (typeof x !== 'number' || typeof y !== 'number') {
|
|
@@ -39,8 +92,88 @@ function parseButton(input) {
|
|
|
39
92
|
}
|
|
40
93
|
return 'left';
|
|
41
94
|
}
|
|
95
|
+
function printDesktopList({ displays }) {
|
|
96
|
+
const rows = displays.map((display) => {
|
|
97
|
+
return {
|
|
98
|
+
desktop: `#${display.index}`,
|
|
99
|
+
primary: display.isPrimary ? pc.green('yes') : 'no',
|
|
100
|
+
size: `${display.width}x${display.height}`,
|
|
101
|
+
position: `${display.x},${display.y}`,
|
|
102
|
+
id: String(display.id),
|
|
103
|
+
scale: String(display.scale),
|
|
104
|
+
name: display.name,
|
|
105
|
+
};
|
|
106
|
+
});
|
|
107
|
+
const lines = renderAlignedTable({
|
|
108
|
+
rows,
|
|
109
|
+
columns: [
|
|
110
|
+
{ header: pc.bold('desktop'), value: (row) => { return row.desktop; } },
|
|
111
|
+
{ header: pc.bold('primary'), value: (row) => { return row.primary; } },
|
|
112
|
+
{ header: pc.bold('size'), value: (row) => { return row.size; }, align: 'right' },
|
|
113
|
+
{ header: pc.bold('position'), value: (row) => { return row.position; }, align: 'right' },
|
|
114
|
+
{ header: pc.bold('id'), value: (row) => { return row.id; }, align: 'right' },
|
|
115
|
+
{ header: pc.bold('scale'), value: (row) => { return row.scale; }, align: 'right' },
|
|
116
|
+
{ header: pc.bold('name'), value: (row) => { return row.name; } },
|
|
117
|
+
],
|
|
118
|
+
});
|
|
119
|
+
lines.forEach((line) => {
|
|
120
|
+
printLine(line);
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
function mapWindowsByDesktopIndex({ windows, }) {
|
|
124
|
+
return windows.reduce((acc, window) => {
|
|
125
|
+
const list = acc.get(window.desktopIndex) ?? [];
|
|
126
|
+
list.push(window);
|
|
127
|
+
acc.set(window.desktopIndex, list);
|
|
128
|
+
return acc;
|
|
129
|
+
}, new Map());
|
|
130
|
+
}
|
|
131
|
+
function printDesktopListWithWindows({ displays, windows, }) {
|
|
132
|
+
const windowsByDesktop = mapWindowsByDesktopIndex({ windows });
|
|
133
|
+
printDesktopList({ displays });
|
|
134
|
+
displays.forEach((display) => {
|
|
135
|
+
printLine('');
|
|
136
|
+
printLine(pc.bold(pc.cyan(`desktop #${display.index} windows`)));
|
|
137
|
+
const desktopWindows = windowsByDesktop.get(display.index) ?? [];
|
|
138
|
+
if (desktopWindows.length === 0) {
|
|
139
|
+
printLine(pc.dim('none'));
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const lines = renderAlignedTable({
|
|
143
|
+
rows: desktopWindows,
|
|
144
|
+
columns: [
|
|
145
|
+
{ header: pc.bold('id'), value: (row) => { return String(row.id); }, align: 'right' },
|
|
146
|
+
{ header: pc.bold('app'), value: (row) => { return row.ownerName; } },
|
|
147
|
+
{ header: pc.bold('pid'), value: (row) => { return String(row.ownerPid); }, align: 'right' },
|
|
148
|
+
{ header: pc.bold('size'), value: (row) => { return `${row.width}x${row.height}`; }, align: 'right' },
|
|
149
|
+
{ header: pc.bold('position'), value: (row) => { return `${row.x},${row.y}`; }, align: 'right' },
|
|
150
|
+
{ header: pc.bold('title'), value: (row) => { return row.title; } },
|
|
151
|
+
],
|
|
152
|
+
});
|
|
153
|
+
lines.forEach((line) => {
|
|
154
|
+
printLine(line);
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
function printWindowList({ windows }) {
|
|
159
|
+
const lines = renderAlignedTable({
|
|
160
|
+
rows: windows,
|
|
161
|
+
columns: [
|
|
162
|
+
{ header: pc.bold('id'), value: (row) => { return String(row.id); }, align: 'right' },
|
|
163
|
+
{ header: pc.bold('desktop'), value: (row) => { return `#${row.desktopIndex}`; }, align: 'right' },
|
|
164
|
+
{ header: pc.bold('app'), value: (row) => { return row.ownerName; } },
|
|
165
|
+
{ header: pc.bold('pid'), value: (row) => { return String(row.ownerPid); }, align: 'right' },
|
|
166
|
+
{ header: pc.bold('size'), value: (row) => { return `${row.width}x${row.height}`; }, align: 'right' },
|
|
167
|
+
{ header: pc.bold('position'), value: (row) => { return `${row.x},${row.y}`; }, align: 'right' },
|
|
168
|
+
{ header: pc.bold('title'), value: (row) => { return row.title; } },
|
|
169
|
+
],
|
|
170
|
+
});
|
|
171
|
+
lines.forEach((line) => {
|
|
172
|
+
printLine(line);
|
|
173
|
+
});
|
|
174
|
+
}
|
|
42
175
|
function notImplemented({ command }) {
|
|
43
|
-
throw new Error(`
|
|
176
|
+
throw new Error(`TODO not implemented: ${command}`);
|
|
44
177
|
}
|
|
45
178
|
export function createCli({ bridge = createBridge() } = {}) {
|
|
46
179
|
const cli = goke('usecomputer');
|
|
@@ -51,18 +184,28 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
51
184
|
This command uses a native Zig backend over macOS APIs.
|
|
52
185
|
`)
|
|
53
186
|
.option('-r, --region [region]', z.string().describe('Capture region as x,y,width,height'))
|
|
54
|
-
.option('--display [display]', z.number().describe('Display index for multi-monitor setups'))
|
|
187
|
+
.option('--display [display]', z.number().describe('Display index for multi-monitor setups (0-based: first display is index 0)'))
|
|
188
|
+
.option('--window [window]', z.number().describe('Capture a specific window by window id'))
|
|
55
189
|
.option('--annotate', 'Annotate screenshot with labels')
|
|
56
190
|
.option('--json', 'Output as JSON')
|
|
57
191
|
.action(async (path, options) => {
|
|
192
|
+
const outputPath = resolveOutputPath({ path });
|
|
193
|
+
ensureParentDirectory({ filePath: outputPath });
|
|
58
194
|
const region = options.region ? parseRegion(options.region) : undefined;
|
|
59
195
|
if (region instanceof Error) {
|
|
60
196
|
throw region;
|
|
61
197
|
}
|
|
198
|
+
if (typeof options.window === 'number' && region) {
|
|
199
|
+
throw new Error('Cannot use --window and --region together');
|
|
200
|
+
}
|
|
201
|
+
if (typeof options.window === 'number' && typeof options.display === 'number') {
|
|
202
|
+
throw new Error('Cannot use --window and --display together');
|
|
203
|
+
}
|
|
62
204
|
const result = await bridge.screenshot({
|
|
63
|
-
path,
|
|
205
|
+
path: outputPath,
|
|
64
206
|
region,
|
|
65
207
|
display: options.display,
|
|
208
|
+
window: options.window,
|
|
66
209
|
annotate: options.annotate,
|
|
67
210
|
});
|
|
68
211
|
if (options.json) {
|
|
@@ -70,14 +213,27 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
70
213
|
return;
|
|
71
214
|
}
|
|
72
215
|
printLine(result.path);
|
|
216
|
+
printLine(result.hint);
|
|
217
|
+
printLine(`desktop-index=${String(result.desktopIndex)}`);
|
|
73
218
|
});
|
|
74
219
|
cli
|
|
75
|
-
.command('click [target]',
|
|
76
|
-
|
|
77
|
-
|
|
220
|
+
.command('click [target]', dedent `
|
|
221
|
+
Click at coordinates.
|
|
222
|
+
|
|
223
|
+
When you are clicking from a screenshot, use the exact pixel coordinates
|
|
224
|
+
of the target in that screenshot image and always pass the exact
|
|
225
|
+
--coord-map value printed by usecomputer screenshot. The coord map
|
|
226
|
+
scales screenshot-space pixels back into the real captured desktop or
|
|
227
|
+
window rectangle before sending the native click.
|
|
228
|
+
`)
|
|
229
|
+
.option('-x [x]', z.number().describe('X coordinate. When using --coord-map, this must be the exact pixel from the screenshot image'))
|
|
230
|
+
.option('-y [y]', z.number().describe('Y coordinate. When using --coord-map, this must be the exact pixel from the screenshot image'))
|
|
78
231
|
.option('--button [button]', z.enum(['left', 'right', 'middle']).default('left').describe('Mouse button'))
|
|
79
232
|
.option('--count [count]', z.number().default(1).describe('Number of clicks'))
|
|
80
233
|
.option('--modifiers [modifiers]', z.string().describe('Modifiers as ctrl,shift,alt,meta'))
|
|
234
|
+
.option('--coord-map [coordMap]', z.string().describe('Map exact screenshot-space pixels back into the real captured desktop or window rectangle'))
|
|
235
|
+
.example('# Click the exact pixel you saw in a screenshot')
|
|
236
|
+
.example('usecomputer click -x 155 -y 446 --coord-map "0,0,1720,1440,1568,1313"')
|
|
81
237
|
.action(async (target, options) => {
|
|
82
238
|
const point = resolvePointInput({
|
|
83
239
|
x: options.x,
|
|
@@ -85,23 +241,144 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
85
241
|
target,
|
|
86
242
|
command: 'click',
|
|
87
243
|
});
|
|
244
|
+
const coordMap = parseCoordMapOrThrow(options.coordMap);
|
|
88
245
|
await bridge.click({
|
|
89
|
-
point,
|
|
246
|
+
point: mapPointFromCoordMap({ point, coordMap }),
|
|
90
247
|
button: options.button,
|
|
91
248
|
count: options.count,
|
|
92
249
|
modifiers: parseModifiers(options.modifiers),
|
|
93
250
|
});
|
|
94
251
|
});
|
|
95
252
|
cli
|
|
96
|
-
.command('
|
|
97
|
-
|
|
253
|
+
.command('debug-point [target]', dedent `
|
|
254
|
+
Capture a screenshot and draw a red marker where a click would land.
|
|
255
|
+
|
|
256
|
+
Pass the same --coord-map you plan to use for click. This validates
|
|
257
|
+
screenshot-space coordinates before you send a real click. When
|
|
258
|
+
--coord-map is present, debug-point captures that same region so the
|
|
259
|
+
overlay matches the screenshot you are targeting.
|
|
260
|
+
`)
|
|
261
|
+
.option('-x [x]', z.number().describe('X coordinate'))
|
|
262
|
+
.option('-y [y]', z.number().describe('Y coordinate'))
|
|
263
|
+
.option('--coord-map [coordMap]', z.string().describe('Map input coordinates from screenshot space'))
|
|
264
|
+
.option('--output [path]', z.string().describe('Write the annotated screenshot to this path'))
|
|
265
|
+
.option('--json', 'Output as JSON')
|
|
266
|
+
.example('# Validate the same coordinates you plan to click')
|
|
267
|
+
.example('usecomputer debug-point -x 210 -y 560 --coord-map "0,0,1720,1440,1568,1313"')
|
|
268
|
+
.action(async (target, options) => {
|
|
269
|
+
const point = resolvePointInput({
|
|
270
|
+
x: options.x,
|
|
271
|
+
y: options.y,
|
|
272
|
+
target,
|
|
273
|
+
command: 'debug-point',
|
|
274
|
+
});
|
|
275
|
+
const inputCoordMap = parseCoordMapOrThrow(options.coordMap);
|
|
276
|
+
const desktopPoint = mapPointFromCoordMap({ point, coordMap: inputCoordMap });
|
|
277
|
+
const outputPath = resolveOutputPath({ path: options.output ?? './tmp/debug-point.png' });
|
|
278
|
+
ensureParentDirectory({ filePath: outputPath });
|
|
279
|
+
const screenshotRegion = getRegionFromCoordMap({ coordMap: inputCoordMap });
|
|
280
|
+
const screenshot = await bridge.screenshot({
|
|
281
|
+
path: outputPath,
|
|
282
|
+
region: screenshotRegion,
|
|
283
|
+
});
|
|
284
|
+
const screenshotCoordMap = parseCoordMapOrThrow(screenshot.coordMap);
|
|
285
|
+
const screenshotPoint = mapPointToCoordMap({ point: desktopPoint, coordMap: screenshotCoordMap });
|
|
286
|
+
await drawDebugPointOnImage({
|
|
287
|
+
imagePath: screenshot.path,
|
|
288
|
+
point: screenshotPoint,
|
|
289
|
+
imageWidth: screenshot.imageWidth,
|
|
290
|
+
imageHeight: screenshot.imageHeight,
|
|
291
|
+
});
|
|
292
|
+
if (options.json) {
|
|
293
|
+
printJson({
|
|
294
|
+
path: screenshot.path,
|
|
295
|
+
inputPoint: point,
|
|
296
|
+
desktopPoint,
|
|
297
|
+
screenshotPoint,
|
|
298
|
+
inputCoordMap: options.coordMap ?? null,
|
|
299
|
+
screenshotCoordMap: screenshot.coordMap,
|
|
300
|
+
hint: screenshot.hint,
|
|
301
|
+
});
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
printLine(screenshot.path);
|
|
305
|
+
printLine(`input-point=${point.x},${point.y}`);
|
|
306
|
+
printLine(`desktop-point=${desktopPoint.x},${desktopPoint.y}`);
|
|
307
|
+
printLine(`screenshot-point=${screenshotPoint.x},${screenshotPoint.y}`);
|
|
308
|
+
printLine(screenshot.hint);
|
|
309
|
+
});
|
|
310
|
+
cli
|
|
311
|
+
.command('type [text]', dedent `
|
|
312
|
+
Type text in the currently focused input.
|
|
313
|
+
|
|
314
|
+
Supports direct text arguments or --stdin for long/multiline content.
|
|
315
|
+
For very long text, use --chunk-size to split input into multiple native
|
|
316
|
+
type calls so shells and apps are less likely to drop input.
|
|
317
|
+
`)
|
|
318
|
+
.option('--stdin', 'Read text from stdin instead of [text] argument')
|
|
319
|
+
.option('--delay [delay]', z.number().describe('Delay in milliseconds between typed characters'))
|
|
320
|
+
.option('--chunk-size [size]', z.number().describe('Split text into fixed-size chunks before typing'))
|
|
321
|
+
.option('--chunk-delay [delay]', z.number().describe('Delay in milliseconds between chunks'))
|
|
322
|
+
.option('--max-length [length]', z.number().describe('Fail when input text exceeds this maximum length'))
|
|
323
|
+
.example('# Type a short string')
|
|
324
|
+
.example('usecomputer type "hello"')
|
|
325
|
+
.example('# Type multiline text from a file')
|
|
326
|
+
.example('cat ./notes.txt | usecomputer type --stdin --chunk-size 4000 --chunk-delay 15')
|
|
98
327
|
.action(async (text, options) => {
|
|
99
|
-
|
|
328
|
+
const fromStdin = Boolean(options.stdin);
|
|
329
|
+
if (fromStdin && text) {
|
|
330
|
+
throw new Error('Use either [text] or --stdin, not both');
|
|
331
|
+
}
|
|
332
|
+
if (!fromStdin && !text) {
|
|
333
|
+
throw new Error('Command "type" requires [text] or --stdin');
|
|
334
|
+
}
|
|
335
|
+
const sourceText = fromStdin ? readTextFromStdin() : text ?? '';
|
|
336
|
+
const chunkSize = parsePositiveInteger({
|
|
337
|
+
value: options.chunkSize,
|
|
338
|
+
option: '--chunk-size',
|
|
339
|
+
});
|
|
340
|
+
const maxLength = parsePositiveInteger({
|
|
341
|
+
value: options.maxLength,
|
|
342
|
+
option: '--max-length',
|
|
343
|
+
});
|
|
344
|
+
const chunkDelay = parsePositiveInteger({
|
|
345
|
+
value: options.chunkDelay,
|
|
346
|
+
option: '--chunk-delay',
|
|
347
|
+
});
|
|
348
|
+
if (typeof maxLength === 'number' && sourceText.length > maxLength) {
|
|
349
|
+
throw new Error(`Input text length ${String(sourceText.length)} exceeds --max-length ${String(maxLength)}`);
|
|
350
|
+
}
|
|
351
|
+
const chunks = splitIntoChunks({
|
|
352
|
+
text: sourceText,
|
|
353
|
+
chunkSize,
|
|
354
|
+
});
|
|
355
|
+
await chunks.reduce(async (previousChunk, chunk, index) => {
|
|
356
|
+
await previousChunk;
|
|
357
|
+
await bridge.typeText({
|
|
358
|
+
text: chunk,
|
|
359
|
+
delayMs: options.delay,
|
|
360
|
+
});
|
|
361
|
+
if (typeof chunkDelay === 'number' && index < chunks.length - 1) {
|
|
362
|
+
await sleep({ ms: chunkDelay });
|
|
363
|
+
}
|
|
364
|
+
}, Promise.resolve());
|
|
100
365
|
});
|
|
101
366
|
cli
|
|
102
|
-
.command('press <key>',
|
|
367
|
+
.command('press <key>', dedent `
|
|
368
|
+
Press a key or key combo in the focused app.
|
|
369
|
+
|
|
370
|
+
Key combos use plus syntax such as cmd+s or ctrl+shift+p.
|
|
371
|
+
Platform behavior: cmd maps to Command on macOS, Win/Super on
|
|
372
|
+
Windows/Linux. For cross-platform app shortcuts, prefer ctrl+... .
|
|
373
|
+
`)
|
|
103
374
|
.option('--count [count]', z.number().default(1).describe('How many times to press'))
|
|
104
375
|
.option('--delay [delay]', z.number().describe('Delay between presses in milliseconds'))
|
|
376
|
+
.example('# Save in the current app on macOS')
|
|
377
|
+
.example('usecomputer press "cmd+s"')
|
|
378
|
+
.example('# Portable save shortcut across most apps')
|
|
379
|
+
.example('usecomputer press "ctrl+s"')
|
|
380
|
+
.example('# Open command palette in many editors')
|
|
381
|
+
.example('usecomputer press "cmd+shift+p"')
|
|
105
382
|
.action(async (key, options) => {
|
|
106
383
|
await bridge.press({ key, count: options.count, delayMs: options.delay });
|
|
107
384
|
});
|
|
@@ -128,10 +405,12 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
128
405
|
.command('drag <from> <to>', 'Drag from one coordinate to another')
|
|
129
406
|
.option('--duration [duration]', z.number().describe('Duration in milliseconds'))
|
|
130
407
|
.option('--button [button]', z.enum(['left', 'right', 'middle']).default('left').describe('Mouse button'))
|
|
408
|
+
.option('--coord-map [coordMap]', z.string().describe('Map input coordinates from screenshot space'))
|
|
131
409
|
.action(async (from, to, options) => {
|
|
410
|
+
const coordMap = parseCoordMapOrThrow(options.coordMap);
|
|
132
411
|
await bridge.drag({
|
|
133
|
-
from: parsePointOrThrow(from),
|
|
134
|
-
to: parsePointOrThrow(to),
|
|
412
|
+
from: mapPointFromCoordMap({ point: parsePointOrThrow(from), coordMap }),
|
|
413
|
+
to: mapPointFromCoordMap({ point: parsePointOrThrow(to), coordMap }),
|
|
135
414
|
durationMs: options.duration,
|
|
136
415
|
button: options.button,
|
|
137
416
|
});
|
|
@@ -140,6 +419,7 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
140
419
|
.command('hover [target]', 'Move mouse cursor to coordinates without clicking')
|
|
141
420
|
.option('-x [x]', z.number().describe('X coordinate'))
|
|
142
421
|
.option('-y [y]', z.number().describe('Y coordinate'))
|
|
422
|
+
.option('--coord-map [coordMap]', z.string().describe('Map input coordinates from screenshot space'))
|
|
143
423
|
.action(async (target, options) => {
|
|
144
424
|
const point = resolvePointInput({
|
|
145
425
|
x: options.x,
|
|
@@ -147,12 +427,14 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
147
427
|
target,
|
|
148
428
|
command: 'hover',
|
|
149
429
|
});
|
|
150
|
-
|
|
430
|
+
const coordMap = parseCoordMapOrThrow(options.coordMap);
|
|
431
|
+
await bridge.hover(mapPointFromCoordMap({ point, coordMap }));
|
|
151
432
|
});
|
|
152
433
|
cli
|
|
153
|
-
.command('mouse move [x] [y]', 'Move mouse cursor to absolute coordinates')
|
|
434
|
+
.command('mouse move [x] [y]', 'Move mouse cursor to absolute coordinates (optional before click; click can target coordinates directly)')
|
|
154
435
|
.option('-x [x]', z.number().describe('X coordinate'))
|
|
155
436
|
.option('-y [y]', z.number().describe('Y coordinate'))
|
|
437
|
+
.option('--coord-map [coordMap]', z.string().describe('Map input coordinates from screenshot space'))
|
|
156
438
|
.action(async (x, y, options) => {
|
|
157
439
|
const point = resolvePointInput({
|
|
158
440
|
x: options.x,
|
|
@@ -160,7 +442,8 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
160
442
|
target: x && y ? `${x},${y}` : undefined,
|
|
161
443
|
command: 'mouse move',
|
|
162
444
|
});
|
|
163
|
-
|
|
445
|
+
const coordMap = parseCoordMapOrThrow(options.coordMap);
|
|
446
|
+
await bridge.mouseMove(mapPointFromCoordMap({ point, coordMap }));
|
|
164
447
|
});
|
|
165
448
|
cli
|
|
166
449
|
.command('mouse down', 'Press and hold mouse button')
|
|
@@ -194,10 +477,28 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
194
477
|
printJson(displays);
|
|
195
478
|
return;
|
|
196
479
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
480
|
+
printDesktopList({ displays });
|
|
481
|
+
});
|
|
482
|
+
cli
|
|
483
|
+
.command('desktop list', 'List desktops as display indexes and sizes (#0 is the primary display)')
|
|
484
|
+
.option('--windows', 'Include available windows grouped by desktop index')
|
|
485
|
+
.option('--json', 'Output as JSON')
|
|
486
|
+
.action(async (options) => {
|
|
487
|
+
const displays = await bridge.displayList();
|
|
488
|
+
const windows = options.windows ? await bridge.windowList() : [];
|
|
489
|
+
if (options.json) {
|
|
490
|
+
if (options.windows) {
|
|
491
|
+
printJson({ displays, windows });
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
printJson(displays);
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
if (options.windows) {
|
|
498
|
+
printDesktopListWithWindows({ displays, windows });
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
printDesktopList({ displays });
|
|
201
502
|
});
|
|
202
503
|
cli
|
|
203
504
|
.command('clipboard get', 'Print clipboard text')
|
|
@@ -228,8 +529,13 @@ export function createCli({ bridge = createBridge() } = {}) {
|
|
|
228
529
|
cli.command('get focused').action(() => {
|
|
229
530
|
notImplemented({ command: 'get focused' });
|
|
230
531
|
});
|
|
231
|
-
cli.command('window list').action(() => {
|
|
232
|
-
|
|
532
|
+
cli.command('window list').option('--json', 'Output as JSON').action(async (options) => {
|
|
533
|
+
const windows = await bridge.windowList();
|
|
534
|
+
if (options.json) {
|
|
535
|
+
printJson(windows);
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
printWindowList({ windows });
|
|
233
539
|
});
|
|
234
540
|
cli.command('window focus <target>').action(() => {
|
|
235
541
|
notImplemented({ command: 'window focus' });
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { CoordMap, Point, Region } from './types.js';
|
|
2
|
+
export declare function parseCoordMapOrThrow(input?: string): CoordMap | undefined;
|
|
3
|
+
export declare function mapPointFromCoordMap({ point, coordMap, }: {
|
|
4
|
+
point: Point;
|
|
5
|
+
coordMap?: CoordMap;
|
|
6
|
+
}): Point;
|
|
7
|
+
export declare function mapPointToCoordMap({ point, coordMap, }: {
|
|
8
|
+
point: Point;
|
|
9
|
+
coordMap?: CoordMap;
|
|
10
|
+
}): Point;
|
|
11
|
+
export declare function getRegionFromCoordMap({ coordMap, }: {
|
|
12
|
+
coordMap?: CoordMap;
|
|
13
|
+
}): Region | undefined;
|
|
14
|
+
//# sourceMappingURL=coord-map.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coord-map.d.ts","sourceRoot":"","sources":["../src/coord-map.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAEzD,wBAAgB,oBAAoB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,CA2BzE;AAED,wBAAgB,oBAAoB,CAAC,EACnC,KAAK,EACL,QAAQ,GACT,EAAE;IACD,KAAK,EAAE,KAAK,CAAA;IACZ,QAAQ,CAAC,EAAE,QAAQ,CAAA;CACpB,GAAG,KAAK,CAoBR;AAED,wBAAgB,kBAAkB,CAAC,EACjC,KAAK,EACL,QAAQ,GACT,EAAE;IACD,KAAK,EAAE,KAAK,CAAA;IACZ,QAAQ,CAAC,EAAE,QAAQ,CAAA;CACpB,GAAG,KAAK,CAoBR;AAED,wBAAgB,qBAAqB,CAAC,EACpC,QAAQ,GACT,EAAE;IACD,QAAQ,CAAC,EAAE,QAAQ,CAAA;CACpB,GAAG,MAAM,GAAG,SAAS,CAWrB"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Shared coord-map helpers for converting screenshot-space pixels to desktop coordinates.
|
|
2
|
+
export function parseCoordMapOrThrow(input) {
|
|
3
|
+
if (!input) {
|
|
4
|
+
return undefined;
|
|
5
|
+
}
|
|
6
|
+
const values = input.split(',').map((value) => {
|
|
7
|
+
return Number(value.trim());
|
|
8
|
+
});
|
|
9
|
+
if (values.length !== 6 || values.some((value) => {
|
|
10
|
+
return !Number.isFinite(value);
|
|
11
|
+
})) {
|
|
12
|
+
throw new Error('Option --coord-map must be x,y,width,height,imageWidth,imageHeight');
|
|
13
|
+
}
|
|
14
|
+
const [captureX, captureY, captureWidth, captureHeight, imageWidth, imageHeight] = values;
|
|
15
|
+
if (captureWidth <= 0 || captureHeight <= 0 || imageWidth <= 0 || imageHeight <= 0) {
|
|
16
|
+
throw new Error('Option --coord-map must have positive width and height values');
|
|
17
|
+
}
|
|
18
|
+
return {
|
|
19
|
+
captureX,
|
|
20
|
+
captureY,
|
|
21
|
+
captureWidth,
|
|
22
|
+
captureHeight,
|
|
23
|
+
imageWidth,
|
|
24
|
+
imageHeight,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export function mapPointFromCoordMap({ point, coordMap, }) {
|
|
28
|
+
if (!coordMap) {
|
|
29
|
+
return point;
|
|
30
|
+
}
|
|
31
|
+
const imageWidthSpan = Math.max(coordMap.imageWidth - 1, 1);
|
|
32
|
+
const imageHeightSpan = Math.max(coordMap.imageHeight - 1, 1);
|
|
33
|
+
const captureWidthSpan = Math.max(coordMap.captureWidth - 1, 0);
|
|
34
|
+
const captureHeightSpan = Math.max(coordMap.captureHeight - 1, 0);
|
|
35
|
+
const maxCaptureX = coordMap.captureX + captureWidthSpan;
|
|
36
|
+
const maxCaptureY = coordMap.captureY + captureHeightSpan;
|
|
37
|
+
const mappedX = coordMap.captureX + (point.x / imageWidthSpan) * captureWidthSpan;
|
|
38
|
+
const mappedY = coordMap.captureY + (point.y / imageHeightSpan) * captureHeightSpan;
|
|
39
|
+
const clampedX = Math.max(coordMap.captureX, Math.min(maxCaptureX, mappedX));
|
|
40
|
+
const clampedY = Math.max(coordMap.captureY, Math.min(maxCaptureY, mappedY));
|
|
41
|
+
return {
|
|
42
|
+
x: Math.round(clampedX),
|
|
43
|
+
y: Math.round(clampedY),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export function mapPointToCoordMap({ point, coordMap, }) {
|
|
47
|
+
if (!coordMap) {
|
|
48
|
+
return point;
|
|
49
|
+
}
|
|
50
|
+
const captureWidthSpan = Math.max(coordMap.captureWidth - 1, 1);
|
|
51
|
+
const captureHeightSpan = Math.max(coordMap.captureHeight - 1, 1);
|
|
52
|
+
const imageWidthSpan = Math.max(coordMap.imageWidth - 1, 0);
|
|
53
|
+
const imageHeightSpan = Math.max(coordMap.imageHeight - 1, 0);
|
|
54
|
+
const relativeX = (point.x - coordMap.captureX) / captureWidthSpan;
|
|
55
|
+
const relativeY = (point.y - coordMap.captureY) / captureHeightSpan;
|
|
56
|
+
const mappedX = relativeX * imageWidthSpan;
|
|
57
|
+
const mappedY = relativeY * imageHeightSpan;
|
|
58
|
+
const clampedX = Math.max(0, Math.min(imageWidthSpan, mappedX));
|
|
59
|
+
const clampedY = Math.max(0, Math.min(imageHeightSpan, mappedY));
|
|
60
|
+
return {
|
|
61
|
+
x: Math.round(clampedX),
|
|
62
|
+
y: Math.round(clampedY),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
export function getRegionFromCoordMap({ coordMap, }) {
|
|
66
|
+
if (!coordMap) {
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
x: coordMap.captureX,
|
|
71
|
+
y: coordMap.captureY,
|
|
72
|
+
width: coordMap.captureWidth,
|
|
73
|
+
height: coordMap.captureHeight,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coord-map.test.d.ts","sourceRoot":"","sources":["../src/coord-map.test.ts"],"names":[],"mappings":""}
|