usecomputer 0.0.4 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/build.zig +100 -11
- package/build.zig.zon +5 -0
- package/dist/bridge-contract.test.js +42 -109
- package/dist/darwin-arm64/usecomputer.node +0 -0
- package/dist/darwin-x64/usecomputer.node +0 -0
- package/dist/linux-x64/usecomputer.node +0 -0
- package/package.json +14 -11
- package/src/bridge-contract.test.ts +44 -120
- package/zig/src/lib.zig +799 -94
- package/zig/src/main.zig +382 -0
- package/LICENSE +0 -21
- package/dist/native-lib.cjs +0 -33
- package/dist/native-lib.d.cts +0 -7
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,78 @@
|
|
|
4
4
|
|
|
5
5
|
All notable changes to `usecomputer` will be documented in this file.
|
|
6
6
|
|
|
7
|
+
## 0.1.1
|
|
8
|
+
|
|
9
|
+
1. **Fixed Linux native builds** — standalone executable now links libc correctly on Linux, fixing "C allocator is only available when linking against libc" errors.
|
|
10
|
+
2. **Fixed native host builds** — build script now omits `-Dtarget` when building for the host platform so Zig finds system libraries (X11, libpng, etc).
|
|
11
|
+
|
|
12
|
+
## 0.1.0
|
|
13
|
+
|
|
14
|
+
1. **Standalone executable** — `usecomputer` now ships as a self-contained binary.
|
|
15
|
+
Install once and run anywhere without needing Node.js at runtime:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
npm install -g usecomputer
|
|
19
|
+
usecomputer screenshot ./shot.png --json
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
2. **Linux X11 screenshot support** — capture screens on Linux desktops via XShm
|
|
23
|
+
(with automatic fallback to XGetImage on XWayland). Returns the same JSON
|
|
24
|
+
output shape as macOS:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
usecomputer screenshot ./shot.png --json
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
3. **Screenshot coord-map and scaling** — screenshots are scaled so the longest edge
|
|
31
|
+
is at most 1568 px (model-friendly size). Output includes a `coordMap` field
|
|
32
|
+
for accurate pointer remapping:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
usecomputer screenshot ./shot.png --json
|
|
36
|
+
# use the emitted coord-map for all subsequent pointer commands
|
|
37
|
+
usecomputer click -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
4. **New `debug-point` command** — validate a click target before clicking. Captures
|
|
41
|
+
a screenshot and draws a red marker at the mapped coordinate:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
usecomputer debug-point -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
5. **Keyboard synthesis** — new `type` and `press` commands for text input and key
|
|
48
|
+
chords:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
usecomputer type "hello from usecomputer"
|
|
52
|
+
usecomputer press "cmd+s"
|
|
53
|
+
usecomputer press "down" --count 10 --delay 30
|
|
54
|
+
cat ./notes.txt | usecomputer type --stdin --chunk-size 4000
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
6. **Native scroll support** — scroll in any direction at any position:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
usecomputer scroll --direction down --amount 5
|
|
61
|
+
usecomputer scroll --direction up --amount 3 -x 800 -y 400
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
7. **Library exports** — import `usecomputer` as a Node.js library to reuse all
|
|
65
|
+
commands in your own agent harness:
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
import * as usecomputer from 'usecomputer'
|
|
69
|
+
|
|
70
|
+
const shot = await usecomputer.screenshot({ path: './shot.png', display: null, window: null, region: null, annotate: null })
|
|
71
|
+
const coordMap = usecomputer.parseCoordMapOrThrow(shot.coordMap)
|
|
72
|
+
await usecomputer.click({ point: usecomputer.mapPointFromCoordMap({ point: { x: 400, y: 220 }, coordMap }), button: 'left', count: 1 })
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
8. **OpenAI and Anthropic computer-use examples** — README now includes full
|
|
76
|
+
agentic loop examples for both providers showing screenshot → action → result
|
|
77
|
+
cycles.
|
|
78
|
+
|
|
7
79
|
## 0.0.3
|
|
8
80
|
|
|
9
81
|
- Implement real screenshot capture + PNG file writing on macOS.
|
package/build.zig
CHANGED
|
@@ -1,36 +1,64 @@
|
|
|
1
|
-
// Build script for usecomputer
|
|
1
|
+
// Build script for usecomputer — produces both:
|
|
2
|
+
// 1. Dynamic library (.node) for N-API consumption from Node.js
|
|
3
|
+
// 2. Standalone executable CLI (no Node.js required, uses zeke)
|
|
2
4
|
|
|
3
5
|
const std = @import("std");
|
|
4
6
|
const napigen = @import("napigen");
|
|
5
7
|
|
|
6
8
|
const LIB_NAME = "usecomputer";
|
|
7
9
|
|
|
10
|
+
/// Link platform-specific libraries needed by the native core.
|
|
11
|
+
fn linkPlatformDeps(mod: *std.Build.Module, target_os: std.Target.Os.Tag) void {
|
|
12
|
+
if (target_os == .macos) {
|
|
13
|
+
mod.linkFramework("CoreGraphics", .{});
|
|
14
|
+
mod.linkFramework("CoreFoundation", .{});
|
|
15
|
+
mod.linkFramework("ImageIO", .{});
|
|
16
|
+
}
|
|
17
|
+
if (target_os == .linux) {
|
|
18
|
+
mod.linkSystemLibrary("X11", .{});
|
|
19
|
+
mod.linkSystemLibrary("Xext", .{});
|
|
20
|
+
mod.linkSystemLibrary("Xtst", .{});
|
|
21
|
+
mod.linkSystemLibrary("png", .{});
|
|
22
|
+
}
|
|
23
|
+
if (target_os == .windows) {
|
|
24
|
+
mod.linkSystemLibrary("user32", .{});
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
8
28
|
pub fn build(b: *std.Build) void {
|
|
9
29
|
const target = b.standardTargetOptions(.{});
|
|
10
30
|
const optimize = b.standardOptimizeOption(.{});
|
|
31
|
+
const target_os = target.result.os.tag;
|
|
32
|
+
|
|
33
|
+
// ── N-API dynamic library (.node) ──
|
|
34
|
+
|
|
35
|
+
// Build options for lib.zig: enable_napigen controls N-API glue
|
|
36
|
+
const lib_options = b.addOptions();
|
|
37
|
+
lib_options.addOption(bool, "enable_napigen", true);
|
|
38
|
+
const lib_options_mod = lib_options.createModule();
|
|
11
39
|
|
|
12
40
|
const lib_mod = b.createModule(.{
|
|
13
41
|
.root_source_file = b.path("zig/src/lib.zig"),
|
|
14
42
|
.target = target,
|
|
15
43
|
.optimize = optimize,
|
|
16
44
|
});
|
|
45
|
+
lib_mod.addImport("build_options", lib_options_mod);
|
|
17
46
|
lib_mod.addImport("napigen", b.dependency("napigen", .{}).module("napigen"));
|
|
18
|
-
|
|
19
|
-
.
|
|
20
|
-
|
|
21
|
-
|
|
47
|
+
if (target_os == .macos) {
|
|
48
|
+
if (b.lazyDependency("zig_objc", .{
|
|
49
|
+
.target = target,
|
|
50
|
+
.optimize = optimize,
|
|
51
|
+
})) |dep| {
|
|
52
|
+
lib_mod.addImport("objc", dep.module("objc"));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
22
55
|
|
|
23
56
|
const lib = b.addLibrary(.{
|
|
24
57
|
.name = LIB_NAME,
|
|
25
58
|
.root_module = lib_mod,
|
|
26
59
|
.linkage = .dynamic,
|
|
27
60
|
});
|
|
28
|
-
|
|
29
|
-
if (target.result.os.tag == .macos) {
|
|
30
|
-
lib.root_module.linkFramework("CoreGraphics", .{});
|
|
31
|
-
lib.root_module.linkFramework("CoreFoundation", .{});
|
|
32
|
-
lib.root_module.linkFramework("ImageIO", .{});
|
|
33
|
-
}
|
|
61
|
+
linkPlatformDeps(lib.root_module, target_os);
|
|
34
62
|
|
|
35
63
|
napigen.setup(lib);
|
|
36
64
|
b.installArtifact(lib);
|
|
@@ -38,16 +66,77 @@ pub fn build(b: *std.Build) void {
|
|
|
38
66
|
const copy_node_step = b.addInstallLibFile(lib.getEmittedBin(), LIB_NAME ++ ".node");
|
|
39
67
|
b.getInstallStep().dependOn(©_node_step.step);
|
|
40
68
|
|
|
69
|
+
// ── Standalone executable CLI ──
|
|
70
|
+
//
|
|
71
|
+
// Uses a separate copy of lib.zig WITHOUT napigen so the executable
|
|
72
|
+
// doesn't try to link N-API symbols (those only exist in Node.js).
|
|
73
|
+
|
|
74
|
+
const exe_options = b.addOptions();
|
|
75
|
+
exe_options.addOption(bool, "enable_napigen", false);
|
|
76
|
+
const exe_options_mod = exe_options.createModule();
|
|
77
|
+
|
|
78
|
+
const exe_lib_mod = b.createModule(.{
|
|
79
|
+
.root_source_file = b.path("zig/src/lib.zig"),
|
|
80
|
+
.target = target,
|
|
81
|
+
.optimize = optimize,
|
|
82
|
+
});
|
|
83
|
+
exe_lib_mod.addImport("build_options", exe_options_mod);
|
|
84
|
+
if (target_os == .macos) {
|
|
85
|
+
if (b.lazyDependency("zig_objc", .{
|
|
86
|
+
.target = target,
|
|
87
|
+
.optimize = optimize,
|
|
88
|
+
})) |dep| {
|
|
89
|
+
exe_lib_mod.addImport("objc", dep.module("objc"));
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const exe_mod = b.createModule(.{
|
|
94
|
+
.root_source_file = b.path("zig/src/main.zig"),
|
|
95
|
+
.target = target,
|
|
96
|
+
.optimize = optimize,
|
|
97
|
+
});
|
|
98
|
+
exe_mod.addImport("usecomputer_lib", exe_lib_mod);
|
|
99
|
+
exe_mod.addImport("zeke", b.dependency("zeke", .{
|
|
100
|
+
.target = target,
|
|
101
|
+
.optimize = optimize,
|
|
102
|
+
}).module("zeke"));
|
|
103
|
+
|
|
104
|
+
const exe = b.addExecutable(.{
|
|
105
|
+
.name = LIB_NAME,
|
|
106
|
+
.root_module = exe_mod,
|
|
107
|
+
});
|
|
108
|
+
linkPlatformDeps(exe.root_module, target_os);
|
|
109
|
+
// The standalone exe uses c_allocator and system libs that require libc.
|
|
110
|
+
// The N-API .node lib gets this automatically through napigen, but the
|
|
111
|
+
// exe needs it explicitly — otherwise native builds fail with
|
|
112
|
+
// "C allocator is only available when linking against libc".
|
|
113
|
+
exe.root_module.link_libc = true;
|
|
114
|
+
b.installArtifact(exe);
|
|
115
|
+
|
|
116
|
+
const run_exe = b.addRunArtifact(exe);
|
|
117
|
+
if (b.args) |args| {
|
|
118
|
+
run_exe.addArgs(args);
|
|
119
|
+
}
|
|
120
|
+
const run_step = b.step("run", "Run the CLI");
|
|
121
|
+
run_step.dependOn(&run_exe.step);
|
|
122
|
+
|
|
123
|
+
// ── Tests ──
|
|
124
|
+
|
|
125
|
+
const test_options = b.addOptions();
|
|
126
|
+
test_options.addOption(bool, "enable_napigen", false);
|
|
127
|
+
|
|
41
128
|
const test_mod = b.createModule(.{
|
|
42
129
|
.root_source_file = b.path("zig/src/lib.zig"),
|
|
43
130
|
.target = target,
|
|
44
131
|
.optimize = optimize,
|
|
45
132
|
});
|
|
133
|
+
test_mod.addImport("build_options", test_options.createModule());
|
|
46
134
|
|
|
47
135
|
const test_step = b.step("test", "Run Zig unit tests");
|
|
48
136
|
const test_exe = b.addTest(.{
|
|
49
137
|
.root_module = test_mod,
|
|
50
138
|
});
|
|
139
|
+
linkPlatformDeps(test_exe.root_module, target_os);
|
|
51
140
|
const run_test = b.addRunArtifact(test_exe);
|
|
52
141
|
test_step.dependOn(&run_test.step);
|
|
53
142
|
}
|
package/build.zig.zon
CHANGED
|
@@ -11,6 +11,11 @@
|
|
|
11
11
|
.zig_objc = .{
|
|
12
12
|
.url = "git+https://github.com/mitchellh/zig-objc?ref=main#27d0e03242e7ee6842bf8a86d2e0bb1f586a9847",
|
|
13
13
|
.hash = "zig_objc-0.0.0-Ir_Sp7oUAQC3JpeR9EGUFGcHRSx_33IehitnjBCy-CwD",
|
|
14
|
+
.lazy = true,
|
|
15
|
+
},
|
|
16
|
+
.zeke = .{
|
|
17
|
+
.url = "https://github.com/remorses/zeke/archive/87f8844f4a8d4427671cdb79bce5f501739eb54b.tar.gz",
|
|
18
|
+
.hash = "zeke-0.1.0-fnPIzGwUAQA4utTXwlr6mZo7vVhxTt1_h1MTpsBixLC0",
|
|
14
19
|
},
|
|
15
20
|
},
|
|
16
21
|
.paths = .{
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
// Contract tests for direct native method calls emitted by the TS bridge.
|
|
2
2
|
// These tests intentionally call the real Zig native module.
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import os from 'node:os';
|
|
3
5
|
import { describe, expect, test } from 'vitest';
|
|
4
6
|
import { createBridgeFromNative } from './bridge.js';
|
|
5
7
|
import { native } from './native-lib.js';
|
|
8
|
+
const isMacOS = os.platform() === 'darwin';
|
|
6
9
|
describe('native bridge contract', () => {
|
|
7
10
|
test('bridge calls hit real Zig module', async () => {
|
|
8
11
|
expect(native).toBeTruthy();
|
|
@@ -10,16 +13,9 @@ describe('native bridge contract', () => {
|
|
|
10
13
|
return;
|
|
11
14
|
}
|
|
12
15
|
const bridge = createBridgeFromNative({ nativeModule: native });
|
|
13
|
-
const safeTarget = {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
};
|
|
17
|
-
await bridge.click({
|
|
18
|
-
point: safeTarget,
|
|
19
|
-
button: 'left',
|
|
20
|
-
count: 1,
|
|
21
|
-
modifiers: [],
|
|
22
|
-
});
|
|
16
|
+
const safeTarget = { x: 0, y: 0 };
|
|
17
|
+
// -- Mouse commands --
|
|
18
|
+
await bridge.click({ point: safeTarget, button: 'left', count: 1, modifiers: [] });
|
|
23
19
|
await bridge.hover(safeTarget);
|
|
24
20
|
await bridge.mouseMove(safeTarget);
|
|
25
21
|
await bridge.mouseDown({ button: 'left' });
|
|
@@ -30,106 +26,43 @@ describe('native bridge contract', () => {
|
|
|
30
26
|
button: 'left',
|
|
31
27
|
durationMs: 10,
|
|
32
28
|
});
|
|
33
|
-
|
|
29
|
+
// -- Screenshot --
|
|
30
|
+
const screenshotPath = `${process.cwd()}/tmp/bridge-contract-shot.png`;
|
|
31
|
+
const shot = await bridge.screenshot({ path: screenshotPath });
|
|
32
|
+
expect(shot.captureWidth).toBeGreaterThan(0);
|
|
33
|
+
expect(shot.captureHeight).toBeGreaterThan(0);
|
|
34
|
+
expect(shot.imageWidth).toBeGreaterThan(0);
|
|
35
|
+
expect(shot.imageHeight).toBeGreaterThan(0);
|
|
36
|
+
expect(shot.coordMap.split(',').length).toBe(6);
|
|
37
|
+
expect(shot.hint).toContain('--coord-map');
|
|
38
|
+
expect(fs.existsSync(screenshotPath)).toBe(true);
|
|
39
|
+
const stat = fs.statSync(screenshotPath);
|
|
40
|
+
expect(stat.size).toBeGreaterThan(100);
|
|
41
|
+
// -- Keyboard (works on both platforms) --
|
|
34
42
|
await bridge.typeText({ text: 'h', delayMs: 30 });
|
|
35
43
|
await bridge.press({ key: 'backspace', count: 1 });
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
return value === 'ok' || value.includes('TODO not implemented');
|
|
60
|
-
};
|
|
61
|
-
expect({
|
|
62
|
-
screenshotShape: {
|
|
63
|
-
path: screenshot.path,
|
|
64
|
-
desktopIndex: typeof screenshot.desktopIndex,
|
|
65
|
-
captureX: typeof screenshot.captureX,
|
|
66
|
-
captureY: typeof screenshot.captureY,
|
|
67
|
-
captureWidth: screenshot.captureWidth > 0,
|
|
68
|
-
captureHeight: screenshot.captureHeight > 0,
|
|
69
|
-
imageWidth: screenshot.imageWidth > 0,
|
|
70
|
-
imageHeight: screenshot.imageHeight > 0,
|
|
71
|
-
coordMapHasSixValues: screenshot.coordMap.split(',').length === 6,
|
|
72
|
-
hint: screenshot.hint,
|
|
73
|
-
},
|
|
74
|
-
firstDisplayShape: displays[0]
|
|
75
|
-
? {
|
|
76
|
-
id: typeof displays[0].id,
|
|
77
|
-
index: typeof displays[0].index,
|
|
78
|
-
width: displays[0].width > 0,
|
|
79
|
-
height: displays[0].height > 0,
|
|
80
|
-
}
|
|
81
|
-
: null,
|
|
82
|
-
firstWindowShape: windows[0]
|
|
83
|
-
? {
|
|
84
|
-
id: typeof windows[0].id,
|
|
85
|
-
ownerName: typeof windows[0].ownerName,
|
|
86
|
-
desktopIndex: typeof windows[0].desktopIndex,
|
|
87
|
-
}
|
|
88
|
-
: null,
|
|
89
|
-
optionalCommandOutcomes: {
|
|
90
|
-
scrollResult: isOkOrTodo({ value: scrollResult }),
|
|
91
|
-
scrollAtResult: isOkOrTodo({ value: scrollAtResult }),
|
|
92
|
-
clipboardGetResult: isOkOrTodo({ value: clipboardGetResult }),
|
|
93
|
-
clipboardSetResult: isOkOrTodo({ value: clipboardSetResult }),
|
|
94
|
-
},
|
|
95
|
-
}).toMatchInlineSnapshot(`
|
|
96
|
-
{
|
|
97
|
-
"firstDisplayShape": {
|
|
98
|
-
"height": true,
|
|
99
|
-
"id": "number",
|
|
100
|
-
"index": "number",
|
|
101
|
-
"width": true,
|
|
102
|
-
},
|
|
103
|
-
"firstWindowShape": {
|
|
104
|
-
"desktopIndex": "number",
|
|
105
|
-
"id": "number",
|
|
106
|
-
"ownerName": "string",
|
|
107
|
-
},
|
|
108
|
-
"optionalCommandOutcomes": {
|
|
109
|
-
"clipboardGetResult": true,
|
|
110
|
-
"clipboardSetResult": true,
|
|
111
|
-
"scrollAtResult": true,
|
|
112
|
-
"scrollResult": true,
|
|
113
|
-
},
|
|
114
|
-
"screenshotShape": {
|
|
115
|
-
"captureHeight": true,
|
|
116
|
-
"captureWidth": true,
|
|
117
|
-
"captureX": "number",
|
|
118
|
-
"captureY": "number",
|
|
119
|
-
"coordMapHasSixValues": true,
|
|
120
|
-
"desktopIndex": "number",
|
|
121
|
-
"hint": "ALWAYS pass this exact coord map to click, hover, drag, and mouse move when using coordinates from this screenshot:
|
|
122
|
-
--coord-map "0,0,3440,1440,1568,656"
|
|
123
|
-
|
|
124
|
-
Example:
|
|
125
|
-
usecomputer click -x 400 -y 220 --coord-map "0,0,3440,1440,1568,656"",
|
|
126
|
-
"imageHeight": true,
|
|
127
|
-
"imageWidth": true,
|
|
128
|
-
"path": "/Users/morse/Documents/GitHub/kimakivoice/usecomputer/tmp/bridge-contract-shot.png",
|
|
129
|
-
},
|
|
130
|
-
}
|
|
131
|
-
`);
|
|
132
|
-
expect(displays.length).toBeGreaterThan(0);
|
|
133
|
-
expect(windows.length).toBeGreaterThan(0);
|
|
44
|
+
// -- Scroll --
|
|
45
|
+
await bridge.scroll({ direction: 'down', amount: 1 });
|
|
46
|
+
await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget });
|
|
47
|
+
// -- Display list --
|
|
48
|
+
const displayList = await bridge.displayList();
|
|
49
|
+
expect(displayList.length).toBeGreaterThan(0);
|
|
50
|
+
const firstDisplay = displayList[0];
|
|
51
|
+
expect(firstDisplay.width).toBeGreaterThan(0);
|
|
52
|
+
expect(firstDisplay.height).toBeGreaterThan(0);
|
|
53
|
+
expect(typeof firstDisplay.id).toBe('number');
|
|
54
|
+
expect(typeof firstDisplay.index).toBe('number');
|
|
55
|
+
// -- Window list --
|
|
56
|
+
if (isMacOS) {
|
|
57
|
+
const windowList = await bridge.windowList();
|
|
58
|
+
expect(windowList.length).toBeGreaterThan(0);
|
|
59
|
+
const firstWindow = windowList[0];
|
|
60
|
+
expect(typeof firstWindow.id).toBe('number');
|
|
61
|
+
expect(typeof firstWindow.ownerName).toBe('string');
|
|
62
|
+
expect(typeof firstWindow.desktopIndex).toBe('number');
|
|
63
|
+
}
|
|
64
|
+
// -- Clipboard (TODO on all platforms — Zig returns "TODO not implemented") --
|
|
65
|
+
await expect(bridge.clipboardSet({ text: 'bridge-contract-test' })).rejects.toThrow('TODO not implemented');
|
|
66
|
+
await expect(bridge.clipboardGet()).rejects.toThrow('TODO not implemented');
|
|
134
67
|
});
|
|
135
68
|
});
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "usecomputer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Fast computer automation CLI for AI agents. Control any desktop with accessibility snapshots, clicks, typing, scrolling, and more.",
|
|
6
6
|
"bin": "./bin.js",
|
|
@@ -39,6 +39,16 @@
|
|
|
39
39
|
"README.md",
|
|
40
40
|
"CHANGELOG.md"
|
|
41
41
|
],
|
|
42
|
+
"scripts": {
|
|
43
|
+
"build": "tsc && chmod +x bin.js",
|
|
44
|
+
"build:zig": "zig build",
|
|
45
|
+
"build:native": "tsx scripts/build.ts",
|
|
46
|
+
"build:native:macos": "tsx scripts/build.ts darwin-arm64 darwin-x64",
|
|
47
|
+
"vm": "tsx scripts/vm.ts",
|
|
48
|
+
"test": "vitest --run",
|
|
49
|
+
"typecheck": "tsc --noEmit",
|
|
50
|
+
"prepublishOnly": "[ -n \"$CI\" ] || (pnpm build && pnpm build:native:macos)"
|
|
51
|
+
},
|
|
42
52
|
"keywords": [
|
|
43
53
|
"computer-use",
|
|
44
54
|
"automation",
|
|
@@ -63,7 +73,8 @@
|
|
|
63
73
|
"url": "https://github.com/remorses/kimaki/issues"
|
|
64
74
|
},
|
|
65
75
|
"os": [
|
|
66
|
-
"darwin"
|
|
76
|
+
"darwin",
|
|
77
|
+
"linux"
|
|
67
78
|
],
|
|
68
79
|
"dependencies": {
|
|
69
80
|
"goke": "^6.3.0",
|
|
@@ -79,13 +90,5 @@
|
|
|
79
90
|
},
|
|
80
91
|
"optionalDependencies": {
|
|
81
92
|
"sharp": "^0.34.5"
|
|
82
|
-
},
|
|
83
|
-
"scripts": {
|
|
84
|
-
"build": "tsc && chmod +x bin.js",
|
|
85
|
-
"build:zig": "zig build",
|
|
86
|
-
"build:native": "tsx scripts/build.ts",
|
|
87
|
-
"build:native:macos": "tsx scripts/build.ts darwin-arm64 darwin-x64",
|
|
88
|
-
"test": "vitest --run",
|
|
89
|
-
"typecheck": "tsc --noEmit"
|
|
90
93
|
}
|
|
91
|
-
}
|
|
94
|
+
}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
// Contract tests for direct native method calls emitted by the TS bridge.
|
|
2
2
|
// These tests intentionally call the real Zig native module.
|
|
3
3
|
|
|
4
|
+
import fs from 'node:fs'
|
|
5
|
+
import os from 'node:os'
|
|
4
6
|
import { describe, expect, test } from 'vitest'
|
|
5
7
|
import { createBridgeFromNative } from './bridge.js'
|
|
6
8
|
import { native } from './native-lib.js'
|
|
7
9
|
|
|
10
|
+
const isMacOS = os.platform() === 'darwin'
|
|
11
|
+
|
|
8
12
|
describe('native bridge contract', () => {
|
|
9
13
|
test('bridge calls hit real Zig module', async () => {
|
|
10
14
|
expect(native).toBeTruthy()
|
|
@@ -14,17 +18,10 @@ describe('native bridge contract', () => {
|
|
|
14
18
|
|
|
15
19
|
const bridge = createBridgeFromNative({ nativeModule: native })
|
|
16
20
|
|
|
17
|
-
const safeTarget = {
|
|
18
|
-
x: 0,
|
|
19
|
-
y: 0,
|
|
20
|
-
}
|
|
21
|
+
const safeTarget = { x: 0, y: 0 }
|
|
21
22
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
button: 'left',
|
|
25
|
-
count: 1,
|
|
26
|
-
modifiers: [],
|
|
27
|
-
})
|
|
23
|
+
// -- Mouse commands --
|
|
24
|
+
await bridge.click({ point: safeTarget, button: 'left', count: 1, modifiers: [] })
|
|
28
25
|
await bridge.hover(safeTarget)
|
|
29
26
|
await bridge.mouseMove(safeTarget)
|
|
30
27
|
await bridge.mouseDown({ button: 'left' })
|
|
@@ -36,121 +33,48 @@ describe('native bridge contract', () => {
|
|
|
36
33
|
durationMs: 10,
|
|
37
34
|
})
|
|
38
35
|
|
|
39
|
-
|
|
36
|
+
// -- Screenshot --
|
|
37
|
+
const screenshotPath = `${process.cwd()}/tmp/bridge-contract-shot.png`
|
|
38
|
+
const shot = await bridge.screenshot({ path: screenshotPath })
|
|
39
|
+
expect(shot.captureWidth).toBeGreaterThan(0)
|
|
40
|
+
expect(shot.captureHeight).toBeGreaterThan(0)
|
|
41
|
+
expect(shot.imageWidth).toBeGreaterThan(0)
|
|
42
|
+
expect(shot.imageHeight).toBeGreaterThan(0)
|
|
43
|
+
expect(shot.coordMap.split(',').length).toBe(6)
|
|
44
|
+
expect(shot.hint).toContain('--coord-map')
|
|
45
|
+
expect(fs.existsSync(screenshotPath)).toBe(true)
|
|
46
|
+
const stat = fs.statSync(screenshotPath)
|
|
47
|
+
expect(stat.size).toBeGreaterThan(100)
|
|
40
48
|
|
|
49
|
+
// -- Keyboard (works on both platforms) --
|
|
41
50
|
await bridge.typeText({ text: 'h', delayMs: 30 })
|
|
42
51
|
await bridge.press({ key: 'backspace', count: 1 })
|
|
43
|
-
const scrollResult = await bridge.scroll({ direction: 'down', amount: 1 }).then(
|
|
44
|
-
() => {
|
|
45
|
-
return 'ok'
|
|
46
|
-
},
|
|
47
|
-
(error: unknown) => {
|
|
48
|
-
return error instanceof Error ? error.message : String(error)
|
|
49
|
-
},
|
|
50
|
-
)
|
|
51
|
-
const scrollAtResult = await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget }).then(
|
|
52
|
-
() => {
|
|
53
|
-
return 'ok'
|
|
54
|
-
},
|
|
55
|
-
(error: unknown) => {
|
|
56
|
-
return error instanceof Error ? error.message : String(error)
|
|
57
|
-
},
|
|
58
|
-
)
|
|
59
|
-
const displays = await bridge.displayList()
|
|
60
|
-
const windows = await bridge.windowList()
|
|
61
|
-
const clipboardGetResult = await bridge.clipboardGet().then(
|
|
62
|
-
() => {
|
|
63
|
-
return 'ok'
|
|
64
|
-
},
|
|
65
|
-
(error: unknown) => {
|
|
66
|
-
return error instanceof Error ? error.message : String(error)
|
|
67
|
-
},
|
|
68
|
-
)
|
|
69
|
-
const clipboardSetResult = await bridge.clipboardSet({ text: 'copied' }).then(
|
|
70
|
-
() => {
|
|
71
|
-
return 'ok'
|
|
72
|
-
},
|
|
73
|
-
(error: unknown) => {
|
|
74
|
-
return error instanceof Error ? error.message : String(error)
|
|
75
|
-
},
|
|
76
|
-
)
|
|
77
|
-
const isOkOrTodo = ({ value }: { value: string }): boolean => {
|
|
78
|
-
return value === 'ok' || value.includes('TODO not implemented')
|
|
79
|
-
}
|
|
80
52
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
},
|
|
94
|
-
firstDisplayShape: displays[0]
|
|
95
|
-
? {
|
|
96
|
-
id: typeof displays[0].id,
|
|
97
|
-
index: typeof displays[0].index,
|
|
98
|
-
width: displays[0].width > 0,
|
|
99
|
-
height: displays[0].height > 0,
|
|
100
|
-
}
|
|
101
|
-
: null,
|
|
102
|
-
firstWindowShape: windows[0]
|
|
103
|
-
? {
|
|
104
|
-
id: typeof windows[0].id,
|
|
105
|
-
ownerName: typeof windows[0].ownerName,
|
|
106
|
-
desktopIndex: typeof windows[0].desktopIndex,
|
|
107
|
-
}
|
|
108
|
-
: null,
|
|
109
|
-
optionalCommandOutcomes: {
|
|
110
|
-
scrollResult: isOkOrTodo({ value: scrollResult }),
|
|
111
|
-
scrollAtResult: isOkOrTodo({ value: scrollAtResult }),
|
|
112
|
-
clipboardGetResult: isOkOrTodo({ value: clipboardGetResult }),
|
|
113
|
-
clipboardSetResult: isOkOrTodo({ value: clipboardSetResult }),
|
|
114
|
-
},
|
|
115
|
-
}).toMatchInlineSnapshot(`
|
|
116
|
-
{
|
|
117
|
-
"firstDisplayShape": {
|
|
118
|
-
"height": true,
|
|
119
|
-
"id": "number",
|
|
120
|
-
"index": "number",
|
|
121
|
-
"width": true,
|
|
122
|
-
},
|
|
123
|
-
"firstWindowShape": {
|
|
124
|
-
"desktopIndex": "number",
|
|
125
|
-
"id": "number",
|
|
126
|
-
"ownerName": "string",
|
|
127
|
-
},
|
|
128
|
-
"optionalCommandOutcomes": {
|
|
129
|
-
"clipboardGetResult": true,
|
|
130
|
-
"clipboardSetResult": true,
|
|
131
|
-
"scrollAtResult": true,
|
|
132
|
-
"scrollResult": true,
|
|
133
|
-
},
|
|
134
|
-
"screenshotShape": {
|
|
135
|
-
"captureHeight": true,
|
|
136
|
-
"captureWidth": true,
|
|
137
|
-
"captureX": "number",
|
|
138
|
-
"captureY": "number",
|
|
139
|
-
"coordMapHasSixValues": true,
|
|
140
|
-
"desktopIndex": "number",
|
|
141
|
-
"hint": "ALWAYS pass this exact coord map to click, hover, drag, and mouse move when using coordinates from this screenshot:
|
|
142
|
-
--coord-map "0,0,3440,1440,1568,656"
|
|
53
|
+
// -- Scroll --
|
|
54
|
+
await bridge.scroll({ direction: 'down', amount: 1 })
|
|
55
|
+
await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget })
|
|
56
|
+
|
|
57
|
+
// -- Display list --
|
|
58
|
+
const displayList = await bridge.displayList()
|
|
59
|
+
expect(displayList.length).toBeGreaterThan(0)
|
|
60
|
+
const firstDisplay = displayList[0]!
|
|
61
|
+
expect(firstDisplay.width).toBeGreaterThan(0)
|
|
62
|
+
expect(firstDisplay.height).toBeGreaterThan(0)
|
|
63
|
+
expect(typeof firstDisplay.id).toBe('number')
|
|
64
|
+
expect(typeof firstDisplay.index).toBe('number')
|
|
143
65
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
66
|
+
// -- Window list --
|
|
67
|
+
if (isMacOS) {
|
|
68
|
+
const windowList = await bridge.windowList()
|
|
69
|
+
expect(windowList.length).toBeGreaterThan(0)
|
|
70
|
+
const firstWindow = windowList[0]!
|
|
71
|
+
expect(typeof firstWindow.id).toBe('number')
|
|
72
|
+
expect(typeof firstWindow.ownerName).toBe('string')
|
|
73
|
+
expect(typeof firstWindow.desktopIndex).toBe('number')
|
|
74
|
+
}
|
|
152
75
|
|
|
153
|
-
|
|
154
|
-
expect(
|
|
76
|
+
// -- Clipboard (TODO on all platforms — Zig returns "TODO not implemented") --
|
|
77
|
+
await expect(bridge.clipboardSet({ text: 'bridge-contract-test' })).rejects.toThrow('TODO not implemented')
|
|
78
|
+
await expect(bridge.clipboardGet()).rejects.toThrow('TODO not implemented')
|
|
155
79
|
})
|
|
156
80
|
})
|