usecomputer 0.0.4 → 0.1.0

package/CHANGELOG.md

@@ -4,6 +4,73 @@
 
 All notable changes to `usecomputer` will be documented in this file.
 
+## 0.1.0
+
+1. **Standalone executable** — `usecomputer` now ships as a self-contained binary.
+   Install once and run anywhere without needing Node.js at runtime:
+
+   ```bash
+   npm install -g usecomputer
+   usecomputer screenshot ./shot.png --json
+   ```
+
+2. **Linux X11 screenshot support** — capture screens on Linux desktops via XShm
+   (with automatic fallback to XGetImage on XWayland). Returns the same JSON
+   output shape as macOS:
+
+   ```bash
+   usecomputer screenshot ./shot.png --json
+   ```
+
+3. **Screenshot coord-map and scaling** — screenshots are scaled so the longest edge
+   is at most 1568 px (model-friendly size). Output includes a `coordMap` field
+   for accurate pointer remapping:
+
+   ```bash
+   usecomputer screenshot ./shot.png --json
+   # use the emitted coord-map for all subsequent pointer commands
+   usecomputer click -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+   ```
+
+4. **New `debug-point` command** — validate a click target before clicking. Captures
+   a screenshot and draws a red marker at the mapped coordinate:
+
+   ```bash
+   usecomputer debug-point -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+   ```
+
+5. **Keyboard synthesis** — new `type` and `press` commands for text input and key
+   chords:
+
+   ```bash
+   usecomputer type "hello from usecomputer"
+   usecomputer press "cmd+s"
+   usecomputer press "down" --count 10 --delay 30
+   cat ./notes.txt | usecomputer type --stdin --chunk-size 4000
+   ```
+
+6. **Native scroll support** — scroll in any direction at any position:
+
+   ```bash
+   usecomputer scroll --direction down --amount 5
+   usecomputer scroll --direction up --amount 3 -x 800 -y 400
+   ```
+
+7. **Library exports** — import `usecomputer` as a Node.js library to reuse all
+   commands in your own agent harness:
+
+   ```ts
+   import * as usecomputer from 'usecomputer'
+
+   const shot = await usecomputer.screenshot({ path: './shot.png', display: null, window: null, region: null, annotate: null })
+   const coordMap = usecomputer.parseCoordMapOrThrow(shot.coordMap)
+   await usecomputer.click({ point: usecomputer.mapPointFromCoordMap({ point: { x: 400, y: 220 }, coordMap }), button: 'left', count: 1 })
+   ```
+
+8. **OpenAI and Anthropic computer-use examples** — README now includes full
+   agentic loop examples for both providers showing screenshot → action → result
+   cycles.
+
 ## 0.0.3
 
 - Implement real screenshot capture + PNG file writing on macOS.

package/build.zig

@@ -1,36 +1,64 @@
-// Build script for usecomputer Zig N-API native module artifacts.
+// Build script for usecomputer — produces both:
+// 1. Dynamic library (.node) for N-API consumption from Node.js
+// 2. Standalone executable CLI (no Node.js required, uses zeke)
 
 const std = @import("std");
 const napigen = @import("napigen");
 
 const LIB_NAME = "usecomputer";
 
+/// Link platform-specific libraries needed by the native core.
+fn linkPlatformDeps(mod: *std.Build.Module, target_os: std.Target.Os.Tag) void {
+    if (target_os == .macos) {
+        mod.linkFramework("CoreGraphics", .{});
+        mod.linkFramework("CoreFoundation", .{});
+        mod.linkFramework("ImageIO", .{});
+    }
+    if (target_os == .linux) {
+        mod.linkSystemLibrary("X11", .{});
+        mod.linkSystemLibrary("Xext", .{});
+        mod.linkSystemLibrary("Xtst", .{});
+        mod.linkSystemLibrary("png", .{});
+    }
+    if (target_os == .windows) {
+        mod.linkSystemLibrary("user32", .{});
+    }
+}
+
 pub fn build(b: *std.Build) void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
+    const target_os = target.result.os.tag;
+
+    // ── N-API dynamic library (.node) ──
+
+    // Build options for lib.zig: enable_napigen controls N-API glue
+    const lib_options = b.addOptions();
+    lib_options.addOption(bool, "enable_napigen", true);
+    const lib_options_mod = lib_options.createModule();
 
     const lib_mod = b.createModule(.{
         .root_source_file = b.path("zig/src/lib.zig"),
         .target = target,
         .optimize = optimize,
     });
+    lib_mod.addImport("build_options", lib_options_mod);
     lib_mod.addImport("napigen", b.dependency("napigen", .{}).module("napigen"));
-    lib_mod.addImport("objc", b.dependency("zig_objc", .{
-        .target = target,
-        .optimize = optimize,
-    }).module("objc"));
+    if (target_os == .macos) {
+        if (b.lazyDependency("zig_objc", .{
+            .target = target,
+            .optimize = optimize,
+        })) |dep| {
+            lib_mod.addImport("objc", dep.module("objc"));
+        }
+    }
 
     const lib = b.addLibrary(.{
         .name = LIB_NAME,
         .root_module = lib_mod,
         .linkage = .dynamic,
     });
-
-    if (target.result.os.tag == .macos) {
-        lib.root_module.linkFramework("CoreGraphics", .{});
-        lib.root_module.linkFramework("CoreFoundation", .{});
-        lib.root_module.linkFramework("ImageIO", .{});
-    }
+    linkPlatformDeps(lib.root_module, target_os);
 
     napigen.setup(lib);
     b.installArtifact(lib);
@@ -38,16 +66,72 @@ pub fn build(b: *std.Build) void {
     const copy_node_step = b.addInstallLibFile(lib.getEmittedBin(), LIB_NAME ++ ".node");
     b.getInstallStep().dependOn(&copy_node_step.step);
 
+    // ── Standalone executable CLI ──
+    //
+    // Uses a separate copy of lib.zig WITHOUT napigen so the executable
+    // doesn't try to link N-API symbols (those only exist in Node.js).
+
+    const exe_options = b.addOptions();
+    exe_options.addOption(bool, "enable_napigen", false);
+    const exe_options_mod = exe_options.createModule();
+
+    const exe_lib_mod = b.createModule(.{
+        .root_source_file = b.path("zig/src/lib.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    exe_lib_mod.addImport("build_options", exe_options_mod);
+    if (target_os == .macos) {
+        if (b.lazyDependency("zig_objc", .{
+            .target = target,
+            .optimize = optimize,
+        })) |dep| {
+            exe_lib_mod.addImport("objc", dep.module("objc"));
+        }
+    }
+
+    const exe_mod = b.createModule(.{
+        .root_source_file = b.path("zig/src/main.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    exe_mod.addImport("usecomputer_lib", exe_lib_mod);
+    exe_mod.addImport("zeke", b.dependency("zeke", .{
+        .target = target,
+        .optimize = optimize,
+    }).module("zeke"));
+
+    const exe = b.addExecutable(.{
+        .name = LIB_NAME,
+        .root_module = exe_mod,
+    });
+    linkPlatformDeps(exe.root_module, target_os);
+    b.installArtifact(exe);
+
+    const run_exe = b.addRunArtifact(exe);
+    if (b.args) |args| {
+        run_exe.addArgs(args);
+    }
+    const run_step = b.step("run", "Run the CLI");
+    run_step.dependOn(&run_exe.step);
+
+    // ── Tests ──
+
+    const test_options = b.addOptions();
+    test_options.addOption(bool, "enable_napigen", false);
+
     const test_mod = b.createModule(.{
         .root_source_file = b.path("zig/src/lib.zig"),
         .target = target,
         .optimize = optimize,
     });
+    test_mod.addImport("build_options", test_options.createModule());
 
     const test_step = b.step("test", "Run Zig unit tests");
     const test_exe = b.addTest(.{
         .root_module = test_mod,
     });
+    linkPlatformDeps(test_exe.root_module, target_os);
     const run_test = b.addRunArtifact(test_exe);
     test_step.dependOn(&run_test.step);
 }

package/build.zig.zon

@@ -11,6 +11,11 @@
         .zig_objc = .{
             .url = "git+https://github.com/mitchellh/zig-objc?ref=main#27d0e03242e7ee6842bf8a86d2e0bb1f586a9847",
             .hash = "zig_objc-0.0.0-Ir_Sp7oUAQC3JpeR9EGUFGcHRSx_33IehitnjBCy-CwD",
+            .lazy = true,
+        },
+        .zeke = .{
+            .url = "https://github.com/remorses/zeke/archive/refs/heads/main.tar.gz",
+            .hash = "zeke-0.1.0-fnPIzP2mAADBDhCqMNuyU5TV7PEG9rEb2GDDjwMXCZYN",
         },
     },
     .paths = .{

package/dist/bridge-contract.test.js

@@ -1,8 +1,11 @@
 // Contract tests for direct native method calls emitted by the TS bridge.
 // These tests intentionally call the real Zig native module.
+import fs from 'node:fs';
+import os from 'node:os';
 import { describe, expect, test } from 'vitest';
 import { createBridgeFromNative } from './bridge.js';
 import { native } from './native-lib.js';
+const isMacOS = os.platform() === 'darwin';
 describe('native bridge contract', () => {
     test('bridge calls hit real Zig module', async () => {
         expect(native).toBeTruthy();
@@ -10,16 +13,9 @@ describe('native bridge contract', () => {
             return;
         }
         const bridge = createBridgeFromNative({ nativeModule: native });
-        const safeTarget = {
-            x: 0,
-            y: 0,
-        };
-        await bridge.click({
-            point: safeTarget,
-            button: 'left',
-            count: 1,
-            modifiers: [],
-        });
+        const safeTarget = { x: 0, y: 0 };
+        // -- Mouse commands --
+        await bridge.click({ point: safeTarget, button: 'left', count: 1, modifiers: [] });
         await bridge.hover(safeTarget);
         await bridge.mouseMove(safeTarget);
         await bridge.mouseDown({ button: 'left' });
@@ -30,106 +26,43 @@ describe('native bridge contract', () => {
             button: 'left',
             durationMs: 10,
         });
-        const screenshot = await bridge.screenshot({ path: `${process.cwd()}/tmp/bridge-contract-shot.png` });
+        // -- Screenshot --
+        const screenshotPath = `${process.cwd()}/tmp/bridge-contract-shot.png`;
+        const shot = await bridge.screenshot({ path: screenshotPath });
+        expect(shot.captureWidth).toBeGreaterThan(0);
+        expect(shot.captureHeight).toBeGreaterThan(0);
+        expect(shot.imageWidth).toBeGreaterThan(0);
+        expect(shot.imageHeight).toBeGreaterThan(0);
+        expect(shot.coordMap.split(',').length).toBe(6);
+        expect(shot.hint).toContain('--coord-map');
+        expect(fs.existsSync(screenshotPath)).toBe(true);
+        const stat = fs.statSync(screenshotPath);
+        expect(stat.size).toBeGreaterThan(100);
+        // -- Keyboard (works on both platforms) --
         await bridge.typeText({ text: 'h', delayMs: 30 });
         await bridge.press({ key: 'backspace', count: 1 });
-        const scrollResult = await bridge.scroll({ direction: 'down', amount: 1 }).then(() => {
-            return 'ok';
-        }, (error) => {
-            return error instanceof Error ? error.message : String(error);
-        });
-        const scrollAtResult = await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget }).then(() => {
-            return 'ok';
-        }, (error) => {
-            return error instanceof Error ? error.message : String(error);
-        });
-        const displays = await bridge.displayList();
-        const windows = await bridge.windowList();
-        const clipboardGetResult = await bridge.clipboardGet().then(() => {
-            return 'ok';
-        }, (error) => {
-            return error instanceof Error ? error.message : String(error);
-        });
-        const clipboardSetResult = await bridge.clipboardSet({ text: 'copied' }).then(() => {
-            return 'ok';
-        }, (error) => {
-            return error instanceof Error ? error.message : String(error);
-        });
-        const isOkOrTodo = ({ value }) => {
-            return value === 'ok' || value.includes('TODO not implemented');
-        };
-        expect({
-            screenshotShape: {
-                path: screenshot.path,
-                desktopIndex: typeof screenshot.desktopIndex,
-                captureX: typeof screenshot.captureX,
-                captureY: typeof screenshot.captureY,
-                captureWidth: screenshot.captureWidth > 0,
-                captureHeight: screenshot.captureHeight > 0,
-                imageWidth: screenshot.imageWidth > 0,
-                imageHeight: screenshot.imageHeight > 0,
-                coordMapHasSixValues: screenshot.coordMap.split(',').length === 6,
-                hint: screenshot.hint,
-            },
-            firstDisplayShape: displays[0]
-                ? {
-                    id: typeof displays[0].id,
-                    index: typeof displays[0].index,
-                    width: displays[0].width > 0,
-                    height: displays[0].height > 0,
-                }
-                : null,
-            firstWindowShape: windows[0]
-                ? {
-                    id: typeof windows[0].id,
-                    ownerName: typeof windows[0].ownerName,
-                    desktopIndex: typeof windows[0].desktopIndex,
-                }
-                : null,
-            optionalCommandOutcomes: {
-                scrollResult: isOkOrTodo({ value: scrollResult }),
-                scrollAtResult: isOkOrTodo({ value: scrollAtResult }),
-                clipboardGetResult: isOkOrTodo({ value: clipboardGetResult }),
-                clipboardSetResult: isOkOrTodo({ value: clipboardSetResult }),
-            },
-        }).toMatchInlineSnapshot(`
-      {
-        "firstDisplayShape": {
-          "height": true,
-          "id": "number",
-          "index": "number",
-          "width": true,
-        },
-        "firstWindowShape": {
-          "desktopIndex": "number",
-          "id": "number",
-          "ownerName": "string",
-        },
-        "optionalCommandOutcomes": {
-          "clipboardGetResult": true,
-          "clipboardSetResult": true,
-          "scrollAtResult": true,
-          "scrollResult": true,
-        },
-        "screenshotShape": {
-          "captureHeight": true,
-          "captureWidth": true,
-          "captureX": "number",
-          "captureY": "number",
-          "coordMapHasSixValues": true,
-          "desktopIndex": "number",
-          "hint": "ALWAYS pass this exact coord map to click, hover, drag, and mouse move when using coordinates from this screenshot:
-      --coord-map "0,0,3440,1440,1568,656"
-
-      Example:
-      usecomputer click -x 400 -y 220 --coord-map "0,0,3440,1440,1568,656"",
-          "imageHeight": true,
-          "imageWidth": true,
-          "path": "/Users/morse/Documents/GitHub/kimakivoice/usecomputer/tmp/bridge-contract-shot.png",
-        },
-      }
-    `);
-        expect(displays.length).toBeGreaterThan(0);
-        expect(windows.length).toBeGreaterThan(0);
+        // -- Scroll --
+        await bridge.scroll({ direction: 'down', amount: 1 });
+        await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget });
+        // -- Display list --
+        const displayList = await bridge.displayList();
+        expect(displayList.length).toBeGreaterThan(0);
+        const firstDisplay = displayList[0];
+        expect(firstDisplay.width).toBeGreaterThan(0);
+        expect(firstDisplay.height).toBeGreaterThan(0);
+        expect(typeof firstDisplay.id).toBe('number');
+        expect(typeof firstDisplay.index).toBe('number');
+        // -- Window list --
+        if (isMacOS) {
+            const windowList = await bridge.windowList();
+            expect(windowList.length).toBeGreaterThan(0);
+            const firstWindow = windowList[0];
+            expect(typeof firstWindow.id).toBe('number');
+            expect(typeof firstWindow.ownerName).toBe('string');
+            expect(typeof firstWindow.desktopIndex).toBe('number');
+        }
+        // -- Clipboard (TODO on all platforms — Zig returns "TODO not implemented") --
+        await expect(bridge.clipboardSet({ text: 'bridge-contract-test' })).rejects.toThrow('TODO not implemented');
+        await expect(bridge.clipboardGet()).rejects.toThrow('TODO not implemented');
     });
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "usecomputer",
-  "version": "0.0.4",
+  "version": "0.1.0",
   "type": "module",
   "description": "Fast computer automation CLI for AI agents. Control any desktop with accessibility snapshots, clicks, typing, scrolling, and more.",
   "bin": "./bin.js",
@@ -63,7 +63,8 @@
     "url": "https://github.com/remorses/kimaki/issues"
   },
   "os": [
-    "darwin"
+    "darwin",
+    "linux"
   ],
   "dependencies": {
     "goke": "^6.3.0",
@@ -85,6 +86,7 @@
     "build:zig": "zig build",
     "build:native": "tsx scripts/build.ts",
     "build:native:macos": "tsx scripts/build.ts darwin-arm64 darwin-x64",
+    "vm": "tsx scripts/vm.ts",
     "test": "vitest --run",
     "typecheck": "tsc --noEmit"
   }

package/src/bridge-contract.test.ts

@@ -1,10 +1,14 @@
 // Contract tests for direct native method calls emitted by the TS bridge.
 // These tests intentionally call the real Zig native module.
 
+import fs from 'node:fs'
+import os from 'node:os'
 import { describe, expect, test } from 'vitest'
 import { createBridgeFromNative } from './bridge.js'
 import { native } from './native-lib.js'
 
+const isMacOS = os.platform() === 'darwin'
+
 describe('native bridge contract', () => {
   test('bridge calls hit real Zig module', async () => {
     expect(native).toBeTruthy()
@@ -14,17 +18,10 @@ describe('native bridge contract', () => {
 
     const bridge = createBridgeFromNative({ nativeModule: native })
 
-    const safeTarget = {
-      x: 0,
-      y: 0,
-    }
+    const safeTarget = { x: 0, y: 0 }
 
-    await bridge.click({
-      point: safeTarget,
-      button: 'left',
-      count: 1,
-      modifiers: [],
-    })
+    // -- Mouse commands --
+    await bridge.click({ point: safeTarget, button: 'left', count: 1, modifiers: [] })
     await bridge.hover(safeTarget)
     await bridge.mouseMove(safeTarget)
     await bridge.mouseDown({ button: 'left' })
@@ -36,121 +33,48 @@ describe('native bridge contract', () => {
       durationMs: 10,
     })
 
-    const screenshot = await bridge.screenshot({ path: `${process.cwd()}/tmp/bridge-contract-shot.png` })
+    // -- Screenshot --
+    const screenshotPath = `${process.cwd()}/tmp/bridge-contract-shot.png`
+    const shot = await bridge.screenshot({ path: screenshotPath })
+    expect(shot.captureWidth).toBeGreaterThan(0)
+    expect(shot.captureHeight).toBeGreaterThan(0)
+    expect(shot.imageWidth).toBeGreaterThan(0)
+    expect(shot.imageHeight).toBeGreaterThan(0)
+    expect(shot.coordMap.split(',').length).toBe(6)
+    expect(shot.hint).toContain('--coord-map')
+    expect(fs.existsSync(screenshotPath)).toBe(true)
+    const stat = fs.statSync(screenshotPath)
+    expect(stat.size).toBeGreaterThan(100)
 
+    // -- Keyboard (works on both platforms) --
     await bridge.typeText({ text: 'h', delayMs: 30 })
     await bridge.press({ key: 'backspace', count: 1 })
-    const scrollResult = await bridge.scroll({ direction: 'down', amount: 1 }).then(
-      () => {
-        return 'ok'
-      },
-      (error: unknown) => {
-        return error instanceof Error ? error.message : String(error)
-      },
-    )
-    const scrollAtResult = await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget }).then(
-      () => {
-        return 'ok'
-      },
-      (error: unknown) => {
-        return error instanceof Error ? error.message : String(error)
-      },
-    )
-    const displays = await bridge.displayList()
-    const windows = await bridge.windowList()
-    const clipboardGetResult = await bridge.clipboardGet().then(
-      () => {
-        return 'ok'
-      },
-      (error: unknown) => {
-        return error instanceof Error ? error.message : String(error)
-      },
-    )
-    const clipboardSetResult = await bridge.clipboardSet({ text: 'copied' }).then(
-      () => {
-        return 'ok'
-      },
-      (error: unknown) => {
-        return error instanceof Error ? error.message : String(error)
-      },
-    )
-    const isOkOrTodo = ({ value }: { value: string }): boolean => {
-      return value === 'ok' || value.includes('TODO not implemented')
-    }
 
-    expect({
-      screenshotShape: {
-        path: screenshot.path,
-        desktopIndex: typeof screenshot.desktopIndex,
-        captureX: typeof screenshot.captureX,
-        captureY: typeof screenshot.captureY,
-        captureWidth: screenshot.captureWidth > 0,
-        captureHeight: screenshot.captureHeight > 0,
-        imageWidth: screenshot.imageWidth > 0,
-        imageHeight: screenshot.imageHeight > 0,
-        coordMapHasSixValues: screenshot.coordMap.split(',').length === 6,
-        hint: screenshot.hint,
-      },
-      firstDisplayShape: displays[0]
-        ? {
-            id: typeof displays[0].id,
-            index: typeof displays[0].index,
-            width: displays[0].width > 0,
-            height: displays[0].height > 0,
-          }
-        : null,
-      firstWindowShape: windows[0]
-        ? {
-            id: typeof windows[0].id,
-            ownerName: typeof windows[0].ownerName,
-            desktopIndex: typeof windows[0].desktopIndex,
-          }
-        : null,
-      optionalCommandOutcomes: {
-        scrollResult: isOkOrTodo({ value: scrollResult }),
-        scrollAtResult: isOkOrTodo({ value: scrollAtResult }),
-        clipboardGetResult: isOkOrTodo({ value: clipboardGetResult }),
-        clipboardSetResult: isOkOrTodo({ value: clipboardSetResult }),
-      },
-    }).toMatchInlineSnapshot(`
-      {
-        "firstDisplayShape": {
-          "height": true,
-          "id": "number",
-          "index": "number",
-          "width": true,
-        },
-        "firstWindowShape": {
-          "desktopIndex": "number",
-          "id": "number",
-          "ownerName": "string",
-        },
-        "optionalCommandOutcomes": {
-          "clipboardGetResult": true,
-          "clipboardSetResult": true,
-          "scrollAtResult": true,
-          "scrollResult": true,
-        },
-        "screenshotShape": {
-          "captureHeight": true,
-          "captureWidth": true,
-          "captureX": "number",
-          "captureY": "number",
-          "coordMapHasSixValues": true,
-          "desktopIndex": "number",
-          "hint": "ALWAYS pass this exact coord map to click, hover, drag, and mouse move when using coordinates from this screenshot:
-      --coord-map "0,0,3440,1440,1568,656"
+    // -- Scroll --
+    await bridge.scroll({ direction: 'down', amount: 1 })
+    await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget })
+
+    // -- Display list --
+    const displayList = await bridge.displayList()
+    expect(displayList.length).toBeGreaterThan(0)
+    const firstDisplay = displayList[0]!
+    expect(firstDisplay.width).toBeGreaterThan(0)
+    expect(firstDisplay.height).toBeGreaterThan(0)
+    expect(typeof firstDisplay.id).toBe('number')
+    expect(typeof firstDisplay.index).toBe('number')
 
-      Example:
-      usecomputer click -x 400 -y 220 --coord-map "0,0,3440,1440,1568,656"",
-          "imageHeight": true,
-          "imageWidth": true,
-          "path": "/Users/morse/Documents/GitHub/kimakivoice/usecomputer/tmp/bridge-contract-shot.png",
-        },
-      }
-    `)
+    // -- Window list --
+    if (isMacOS) {
+      const windowList = await bridge.windowList()
+      expect(windowList.length).toBeGreaterThan(0)
+      const firstWindow = windowList[0]!
+      expect(typeof firstWindow.id).toBe('number')
+      expect(typeof firstWindow.ownerName).toBe('string')
+      expect(typeof firstWindow.desktopIndex).toBe('number')
+    }
 
-    expect(displays.length).toBeGreaterThan(0)
-    expect(windows.length).toBeGreaterThan(0)
+    // -- Clipboard (TODO on all platforms — Zig returns "TODO not implemented") --
+    await expect(bridge.clipboardSet({ text: 'bridge-contract-test' })).rejects.toThrow('TODO not implemented')
+    await expect(bridge.clipboardGet()).rejects.toThrow('TODO not implemented')
   })
 })