usecomputer 0.0.3 → 0.1.0

package/CHANGELOG.md

@@ -4,6 +4,73 @@
 
 All notable changes to `usecomputer` will be documented in this file.
 
+## 0.1.0
+
+1. **Standalone executable** — `usecomputer` now ships as a self-contained binary.
+   Install once and run anywhere without needing Node.js at runtime:
+
+   ```bash
+   npm install -g usecomputer
+   usecomputer screenshot ./shot.png --json
+   ```
+
+2. **Linux X11 screenshot support** — capture screens on Linux desktops via XShm
+   (with automatic fallback to XGetImage on XWayland). Returns the same JSON
+   output shape as macOS:
+
+   ```bash
+   usecomputer screenshot ./shot.png --json
+   ```
+
+3. **Screenshot coord-map and scaling** — screenshots are scaled so the longest edge
+   is at most 1568 px (model-friendly size). Output includes a `coordMap` field
+   for accurate pointer remapping:
+
+   ```bash
+   usecomputer screenshot ./shot.png --json
+   # use the emitted coord-map for all subsequent pointer commands
+   usecomputer click -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+   ```
+
+4. **New `debug-point` command** — validate a click target before clicking. Captures
+   a screenshot and draws a red marker at the mapped coordinate:
+
+   ```bash
+   usecomputer debug-point -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+   ```
+
+5. **Keyboard synthesis** — new `type` and `press` commands for text input and key
+   chords:
+
+   ```bash
+   usecomputer type "hello from usecomputer"
+   usecomputer press "cmd+s"
+   usecomputer press "down" --count 10 --delay 30
+   cat ./notes.txt | usecomputer type --stdin --chunk-size 4000
+   ```
+
+6. **Native scroll support** — scroll in any direction at any position:
+
+   ```bash
+   usecomputer scroll --direction down --amount 5
+   usecomputer scroll --direction up --amount 3 -x 800 -y 400
+   ```
+
+7. **Library exports** — import `usecomputer` as a Node.js library to reuse all
+   commands in your own agent harness:
+
+   ```ts
+   import * as usecomputer from 'usecomputer'
+
+   const shot = await usecomputer.screenshot({ path: './shot.png', display: null, window: null, region: null, annotate: null })
+   const coordMap = usecomputer.parseCoordMapOrThrow(shot.coordMap)
+   await usecomputer.click({ point: usecomputer.mapPointFromCoordMap({ point: { x: 400, y: 220 }, coordMap }), button: 'left', count: 1 })
+   ```
+
+8. **OpenAI and Anthropic computer-use examples** — README now includes full
+   agentic loop examples for both providers showing screenshot → action → result
+   cycles.
+
 ## 0.0.3
 
 - Implement real screenshot capture + PNG file writing on macOS.

package/README.md

@@ -7,6 +7,14 @@
 It can move the mouse, click, drag, and query cursor position using native
 Quartz events through a Zig N-API module.
 
+Keyboard synthesis (`type` and `press`) is also available. The native backend
+includes platform-specific key injection paths for macOS, Windows, and Linux
+X11.
+
+The package also exports the native commands as plain library functions, so you
+can `import * as usecomputer from "usecomputer"` and reuse the same screenshot,
+mouse, keyboard, and coord-map behavior from Node.js.
+
 ## Install
 
 ```bash
@@ -24,8 +32,321 @@ npm install -g usecomputer
 usecomputer mouse position --json
 usecomputer mouse move -x 500 -y 500
 usecomputer click -x 500 -y 500 --button left --count 1
+usecomputer type "hello"
+usecomputer press "cmd+s"
+```
+
+## Library usage
+
+```ts
+import * as usecomputer from 'usecomputer'
+
+const screenshot = await usecomputer.screenshot({
+  path: './tmp/shot.png',
+  display: null,
+  window: null,
+  region: null,
+  annotate: null,
+})
+
+const coordMap = usecomputer.parseCoordMapOrThrow(screenshot.coordMap)
+const point = usecomputer.mapPointFromCoordMap({
+  point: { x: 400, y: 220 },
+  coordMap,
+})
+
+await usecomputer.click({
+  point,
+  button: 'left',
+  count: 1,
+})
+```
+
+These exported functions intentionally mirror the native command shapes used by
+the Zig N-API module. Optional native fields are passed as `null` when absent.
+
+## OpenAI computer tool example
+
+```ts
+import fs from 'node:fs'
+import * as usecomputer from 'usecomputer'
+
+async function sendComputerScreenshot() {
+  const screenshot = await usecomputer.screenshot({
+    path: './tmp/computer-tool.png',
+    display: null,
+    window: null,
+    region: null,
+    annotate: null,
+  })
+
+  return {
+    screenshot,
+    imageBase64: await fs.promises.readFile(screenshot.path, 'base64'),
+  }
+}
+
+async function runComputerAction(action, coordMap) {
+  if (action.type === 'click') {
+    await usecomputer.click({
+      point: usecomputer.mapPointFromCoordMap({
+        point: { x: action.x, y: action.y },
+        coordMap: usecomputer.parseCoordMapOrThrow(coordMap),
+      }),
+      button: action.button ?? 'left',
+      count: 1,
+    })
+    return
+  }
+
+  if (action.type === 'double_click') {
+    await usecomputer.click({
+      point: usecomputer.mapPointFromCoordMap({
+        point: { x: action.x, y: action.y },
+        coordMap: usecomputer.parseCoordMapOrThrow(coordMap),
+      }),
+      button: action.button ?? 'left',
+      count: 2,
+    })
+    return
+  }
+
+  if (action.type === 'scroll') {
+    await usecomputer.scroll({
+      direction: action.scrollY && action.scrollY < 0 ? 'up' : 'down',
+      amount: Math.abs(action.scrollY ?? 0),
+      at: typeof action.x === 'number' && typeof action.y === 'number'
+        ? usecomputer.mapPointFromCoordMap({
+            point: { x: action.x, y: action.y },
+            coordMap: usecomputer.parseCoordMapOrThrow(coordMap),
+          })
+        : null,
+    })
+    return
+  }
+
+  if (action.type === 'keypress') {
+    await usecomputer.press({
+      key: action.keys.join('+'),
+      count: 1,
+      delayMs: null,
+    })
+    return
+  }
+
+  if (action.type === 'type') {
+    await usecomputer.typeText({
+      text: action.text,
+      delayMs: null,
+    })
+}
+}
+```
+
+## Anthropic computer use example
+
+Anthropic's computer tool uses action names like `left_click`, `double_click`,
+`mouse_move`, `key`, `type`, `scroll`, and `screenshot`. `usecomputer`
+provides the execution layer for those actions.
+
+```ts
+import fs from 'node:fs'
+import Anthropic from '@anthropic-ai/sdk'
+import type {
+  BetaToolResultBlockParam,
+  BetaToolUseBlock,
+} from '@anthropic-ai/sdk/resources/beta/messages/messages'
+import * as usecomputer from 'usecomputer'
+
+const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY })
+
+const message = await anthropic.beta.messages.create({
+  model: 'claude-opus-4-6',
+  max_tokens: 1024,
+  tools: [
+    {
+      type: 'computer_20251124',
+      name: 'computer',
+      display_width_px: 1024,
+      display_height_px: 768,
+      display_number: 1,
+    },
+  ],
+  messages: [{ role: 'user', content: 'Open Safari and search for usecomputer.' }],
+  betas: ['computer-use-2025-11-24'],
+})
+
+for (const block of message.content) {
+  if (block.type !== 'tool_use' || block.name !== 'computer') {
+    continue
+  }
+
+  const toolUse = block as BetaToolUseBlock
+  await usecomputer.screenshot({
+    path: './tmp/claude-current-screen.png',
+    display: null,
+    window: null,
+    region: null,
+    annotate: null,
+  })
+  const coordinate = Array.isArray(toolUse.input.coordinate)
+    ? toolUse.input.coordinate
+    : null
+  const point = coordinate
+    ? { x: coordinate[0] ?? 0, y: coordinate[1] ?? 0 }
+    : null
+
+  switch (toolUse.input.action) {
+    case 'screenshot': {
+      break
+    }
+    case 'left_click': {
+      if (point) {
+        await usecomputer.click({ point, button: 'left', count: 1 })
+      }
+      break
+    }
+    case 'double_click': {
+      if (point) {
+        await usecomputer.click({ point, button: 'left', count: 2 })
+      }
+      break
+    }
+    case 'mouse_move': {
+      if (point) {
+        await usecomputer.mouseMove(point)
+      }
+      break
+    }
+    case 'type': {
+      if (typeof toolUse.input.text === 'string') {
+        await usecomputer.typeText({ text: toolUse.input.text, delayMs: null })
+      }
+      break
+    }
+    case 'key': {
+      if (typeof toolUse.input.text === 'string') {
+        await usecomputer.press({ key: toolUse.input.text, count: 1, delayMs: null })
+      }
+      break
+    }
+    case 'scroll': {
+      await usecomputer.scroll({
+        direction: toolUse.input.scroll_direction === 'up' || toolUse.input.scroll_direction === 'down' || toolUse.input.scroll_direction === 'left' || toolUse.input.scroll_direction === 'right'
+          ? toolUse.input.scroll_direction
+          : 'down',
+        amount: typeof toolUse.input.scroll_amount === 'number' ? toolUse.input.scroll_amount : 3,
+        at: point,
+      })
+      break
+    }
+    default: {
+      throw new Error(`Unsupported Claude computer action: ${String(toolUse.input.action)}`)
+    }
+  }
+
+  const afterActionScreenshot = await usecomputer.screenshot({
+    path: './tmp/claude-computer-tool.png',
+    display: null,
+    window: null,
+    region: null,
+    annotate: null,
+  })
+  const imageBase64 = await fs.promises.readFile(afterActionScreenshot.path, 'base64')
+  const toolResult: BetaToolResultBlockParam = {
+    type: 'tool_result',
+    tool_use_id: toolUse.id,
+    content: [
+      {
+        type: 'image',
+        source: {
+          type: 'base64',
+          media_type: 'image/png',
+          data: imageBase64,
+        },
+      },
+    ],
+  }
+  // Append toolResult to the next user message in your agent loop.
+}
+```
+
+## Screenshot scaling and coord-map
+
+`usecomputer screenshot` always scales the output image so the longest edge is
+at most `1568` px. This keeps screenshots in a model-friendly size for
+computer-use agents.
+
+Screenshot output includes:
+
+- `desktopIndex` (display index used for capture)
+- `coordMap` in the form `captureX,captureY,captureWidth,captureHeight,imageWidth,imageHeight`
+- `hint` with usage text for coordinate mapping
+
+Always pass the exact `--coord-map` value emitted by `usecomputer screenshot`
+to pointer commands when you are clicking coordinates from that screenshot.
+This maps screenshot-space coordinates back to real screen coordinates:
+
+```bash
+usecomputer screenshot ./shot.png --json
+usecomputer click -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+usecomputer mouse move -x 100 -y 80 --coord-map "0,0,1600,900,1568,882"
+```
+
+To validate a target before clicking, use `debug-point`. It takes the same
+coordinates and `--coord-map`, captures a fresh full-desktop screenshot, and
+draws a red marker where the click would land. When `--coord-map` is present,
+it captures that same region so the overlay matches the screenshot you are
+targeting:
+
+```bash
+usecomputer debug-point -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+```
+
+## Keyboard commands
+
+### Type text
+
+```bash
+# Short text
+usecomputer type "hello from usecomputer"
+
+# Type from stdin (good for multiline or very long text)
+cat ./notes.txt | usecomputer type --stdin --chunk-size 4000 --chunk-delay 15
+
+# Simulate slower typing for apps that drop fast input
+usecomputer type "hello" --delay 20
 ```
 
+`--delay` is the per-character delay in milliseconds.
+
+For very long text, prefer `--stdin` + `--chunk-size` so shell argument limits
+and app input buffers are less likely to cause dropped characters.
+
+### Press keys and shortcuts
+
+```bash
+# Single key
+usecomputer press "enter"
+
+# Chords
+usecomputer press "cmd+s"
+usecomputer press "cmd+shift+p"
+usecomputer press "ctrl+s"
+
+# Repeats
+usecomputer press "down" --count 10 --delay 30
+```
+
+Modifier aliases: `cmd`/`command`/`meta`, `ctrl`/`control`, `alt`/`option`,
+`shift`, `fn`.
+
+Platform note:
+
+- macOS: `cmd` maps to Command.
+- Windows/Linux: `cmd` maps to Win/Super.
+- For app shortcuts that should work on Windows/Linux too, prefer `ctrl+...`.
+
 ## Coordinate options
 
 Commands that target coordinates accept `-x` and `-y` flags:
@@ -34,6 +355,9 @@ Commands that target coordinates accept `-x` and `-y` flags:
 - `usecomputer hover -x <n> -y <n>`
 - `usecomputer mouse move -x <n> -y <n>`
 
+`mouse move` is optional before `click` when click coordinates are already
+provided.
+
 Legacy coordinate forms are also accepted where available.
 
 ## Display index options

package/build.zig

@@ -1,36 +1,64 @@
-// Build script for usecomputer Zig N-API native module artifacts.
+// Build script for usecomputer — produces both:
+// 1. Dynamic library (.node) for N-API consumption from Node.js
+// 2. Standalone executable CLI (no Node.js required, uses zeke)
 
 const std = @import("std");
 const napigen = @import("napigen");
 
 const LIB_NAME = "usecomputer";
 
+/// Link platform-specific libraries needed by the native core.
+fn linkPlatformDeps(mod: *std.Build.Module, target_os: std.Target.Os.Tag) void {
+    if (target_os == .macos) {
+        mod.linkFramework("CoreGraphics", .{});
+        mod.linkFramework("CoreFoundation", .{});
+        mod.linkFramework("ImageIO", .{});
+    }
+    if (target_os == .linux) {
+        mod.linkSystemLibrary("X11", .{});
+        mod.linkSystemLibrary("Xext", .{});
+        mod.linkSystemLibrary("Xtst", .{});
+        mod.linkSystemLibrary("png", .{});
+    }
+    if (target_os == .windows) {
+        mod.linkSystemLibrary("user32", .{});
+    }
+}
+
 pub fn build(b: *std.Build) void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
+    const target_os = target.result.os.tag;
+
+    // ── N-API dynamic library (.node) ──
+
+    // Build options for lib.zig: enable_napigen controls N-API glue
+    const lib_options = b.addOptions();
+    lib_options.addOption(bool, "enable_napigen", true);
+    const lib_options_mod = lib_options.createModule();
 
     const lib_mod = b.createModule(.{
         .root_source_file = b.path("zig/src/lib.zig"),
         .target = target,
         .optimize = optimize,
     });
+    lib_mod.addImport("build_options", lib_options_mod);
     lib_mod.addImport("napigen", b.dependency("napigen", .{}).module("napigen"));
-    lib_mod.addImport("objc", b.dependency("zig_objc", .{
-        .target = target,
-        .optimize = optimize,
-    }).module("objc"));
+    if (target_os == .macos) {
+        if (b.lazyDependency("zig_objc", .{
+            .target = target,
+            .optimize = optimize,
+        })) |dep| {
+            lib_mod.addImport("objc", dep.module("objc"));
+        }
+    }
 
     const lib = b.addLibrary(.{
         .name = LIB_NAME,
         .root_module = lib_mod,
         .linkage = .dynamic,
     });
-
-    if (target.result.os.tag == .macos) {
-        lib.root_module.linkFramework("CoreGraphics", .{});
-        lib.root_module.linkFramework("CoreFoundation", .{});
-        lib.root_module.linkFramework("ImageIO", .{});
-    }
+    linkPlatformDeps(lib.root_module, target_os);
 
     napigen.setup(lib);
     b.installArtifact(lib);
@@ -38,16 +66,72 @@ pub fn build(b: *std.Build) void {
     const copy_node_step = b.addInstallLibFile(lib.getEmittedBin(), LIB_NAME ++ ".node");
     b.getInstallStep().dependOn(&copy_node_step.step);
 
+    // ── Standalone executable CLI ──
+    //
+    // Uses a separate copy of lib.zig WITHOUT napigen so the executable
+    // doesn't try to link N-API symbols (those only exist in Node.js).
+
+    const exe_options = b.addOptions();
+    exe_options.addOption(bool, "enable_napigen", false);
+    const exe_options_mod = exe_options.createModule();
+
+    const exe_lib_mod = b.createModule(.{
+        .root_source_file = b.path("zig/src/lib.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    exe_lib_mod.addImport("build_options", exe_options_mod);
+    if (target_os == .macos) {
+        if (b.lazyDependency("zig_objc", .{
+            .target = target,
+            .optimize = optimize,
+        })) |dep| {
+            exe_lib_mod.addImport("objc", dep.module("objc"));
+        }
+    }
+
+    const exe_mod = b.createModule(.{
+        .root_source_file = b.path("zig/src/main.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    exe_mod.addImport("usecomputer_lib", exe_lib_mod);
+    exe_mod.addImport("zeke", b.dependency("zeke", .{
+        .target = target,
+        .optimize = optimize,
+    }).module("zeke"));
+
+    const exe = b.addExecutable(.{
+        .name = LIB_NAME,
+        .root_module = exe_mod,
+    });
+    linkPlatformDeps(exe.root_module, target_os);
+    b.installArtifact(exe);
+
+    const run_exe = b.addRunArtifact(exe);
+    if (b.args) |args| {
+        run_exe.addArgs(args);
+    }
+    const run_step = b.step("run", "Run the CLI");
+    run_step.dependOn(&run_exe.step);
+
+    // ── Tests ──
+
+    const test_options = b.addOptions();
+    test_options.addOption(bool, "enable_napigen", false);
+
     const test_mod = b.createModule(.{
         .root_source_file = b.path("zig/src/lib.zig"),
         .target = target,
         .optimize = optimize,
     });
+    test_mod.addImport("build_options", test_options.createModule());
 
     const test_step = b.step("test", "Run Zig unit tests");
     const test_exe = b.addTest(.{
         .root_module = test_mod,
     });
+    linkPlatformDeps(test_exe.root_module, target_os);
     const run_test = b.addRunArtifact(test_exe);
     test_step.dependOn(&run_test.step);
 }

package/build.zig.zon

@@ -11,6 +11,11 @@
         .zig_objc = .{
             .url = "git+https://github.com/mitchellh/zig-objc?ref=main#27d0e03242e7ee6842bf8a86d2e0bb1f586a9847",
             .hash = "zig_objc-0.0.0-Ir_Sp7oUAQC3JpeR9EGUFGcHRSx_33IehitnjBCy-CwD",
+            .lazy = true,
+        },
+        .zeke = .{
+            .url = "https://github.com/remorses/zeke/archive/refs/heads/main.tar.gz",
+            .hash = "zeke-0.1.0-fnPIzP2mAADBDhCqMNuyU5TV7PEG9rEb2GDDjwMXCZYN",
         },
     },
     .paths = .{