usecomputer 0.0.3 → 0.0.4

package/README.md

@@ -7,6 +7,14 @@
 It can move the mouse, click, drag, and query cursor position using native
 Quartz events through a Zig N-API module.
 
+Keyboard synthesis (`type` and `press`) is also available. The native backend
+includes platform-specific key injection paths for macOS, Windows, and Linux
+X11.
+
+The package also exports the native commands as plain library functions, so you
+can `import * as usecomputer from "usecomputer"` and reuse the same screenshot,
+mouse, keyboard, and coord-map behavior from Node.js.
+
 ## Install
 
 ```bash
@@ -24,8 +32,321 @@ npm install -g usecomputer
 usecomputer mouse position --json
 usecomputer mouse move -x 500 -y 500
 usecomputer click -x 500 -y 500 --button left --count 1
+usecomputer type "hello"
+usecomputer press "cmd+s"
+```
+
+## Library usage
+
+```ts
+import * as usecomputer from 'usecomputer'
+
+const screenshot = await usecomputer.screenshot({
+  path: './tmp/shot.png',
+  display: null,
+  window: null,
+  region: null,
+  annotate: null,
+})
+
+const coordMap = usecomputer.parseCoordMapOrThrow(screenshot.coordMap)
+const point = usecomputer.mapPointFromCoordMap({
+  point: { x: 400, y: 220 },
+  coordMap,
+})
+
+await usecomputer.click({
+  point,
+  button: 'left',
+  count: 1,
+})
+```
+
+These exported functions intentionally mirror the native command shapes used by
+the Zig N-API module. Optional native fields are passed as `null` when absent.
+
+## OpenAI computer tool example
+
+```ts
+import fs from 'node:fs'
+import * as usecomputer from 'usecomputer'
+
+async function sendComputerScreenshot() {
+  const screenshot = await usecomputer.screenshot({
+    path: './tmp/computer-tool.png',
+    display: null,
+    window: null,
+    region: null,
+    annotate: null,
+  })
+
+  return {
+    screenshot,
+    imageBase64: await fs.promises.readFile(screenshot.path, 'base64'),
+  }
+}
+
+async function runComputerAction(action, coordMap) {
+  if (action.type === 'click') {
+    await usecomputer.click({
+      point: usecomputer.mapPointFromCoordMap({
+        point: { x: action.x, y: action.y },
+        coordMap: usecomputer.parseCoordMapOrThrow(coordMap),
+      }),
+      button: action.button ?? 'left',
+      count: 1,
+    })
+    return
+  }
+
+  if (action.type === 'double_click') {
+    await usecomputer.click({
+      point: usecomputer.mapPointFromCoordMap({
+        point: { x: action.x, y: action.y },
+        coordMap: usecomputer.parseCoordMapOrThrow(coordMap),
+      }),
+      button: action.button ?? 'left',
+      count: 2,
+    })
+    return
+  }
+
+  if (action.type === 'scroll') {
+    await usecomputer.scroll({
+      direction: action.scrollY && action.scrollY < 0 ? 'up' : 'down',
+      amount: Math.abs(action.scrollY ?? 0),
+      at: typeof action.x === 'number' && typeof action.y === 'number'
+        ? usecomputer.mapPointFromCoordMap({
+            point: { x: action.x, y: action.y },
+            coordMap: usecomputer.parseCoordMapOrThrow(coordMap),
+          })
+        : null,
+    })
+    return
+  }
+
+  if (action.type === 'keypress') {
+    await usecomputer.press({
+      key: action.keys.join('+'),
+      count: 1,
+      delayMs: null,
+    })
+    return
+  }
+
+  if (action.type === 'type') {
+    await usecomputer.typeText({
+      text: action.text,
+      delayMs: null,
+    })
+}
+}
+```
+
+## Anthropic computer use example
+
+Anthropic's computer tool uses action names like `left_click`, `double_click`,
+`mouse_move`, `key`, `type`, `scroll`, and `screenshot`. `usecomputer`
+provides the execution layer for those actions.
+
+```ts
+import fs from 'node:fs'
+import Anthropic from '@anthropic-ai/sdk'
+import type {
+  BetaToolResultBlockParam,
+  BetaToolUseBlock,
+} from '@anthropic-ai/sdk/resources/beta/messages/messages'
+import * as usecomputer from 'usecomputer'
+
+const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY })
+
+const message = await anthropic.beta.messages.create({
+  model: 'claude-opus-4-6',
+  max_tokens: 1024,
+  tools: [
+    {
+      type: 'computer_20251124',
+      name: 'computer',
+      display_width_px: 1024,
+      display_height_px: 768,
+      display_number: 1,
+    },
+  ],
+  messages: [{ role: 'user', content: 'Open Safari and search for usecomputer.' }],
+  betas: ['computer-use-2025-11-24'],
+})
+
+for (const block of message.content) {
+  if (block.type !== 'tool_use' || block.name !== 'computer') {
+    continue
+  }
+
+  const toolUse = block as BetaToolUseBlock
+  await usecomputer.screenshot({
+    path: './tmp/claude-current-screen.png',
+    display: null,
+    window: null,
+    region: null,
+    annotate: null,
+  })
+  const coordinate = Array.isArray(toolUse.input.coordinate)
+    ? toolUse.input.coordinate
+    : null
+  const point = coordinate
+    ? { x: coordinate[0] ?? 0, y: coordinate[1] ?? 0 }
+    : null
+
+  switch (toolUse.input.action) {
+    case 'screenshot': {
+      break
+    }
+    case 'left_click': {
+      if (point) {
+        await usecomputer.click({ point, button: 'left', count: 1 })
+      }
+      break
+    }
+    case 'double_click': {
+      if (point) {
+        await usecomputer.click({ point, button: 'left', count: 2 })
+      }
+      break
+    }
+    case 'mouse_move': {
+      if (point) {
+        await usecomputer.mouseMove(point)
+      }
+      break
+    }
+    case 'type': {
+      if (typeof toolUse.input.text === 'string') {
+        await usecomputer.typeText({ text: toolUse.input.text, delayMs: null })
+      }
+      break
+    }
+    case 'key': {
+      if (typeof toolUse.input.text === 'string') {
+        await usecomputer.press({ key: toolUse.input.text, count: 1, delayMs: null })
+      }
+      break
+    }
+    case 'scroll': {
+      await usecomputer.scroll({
+        direction: toolUse.input.scroll_direction === 'up' || toolUse.input.scroll_direction === 'down' || toolUse.input.scroll_direction === 'left' || toolUse.input.scroll_direction === 'right'
+          ? toolUse.input.scroll_direction
+          : 'down',
+        amount: typeof toolUse.input.scroll_amount === 'number' ? toolUse.input.scroll_amount : 3,
+        at: point,
+      })
+      break
+    }
+    default: {
+      throw new Error(`Unsupported Claude computer action: ${String(toolUse.input.action)}`)
+    }
+  }
+
+  const afterActionScreenshot = await usecomputer.screenshot({
+    path: './tmp/claude-computer-tool.png',
+    display: null,
+    window: null,
+    region: null,
+    annotate: null,
+  })
+  const imageBase64 = await fs.promises.readFile(afterActionScreenshot.path, 'base64')
+  const toolResult: BetaToolResultBlockParam = {
+    type: 'tool_result',
+    tool_use_id: toolUse.id,
+    content: [
+      {
+        type: 'image',
+        source: {
+          type: 'base64',
+          media_type: 'image/png',
+          data: imageBase64,
+        },
+      },
+    ],
+  }
+  // Append toolResult to the next user message in your agent loop.
+}
+```
+
+## Screenshot scaling and coord-map
+
+`usecomputer screenshot` always scales the output image so the longest edge is
+at most `1568` px. This keeps screenshots in a model-friendly size for
+computer-use agents.
+
+Screenshot output includes:
+
+- `desktopIndex` (display index used for capture)
+- `coordMap` in the form `captureX,captureY,captureWidth,captureHeight,imageWidth,imageHeight`
+- `hint` with usage text for coordinate mapping
+
+Always pass the exact `--coord-map` value emitted by `usecomputer screenshot`
+to pointer commands when you are clicking coordinates from that screenshot.
+This maps screenshot-space coordinates back to real screen coordinates:
+
+```bash
+usecomputer screenshot ./shot.png --json
+usecomputer click -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+usecomputer mouse move -x 100 -y 80 --coord-map "0,0,1600,900,1568,882"
+```
+
+To validate a target before clicking, use `debug-point`. It takes the same
+coordinates and `--coord-map`, captures a fresh full-desktop screenshot, and
+draws a red marker where the click would land. When `--coord-map` is present,
+it captures that same region so the overlay matches the screenshot you are
+targeting:
+
+```bash
+usecomputer debug-point -x 400 -y 220 --coord-map "0,0,1600,900,1568,882"
+```
+
+## Keyboard commands
+
+### Type text
+
+```bash
+# Short text
+usecomputer type "hello from usecomputer"
+
+# Type from stdin (good for multiline or very long text)
+cat ./notes.txt | usecomputer type --stdin --chunk-size 4000 --chunk-delay 15
+
+# Simulate slower typing for apps that drop fast input
+usecomputer type "hello" --delay 20
 ```
 
+`--delay` is the per-character delay in milliseconds.
+
+For very long text, prefer `--stdin` + `--chunk-size` so shell argument limits
+and app input buffers are less likely to cause dropped characters.
+
+### Press keys and shortcuts
+
+```bash
+# Single key
+usecomputer press "enter"
+
+# Chords
+usecomputer press "cmd+s"
+usecomputer press "cmd+shift+p"
+usecomputer press "ctrl+s"
+
+# Repeats
+usecomputer press "down" --count 10 --delay 30
+```
+
+Modifier aliases: `cmd`/`command`/`meta`, `ctrl`/`control`, `alt`/`option`,
+`shift`, `fn`.
+
+Platform note:
+
+- macOS: `cmd` maps to Command.
+- Windows/Linux: `cmd` maps to Win/Super.
+- For app shortcuts that should work on Windows/Linux too, prefer `ctrl+...`.
+
 ## Coordinate options
 
 Commands that target coordinates accept `-x` and `-y` flags:
@@ -34,6 +355,9 @@ Commands that target coordinates accept `-x` and `-y` flags:
 - `usecomputer hover -x <n> -y <n>`
 - `usecomputer mouse move -x <n> -y <n>`
 
+`mouse move` is optional before `click` when click coordinates are already
+provided.
+
 Legacy coordinate forms are also accepted where available.
 
 ## Display index options

package/dist/bridge-contract.test.js

@@ -1,74 +1,135 @@
-// Contract tests for command names and payloads emitted to the native bridge.
+// Contract tests for direct native method calls emitted by the TS bridge.
+// These tests intentionally call the real Zig native module.
 import { describe, expect, test } from 'vitest';
 import { createBridgeFromNative } from './bridge.js';
-function createFakeNative({ calls }) {
-    return {
-        execute(command, payloadJson) {
-            const payload = JSON.parse(payloadJson);
-            calls.push({ command, payload });
-            if (command === 'mouse-position') {
-                return JSON.stringify({ ok: true, data: { x: 10, y: 20 } });
-            }
-            if (command === 'display-list') {
-                return JSON.stringify({
-                    ok: true,
-                    data: [
-                        { id: 1, name: 'Built-in', x: 0, y: 0, width: 1512, height: 982, scale: 2, isPrimary: true },
-                    ],
-                });
-            }
-            if (command === 'clipboard-get') {
-                return JSON.stringify({ ok: true, data: { text: 'hello' } });
-            }
-            if (command === 'screenshot') {
-                return JSON.stringify({ ok: true, data: { path: '/tmp/test.png' } });
-            }
-            return JSON.stringify({ ok: true, data: null });
-        },
-    };
-}
+import { native } from './native-lib.js';
 describe('native bridge contract', () => {
-    test('maps high-level calls to native commands', async () => {
-        const calls = [];
-        const bridge = createBridgeFromNative({ nativeModule: createFakeNative({ calls }) });
+    test('bridge calls hit real Zig module', async () => {
+        expect(native).toBeTruthy();
+        if (!native) {
+            return;
+        }
+        const bridge = createBridgeFromNative({ nativeModule: native });
+        const safeTarget = {
+            x: 0,
+            y: 0,
+        };
         await bridge.click({
-            point: { x: 100, y: 200 },
+            point: safeTarget,
             button: 'left',
-            count: 2,
-            modifiers: ['cmd'],
+            count: 1,
+            modifiers: [],
         });
-        await bridge.typeText({ text: 'hello', delayMs: 30 });
-        await bridge.press({ key: 'enter', count: 1 });
-        await bridge.scroll({ direction: 'down', amount: 300 });
-        await bridge.drag({ from: { x: 10, y: 10 }, to: { x: 200, y: 120 }, button: 'left' });
-        await bridge.hover({ x: 33, y: 44 });
-        await bridge.mouseMove({ x: 60, y: 70 });
+        await bridge.hover(safeTarget);
+        await bridge.mouseMove(safeTarget);
         await bridge.mouseDown({ button: 'left' });
         await bridge.mouseUp({ button: 'left' });
-        await bridge.mousePosition();
-        await bridge.displayList();
-        await bridge.clipboardGet();
-        await bridge.clipboardSet({ text: 'copied' });
-        await bridge.screenshot({ path: './out.png' });
-        expect(calls.map((call) => {
-            return call.command;
-        })).toMatchInlineSnapshot(`
-      [
-        "click",
-        "type-text",
-        "press",
-        "scroll",
-        "drag",
-        "hover",
-        "mouse-move",
-        "mouse-down",
-        "mouse-up",
-        "mouse-position",
-        "display-list",
-        "clipboard-get",
-        "clipboard-set",
-        "screenshot",
-      ]
+        await bridge.drag({
+            from: safeTarget,
+            to: { x: safeTarget.x + 6, y: safeTarget.y + 6 },
+            button: 'left',
+            durationMs: 10,
+        });
+        const screenshot = await bridge.screenshot({ path: `${process.cwd()}/tmp/bridge-contract-shot.png` });
+        await bridge.typeText({ text: 'h', delayMs: 30 });
+        await bridge.press({ key: 'backspace', count: 1 });
+        const scrollResult = await bridge.scroll({ direction: 'down', amount: 1 }).then(() => {
+            return 'ok';
+        }, (error) => {
+            return error instanceof Error ? error.message : String(error);
+        });
+        const scrollAtResult = await bridge.scroll({ direction: 'right', amount: 1, at: safeTarget }).then(() => {
+            return 'ok';
+        }, (error) => {
+            return error instanceof Error ? error.message : String(error);
+        });
+        const displays = await bridge.displayList();
+        const windows = await bridge.windowList();
+        const clipboardGetResult = await bridge.clipboardGet().then(() => {
+            return 'ok';
+        }, (error) => {
+            return error instanceof Error ? error.message : String(error);
+        });
+        const clipboardSetResult = await bridge.clipboardSet({ text: 'copied' }).then(() => {
+            return 'ok';
+        }, (error) => {
+            return error instanceof Error ? error.message : String(error);
+        });
+        const isOkOrTodo = ({ value }) => {
+            return value === 'ok' || value.includes('TODO not implemented');
+        };
+        expect({
+            screenshotShape: {
+                path: screenshot.path,
+                desktopIndex: typeof screenshot.desktopIndex,
+                captureX: typeof screenshot.captureX,
+                captureY: typeof screenshot.captureY,
+                captureWidth: screenshot.captureWidth > 0,
+                captureHeight: screenshot.captureHeight > 0,
+                imageWidth: screenshot.imageWidth > 0,
+                imageHeight: screenshot.imageHeight > 0,
+                coordMapHasSixValues: screenshot.coordMap.split(',').length === 6,
+                hint: screenshot.hint,
+            },
+            firstDisplayShape: displays[0]
+                ? {
+                    id: typeof displays[0].id,
+                    index: typeof displays[0].index,
+                    width: displays[0].width > 0,
+                    height: displays[0].height > 0,
+                }
+                : null,
+            firstWindowShape: windows[0]
+                ? {
+                    id: typeof windows[0].id,
+                    ownerName: typeof windows[0].ownerName,
+                    desktopIndex: typeof windows[0].desktopIndex,
+                }
+                : null,
+            optionalCommandOutcomes: {
+                scrollResult: isOkOrTodo({ value: scrollResult }),
+                scrollAtResult: isOkOrTodo({ value: scrollAtResult }),
+                clipboardGetResult: isOkOrTodo({ value: clipboardGetResult }),
+                clipboardSetResult: isOkOrTodo({ value: clipboardSetResult }),
+            },
+        }).toMatchInlineSnapshot(`
+      {
+        "firstDisplayShape": {
+          "height": true,
+          "id": "number",
+          "index": "number",
+          "width": true,
+        },
+        "firstWindowShape": {
+          "desktopIndex": "number",
+          "id": "number",
+          "ownerName": "string",
+        },
+        "optionalCommandOutcomes": {
+          "clipboardGetResult": true,
+          "clipboardSetResult": true,
+          "scrollAtResult": true,
+          "scrollResult": true,
+        },
+        "screenshotShape": {
+          "captureHeight": true,
+          "captureWidth": true,
+          "captureX": "number",
+          "captureY": "number",
+          "coordMapHasSixValues": true,
+          "desktopIndex": "number",
+          "hint": "ALWAYS pass this exact coord map to click, hover, drag, and mouse move when using coordinates from this screenshot:
+      --coord-map "0,0,3440,1440,1568,656"
+
+      Example:
+      usecomputer click -x 400 -y 220 --coord-map "0,0,3440,1440,1568,656"",
+          "imageHeight": true,
+          "imageWidth": true,
+          "path": "/Users/morse/Documents/GitHub/kimakivoice/usecomputer/tmp/bridge-contract-shot.png",
+        },
+      }
     `);
+        expect(displays.length).toBeGreaterThan(0);
+        expect(windows.length).toBeGreaterThan(0);
     });
 });

package/dist/bridge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bridge.d.ts","sourceRoot":"","sources":["../src/bridge.ts"],"names":[],"mappings":"AAEA,OAAO,EAAU,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC3D,OAAO,KAAK,EASV,iBAAiB,EAElB,MAAM,YAAY,CAAA;AA6CnB,wBAAgB,sBAAsB,CAAC,EAAE,YAAY,EAAE,EAAE;IAAE,YAAY,EAAE,YAAY,GAAG,IAAI,CAAA;CAAE,GAAG,iBAAiB,CA+FjH;AAED,wBAAgB,YAAY,IAAI,iBAAiB,CAEhD"}
+{"version":3,"file":"bridge.d.ts","sourceRoot":"","sources":["../src/bridge.ts"],"names":[],"mappings":"AAEA,OAAO,EAAU,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAE3D,OAAO,KAAK,EAaV,iBAAiB,EAElB,MAAM,YAAY,CAAA;AAsHnB,wBAAgB,sBAAsB,CAAC,EAAE,YAAY,EAAE,EAAE;IAAE,YAAY,EAAE,YAAY,GAAG,IAAI,CAAA;CAAE,GAAG,iBAAiB,CAiQjH;AAED,wBAAgB,YAAY,IAAI,iBAAiB,CAEhD"}