usecomputer 0.0.2 → 0.0.4

package/src/coord-map.test.ts

@@ -0,0 +1,178 @@
+// Validates screenshot coord-map parsing and reverse mapping edge cases.
+
+import { describe, expect, test } from 'vitest'
+import { mapPointFromCoordMap, mapPointToCoordMap, parseCoordMapOrThrow } from './coord-map.js'
+
+describe('coord-map reverse mapping', () => {
+  test('maps full-display scaled screenshot coordinates to desktop coordinates', () => {
+    const coordMap = parseCoordMapOrThrow('0,0,1600,900,1568,882')
+
+    const mapped = [
+      mapPointFromCoordMap({ point: { x: 0, y: 0 }, coordMap }),
+      mapPointFromCoordMap({ point: { x: 1567, y: 881 }, coordMap }),
+      mapPointFromCoordMap({ point: { x: 784, y: 441 }, coordMap }),
+    ]
+
+    expect(mapped).toMatchInlineSnapshot(`
+      [
+        {
+          "x": 0,
+          "y": 0,
+        },
+        {
+          "x": 1599,
+          "y": 899,
+        },
+        {
+          "x": 800,
+          "y": 450,
+        },
+      ]
+    `)
+  })
+
+  test('maps correctly when display origin is non-zero', () => {
+    const coordMap = parseCoordMapOrThrow('-1728,120,1728,1117,1568,1014')
+
+    const mapped = [
+      mapPointFromCoordMap({ point: { x: 0, y: 0 }, coordMap }),
+      mapPointFromCoordMap({ point: { x: 1567, y: 1013 }, coordMap }),
+    ]
+
+    expect(mapped).toMatchInlineSnapshot(`
+      [
+        {
+          "x": -1728,
+          "y": 120,
+        },
+        {
+          "x": -1,
+          "y": 1236,
+        },
+      ]
+    `)
+  })
+
+  test('maps region capture coordinates including display offset', () => {
+    const coordMap = parseCoordMapOrThrow('2200,80,640,360,640,360')
+
+    const mapped = [
+      mapPointFromCoordMap({ point: { x: 0, y: 0 }, coordMap }),
+      mapPointFromCoordMap({ point: { x: 639, y: 359 }, coordMap }),
+      mapPointFromCoordMap({ point: { x: 320, y: 180 }, coordMap }),
+    ]
+
+    expect(mapped).toMatchInlineSnapshot(`
+      [
+        {
+          "x": 2200,
+          "y": 80,
+        },
+        {
+          "x": 2839,
+          "y": 439,
+        },
+        {
+          "x": 2520,
+          "y": 260,
+        },
+      ]
+    `)
+  })
+
+  test('clamps out-of-bounds screenshot coordinates to capture bounds', () => {
+    const coordMap = parseCoordMapOrThrow('500,400,300,200,150,100')
+
+    const mapped = [
+      mapPointFromCoordMap({ point: { x: -10, y: -20 }, coordMap }),
+      mapPointFromCoordMap({ point: { x: 150, y: 100 }, coordMap }),
+      mapPointFromCoordMap({ point: { x: 200, y: 1000 }, coordMap }),
+    ]
+
+    expect(mapped).toMatchInlineSnapshot(`
+      [
+        {
+          "x": 500,
+          "y": 400,
+        },
+        {
+          "x": 799,
+          "y": 599,
+        },
+        {
+          "x": 799,
+          "y": 599,
+        },
+      ]
+    `)
+  })
+
+  test('maps desktop coordinates back into screenshot image coordinates', () => {
+    const coordMap = parseCoordMapOrThrow('0,0,1720,1440,1568,1313')
+
+    const mapped = [
+      mapPointToCoordMap({ point: { x: 0, y: 0 }, coordMap }),
+      mapPointToCoordMap({ point: { x: 1719, y: 1439 }, coordMap }),
+      mapPointToCoordMap({ point: { x: 230, y: 614 }, coordMap }),
+    ]
+
+    expect(mapped).toMatchInlineSnapshot(`
+      [
+        {
+          "x": 0,
+          "y": 0,
+        },
+        {
+          "x": 1567,
+          "y": 1312,
+        },
+        {
+          "x": 210,
+          "y": 560,
+        },
+      ]
+    `)
+  })
+
+  test('round-trips screenshot coordinates through desktop space', () => {
+    const coordMap = parseCoordMapOrThrow('0,0,1720,1440,1568,1313')
+
+    const roundTrip = [
+      { x: 0, y: 0 },
+      { x: 210, y: 560 },
+      { x: 1567, y: 1312 },
+    ].map((point) => {
+      return mapPointToCoordMap({
+        point: mapPointFromCoordMap({ point, coordMap }),
+        coordMap,
+      })
+    })
+
+    expect(roundTrip).toMatchInlineSnapshot(`
+      [
+        {
+          "x": 0,
+          "y": 0,
+        },
+        {
+          "x": 210,
+          "y": 560,
+        },
+        {
+          "x": 1567,
+          "y": 1312,
+        },
+      ]
+    `)
+  })
+
+  test('rejects invalid coord-map payloads', () => {
+    expect(() => {
+      parseCoordMapOrThrow('0,0,10,10,20')
+    }).toThrowError('Option --coord-map must be x,y,width,height,imageWidth,imageHeight')
+
+    expect(() => {
+      parseCoordMapOrThrow('0,0,0,10,20,20')
+    }).toThrowError('Option --coord-map must have positive width and height values')
+  })
+})

package/src/coord-map.ts

@@ -0,0 +1,105 @@
+// Shared coord-map helpers for converting screenshot-space pixels to desktop coordinates.
+
+import type { CoordMap, Point, Region } from './types.js'
+
+export function parseCoordMapOrThrow(input?: string): CoordMap | undefined {
+  if (!input) {
+    return undefined
+  }
+
+  const values = input.split(',').map((value) => {
+    return Number(value.trim())
+  })
+  if (values.length !== 6 || values.some((value) => {
+    return !Number.isFinite(value)
+  })) {
+    throw new Error('Option --coord-map must be x,y,width,height,imageWidth,imageHeight')
+  }
+
+  const [captureX, captureY, captureWidth, captureHeight, imageWidth, imageHeight] = values
+  if (captureWidth <= 0 || captureHeight <= 0 || imageWidth <= 0 || imageHeight <= 0) {
+    throw new Error('Option --coord-map must have positive width and height values')
+  }
+
+  return {
+    captureX,
+    captureY,
+    captureWidth,
+    captureHeight,
+    imageWidth,
+    imageHeight,
+  }
+}
+
+export function mapPointFromCoordMap({
+  point,
+  coordMap,
+}: {
+  point: Point
+  coordMap?: CoordMap
+}): Point {
+  if (!coordMap) {
+    return point
+  }
+
+  const imageWidthSpan = Math.max(coordMap.imageWidth - 1, 1)
+  const imageHeightSpan = Math.max(coordMap.imageHeight - 1, 1)
+  const captureWidthSpan = Math.max(coordMap.captureWidth - 1, 0)
+  const captureHeightSpan = Math.max(coordMap.captureHeight - 1, 0)
+  const maxCaptureX = coordMap.captureX + captureWidthSpan
+  const maxCaptureY = coordMap.captureY + captureHeightSpan
+  const mappedX = coordMap.captureX + (point.x / imageWidthSpan) * captureWidthSpan
+  const mappedY = coordMap.captureY + (point.y / imageHeightSpan) * captureHeightSpan
+  const clampedX = Math.max(coordMap.captureX, Math.min(maxCaptureX, mappedX))
+  const clampedY = Math.max(coordMap.captureY, Math.min(maxCaptureY, mappedY))
+
+  return {
+    x: Math.round(clampedX),
+    y: Math.round(clampedY),
+  }
+}
+
+export function mapPointToCoordMap({
+  point,
+  coordMap,
+}: {
+  point: Point
+  coordMap?: CoordMap
+}): Point {
+  if (!coordMap) {
+    return point
+  }
+
+  const captureWidthSpan = Math.max(coordMap.captureWidth - 1, 1)
+  const captureHeightSpan = Math.max(coordMap.captureHeight - 1, 1)
+  const imageWidthSpan = Math.max(coordMap.imageWidth - 1, 0)
+  const imageHeightSpan = Math.max(coordMap.imageHeight - 1, 0)
+  const relativeX = (point.x - coordMap.captureX) / captureWidthSpan
+  const relativeY = (point.y - coordMap.captureY) / captureHeightSpan
+  const mappedX = relativeX * imageWidthSpan
+  const mappedY = relativeY * imageHeightSpan
+  const clampedX = Math.max(0, Math.min(imageWidthSpan, mappedX))
+  const clampedY = Math.max(0, Math.min(imageHeightSpan, mappedY))
+
+  return {
+    x: Math.round(clampedX),
+    y: Math.round(clampedY),
+  }
+}
+
+export function getRegionFromCoordMap({
+  coordMap,
+}: {
+  coordMap?: CoordMap
+}): Region | undefined {
+  if (!coordMap) {
+    return undefined
+  }
+
+  return {
+    x: coordMap.captureX,
+    y: coordMap.captureY,
+    width: coordMap.captureWidth,
+    height: coordMap.captureHeight,
+  }
+}

package/src/debug-point-image.test.ts

@@ -0,0 +1,50 @@
+// Validates that debug-point image overlays draw a visible red marker.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import { describe, expect, test } from 'vitest'
+import { drawDebugPointOnImage } from './debug-point-image.js'
+
+describe('drawDebugPointOnImage', () => {
+  test('draws a red marker at the requested point', async () => {
+    const sharpModule = await import('sharp')
+    const sharp = sharpModule.default
+    const filePath = path.join(process.cwd(), 'tmp', 'debug-point-image-test.png')
+
+    fs.mkdirSync(path.dirname(filePath), { recursive: true })
+    const baseImage = await sharp({
+      create: {
+        width: 40,
+        height: 30,
+        channels: 4,
+        background: { r: 255, g: 255, b: 255, alpha: 1 },
+      },
+    })
+      .png()
+      .toBuffer()
+    fs.writeFileSync(filePath, baseImage)
+
+    await drawDebugPointOnImage({
+      imagePath: filePath,
+      point: { x: 20, y: 15 },
+      imageWidth: 40,
+      imageHeight: 30,
+    })
+
+    const result = await sharp(filePath)
+      .raw()
+      .toBuffer({ resolveWithObject: true })
+    const channels = result.info.channels
+    const index = (15 * result.info.width + 20) * channels
+    const pixel = Array.from(result.data.slice(index, index + channels))
+
+    expect(pixel).toMatchInlineSnapshot(`
+      [
+        255,
+        45,
+        45,
+        255,
+      ]
+    `)
+  })
+})

package/src/debug-point-image.ts

@@ -0,0 +1,69 @@
+// Draws visible debug markers onto screenshots to validate coord-map targeting.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import { createRequire } from 'node:module'
+import type { Point } from './types.js'
+
+type SharpModule = typeof import('sharp')
+const require = createRequire(import.meta.url)
+
+async function loadSharp(): Promise<SharpModule> {
+  try {
+    return require('sharp') as SharpModule
+  } catch (error) {
+    throw new Error('Optional dependency `sharp` is required for `debug-point`. Install it with `pnpm add sharp --save-optional`.', {
+      cause: error,
+    })
+  }
+}
+
+function createMarkerSvg({
+  point,
+  imageWidth,
+  imageHeight,
+}: {
+  point: Point
+  imageWidth: number
+  imageHeight: number
+}): string {
+  const radius = 10
+  const crosshairRadius = 22
+  const ringRadius = 18
+
+  return [
+    `<svg width="${String(imageWidth)}" height="${String(imageHeight)}" xmlns="http://www.w3.org/2000/svg">`,
+    '  <g>',
+    `    <circle cx="${String(point.x)}" cy="${String(point.y)}" r="${String(ringRadius)}" fill="none" stroke="white" stroke-width="4" opacity="0.95" />`,
+    `    <line x1="${String(point.x - crosshairRadius)}" y1="${String(point.y)}" x2="${String(point.x + crosshairRadius)}" y2="${String(point.y)}" stroke="white" stroke-width="5" stroke-linecap="round" opacity="0.95" />`,
+    `    <line x1="${String(point.x)}" y1="${String(point.y - crosshairRadius)}" x2="${String(point.x)}" y2="${String(point.y + crosshairRadius)}" stroke="white" stroke-width="5" stroke-linecap="round" opacity="0.95" />`,
+    `    <circle cx="${String(point.x)}" cy="${String(point.y)}" r="${String(ringRadius)}" fill="none" stroke="#ff2d2d" stroke-width="2" />`,
+    `    <line x1="${String(point.x - crosshairRadius)}" y1="${String(point.y)}" x2="${String(point.x + crosshairRadius)}" y2="${String(point.y)}" stroke="#ff2d2d" stroke-width="3" stroke-linecap="round" />`,
+    `    <line x1="${String(point.x)}" y1="${String(point.y - crosshairRadius)}" x2="${String(point.x)}" y2="${String(point.y + crosshairRadius)}" stroke="#ff2d2d" stroke-width="3" stroke-linecap="round" />`,
+    `    <circle cx="${String(point.x)}" cy="${String(point.y)}" r="${String(radius)}" fill="#ff2d2d" stroke="white" stroke-width="3" />`,
+    '  </g>',
+    '</svg>',
+  ].join('\n')
+}
+
+export async function drawDebugPointOnImage({
+  imagePath,
+  point,
+  imageWidth,
+  imageHeight,
+}: {
+  imagePath: string
+  point: Point
+  imageWidth: number
+  imageHeight: number
+}): Promise<void> {
+  const sharpModule = await loadSharp()
+  const markerSvg = createMarkerSvg({ point, imageWidth, imageHeight })
+  const output = await sharpModule(imagePath)
+    .composite([{ input: Buffer.from(markerSvg) }])
+    .png()
+    .toBuffer()
+
+  fs.mkdirSync(path.dirname(imagePath), { recursive: true })
+  fs.writeFileSync(imagePath, output)
+}

package/src/index.ts

@@ -1,6 +1,8 @@
-// Public API exports for usecomputer parser, bridge, and CLI modules.
+// Public API exports for usecomputer library helpers, parser, bridge, and CLI modules.
 
 export { createCli } from './cli.js'
 export { createBridge, createBridgeFromNative } from './bridge.js'
+export * from './lib.js'
+export * from './coord-map.js'
 export * from './types.js'
 export * from './command-parsers.js'

package/src/lib.ts

@@ -0,0 +1,125 @@
+// Public library helpers that expose the native automation commands as plain functions.
+
+import { createBridge } from './bridge.js'
+import type { NativeModule } from './native-lib.js'
+import type {
+  DisplayInfo,
+  MouseButton,
+  Point,
+  ScreenshotResult,
+  WindowInfo,
+} from './types.js'
+
+const bridge = createBridge()
+
+export type NativeScreenshotInput = Parameters<NativeModule['screenshot']>[0]
+export type NativeClickInput = Parameters<NativeModule['click']>[0]
+export type NativeTypeTextInput = Parameters<NativeModule['typeText']>[0]
+export type NativePressInput = Parameters<NativeModule['press']>[0]
+export type NativeScrollInput = Parameters<NativeModule['scroll']>[0]
+export type NativeDragInput = Parameters<NativeModule['drag']>[0]
+export type NativeMouseButtonInput = Parameters<NativeModule['mouseDown']>[0]
+export type NativeClipboardSetInput = Parameters<NativeModule['clipboardSet']>[0]
+
+export async function screenshot(input: NativeScreenshotInput): Promise<ScreenshotResult> {
+  return bridge.screenshot({
+    path: input.path ?? undefined,
+    display: input.display ?? undefined,
+    window: input.window ?? undefined,
+    region: input.region ?? undefined,
+    annotate: input.annotate ?? undefined,
+  })
+}
+
+export async function click(input: NativeClickInput): Promise<void> {
+  return bridge.click({
+    point: input.point,
+    button: normalizeMouseButton(input.button),
+    count: input.count ?? 1,
+    modifiers: [],
+  })
+}
+
+export async function typeText(input: NativeTypeTextInput): Promise<void> {
+  return bridge.typeText({
+    text: input.text,
+    delayMs: input.delayMs ?? undefined,
+  })
+}
+
+export async function press(input: NativePressInput): Promise<void> {
+  return bridge.press({
+    key: input.key,
+    count: input.count ?? 1,
+    delayMs: input.delayMs ?? undefined,
+  })
+}
+
+export async function scroll(input: NativeScrollInput): Promise<void> {
+  return bridge.scroll({
+    direction: normalizeDirection(input.direction),
+    amount: input.amount,
+    at: input.at ?? undefined,
+  })
+}
+
+export async function drag(input: NativeDragInput): Promise<void> {
+  return bridge.drag({
+    from: input.from,
+    to: input.to,
+    durationMs: input.durationMs ?? undefined,
+    button: normalizeMouseButton(input.button),
+  })
+}
+
+export async function hover(input: Point): Promise<void> {
+  return bridge.hover(input)
+}
+
+export async function mouseMove(input: Point): Promise<void> {
+  return bridge.mouseMove(input)
+}
+
+export async function mouseDown(input: NativeMouseButtonInput): Promise<void> {
+  return bridge.mouseDown({
+    button: normalizeMouseButton(input.button),
+  })
+}
+
+export async function mouseUp(input: NativeMouseButtonInput): Promise<void> {
+  return bridge.mouseUp({
+    button: normalizeMouseButton(input.button),
+  })
+}
+
+export async function mousePosition(): Promise<Point> {
+  return bridge.mousePosition()
+}
+
+export async function displayList(): Promise<DisplayInfo[]> {
+  return bridge.displayList()
+}
+
+export async function windowList(): Promise<WindowInfo[]> {
+  return bridge.windowList()
+}
+
+export async function clipboardGet(): Promise<string> {
+  return bridge.clipboardGet()
+}
+
+export async function clipboardSet(input: NativeClipboardSetInput): Promise<void> {
+  return bridge.clipboardSet(input)
+}
+
+function normalizeMouseButton(input: MouseButton | null): MouseButton {
+  return input ?? 'left'
+}
+
+function normalizeDirection(input: string): 'up' | 'down' | 'left' | 'right' {
+  if (input === 'up' || input === 'down' || input === 'left' || input === 'right') {
+    return input
+  }
+
+  throw new Error(`Invalid direction "${input}". Expected up, down, left, or right`)
+}