ucu-mcp 0.1.2 → 0.2.0

package/dist/src/util/errors.d.ts

@@ -34,6 +34,12 @@ export declare class PermissionError extends UcuError {
 export declare class WindowNotFoundError extends UcuError {
     constructor(windowId: string);
 }
+/**
+ * Requested accessibility element ID no longer resolves.
+ */
+export declare class ElementNotFoundError extends UcuError {
+    constructor(elementId: string);
+}
 /**
  * Click/scroll target is outside screen bounds.
  */

package/dist/src/util/errors.js

@@ -68,6 +68,14 @@ export class WindowNotFoundError extends UcuError {
         super(`Window ${windowId} not found. It may have been closed. Run list_windows to get fresh IDs.`, "WINDOW_NOT_FOUND", false);
     }
 }
+/**
+ * Requested accessibility element ID no longer resolves.
+ */
+export class ElementNotFoundError extends UcuError {
+    constructor(elementId) {
+        super(`Element ${elementId} not found. It may have been removed or invalidated. Run find_element to get a fresh ID.`, "ELEMENT_NOT_FOUND", false);
+    }
+}
 // ---------------------------------------------------------------------------
 // Input Errors
 // ---------------------------------------------------------------------------

package/dist/src/utils/input.js

@@ -9,10 +9,51 @@
  * Windows: Uses SendInput (stub).
  * Linux: Uses xdotool (stub).
  */
-import { execFile } from "node:child_process";
+import { execFile, execFileSync } from "node:child_process";
 import { promisify } from "node:util";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
 import { logger } from "../util/logger.js";
 const execFileAsync = promisify(execFile);
+// ── Native CGEvent helper (macOS) ──────────────────────────────────────
+// JXA (osascript -l JavaScript) cannot call CGEventPost without segfault.
+// We ship a small Swift binary that does native CGEvent injection instead.
+const __dirname = dirname(fileURLToPath(import.meta.url));
+// In dev: src/utils/input.ts → native/cgevent/cgevent-helper
+// In prod: dist/src/utils/input.js → dist/native/cgevent/cgevent-helper
+const nativeHelperPath = join(__dirname, "..", "..", "..", "native", "cgevent", "cgevent-helper");
+// Fallback: try from project root (dev mode)
+const nativeHelperPathAlt = join(__dirname, "..", "..", "native", "cgevent", "cgevent-helper");
+import { existsSync } from "node:fs";
+const resolvedNativePath = existsSync(nativeHelperPath) ? nativeHelperPath : nativeHelperPathAlt;
+let _nativeAvailable;
+function isNativeAvailable() {
+    if (_nativeAvailable !== undefined)
+        return _nativeAvailable;
+    try {
+        const stdout = execFileSync(resolvedNativePath, [], {
+            input: '{"command":"ping"}',
+            encoding: "utf8",
+            timeout: 3000,
+        });
+        _nativeAvailable = stdout.includes('"ok"');
+    }
+    catch {
+        _nativeAvailable = false;
+    }
+    return _nativeAvailable;
+}
+function runNativeChecked(payload) {
+    const raw = execFileSync(resolvedNativePath, [], {
+        input: JSON.stringify(payload),
+        encoding: "utf8",
+        timeout: 10000,
+    }).trim();
+    const resp = JSON.parse(raw);
+    if (resp.error) {
+        throw new Error(`native helper error: ${resp.error}`);
+    }
+}
 // ── Dry-run mode ──────────────────────────────────────────────────────────
 const isDryRun = () => process.env.UCU_DRY_RUN === "true";
 function logDryRun(action, details) {
@@ -63,6 +104,10 @@ export async function click(x, y, button = "left", _platform = process.platform)
         return;
     }
     if (_platform === "darwin") {
+        if (isNativeAvailable()) {
+            runNativeChecked({ command: "click", x, y, button });
+            return;
+        }
         const btnType = { left: 0, right: 1, middle: 2 }[button];
         await runJXA(`
       ObjC.import('CoreGraphics');
@@ -91,6 +136,10 @@ export async function doubleClick(x, y, button = "left", _platform = process.pla
         return;
     }
     if (_platform === "darwin") {
+        if (isNativeAvailable()) {
+            runNativeChecked({ command: "doubleClick", x, y, button });
+            return;
+        }
         const btnType = { left: 0, right: 1, middle: 2 }[button];
         await runJXA(`
       ObjC.import('CoreGraphics');
@@ -125,6 +174,10 @@ export async function move(x, y, _platform = process.platform) {
         return;
     }
     if (_platform === "darwin") {
+        if (isNativeAvailable()) {
+            runNativeChecked({ command: "move", x, y });
+            return;
+        }
         await runJXA(`
       ObjC.import('CoreGraphics');
       var loc = $.CGPointMake(${x}, ${y});
@@ -146,6 +199,10 @@ export async function drag(fromX, fromY, toX, toY, button = "left", duration = 3
         return;
     }
     if (_platform === "darwin") {
+        if (isNativeAvailable()) {
+            runNativeChecked({ command: "drag", fromX, fromY, toX, toY, button, durationMs: duration });
+            return;
+        }
         const btnType = { left: 0, right: 1, middle: 2 }[button];
         const steps = Math.max(2, Math.min(60, Math.ceil(duration / 16)));
         const delayMicros = Math.max(0, Math.floor((duration * 1000) / steps));
@@ -192,6 +249,10 @@ export async function scroll(x, y, deltaX, deltaY, _platform = process.platform)
         return;
     }
     if (_platform === "darwin") {
+        if (isNativeAvailable()) {
+            runNativeChecked({ command: "scroll", x, y, deltaX, deltaY });
+            return;
+        }
         const verticalDelta = -deltaY;
         const horizontalDelta = deltaX;
         await runJXA(`
@@ -314,23 +375,28 @@ export async function typeText(text, delay = 20, _platform = process.platform) {
         // Process each batch
         for (const batch of batches) {
             if (batch.cgEvent && Array.isArray(batch.chars)) {
-                // Build a single JXA script that types all chars in this CGEvent batch
-                const keyStatements = batch.chars.map(({ code, shift }) => {
-                    const flags = shift ? SHIFT_FLAG : 0;
-                    return `
-            kd = $.CGEventCreateKeyboardEvent(null, ${code}, true);
-            ku = $.CGEventCreateKeyboardEvent(null, ${code}, false);
-            if (${flags}) { $.CGEventSetFlags(kd, ${flags}); $.CGEventSetFlags(ku, ${flags}); }
-            $.CGEventPost(0, kd);
-            $.CGEventPost(0, ku);
-            $.CFRelease(kd);
-            $.CFRelease(ku);`;
-                }).join("\n");
-                await runJXA(`
-          ObjC.import('CoreGraphics');
-          var kd, ku;
-          ${keyStatements}
-        `);
+                if (isNativeAvailable()) {
+                    runNativeChecked({ command: "typeBatch", keys: batch.chars });
+                }
+                else {
+                    // Build a single JXA script that types all chars in this CGEvent batch
+                    const keyStatements = batch.chars.map(({ code, shift }) => {
+                        const flags = shift ? SHIFT_FLAG : 0;
+                        return `
+              kd = $.CGEventCreateKeyboardEvent(null, ${code}, true);
+              ku = $.CGEventCreateKeyboardEvent(null, ${code}, false);
+              if (${flags}) { $.CGEventSetFlags(kd, ${flags}); $.CGEventSetFlags(ku, ${flags}); }
+              $.CGEventPost(0, kd);
+              $.CGEventPost(0, ku);
+              $.CFRelease(kd);
+              $.CFRelease(ku);`;
+                    }).join("\n");
+                    await runJXA(`
+            ObjC.import('CoreGraphics');
+            var kd, ku;
+            ${keyStatements}
+          `);
+                }
             }
             else {
                 // Fallback: use osascript keystroke for unsupported chars (emoji, CJK, etc.)
@@ -369,6 +435,10 @@ export async function pressKey(key, modifiers = [], _platform = process.platform
             }
             flags |= flag;
         }
+        if (isNativeAvailable()) {
+            runNativeChecked({ command: "pressKey", keyCode, flags });
+            return;
+        }
         await runJXA(`
       ObjC.import('CoreGraphics');
       var flags = ${flags};

package/native/cgevent/main.swift

@@ -0,0 +1,126 @@
+import CoreGraphics
+import Foundation
+
+// Simple flat params struct — avoids recursive enum decoding issues
+struct Input: Decodable {
+    let command: String
+    let x: Double?
+    let y: Double?
+    let fromX: Double?
+    let fromY: Double?
+    let toX: Double?
+    let toY: Double?
+    let button: String?
+    let durationMs: Int?
+    let deltaX: Int?
+    let deltaY: Int?
+    let keyCode: Int?
+    let flags: Int64?
+    let keys: [KeyEntry]?
+    struct KeyEntry: Decodable { let code: Int; let shift: Bool? }
+}
+
+func out(_ json: String) { FileHandle.standardOutput.write((json + "\n").data(using: .utf8)!); fflush(stdout) }
+func post(_ event: CGEvent) { event.post(tap: CGEventTapLocation.cghidEventTap) }
+
+func btn(_ s: String?) -> CGMouseButton {
+    switch s { case "right": return .right; case "middle": return .center; default: return .left }
+}
+func downT(_ b: CGMouseButton) -> CGEventType {
+    switch b { case .right: return .rightMouseDown; case .center: return .otherMouseDown; default: return .leftMouseDown }
+}
+func upT(_ b: CGMouseButton) -> CGEventType {
+    switch b { case .right: return .rightMouseUp; case .center: return .otherMouseUp; default: return .leftMouseUp }
+}
+func dragT(_ b: CGMouseButton) -> CGEventType {
+    switch b { case .right: return .rightMouseDragged; case .center: return .otherMouseDragged; default: return .leftMouseDragged }
+}
+
+func doClick(_ p: Input) -> String {
+    let loc = CGPoint(x: p.x ?? 0, y: p.y ?? 0); let b = btn(p.button)
+    guard let dn = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: loc, mouseButton: b),
+          let up = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: loc, mouseButton: b)
+    else { return "{\"error\":\"fail\"}" }
+    post(dn); post(up); return "{\"ok\":true}"
+}
+
+func doDoubleClick(_ p: Input) -> String {
+    let loc = CGPoint(x: p.x ?? 0, y: p.y ?? 0); let b = btn(p.button)
+    guard let d1 = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: loc, mouseButton: b),
+          let u1 = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: loc, mouseButton: b),
+          let d2 = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: loc, mouseButton: b),
+          let u2 = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: loc, mouseButton: b)
+    else { return "{\"error\":\"fail\"}" }
+    d1.setIntegerValueField(.mouseEventClickState, value: 1); u1.setIntegerValueField(.mouseEventClickState, value: 1)
+    d2.setIntegerValueField(.mouseEventClickState, value: 2); u2.setIntegerValueField(.mouseEventClickState, value: 2)
+    post(d1); post(u1); post(d2); post(u2); return "{\"ok\":true}"
+}
+
+func doMove(_ p: Input) -> String {
+    let loc = CGPoint(x: p.x ?? 0, y: p.y ?? 0)
+    guard let ev = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: loc, mouseButton: .left)
+    else { return "{\"error\":\"fail\"}" }
+    post(ev); return "{\"ok\":true}"
+}
+
+func doDrag(_ p: Input) -> String {
+    let from = CGPoint(x: p.fromX ?? 0, y: p.fromY ?? 0)
+    let to = CGPoint(x: p.toX ?? 0, y: p.toY ?? 0)
+    let ms = p.durationMs ?? 300; let b = btn(p.button)
+    let steps = max(2, min(60, Int(ceil(Double(ms) / 16.0))))
+    let delay = max(0, (ms * 1000) / steps)
+    guard let dn = CGEvent(mouseEventSource: nil, mouseType: downT(b), mouseCursorPosition: from, mouseButton: b)
+    else { return "{\"error\":\"fail\"}" }
+    post(dn)
+    for n in 1...steps {
+        let t = Double(n) / Double(steps)
+        let pt = CGPoint(x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t)
+        if let ev = CGEvent(mouseEventSource: nil, mouseType: dragT(b), mouseCursorPosition: pt, mouseButton: b) { post(ev) }
+        if delay > 0 && n < steps { usleep(UInt32(delay)) }
+    }
+    if let up = CGEvent(mouseEventSource: nil, mouseType: upT(b), mouseCursorPosition: to, mouseButton: b) { post(up) }
+    return "{\"ok\":true}"
+}
+
+func doScroll(_ p: Input) -> String {
+    let dy = Int32(-(p.deltaY ?? 0)); let dx = Int32(p.deltaX ?? 0)
+    guard let ev = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: dy, wheel2: dx, wheel3: 0)
+    else { return "{\"error\":\"fail\"}" }
+    post(ev); return "{\"ok\":true}"
+}
+
+func doPressKey(_ p: Input) -> String {
+    let code = UInt16(p.keyCode ?? 0); let flags = CGEventFlags(rawValue: UInt64(p.flags ?? 0))
+    guard let dn = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
+          let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false)
+    else { return "{\"error\":\"fail\"}" }
+    dn.flags = flags; up.flags = flags; post(dn); post(up); return "{\"ok\":true}"
+}
+
+func doTypeBatch(_ p: Input) -> String {
+    guard let keys = p.keys else { return "{\"error\":\"missing keys\"}" }
+    let SHIFT = CGEventFlags(rawValue: 0x00020000)
+    for entry in keys {
+        let code = UInt16(entry.code); let flags: CGEventFlags = (entry.shift ?? false) ? SHIFT : []
+        guard let dn = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
+              let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false) else { continue }
+        dn.flags = flags; up.flags = flags; post(dn); post(up)
+    }
+    return "{\"ok\":true}"
+}
+
+guard let line = readLine(), let data = line.data(using: .utf8),
+      let input = try? JSONDecoder().decode(Input.self, from: data)
+else { out("{\"error\":\"invalid JSON\"}"); exit(1) }
+
+switch input.command {
+case "click": out(doClick(input))
+case "doubleClick": out(doDoubleClick(input))
+case "move": out(doMove(input))
+case "drag": out(doDrag(input))
+case "scroll": out(doScroll(input))
+case "pressKey": out(doPressKey(input))
+case "typeBatch": out(doTypeBatch(input))
+case "ping": out("{\"ok\":true}")
+default: out("{\"error\":\"unknown\"}")
+}

package/native/ocr/main.swift

@@ -0,0 +1,89 @@
+import Foundation
+import Vision
+import AppKit
+
+struct OCRInput: Decodable {
+    let imagePath: String
+}
+
+struct OCRElement: Encodable {
+    let text: String
+    let x: Int
+    let y: Int
+    let width: Int
+    let height: Int
+    let confidence: Double
+}
+
+struct OCROutput: Encodable {
+    let elements: [OCRElement]
+    let fullText: String
+    let error: String?
+}
+
+// Read all of stdin as JSON
+let stdinData = FileHandle.standardInput.readDataToEndOfFile()
+guard let input = try? JSONDecoder().decode(OCRInput.self, from: stdinData) else {
+    let err = OCROutput(elements: [], fullText: "", error: "Failed to decode input JSON")
+    let d = try! JSONEncoder().encode(err)
+    FileHandle.standardOutput.write(d)
+    exit(1)
+}
+
+let url = URL(fileURLWithPath: input.imagePath)
+guard let image = NSImage(contentsOf: url), image.isValid else {
+    let err = OCROutput(elements: [], fullText: "", error: "Failed to load image: \(input.imagePath)")
+    let d = try! JSONEncoder().encode(err)
+    FileHandle.standardOutput.write(d)
+    exit(1)
+}
+
+var proposedRect = NSRect.zero
+guard let cgImage = image.cgImage(forProposedRect: &proposedRect, context: nil, hints: nil) else {
+    let err = OCROutput(elements: [], fullText: "", error: "Failed to get CGImage")
+    let d = try! JSONEncoder().encode(err)
+    FileHandle.standardOutput.write(d)
+    exit(1)
+}
+
+let request = VNRecognizeTextRequest()
+request.recognitionLevel = .accurate
+request.usesLanguageCorrection = true
+
+let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+
+do {
+    try handler.perform([request])
+} catch {
+    let err = OCROutput(elements: [], fullText: "", error: "OCR failed: \(error.localizedDescription)")
+    let d = try! JSONEncoder().encode(err)
+    FileHandle.standardOutput.write(d)
+    exit(1)
+}
+
+guard let observations = request.results else {
+    let err = OCROutput(elements: [], fullText: "", error: "No OCR results")
+    let d = try! JSONEncoder().encode(err)
+    FileHandle.standardOutput.write(d)
+    exit(1)
+}
+
+let imgWidth = CGFloat(cgImage.width)
+let imgHeight = CGFloat(cgImage.height)
+var elements: [OCRElement] = []
+var fullTextParts: [String] = []
+
+for obs in observations {
+    guard let candidate = obs.topCandidates(1).first else { continue }
+    let bbox = obs.boundingBox
+    let bx = Int(bbox.origin.x * imgWidth)
+    let by = Int((1 - bbox.origin.y - bbox.height) * imgHeight)
+    let bw = Int(bbox.width * imgWidth)
+    let bh = Int(bbox.height * imgHeight)
+    elements.append(OCRElement(text: candidate.string, x: bx, y: by, width: bw, height: bh, confidence: Double(candidate.confidence)))
+    fullTextParts.append(candidate.string)
+}
+
+let output = OCROutput(elements: elements, fullText: fullTextParts.joined(separator: "\n"), error: nil)
+let encoded = try! JSONEncoder().encode(output)
+FileHandle.standardOutput.write(encoded)

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "ucu-mcp",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "MCP server for Universal Computer Use — desktop automation for AI agents via Model Context Protocol",
   "type": "module",
   "bin": {
     "ucu-mcp": "dist/bin/ucu-mcp.js"
   },
   "files": [
+    "native/",
     "dist/bin/",
     "dist/src/",
     "dist/index.js",
@@ -17,13 +18,15 @@
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && npm run build:native",
     "start": "node dist/bin/ucu-mcp.js",
     "dev": "tsx bin/ucu-mcp.ts",
     "test": "vitest run",
     "test:watch": "vitest",
     "test:integration": "vitest run tests/integration/",
-    "test:macos-gui": "UCU_MACOS_GUI_SMOKE=1 vitest run tests/integration/macos-gui-smoke.test.ts"
+    "test:macos-gui": "UCU_MACOS_GUI_SMOKE=1 vitest run tests/integration/macos-gui-smoke.test.ts",
+    "test:client-cli": "UCU_CLIENT_CLI_SMOKE=1 vitest run tests/integration/client-cli-smoke.test.ts",
+    "build:native": "cd native/cgevent && swiftc -O -o cgevent-helper main.swift -framework CoreGraphics -framework Foundation && cd ../ocr && swiftc -O -o ocr-helper main.swift -framework Vision -framework AppKit"
   },
   "keywords": [
     "mcp",