ucu-mcp 0.1.3 → 0.2.0

package/CHANGELOG.md

@@ -5,7 +5,42 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.1.0] - 2025-06-02
+## [0.2.0] - 2026-06-05
+
+### Changed
+
+- Replaced JXA keyboard/mouse input with native Swift CGEvent helper (`native/cgevent/cgevent-helper`), eliminating SIGSEGV crashes on macOS Sequoia+
+- `listWindows` switched from `CGWindowListCopyWindowInfo` to System Events for reliable window enumeration
+- `getWindowState` adapted to use System Events window IDs instead of CGWindow IDs
+- Fixed OCR JXA script — `isValid` guard now correctly handles missing/broken references
+
+### Fixed
+
+- `typeInElement` now properly escapes `$` in text to prevent JXA template-literal interpolation errors
+- AX element cache now refetches stale references instead of throwing
+- MCP server version now resolves from `package.json` instead of advertising stale `0.1.0`
+- `screenshot.maxWidth`, `screenshot.windowId`, and action `captureAfter` encode options now reach the execution path
+- `captureAfter` now returns a separate MCP image content item instead of embedding screenshot bytes in JSON text
+- Window-relative coordinate tools now reject stale `windowId` values instead of falling back to raw screen coordinates
+- Real input actions no longer use the shared retry wrapper after a partial failure
+- macOS AX traversal now uses `uiElements()` with `elements()` fallback, fixing TextEdit `AXTextArea` discovery
+- User activity monitoring now starts with the MCP server and initializes the cursor baseline before polling
+- Added client-friendly aliases and defaults: `press_key.modifiers`, `scroll.deltaX=0`, `wait_for_element.timeoutMs/intervalMs`, and `move.captureAfter`
+- README tool tables and OCR/captureAfter response examples now match the live MCP schema
+- macOS platform failures now use structured `UcuError` subclasses for screenshots, window lookup, AX permissions, stale elements, cursor queries, and input synthesis
+- MCP tool failures now return `isError: true` with JSON `error.name`, `error.code`, `error.retryable`, `error.message`, and `error.recovery` instead of forcing clients to parse plain text
+- `wait_for_element` no longer masks Accessibility/platform failures as ordinary timeouts; missing elements still time out, but real lookup failures surface through the structured MCP error response
+- macOS `listWindows` now uses a short defensive-copy cache for repeated window lookups, reducing back-to-back window resolution calls from seconds to near-zero while `focusApp` still invalidates before activating a target app
+- Added optional real client CLI smoke coverage for Claude Code CLI, Codex CLI, and OpenCode MCP visibility
+- README now includes verified `claude mcp add`, `codex mcp add`, and OpenCode `opencode.json` setup paths
+
+### Tests
+
+- Unit test count grew from 83 → 161
+- Optional client CLI smoke: 3/3 passing with `npm run test:client-cli`
+- GUI smoke tests 6/6 passing (`UCU_MACOS_GUI_SMOKE=1`)
+
+## [0.1.0] - 2026-06-02
 
 ### Added
 

package/README.md

@@ -78,10 +78,10 @@ UCU-MCP provides 22 tools across five categories:
 
 | Tool | Description | Key Parameters |
 |------|-------------|----------------|
-| `screenshot` | Capture screen, window, or region as base64 PNG/JPEG | `display?`, `windowId?`, `region?`, `maxWidth?`, `format?` |
+| `screenshot` | Capture screen, window, or region as PNG/JPEG image content | `display?`, `windowId?`, `region?`, `maxWidth?`, `format?` |
 | `list_windows` | List all on-screen windows with IDs, titles, bounds | `includeMinimized?` |
 | `list_apps` | List visible macOS apps with pid, frontmost state, and window count | — |
-| `focus_app` | Select an app/window target context without raising it | `app` |
+| `focus_app` | Select an app/window target context for later AX tools | `app` |
 | `get_window_state` | Get accessibility tree of a window, or the prior focus_app target when windowId is omitted | `windowId?`, `depth?`, `includeBounds?` |
 | `get_screen_size` | Get screen dimensions | `display?` |
 | `ocr` | Perform OCR on screen or region; returns text with bounding boxes and confidence | `display?`, `region?` |
@@ -92,9 +92,9 @@ UCU-MCP provides 22 tools across five categories:
 |------|-------------|----------------|
 | `click` | Click at screen coordinates (non-invasive) | `x`, `y`, `windowId?`, `button?` |
 | `double_click` | Double-click at screen coordinates | `x`, `y`, `windowId?`, `button?` |
-| `scroll` | Scroll at a position (vertical/horizontal) | `x`, `y`, `deltaX?`, `deltaY`, `captureAfter?` |
-| `drag` | Drag from one position to another | `startX`, `startY`, `endX`, `endY`, `duration?`, `button?`, `captureAfter?` |
-| `move` | Move the physical cursor to a position (invasive) | `x`, `y` |
+| `scroll` | Scroll at a position (vertical/horizontal) | `x`, `y`, `deltaX?`, `deltaY`, `windowId?`, `captureAfter?` |
+| `drag` | Drag from one position to another | `startX`, `startY`, `endX`, `endY`, `windowId?`, `duration?`, `button?`, `captureAfter?` |
+| `move` | Move the physical cursor to a position (invasive) | `x`, `y`, `windowId?`, `captureAfter?` |
 | `get_cursor_position` | Get current cursor position | — |
 
 ### Keyboard
@@ -102,7 +102,7 @@ UCU-MCP provides 22 tools across five categories:
 | Tool | Description | Key Parameters |
 |------|-------------|----------------|
 | `type_text` | Type text into the currently focused element via OS key events (not clipboard) | `text`, `delay?`, `captureAfter?` |
-| `press_key` | Press key or keyboard shortcut in the focused window | `key`, `modifiers?`, `captureAfter?` |
+| `press_key` | Press key or keyboard shortcut in the focused window | `key?`, `modifiers?`, `keys?`, `captureAfter?` |
 
 ### AX Element Interaction
 
@@ -118,10 +118,10 @@ UCU-MCP provides 22 tools across five categories:
 | Tool | Description | Key Parameters |
 |------|-------------|----------------|
 | `doctor` | Check platform readiness, permissions, lock-screen state, and client integration hints | — |
-| `wait` | Wait for UI state to settle after launches, animations, or navigation | `ms?` |
-| `wait_for_element` | Poll the AX tree until a matching element appears | `text?`, `role?`, `app?`, `timeoutMs?`, `intervalMs?` |
+| `wait` | Wait for UI state to settle after launches, animations, or navigation | `ms` |
+| `wait_for_element` | Poll the AX tree until a matching element appears | `text?`, `role?`, `app?`, `timeout?`, `timeoutMs?`, `interval?`, `intervalMs?` |
 
-Action tools accept `captureAfter`, `captureMaxWidth`, and `captureFormat` so an agent can receive a post-action screenshot in the same MCP response instead of spending another round trip on `screenshot`.
+Action tools accept `captureAfter`, `captureMaxWidth`, and `captureFormat` so an agent can receive a post-action screenshot as a second MCP image content item in the same response instead of spending another round trip on `screenshot`.
 
 For fast AX discovery on large windows, use `find_element` with `includeBounds=false` and a small `maxResults`. Keep bounds enabled when the result may be used for coordinate fallback.
 
@@ -153,11 +153,14 @@ The `ocr` tool captures a screenshot and runs optical character recognition, ret
 
 ```json
 {
-  "text": "Detected text here",
+  "fullText": "Detected text here",
   "elements": [
     {
       "text": "Hello",
-      "bounds": { "x": 120, "y": 210, "width": 80, "height": 24 },
+      "x": 120,
+      "y": 210,
+      "width": 80,
+      "height": 24,
       "confidence": 0.97
     }
   ]
@@ -245,10 +248,20 @@ UCU-MCP runs as a stdio MCP server. This is the common integration path for Clau
 
 ### Claude Code CLI
 
+Verified CLI setup:
+
+```bash
+claude mcp add --scope user ucu -- ucu-mcp
+claude mcp list
+```
+
+Equivalent config shape:
+
 ```json
 {
   "mcpServers": {
     "ucu": {
+      "type": "stdio",
       "command": "ucu-mcp"
     }
   }
@@ -263,25 +276,51 @@ Use the same local MCP server shape as Claude Desktop. Grant Accessibility and S
 {
   "mcpServers": {
     "ucu": {
+      "type": "stdio",
       "command": "ucu-mcp"
     }
   }
 }
 ```
 
+### Codex CLI
+
+Verified CLI setup:
+
+```bash
+codex mcp add ucu -- ucu-mcp
+codex mcp list
+```
+
+Equivalent `~/.codex/config.toml` shape:
+
+```toml
+[mcp_servers.ucu]
+command = "ucu-mcp"
+```
+
 ### OpenCode
 
+OpenCode reads MCP servers from `~/.config/opencode/opencode.json`.
+
 ```json
 {
   "mcp": {
-    "ucu": {
+    "ucu-mcp": {
       "type": "local",
+      "enabled": true,
       "command": ["ucu-mcp"]
     }
   }
 }
 ```
 
+Verify with:
+
+```bash
+opencode mcp list
+```
+
 ### Runtime Doctor
 
 ```bash
@@ -362,15 +401,32 @@ src/
 
 ## Error Handling
 
+Tool execution failures return standard MCP tool results with `isError: true`. The first content item is JSON text so clients can make policy decisions without string matching:
+
+```json
+{
+  "error": {
+    "name": "WindowNotFoundError",
+    "code": "WINDOW_NOT_FOUND",
+    "retryable": false,
+    "message": "Window win-1 not found. It may have been closed. Run list_windows to get fresh IDs.",
+    "recovery": "Run list_windows again, then retry with a fresh windowId or omit windowId for screen coordinates."
+  }
+}
+```
+
 | Error Code | Description | Retryable |
 |------------|-------------|-----------|
 | `PLATFORM_ERROR` | Platform API call failed | Yes |
 | `PERMISSION_DENIED` | Missing system permission | No |
 | `SAFETY_BLOCKED` | Blocked by safety rule | No |
 | `WINDOW_NOT_FOUND` | Window does not exist | No |
+| `ELEMENT_NOT_FOUND` | Accessibility element is stale or missing | No |
+| `UNSUPPORTED_PARAMETER` | Valid JSON requested an unsupported parameter combination | No |
 | `COORDINATE_OUT_OF_BOUNDS` | Coordinate outside screen | No |
 | `INPUT_FAILED` | Input synthesis failed | Yes |
 | `CAPTURE_FAILED` | Screenshot/OCR capture failed | Yes |
+| `UNKNOWN_ERROR` | Unexpected internal failure | No |
 
 ## Development
 

package/dist/src/mcp/server.js

@@ -1,6 +1,9 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { createStdioTransport } from "./transport.js";
-import { registerTools } from "./tools.js";
+import { registerTools, startUserActivityMonitor } from "./tools.js";
 const UCU_MCP_INSTRUCTIONS = `
 UCU-MCP is a cross-client computer-use server for Claude Code CLI, Claude Code Desktop, OpenCode, and other MCP clients.
 
@@ -12,14 +15,27 @@ Safety model: actions are blocked while macOS is locked, dangerous shortcuts and
 
 For Claude Code CLI/Desktop and OpenCode configs, run the ucu-mcp executable over stdio. If tools fail on macOS, run doctor first to check Accessibility and Screen Recording permissions. Windows and Linux adapters are explicit stubs until their native backends are implemented.
 `.trim();
+function getPackageVersion() {
+    let dir = dirname(fileURLToPath(import.meta.url));
+    for (let i = 0; i < 6; i++) {
+        const path = join(dir, "package.json");
+        if (existsSync(path)) {
+            const parsed = JSON.parse(readFileSync(path, "utf-8"));
+            return parsed.version ?? "0.0.0";
+        }
+        dir = dirname(dir);
+    }
+    return "0.0.0";
+}
 export async function startServer() {
     const server = new McpServer({
         name: "ucu-mcp",
-        version: "0.1.0",
+        version: getPackageVersion(),
     }, {
         instructions: UCU_MCP_INSTRUCTIONS,
     });
     registerTools(server);
+    startUserActivityMonitor();
     const transport = createStdioTransport();
     await server.connect(transport);
     console.error("ucu-mcp server started on stdio");

package/dist/src/mcp/tools.d.ts

@@ -6,6 +6,7 @@
  */
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 export declare function startUserActivityMonitor(): void;
+export declare function stopUserActivityMonitor(): void;
 export declare function registerTools(server: McpServer): void;
 export declare class ToolRegistry {
     private static _instance;