ucu-mcp 0.3.8 → 0.3.9

package/dist/src/mcp/tools.js

@@ -306,7 +306,7 @@ export function registerTools(server) {
             const axNote = accessibility === "denied"
                 ? "Accessibility is currently denied to this terminal — grant it via System Settings > Privacy & Security > Accessibility, then retry."
                 : accessibility === "granted"
-                    ? "Accessibility is granted. If you expected a specific app to appear here, it is likely an Electron app whose AX tree is not exposed to System Events; try modifying its config file or database directly rather than driving the UI."
+                    ? "Accessibility is granted. If you expected a specific app to appear here, it is likely an Electron app whose AX tree is not exposed to System Events. Pixel-level workaround: call screenshot, then ocr to locate the target UI text and get its bounding box, then click(x, y) at those screen coordinates. Alternatively, modify the app's config file or database directly."
                     : "Accessibility status is unknown. Run `doctor` first to verify.";
             diagnostics = { hint: `list_windows returned 0 windows. ${axNote}`, accessibility };
         }
@@ -471,9 +471,11 @@ export function registerTools(server) {
         if (process.platform === "darwin") {
             const cgevent = resolveHelperPath(["native", "cgevent", "cgevent-helper"]);
             const ocr = resolveHelperPath(["native", "ocr", "ocr-helper"]);
+            const windowlist = resolveHelperPath(["native", "windowlist", "windowlist-helper"]);
             nativeHelpers = {
                 cgevent: { ok: cgevent.path !== null, path: cgevent.path, tried: cgevent.tried.slice(0, 3) },
                 ocr: { ok: ocr.path !== null, path: ocr.path, tried: ocr.tried.slice(0, 3) },
+                windowlist: { ok: windowlist.path !== null, path: windowlist.path, tried: windowlist.tried.slice(0, 3) },
             };
         }
         let readiness = "ready";
@@ -497,6 +499,10 @@ export function registerTools(server) {
                 readiness = readiness === "ready" ? "degraded" : readiness;
                 issues.push("Native OCR helper not found (OCR may fail on macOS Sequoia+). Run `npm run build` to compile it, or reinstall ucu-mcp so the helper ships from the tarball.");
             }
+            if (!nativeHelpers.windowlist.ok) {
+                readiness = readiness === "ready" ? "degraded" : readiness;
+                issues.push("Native windowlist helper not found (window enumeration will fall back to slow JXA). Run `npm run build` to compile it.");
+            }
         }
         // Heuristic AX hint: if Accessibility is granted but list_windows consistently
         // returns empty for the only app the model cared about, the model has likely
@@ -504,7 +510,7 @@ export function registerTools(server) {
         // Events unless Accessibility is also granted to the Electron process itself,
         // and the app has accessibility features enabled). This block is read-only —
         // we never hit JXA here because the doctor must stay fast and side-effect free.
-        const electronHint = "If the target app is Electron (e.g. CC Switch, VS Code, Discord), list_windows may return [] even with Accessibility granted to your terminal. Grant Accessibility to the Electron app itself in System Settings > Privacy & Security > Accessibility, and restart the app. As a workaround, modify the app\'s config file or database directly rather than driving the UI.";
+        const electronHint = "If the target app is Electron (e.g. CC Switch, VS Code, Discord), list_windows may return [] even with Accessibility granted to your terminal. Grant Accessibility to the Electron app itself in System Settings > Privacy & Security > Accessibility, and restart the app. Pixel-level workaround: use screenshot + ocr to locate UI elements by text, then click(x, y) at the detected bounding box coordinates. Alternatively, modify the app\'s config file or database directly.";
         const clients = {};
         for (const bin of ["claude", "codex", "opencode", "npx"]) {
             try {
@@ -526,7 +532,10 @@ export function registerTools(server) {
         }
         if (readiness !== "ready") {
             if (process.platform === "darwin" && nativeHelpers && (!nativeHelpers.cgevent.ok || !nativeHelpers.ocr.ok)) {
-                recommendations.push("Run `npm run build` in the ucu-mcp project to compile native Swift helpers (cgevent-helper, ocr-helper).");
+                recommendations.push("Run `npm run build` in the ucu-mcp project to compile native Swift helpers (cgevent-helper, ocr-helper, windowlist-helper).");
+            }
+            if (process.platform === "darwin" && nativeHelpers && !nativeHelpers.windowlist.ok) {
+                recommendations.push("windowlist helper missing — list_windows will fall back to JXA (~3-6s, unreliable for Electron). Run `npm run build`.");
             }
         }
         if (readiness === "ready") {

package/dist/src/platform/macos.d.ts

@@ -1,5 +1,15 @@
 import type { Platform, ScreenRegion, ScreenSize, CursorPosition, WindowInfo, WindowState, OcrResult, FindElementOptions, FindElementResponse, AppInfo, AppTarget, BrowserContext, ScreenshotOptions } from "./base.js";
+export interface MacOSPlatformOptions {
+    /**
+     * Override native helper resolution.
+     * - Map of folder name to absolute binary path to inject a specific helper.
+     * - Set a value to null to skip that helper (force JXA fallback).
+     * Used by tests to control native helper behavior without filesystem tricks.
+     */
+    nativeHelperPaths?: Record<string, string | null>;
+}
 export declare class MacOSPlatform implements Platform {
+    private readonly _nativeHelperPaths;
     private readonly elementCache;
     private readonly elementCacheTtlMs;
     private readonly elementCacheMaxSize;
@@ -7,6 +17,7 @@ export declare class MacOSPlatform implements Platform {
     private windowCache;
     private activeTarget;
     private savedFocus;
+    constructor(options?: MacOSPlatformOptions);
     /** Remove expired entries from the element cache. */
     private evictExpiredCacheEntries;
     /** Evict oldest entries when cache exceeds the maximum size (LRU-style). */
@@ -27,6 +38,9 @@ export declare class MacOSPlatform implements Platform {
     focusApp(app: string): Promise<AppTarget>;
     getActiveBrowserContext(app?: string): Promise<BrowserContext | undefined>;
     listWindows(_includeMinimized?: boolean): Promise<WindowInfo[]>;
+    private listWindowsNative;
+    private resolveNativeHelper;
+    private listWindowsJxa;
     getWindowState(windowId?: string, depth?: number, includeBounds?: boolean): Promise<WindowState>;
     click(x: number, y: number, button?: "left" | "right" | "middle", doubleClick?: boolean): Promise<void>;
     move(x: number, y: number): Promise<void>;

package/dist/src/platform/macos.js

@@ -4,6 +4,10 @@ import { promisify } from "node:util";
 import { captureFullScreen, captureRegion } from "../utils/screenshot.js";
 import { click as inputClick, doubleClick as inputDoubleClick, move as inputMove, drag as inputDrag, scroll as inputScroll, typeText, pressShortcut } from "../utils/input.js";
 import { CaptureError, ElementNotFoundError, InputSynthesisError, PermissionError, PlatformError, TargetStaleError, UcuError, WindowNotFoundError } from "../util/errors.js";
+import { existsSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+const __macosDirname = dirname(fileURLToPath(import.meta.url));
 const execFileAsync = promisify(execFile);
 function errorMessage(error) {
     return error instanceof Error ? error.message : String(error);
@@ -59,6 +63,7 @@ function selectWindowForApp(windows, requestedApp) {
         windows.find((window) => appNameMatches(window.processName, requestedApp));
 }
 export class MacOSPlatform {
+    _nativeHelperPaths;
     elementCache = new Map();
     elementCacheTtlMs = 30_000;
     elementCacheMaxSize = 100;
@@ -66,6 +71,9 @@ export class MacOSPlatform {
     windowCache;
     activeTarget;
     savedFocus;
+    constructor(options) {
+        this._nativeHelperPaths = options?.nativeHelperPaths;
+    }
     // ── Element Cache Management ────────────────────────────────────────────
     /** Remove expired entries from the element cache. */
     evictExpiredCacheEntries() {
@@ -260,8 +268,10 @@ export class MacOSPlatform {
                     "the most likely cause is that it is an Electron app whose AX tree is " +
                     "not exposed to System Events (System Settings > Privacy & Security > " +
                     "Accessibility must be granted to the Electron process itself, not just " +
-                    "to the host terminal). As a workaround, modify the app's config file " +
-                    "or database directly.";
+                    "to the host terminal). Pixel-level workaround: call screenshot to " +
+                    "capture the screen, then ocr to locate UI text and get its bounding " +
+                    "box coordinates, then click(x, y) at those screen coordinates. " +
+                    "Alternatively, modify the app's config file or database directly.";
             throw err;
         }
         this.activeTarget = {
@@ -334,11 +344,81 @@ export class MacOSPlatform {
             }));
         }
         try {
-            // Use System Events instead of CGWindowListCopyWindowInfo.
-            // The CoreGraphics API returns CFArrayRef/CFDictionaryRef which JXA
-            // cannot iterate reliably — CFArrayGetCount works but objectAtIndex
-            // does not. System Events JXA is slower (~3-6s) but correct.
-            const jxaScript = `
+            // Try native Swift helper first (CGWindowListCopyWindowInfo, ~1ms).
+            // Falls back to JXA System Events if the helper is not available.
+            // The native helper reliably enumerates ALL windows including Electron
+            // apps, whereas JXA relies on System Events AX which is inconsistent
+            // for Chromium-rendered windows.
+            let windows;
+            const nativeResult = this.listWindowsNative();
+            if (nativeResult !== null) {
+                windows = nativeResult;
+            }
+            else {
+                windows = await this.listWindowsJxa();
+            }
+            this.windowCache = {
+                cachedAt: Date.now(),
+                windows: windows.map((window) => ({
+                    ...window,
+                    bounds: { ...window.bounds },
+                })),
+            };
+            return windows;
+        }
+        catch {
+            // Fallback: return empty list if both methods fail
+            return [];
+        }
+    }
+    listWindowsNative() {
+        try {
+            const helperPath = this.resolveNativeHelper("windowlist", "windowlist-helper");
+            if (!helperPath)
+                return null;
+            const out = execFileSync(helperPath, [], {
+                encoding: "utf-8",
+                timeout: 5000,
+            });
+            const parsed = JSON.parse(out.trim());
+            if (parsed.error)
+                return null;
+            return parsed.windows.map(w => ({
+                id: w.id,
+                title: w.title,
+                processName: w.processName,
+                pid: w.pid,
+                bounds: w.bounds,
+                isMinimized: !w.isOnScreen,
+                isOnScreen: w.isOnScreen,
+            }));
+        }
+        catch {
+            return null;
+        }
+    }
+    resolveNativeHelper(folder, binary) {
+        // Test injection: if the caller provided explicit paths, use those
+        // instead of hitting the filesystem.
+        if (this._nativeHelperPaths && folder in this._nativeHelperPaths) {
+            const override = this._nativeHelperPaths[folder];
+            // null means "skip native, force JXA fallback"
+            return override === null ? null : override;
+        }
+        // dev: src/platform/macos.ts → native/<folder>/<binary>
+        // prod: dist/src/platform/macos.js → native/<folder>/<binary>
+        const candidates = [
+            join(__macosDirname, "..", "..", "native", folder, binary),
+            join(__macosDirname, "..", "..", "..", "native", folder, binary),
+        ];
+        for (const p of candidates) {
+            if (existsSync(p))
+                return p;
+        }
+        return null;
+    }
+    async listWindowsJxa() {
+        const jxaScript = `
         var se = Application('System Events');
         var result = [];
         var procs = se.processes();
@@ -372,24 +452,11 @@ export class MacOSPlatform {
         }
         JSON.stringify(result);
       `;
-            const jxaOut = execFileSync("osascript", [
-                "-l", "JavaScript",
-                "-e", jxaScript
-            ], { encoding: "utf-8", timeout: 15000 });
-            const windows = JSON.parse(jxaOut.trim());
-            this.windowCache = {
-                cachedAt: Date.now(),
-                windows: windows.map((window) => ({
-                    ...window,
-                    bounds: { ...window.bounds },
-                })),
-            };
-            return windows;
-        }
-        catch {
-            // Fallback: return empty list if JXA fails
-            return [];
-        }
+        const jxaOut = execFileSync("osascript", [
+            "-l", "JavaScript",
+            "-e", jxaScript
+        ], { encoding: "utf-8", timeout: 15000 });
+        return JSON.parse(jxaOut.trim());
     }
     async getWindowState(windowId, depth, includeBounds = true) {
         if (!windowId || windowId === this.activeTarget?.windowId) {

package/native/windowlist/main.swift

@@ -0,0 +1,74 @@
+import CoreGraphics
+import Foundation
+
+// Window enumeration using CGWindowListCopyWindowInfo (WindowServer-level).
+// Replaces JXA System Events enumeration which is slow (3-6s) and unreliable
+// for Electron apps. CGWindowListCopyWindowInfo sees ALL windows regardless
+// of whether the app exposes an AX tree.
+
+struct WinInfo: Encodable {
+    let id: String
+    let title: String
+    let processName: String
+    let pid: Int32
+    let bounds: Bounds
+    let isOnScreen: Bool
+    let windowNumber: Int
+}
+
+struct Bounds: Encodable {
+    let x: Double
+    let y: Double
+    let width: Double
+    let height: Double
+}
+
+struct Output: Encodable {
+    let windows: [WinInfo]
+    let error: String?
+}
+
+let options: CGWindowListOption = .optionOnScreenOnly
+guard let windowList = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else {
+    let out = Output(windows: [], error: "CGWindowListCopyWindowInfo returned nil")
+    FileHandle.standardOutput.write(try! JSONEncoder().encode(out))
+    exit(0)
+}
+
+var results: [WinInfo] = []
+
+for info in windowList {
+    guard let pid = info[kCGWindowOwnerPID as String] as? Int32,
+          let windowNumber = info[kCGWindowNumber as String] as? Int,
+          let layer = info[kCGWindowLayer as String] as? Int
+    else { continue }
+
+    // Skip non-normal layers (overlay, screen saver, etc.)
+    if layer != 0 { continue }
+
+    let boundsDict = info[kCGWindowBounds as String] as? [String: Any]
+    let w = boundsDict?["Width"] as? Double ?? 0
+    let h = boundsDict?["Height"] as? Double ?? 0
+    if w == 0 || h == 0 { continue }
+
+    let processName = (info[kCGWindowOwnerName as String] as? String) ?? ""
+    if processName.isEmpty { continue }
+
+    let title = (info[kCGWindowName as String] as? String) ?? ""
+    let isOnScreen = (info[kCGWindowIsOnscreen as String] as? Bool) ?? true
+    let x = (boundsDict?["X"] as? Double) ?? 0
+    let y = (boundsDict?["Y"] as? Double) ?? 0
+
+    results.append(WinInfo(
+        id: "\(processName)/win\(windowNumber)",
+        title: title,
+        processName: processName,
+        pid: pid,
+        bounds: Bounds(x: x, y: y, width: w, height: h),
+        isOnScreen: isOnScreen,
+        windowNumber: windowNumber
+    ))
+}
+
+let output = Output(windows: results, error: nil)
+FileHandle.standardOutput.write(try! JSONEncoder().encode(output))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ucu-mcp",
-  "version": "0.3.8",
+  "version": "0.3.9",
   "description": "MCP server for Universal Computer Use — desktop automation for AI agents via Model Context Protocol",
   "type": "module",
   "bin": {
@@ -27,7 +27,7 @@
     "test:macos-gui": "UCU_MACOS_GUI_SMOKE=1 vitest run tests/integration/macos-gui-smoke.test.ts",
     "test:client-cli": "UCU_CLIENT_CLI_SMOKE=1 vitest run tests/integration/client-cli-smoke.test.ts",
     "prepublishOnly": "npx vitest run tests/unit/ && npm run build",
-    "build:native": "cd native/cgevent && swiftc -O -o cgevent-helper main.swift -framework CoreGraphics -framework Foundation && cd ../ocr && swiftc -O -o ocr-helper main.swift -framework Vision -framework AppKit"
+    "build:native": "cd native/cgevent && swiftc -O -o cgevent-helper main.swift -framework CoreGraphics -framework Foundation && cd ../ocr && swiftc -O -o ocr-helper main.swift -framework Vision -framework AppKit && cd ../windowlist && swiftc -O -o windowlist-helper main.swift -framework CoreGraphics -framework Foundation"
   },
   "keywords": [
     "mcp",