ucu-mcp 0.1.3 → 0.3.0

package/dist/src/platform/macos.js

@@ -1,13 +1,69 @@
 import { execFile, execFileSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { promisify } from "node:util";
-import { captureFullScreen, captureRegion, captureWindow } from "../utils/screenshot.js";
+import { captureFullScreen, captureRegion } from "../utils/screenshot.js";
 import { click as inputClick, doubleClick as inputDoubleClick, move as inputMove, drag as inputDrag, scroll as inputScroll, typeText, pressShortcut } from "../utils/input.js";
+import { CaptureError, ElementNotFoundError, InputSynthesisError, PermissionError, PlatformError, TargetStaleError, UcuError, WindowNotFoundError } from "../util/errors.js";
 const execFileAsync = promisify(execFile);
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function isAccessibilityPermissionError(error) {
+    return /not allowed|permission|assistive|accessibility/i.test(errorMessage(error));
+}
+function rethrowCaptureError(error, operation) {
+    if (error instanceof UcuError)
+        throw error;
+    throw new CaptureError(`${operation} failed: ${errorMessage(error)}`);
+}
+function rethrowAccessibilityError(error, operation) {
+    if (error instanceof UcuError)
+        throw error;
+    if (isAccessibilityPermissionError(error)) {
+        throw new PermissionError("accessibility", "darwin");
+    }
+    throw new PlatformError(`${operation} failed: ${errorMessage(error)}`);
+}
+function rethrowElementActionError(error, operation, elementId) {
+    if (error instanceof UcuError)
+        throw error;
+    if (isAccessibilityPermissionError(error)) {
+        throw new PermissionError("accessibility", "darwin");
+    }
+    if (/element not found/i.test(errorMessage(error))) {
+        throw new ElementNotFoundError(elementId);
+    }
+    throw new PlatformError(`${operation} failed: ${errorMessage(error)}`);
+}
+function rethrowInputError(error, operation) {
+    if (error instanceof UcuError)
+        throw error;
+    throw new InputSynthesisError(`${operation} failed: ${errorMessage(error)}`);
+}
+function normalizeAppName(name) {
+    return name.trim().toLowerCase();
+}
+function appNameMatches(processName, requestedApp) {
+    const process = normalizeAppName(processName);
+    const requested = normalizeAppName(requestedApp);
+    if (!process || !requested)
+        return false;
+    return process === requested ||
+        process.startsWith(`${requested} `) ||
+        process.startsWith(`${requested}-`) ||
+        process.includes(` ${requested} `);
+}
+function selectWindowForApp(windows, requestedApp) {
+    const requested = normalizeAppName(requestedApp);
+    return windows.find((window) => normalizeAppName(window.processName) === requested) ??
+        windows.find((window) => appNameMatches(window.processName, requestedApp));
+}
 export class MacOSPlatform {
     elementCache = new Map();
     elementCacheTtlMs = 30_000;
     elementCacheMaxSize = 100;
+    windowCacheTtlMs = 300;
+    windowCache;
     activeTarget;
     savedFocus;
     // ── Element Cache Management ────────────────────────────────────────────
@@ -43,6 +99,18 @@ export class MacOSPlatform {
     isCacheEntryExpired(descriptor) {
         return Date.now() - descriptor.cachedAt > this.elementCacheTtlMs;
     }
+    // ── Target Validation ────────────────────────────────────────────────────
+    /** Validate that the active target window still exists. */
+    async validateActiveTarget() {
+        if (!this.activeTarget?.windowId)
+            return;
+        this.windowCache = undefined; // Bypass cache — stale detection must use fresh data
+        const windows = await this.listWindows(true);
+        const stillExists = windows.some(w => w.id === this.activeTarget.windowId);
+        if (!stillExists) {
+            throw new TargetStaleError(this.activeTarget.windowId);
+        }
+    }
     // ── Focus Management ────────────────────────────────────────────────────
     /** Save the current frontmost app/window so we can restore after an action. */
     async saveFocus() {
@@ -79,14 +147,22 @@ export class MacOSPlatform {
     }
     // ── Screenshot ──────────────────────────────────────────────────────────
     async screenshot(_display, region, options) {
-        const base64 = region
-            ? await captureRegion(region.x, region.y, region.width, region.height, options)
-            : await captureFullScreen(options);
-        return Buffer.from(base64, "base64");
+        try {
+            const base64 = region
+                ? await captureRegion(region.x, region.y, region.width, region.height, options)
+                : await captureFullScreen(options);
+            return Buffer.from(base64, "base64");
+        }
+        catch (error) {
+            rethrowCaptureError(error, region ? "capture region" : "capture full screen");
+        }
     }
     async screenshotWindow(windowId, options) {
-        const base64 = await captureWindow(windowId, options);
-        return Buffer.from(base64, "base64");
+        const win = (await this.listWindows(true)).find((w) => w.id === windowId);
+        if (!win) {
+            throw new WindowNotFoundError(windowId);
+        }
+        return this.screenshot(undefined, win.bounds, options);
     }
     // ── Screen Info ─────────────────────────────────────────────────────────
     getScreenSize(display) {
@@ -153,17 +229,34 @@ export class MacOSPlatform {
         return JSON.parse(out);
     }
     async focusApp(app) {
-        const appLower = app.toLowerCase();
-        const windows = await this.listWindows(true);
-        const target = windows.find((w) => w.processName.toLowerCase().includes(appLower));
+        const escapedApp = app.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+        this.windowCache = undefined;
+        try {
+            execFileSync("osascript", ["-e", `tell application "${escapedApp}" to activate`], { timeout: 5000 });
+        }
+        catch {
+            // Some app names are process labels rather than AppleScript application names.
+            // Continue with the AX window lookup below so existing callers still work.
+        }
+        let target;
+        const deadline = Date.now() + 3000;
+        do {
+            const windows = await this.listWindows(true);
+            target = selectWindowForApp(windows, app);
+            if (target)
+                break;
+            await new Promise((resolve) => setTimeout(resolve, 150));
+        } while (Date.now() < deadline);
         if (!target) {
-            throw new Error(`No on-screen window found for app "${app}". Use list_apps to inspect localized macOS app names.`);
+            throw new WindowNotFoundError(app);
         }
         this.activeTarget = {
+            targetId: randomUUID(),
             appName: target.processName,
             pid: target.pid,
             windowId: target.id,
             title: target.title,
+            capturedAt: new Date().toISOString(),
         };
         return this.activeTarget;
     }
@@ -219,44 +312,49 @@ export class MacOSPlatform {
         }
     }
     async listWindows(_includeMinimized) {
+        const now = Date.now();
+        if (this.windowCache && now - this.windowCache.cachedAt <= this.windowCacheTtlMs) {
+            return this.windowCache.windows.map((window) => ({
+                ...window,
+                bounds: { ...window.bounds },
+            }));
+        }
         try {
+            // Use System Events instead of CGWindowListCopyWindowInfo.
+            // The CoreGraphics API returns CFArrayRef/CFDictionaryRef which JXA
+            // cannot iterate reliably — CFArrayGetCount works but objectAtIndex
+            // does not. System Events JXA is slower (~3-6s) but correct.
             const jxaScript = `
-        ObjC.import('CoreGraphics');
-        ObjC.import('Foundation');
-        var winList = $.CGWindowListCopyWindowInfo(1, 0);
-        var count = winList.count;
+        var se = Application('System Events');
         var result = [];
-        for (var i = 0; i < count; i++) {
-          var w = $(winList).objectAtIndex(i);
-          var bounds = w.objectForKey('kCGWindowBounds');
-          var numberVal = w.objectForKey('kCGWindowNumber');
-          var nameVal = w.objectForKey('kCGWindowName');
-          var ownerVal = w.objectForKey('kCGWindowOwnerName');
-          var pidVal = w.objectForKey('kCGWindowOwnerPID');
-          var onScreenVal = w.objectForKey('kCGWindowIsOnscreen');
-          var layerVal = w.objectForKey('kCGWindowLayer');
-
-          // Skip windows at layer > 0 (menus, overlays, etc.)
-          if (layerVal && layerVal.intValue > 0) continue;
-
-          var bx = 0, by = 0, bw = 0, bh = 0;
-          try { bx = $(bounds).objectForKey('X').intValue; } catch(e) {}
-          try { by = $(bounds).objectForKey('Y').intValue; } catch(e) {}
-          try { bw = $(bounds).objectForKey('Width').intValue; } catch(e) {}
-          try { bh = $(bounds).objectForKey('Height').intValue; } catch(e) {}
-
-          // Skip zero-size windows
-          if (bw === 0 && bh === 0) continue;
-
-          result.push({
-            id: String(numberVal ? numberVal.intValue : 0),
-            title: nameVal ? String(nameVal) : '',
-            processName: ownerVal ? String(ownerVal) : '',
-            pid: pidVal ? pidVal.intValue : 0,
-            bounds: { x: bx, y: by, width: bw, height: bh },
-            isMinimized: false,
-            isOnScreen: onScreenVal ? onScreenVal.boolValue : true
-          });
+        var procs = se.processes();
+        for (var i = 0; i < procs.length; i++) {
+          var p = procs[i];
+          var pName = '';
+          var pPid = 0;
+          try { pName = p.name(); } catch(e) {}
+          try { pPid = p.unixId(); } catch(e) {}
+          try {
+            var wins = p.windows();
+            for (var j = 0; j < wins.length; j++) {
+              var w = wins[j];
+              var pos, sz;
+              try { pos = w.position(); } catch(e) { pos = [0, 0]; }
+              try { sz = w.size(); } catch(e) { sz = [0, 0]; }
+              if (sz[0] === 0 && sz[1] === 0) continue;
+              var title = '';
+              try { title = w.name() || ''; } catch(e) {}
+              result.push({
+                id: pName + '/win' + j,
+                title: title,
+                processName: pName,
+                pid: pPid,
+                bounds: { x: pos[0], y: pos[1], width: sz[0], height: sz[1] },
+                isMinimized: false,
+                isOnScreen: true
+              });
+            }
+          } catch(e) {}
         }
         JSON.stringify(result);
       `;
@@ -264,7 +362,15 @@ export class MacOSPlatform {
                 "-l", "JavaScript",
                 "-e", jxaScript
             ], { encoding: "utf-8", timeout: 15000 });
-            return JSON.parse(jxaOut.trim());
+            const windows = JSON.parse(jxaOut.trim());
+            this.windowCache = {
+                cachedAt: Date.now(),
+                windows: windows.map((window) => ({
+                    ...window,
+                    bounds: { ...window.bounds },
+                })),
+            };
+            return windows;
         }
         catch {
             // Fallback: return empty list if JXA fails
@@ -272,9 +378,12 @@ export class MacOSPlatform {
         }
     }
     async getWindowState(windowId, depth, includeBounds = true) {
+        if (!windowId || windowId === this.activeTarget?.windowId) {
+            await this.validateActiveTarget();
+        }
         const resolvedWindowId = windowId || this.activeTarget?.windowId;
         if (!resolvedWindowId) {
-            throw new Error("getWindowState requires windowId or a prior focus_app target");
+            throw new WindowNotFoundError("active target");
         }
         const maxDepth = Math.min(depth || 3, 10);
         const maxElements = 50;
@@ -285,6 +394,11 @@ export class MacOSPlatform {
             const jxaScript = `
         ObjC.import('AppKit');
         var se = Application('System Events');
+      function childElements(elem) {
+        try { return elem.uiElements(); } catch(e1) {
+          try { return elem.elements(); } catch(e2) { return []; }
+        }
+      }
         var result = {window: null, focusedElement: null, tree: null, error: null};
         var target = ${targetJson};
         var includeBounds = ${includeBounds ? "true" : "false"};
@@ -319,25 +433,47 @@ export class MacOSPlatform {
           return false;
         }
 
+        var foundWin = null;
+        var foundProc = null;
+
+        // Fast path: resolve "ProcessName/winN" format directly
+        var idParts = "${escapedWindowId}".split('/');
+        if (idParts.length >= 2 && idParts[0]) {
+          var procName = idParts[0];
+          var winIdx = 0;
+          var winMatch = idParts[1].match(/^win(\d+)$/);
+          if (winMatch) winIdx = parseInt(winMatch[1]);
+          try {
+            var proc = se.processes[procName]();
+            var ws = proc.windows();
+            if (winIdx < ws.length) {
+              foundWin = ws[winIdx];
+              foundProc = proc;
+            }
+          } catch(e) {}
+        }
+
         try {
-          var foundWin = null;
-          var foundProc = null;
-          var procs = se.processes();
-          for (var p = 0; p < procs.length; p++) {
-            var proc = procs[p];
-            try {
-              var wins = proc.windows();
-              for (var w = 0; w < wins.length; w++) {
-                if (windowMatches(wins[w], proc)) {
-                  foundWin = wins[w];
-                  foundProc = proc;
-                  break;
+          if (!foundWin) {
+            var procs = se.processes();
+            for (var p = 0; p < procs.length; p++) {
+              var proc = procs[p];
+              try {
+                var wins = proc.windows();
+                for (var w = 0; w < wins.length; w++) {
+                  if (windowMatches(wins[w], proc)) {
+                    foundWin = wins[w];
+                    foundProc = proc;
+                    break;
+                  }
                 }
-              }
-            } catch(e) {}
-            if (foundWin) break;
+              } catch(e) {}
+              if (foundWin) break;
+            }
           }
-          if (!foundWin) { result.error = 'Window not found'; JSON.stringify(result); return; }
+          if (!foundWin) {
+            result.error = 'Window not found';
+          } else {
 
           var winPos = foundWin.position();
           var winSize = foundWin.size();
@@ -423,7 +559,7 @@ export class MacOSPlatform {
 
             if (currentDepth < ${maxDepth}) {
               try {
-                var kids = axElem.elements();
+                var kids = childElements(axElem);
                 for (var k = 0; k < kids.length && elemCount[0] < ${maxElements}; k++) {
                   var child = extractElement(kids[k], currentDepth + 1);
                   if (child) info.children.push(child);
@@ -433,7 +569,8 @@ export class MacOSPlatform {
             return info;
           }
 
-          result.tree = extractElement(foundWin, 0);
+            result.tree = extractElement(foundWin, 0);
+          }
         } catch(e) {
           result.error = String(e.message || e);
         }
@@ -445,7 +582,7 @@ export class MacOSPlatform {
             ], { encoding: "utf-8", timeout: 15000 }).trim();
             const parsed = JSON.parse(out);
             if (parsed.error && !parsed.window) {
-                throw new Error(parsed.error);
+                throw new WindowNotFoundError(resolvedWindowId);
             }
             const windowInfo = parsed.window || {
                 id: resolvedWindowId,
@@ -463,31 +600,48 @@ export class MacOSPlatform {
             };
         }
         catch (error) {
-            if (String(error.message || error).includes("not allowed") ||
-                String(error.message || error).includes("permission") ||
-                String(error.message || error).includes("assistive")) {
-                throw new Error(`Accessibility permission required: grant System Events access in System Preferences > Privacy & Accessibility`);
-            }
-            throw new Error(`Window ${resolvedWindowId} not found or Accessibility permission missing`);
+            if (error instanceof WindowNotFoundError)
+                throw error;
+            rethrowAccessibilityError(error, "get_window_state");
         }
     }
     // ── Mouse ───────────────────────────────────────────────────────────────
     async click(x, y, button, doubleClick) {
-        if (doubleClick) {
-            await inputDoubleClick(x, y, button);
+        try {
+            if (doubleClick) {
+                await inputDoubleClick(x, y, button);
+            }
+            else {
+                await inputClick(x, y, button);
+            }
         }
-        else {
-            await inputClick(x, y, button);
+        catch (error) {
+            rethrowInputError(error, doubleClick ? "double_click" : "click");
         }
     }
     async move(x, y) {
-        await inputMove(x, y);
+        try {
+            await inputMove(x, y);
+        }
+        catch (error) {
+            rethrowInputError(error, "move");
+        }
     }
     async drag(startX, startY, endX, endY, button, duration) {
-        await inputDrag(startX, startY, endX, endY, button, duration);
+        try {
+            await inputDrag(startX, startY, endX, endY, button, duration);
+        }
+        catch (error) {
+            rethrowInputError(error, "drag");
+        }
     }
     async scroll(x, y, deltaX, deltaY) {
-        await inputScroll(x, y, deltaX, deltaY);
+        try {
+            await inputScroll(x, y, deltaX, deltaY);
+        }
+        catch (error) {
+            rethrowInputError(error, "scroll");
+        }
     }
     // ── Cursor ──────────────────────────────────────────────────────────────
     getCursorPosition() {
@@ -502,14 +656,12 @@ export class MacOSPlatform {
             return JSON.parse(out);
         }
         catch (error) {
-            throw new Error(`get_cursor_position failed: ${error.message || error}`);
+            throw new PlatformError(`get_cursor_position failed: ${errorMessage(error)}`);
         }
     }
     // ── OCR ──────────────────────────────────────────────────────────────────
     async ocr(display, region) {
-        // Take a screenshot first (reuse existing logic)
         const buf = await this.screenshot(display, region);
-        // Write screenshot to a temp file so Vision framework can read it
         const { writeFile, unlink } = await import("node:fs/promises");
         const { join } = await import("node:path");
         const { tmpdir } = await import("node:os");
@@ -518,93 +670,46 @@ export class MacOSPlatform {
         try {
             const screenSize = this.getScreenSize(display);
             const scaleFactor = screenSize.scaleFactor ?? 2;
-            // Build JXA script that uses Vision framework for OCR
-            // JXA does not allow return statements at global scope, so we wrap in a function
-            const jxaScript = `
-        function run() {
-          ObjC.import('Vision');
-          ObjC.import('AppKit');
-          ObjC.import('Foundation');
-
-          var app = Application.currentApplication();
-          app.includeStandardAdditions = true;
-
-          var path = "${tmpPath.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/`/g, '\\`').replace(/\$/g, '\\$')}";
-          var url = $.NSURL.fileURLWithPath(path);
-          var image = $.NSImage.alloc.initWithContentsOfURL(url);
-
-          if (!image || image.isValid() === false) {
-            return JSON.stringify({error: "Failed to load screenshot image", elements: [], fullText: ""});
-          }
-
-          var cgImage = image.CGImageForProposedRectContextHints(null, null, null);
-          if (!cgImage) {
-            return JSON.stringify({error: "Failed to get CGImage from screenshot", elements: [], fullText: ""});
-          }
-
-          var request = $.VNRecognizeTextRequest.alloc.init;
-          request.recognitionLevel = $.VNRequestTextRecognitionLevelAccurate;
-          request.usesLanguageCorrection = true;
-
-          var handler = $.VNImageRequestHandler.alloc.initWithCGImageOptions(cgImage, null);
-          var performError = $();
-
-          var success = handler.performRequestsError([request], performError);
-          if (!success) {
-            return JSON.stringify({error: "OCR request failed", elements: [], fullText: ""});
-          }
-
-          var results = request.results;
-          var elements = [];
-          var fullTextParts = [];
-
-          var imgWidth = cgImage.width;
-          var imgHeight = cgImage.height;
-
-          for (var i = 0; i < results.count; i++) {
-            var obs = $(results).objectAtIndex(i);
-            var candidates = obs.topCandidates(1);
-            if (candidates && candidates.count > 0) {
-              var candidate = $(candidates).objectAtIndex(0);
-              var text = candidate.string.toString();
-              var confidence = candidate.confidence;
-              var bbox = obs.boundingBox;
-
-              // Vision boundingBox is normalized (0-1) with origin at bottom-left
-              // Convert to screen coordinates (origin at top-left)
-              var bx = bbox.origin.x * imgWidth;
-              var by = (1 - bbox.origin.y - bbox.size.height) * imgHeight;
-              var bw = bbox.size.width * imgWidth;
-              var bh = bbox.size.height * imgHeight;
-
-              elements.push({
-                text: text,
-                x: Math.round(bx),
-                y: Math.round(by),
-                width: Math.round(bw),
-                height: Math.round(bh),
-                confidence: confidence
-              });
-              fullTextParts.push(text);
+            // Try native Swift OCR helper first (avoids JXA ObjC bridge bugs on macOS Sequoia+)
+            const nativeResult = await this.ocrNative(tmpPath, scaleFactor, region);
+            if (nativeResult)
+                return nativeResult;
+            // Fallback to JXA Vision framework
+            return await this.ocrJxa(tmpPath, screenSize, scaleFactor, region, buf);
+        }
+        finally {
+            await unlink(tmpPath).catch(() => { });
+        }
+    }
+    async ocrNative(tmpPath, scaleFactor, region) {
+        const { existsSync } = await import("node:fs");
+        const { join, dirname } = await import("node:path");
+        const { fileURLToPath } = await import("node:url");
+        // Resolve native binary path (same pattern as input.ts CGEvent helper)
+        const candidates = [
+            join(dirname(fileURLToPath(import.meta.url)), "..", "..", "native", "ocr", "ocr-helper"),
+            join(dirname(fileURLToPath(import.meta.url)), "..", "native", "ocr", "ocr-helper"),
+            join(process.cwd(), "native", "ocr", "ocr-helper"),
+        ];
+        let binaryPath;
+        for (const p of candidates) {
+            if (existsSync(p)) {
+                binaryPath = p;
+                break;
             }
-          }
-
-          return JSON.stringify({elements: elements, fullText: fullTextParts.join("\\n"), error: null});
         }
-        run();
-      `;
-            const out = execFileSync("osascript", [
-                "-l", "JavaScript",
-                "-e", jxaScript,
-            ], { encoding: "utf-8", timeout: 30000 }).trim();
+        if (!binaryPath)
+            return null;
+        try {
+            const input = JSON.stringify({ imagePath: tmpPath });
+            const out = execFileSync(binaryPath, [], {
+                input,
+                encoding: "utf-8",
+                timeout: 30000,
+            }).trim();
             const parsed = JSON.parse(out);
-            if (parsed.error) {
-                throw new Error(parsed.error);
-            }
-            // Scale coordinates from image space to screen space
-            // The screenshot may be taken at a different resolution than screen coordinates
-            const imgWidth = buf.readUInt32BE(16); // PNG width at offset 16
-            const scaleFactorX = screenSize.width / (region ? region.width : (imgWidth / scaleFactor));
+            if (parsed.error)
+                return null;
             const elements = parsed.elements.map((el) => ({
                 text: el.text,
                 x: Math.round(el.x / scaleFactor) + (region ? region.x : 0),
@@ -613,15 +718,81 @@ export class MacOSPlatform {
                 height: Math.round(el.height / scaleFactor),
                 confidence: el.confidence,
             }));
-            return {
-                elements,
-                fullText: parsed.fullText,
-            };
+            return { elements, fullText: parsed.fullText };
         }
-        finally {
-            await unlink(tmpPath).catch(() => { });
+        catch {
+            return null;
         }
     }
+    async ocrJxa(tmpPath, screenSize, scaleFactor, region, buf) {
+        const escapedPath = tmpPath.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/`/g, "\\`").replace(/$/g, "\\$");
+        const jxaScript = `
+      function run() {
+        ObjC.import('Vision');
+        ObjC.import('AppKit');
+        ObjC.import('Foundation');
+        var app = Application.currentApplication();
+        app.includeStandardAdditions = true;
+        var path = "${escapedPath}";
+        var url = $.NSURL.fileURLWithPath(path);
+        var image = $.NSImage.alloc.initWithContentsOfURL(url);
+        if (!image || !image.isValid) {
+          return JSON.stringify({error: "Failed to load screenshot image", elements: [], fullText: ""});
+        }
+        var cgImage = image.CGImageForProposedRectContextHints(null, null, null);
+        if (!cgImage) {
+          return JSON.stringify({error: "Failed to get CGImage from screenshot", elements: [], fullText: ""});
+        }
+        var request = $.VNRecognizeTextRequest.alloc.init;
+        request.recognitionLevel = $.VNRequestTextRecognitionLevelAccurate;
+        request.usesLanguageCorrection = true;
+        var handler = $.VNImageRequestHandler.alloc.initWithCGImageOptions(cgImage, null);
+        var performError = $();
+        var success = handler.performRequestsError([request], performError);
+        if (!success) {
+          return JSON.stringify({error: "OCR request failed", elements: [], fullText: ""});
+        }
+        var results = request.results;
+        var elements = [];
+        var fullTextParts = [];
+        var imgWidth = cgImage.width;
+        var imgHeight = cgImage.height;
+        for (var i = 0; i < results.count; i++) {
+          var obs = $(results).objectAtIndex(i);
+          var candidates = obs.topCandidates(1);
+          if (candidates && candidates.count > 0) {
+            var candidate = $(candidates).objectAtIndex(0);
+            var text = candidate.string.toString();
+            var confidence = candidate.confidence;
+            var bbox = obs.boundingBox;
+            var bx = bbox.origin.x * imgWidth;
+            var by = (1 - bbox.origin.y - bbox.size.height) * imgHeight;
+            var bw = bbox.size.width * imgWidth;
+            var bh = bbox.size.height * imgHeight;
+            elements.push({text:text,x:Math.round(bx),y:Math.round(by),width:Math.round(bw),height:Math.round(bh),confidence:confidence});
+            fullTextParts.push(text);
+          }
+        }
+        return JSON.stringify({elements:elements,fullText:fullTextParts.join("\\n"),error:null});
+      }
+      run();
+    `;
+        const out = execFileSync("osascript", ["-l", "JavaScript", "-e", jxaScript], { encoding: "utf-8", timeout: 30000 }).trim();
+        const parsed = JSON.parse(out);
+        if (parsed.error)
+            throw new CaptureError(`ocr failed: ${parsed.error}`);
+        const imgWidth = buf.readUInt32BE(16);
+        const scaleFactorX = screenSize.width / (region ? region.width : (imgWidth / scaleFactor));
+        const elements = parsed.elements.map((el) => ({
+            text: el.text,
+            x: Math.round(el.x / scaleFactor) + (region ? region.x : 0),
+            y: Math.round(el.y / scaleFactor) + (region ? region.y : 0),
+            width: Math.round(el.width / scaleFactor),
+            height: Math.round(el.height / scaleFactor),
+            confidence: el.confidence,
+        }));
+        return { elements, fullText: parsed.fullText };
+    }
     // ── Keyboard ────────────────────────────────────────────────────────────
     async type(text, delay) {
         await typeText(text, delay);
@@ -632,24 +803,98 @@ export class MacOSPlatform {
     // ── Accessibility (AX) Element Actions ───────────────────────────────────
     async findElement(options) {
         this.evictExpiredCacheEntries();
-        const { text, role, app, depth, includeBounds = true } = options;
+        const { text, role, app, depth, includeBounds = true, textMode = "contains", visibleOnly = false, value } = options;
         const effectiveApp = app || this.activeTarget?.appName;
         const maxDepth = Math.min(depth || 5, 10);
         const maxResults = Math.min(Math.max(options.maxResults ?? 50, 1), 200);
         const escapedApp = (effectiveApp || "").replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/`/g, '\\`').replace(/\$/g, '\\$');
         const escapedText = text ? text.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/`/g, '\\`').replace(/\$/g, '\\$') : "";
         const escapedRole = role ? role.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/`/g, '\\`').replace(/\$/g, '\\$') : "";
+        const escapedValue = value ? value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/`/g, '\\`').replace(/\$/g, '\\$') : "";
+        // Pre-compile regex on TS side to validate syntax before passing to JXA
+        if (text && textMode === "regex") {
+            try {
+                new RegExp(text);
+            }
+            catch {
+                throw new PlatformError(`Invalid regex pattern: ${text}`);
+            }
+        }
+        const startTime = Date.now();
         const jxaScript = `
-      var se = Application('System Events');
-      var results = [];
+        var se = Application('System Events');
+      function childElements(elem) {
+        try { return elem.uiElements(); } catch(e1) {
+          try { return elem.elements(); } catch(e2) { return []; }
+        }
+      }
+        var results = [];
+      var scannedCount = 0;
+      var matchedCount = 0;
       var resultCount = [0];
       var maxResults = ${maxResults};
       var includeBounds = ${includeBounds ? "true" : "false"};
+      var visibleOnly = ${visibleOnly ? "true" : "false"};
+      var textMode = "${textMode}";
 
       var textFilter = ${text ? `"${escapedText}"` : "null"};
       var roleFilter = ${role ? `"${escapedRole}"` : "null"};
+      var valueFilter = ${value ? `"${escapedValue}"` : "null"};
+
+      function isVisible(elem) {
+        try {
+          var pos = elem.position();
+          var sz = elem.size();
+          if (!pos || !sz) return false;
+          return sz[0] > 0 && sz[1] > 0 && pos[0] > -10000 && pos[1] > -10000;
+        } catch(e) {
+          return false;
+        }
+      }
+
+      function textMatches(elemName, elemValue, elemDesc) {
+        if (textFilter === null) return true;
+        var sources = [elemName, elemValue, elemDesc];
+        if (textMode === "exact") {
+          var t = textFilter.toLowerCase();
+          for (var i = 0; i < sources.length; i++) {
+            if (sources[i].toLowerCase() === t) return true;
+          }
+          return false;
+        } else if (textMode === "regex") {
+          try {
+            var re = new RegExp(textFilter, "i");
+            for (var i = 0; i < sources.length; i++) {
+              if (re.test(sources[i])) return true;
+            }
+          } catch(e) {}
+          return false;
+        } else {
+          // contains (default)
+          var t = textFilter.toLowerCase();
+          for (var i = 0; i < sources.length; i++) {
+            if (sources[i].toLowerCase().indexOf(t) !== -1) return true;
+          }
+          return false;
+        }
+      }
+
+      function valueMatches(elemValue) {
+        if (valueFilter === null) return true;
+        if (textMode === "exact") {
+          return elemValue.toLowerCase() === valueFilter.toLowerCase();
+        } else if (textMode === "regex") {
+          try {
+            return new RegExp(valueFilter, "i").test(elemValue);
+          } catch(e) { return false; }
+        } else {
+          // contains (default)
+          return elemValue.toLowerCase().indexOf(valueFilter.toLowerCase()) !== -1;
+        }
+      }
 
       function matches(elem) {
+        scannedCount++;
         var elemName = '';
         var elemRole = '';
         var elemDesc = '';
@@ -659,17 +904,14 @@ export class MacOSPlatform {
         try { elemDesc = elem.description() || ''; } catch(e) {}
         try { var v = elem.value(); elemValue = (v !== undefined && v !== null) ? String(v) : ''; } catch(e) {}
 
-        if (textFilter !== null) {
-          var t = textFilter.toLowerCase();
-          if (elemName.toLowerCase().indexOf(t) === -1 &&
-              elemValue.toLowerCase().indexOf(t) === -1 &&
-              elemDesc.toLowerCase().indexOf(t) === -1) {
-            return false;
-          }
-        }
+        if (visibleOnly && !isVisible(elem)) return false;
+
+        if (!textMatches(elemName, elemValue, elemDesc)) return false;
         if (roleFilter !== null) {
           if (elemRole !== roleFilter) return false;
         }
+        if (!valueMatches(elemValue)) return false;
+        matchedCount++;
         return true;
       }
 
@@ -723,7 +965,7 @@ export class MacOSPlatform {
 
         if (currentDepth < ${maxDepth}) {
           try {
-            var kids = elem.elements();
+            var kids = childElements(elem);
             for (var k = 0; k < kids.length && resultCount[0] < maxResults; k++) {
               traverse(kids[k], path + '/' + k, currentDepth + 1);
             }
@@ -736,7 +978,7 @@ export class MacOSPlatform {
           var proc = se.processes["${escapedApp}"]();
           var wins = proc.windows();
           for (var w = 0; w < wins.length && resultCount[0] < maxResults; w++) {
-            traverse(wins[w], "win" + w, 0);
+            traverse(wins[w], "${escapedApp}/win" + w, 0);
           }
         } else {
           var procs = se.processes();
@@ -752,15 +994,16 @@ export class MacOSPlatform {
         }
       } catch(e) {}
 
-      JSON.stringify(results);
+      JSON.stringify({results: results, scannedCount: scannedCount, matchedCount: matchedCount});
     `;
         try {
             const out = execFileSync("osascript", [
                 "-l", "JavaScript",
                 "-e", jxaScript,
             ], { encoding: "utf-8", timeout: 30000 }).trim();
-            const results = JSON.parse(out);
-            for (const result of results) {
+            const parsed = JSON.parse(out);
+            const durationMs = Date.now() - startTime;
+            for (const result of parsed.results) {
                 const appName = effectiveApp || result.id.split("/")[0] || "";
                 this.elementCache.set(result.id, {
                     elementId: result.id,
@@ -776,15 +1019,35 @@ export class MacOSPlatform {
                 });
             }
             this.evictOverflowCacheEntries();
-            return results;
+            let finalResults = parsed.results;
+            if (options.near) {
+                const nx = options.near.x;
+                const ny = options.near.y;
+                finalResults = [...finalResults].sort((a, b) => {
+                    const acx = (a.bounds?.x ?? 0) + (a.bounds?.width ?? 0) / 2;
+                    const acy = (a.bounds?.y ?? 0) + (a.bounds?.height ?? 0) / 2;
+                    const bcx = (b.bounds?.x ?? 0) + (b.bounds?.width ?? 0) / 2;
+                    const bcy = (b.bounds?.y ?? 0) + (b.bounds?.height ?? 0) / 2;
+                    return Math.hypot(acx - nx, acy - ny) - Math.hypot(bcx - nx, bcy - ny);
+                });
+            }
+            if (typeof options.index === "number") {
+                finalResults = options.index >= 0 && options.index < finalResults.length
+                    ? [finalResults[options.index]]
+                    : [];
+            }
+            return {
+                results: finalResults,
+                metrics: {
+                    scannedCount: parsed.scannedCount,
+                    matchedCount: parsed.matchedCount,
+                    durationMs,
+                    truncated: parsed.results.length >= maxResults,
+                },
+            };
         }
         catch (error) {
-            if (String(error.message || error).includes("not allowed") ||
-                String(error.message || error).includes("permission") ||
-                String(error.message || error).includes("assistive")) {
-                throw new Error("Accessibility permission required: grant System Events access in System Preferences > Privacy & Accessibility");
-            }
-            throw new Error(`find_element failed: ${error.message || error}`);
+            rethrowAccessibilityError(error, "find_element");
         }
     }
     async clickElement(elementId, app) {
@@ -799,6 +1062,11 @@ export class MacOSPlatform {
         const cachedJson = JSON.stringify(this.elementCache.get(elementId) ?? null);
         const jxaScript = `
       var se = Application('System Events');
+      function childElements(elem) {
+        try { return elem.uiElements(); } catch(e1) {
+          try { return elem.elements(); } catch(e2) { return []; }
+        }
+      }
       var elemPath = "${escapedElementId}";
       var appName = "${escapedApp}";
       var cached = ${cachedJson};
@@ -825,7 +1093,7 @@ export class MacOSPlatform {
             var idx = parseInt(parts[i]);
             if (isNaN(idx)) return null;
             try {
-              var kids = current.elements();
+              var kids = childElements(current);
               if (idx >= kids.length) return null;
               current = kids[idx];
             } catch(e) { return null; }
@@ -910,7 +1178,7 @@ export class MacOSPlatform {
             bestScore = score;
           }
           try {
-            var kids = elem.elements();
+            var kids = childElements(elem);
             for (var i = 0; i < kids.length; i++) visit(kids[i], depth + 1);
           } catch(e) {}
         }
@@ -948,7 +1216,7 @@ export class MacOSPlatform {
               var idx = parseInt(parts[i]);
               if (isNaN(idx)) break;
               try {
-                var kids = current.elements();
+                var kids = childElements(current);
                 if (idx >= kids.length) break;
                 current = kids[idx];
               } catch(e) { break; }
@@ -1003,17 +1271,13 @@ export class MacOSPlatform {
             ], { encoding: "utf-8", timeout: 15000 }).trim();
             const result = JSON.parse(out);
             if (!result.success) {
-                throw new Error(result.error || `click_element failed for element ${elementId}`);
+                throw result.error
+                    ? new Error(result.error)
+                    : new ElementNotFoundError(elementId);
             }
         }
         catch (error) {
-            if (error.message && error.message.includes("click_element failed"))
-                throw error;
-            if (String(error.message || error).includes("not allowed") ||
-                String(error.message || error).includes("permission")) {
-                throw new Error("Accessibility permission required: grant System Events access in System Preferences > Privacy & Accessibility");
-            }
-            throw new Error(`click_element failed: ${error.message || error}`);
+            rethrowElementActionError(error, "click_element", elementId);
         }
     }
     async typeInElement(elementId, text, app, clearFirst) {
@@ -1029,6 +1293,11 @@ export class MacOSPlatform {
         const cachedJson = JSON.stringify(this.elementCache.get(elementId) ?? null);
         const jxaScript = `
       var se = Application('System Events');
+      function childElements(elem) {
+        try { return elem.uiElements(); } catch(e1) {
+          try { return elem.elements(); } catch(e2) { return []; }
+        }
+      }
       var elemPath = "${escapedElementId}";
       var appName = "${escapedApp}";
       var textToType = "${escapedText}";
@@ -1057,7 +1326,7 @@ export class MacOSPlatform {
             var idx = parseInt(parts[i]);
             if (isNaN(idx)) return null;
             try {
-              var kids = current.elements();
+              var kids = childElements(current);
               if (idx >= kids.length) return null;
               current = kids[idx];
             } catch(e) { return null; }
@@ -1142,7 +1411,7 @@ export class MacOSPlatform {
             bestScore = score;
           }
           try {
-            var kids = elem.elements();
+            var kids = childElements(elem);
             for (var i = 0; i < kids.length; i++) visit(kids[i], depth + 1);
           } catch(e) {}
         }
@@ -1180,7 +1449,7 @@ export class MacOSPlatform {
               var idx = parseInt(parts[i]);
               if (isNaN(idx)) break;
               try {
-                var kids = current.elements();
+                var kids = childElements(current);
                 if (idx >= kids.length) break;
                 current = kids[idx];
               } catch(e) { break; }
@@ -1245,17 +1514,31 @@ export class MacOSPlatform {
             ], { encoding: "utf-8", timeout: 15000 }).trim();
             const result = JSON.parse(out);
             if (!result.success) {
-                throw new Error(result.error || `type_in_element failed for element ${elementId}`);
+                throw result.error
+                    ? new Error(result.error)
+                    : new ElementNotFoundError(elementId);
             }
         }
         catch (error) {
-            if (error.message && error.message.includes("type_in_element failed"))
-                throw error;
-            if (String(error.message || error).includes("not allowed") ||
-                String(error.message || error).includes("permission")) {
-                throw new Error("Accessibility permission required: grant System Events access in System Preferences > Privacy & Accessibility");
-            }
-            throw new Error(`type_in_element failed: ${error.message || error}`);
+            rethrowElementActionError(error, "type_in_element", elementId);
+        }
+    }
+    // ── Clipboard ───────────────────────────────────────────────────────────
+    async readClipboard() {
+        try {
+            const out = execFileSync("pbpaste", [], { encoding: "utf-8", timeout: 5000 });
+            return out;
+        }
+        catch (error) {
+            throw new PlatformError(`read_clipboard failed: ${errorMessage(error)}`);
+        }
+    }
+    async writeClipboard(text) {
+        try {
+            execFileSync("pbcopy", [], { input: text, encoding: "utf-8", timeout: 5000 });
+        }
+        catch (error) {
+            throw new PlatformError(`write_clipboard failed: ${errorMessage(error)}`);
         }
     }
     async setElementValue(elementId, value, app) {
@@ -1271,6 +1554,11 @@ export class MacOSPlatform {
         const cachedJson = JSON.stringify(this.elementCache.get(elementId) ?? null);
         const jxaScript = `
       var se = Application('System Events');
+      function childElements(elem) {
+        try { return elem.uiElements(); } catch(e1) {
+          try { return elem.elements(); } catch(e2) { return []; }
+        }
+      }
       var elemPath = ${elementIdLiteral};
       var appName = ${appLiteral};
       var valueToSet = ${valueLiteral};
@@ -1296,7 +1584,7 @@ export class MacOSPlatform {
             var idx = parseInt(parts[i]);
             if (isNaN(idx)) return null;
             try {
-              var kids = current.elements();
+              var kids = childElements(current);
               if (idx >= kids.length) return null;
               current = kids[idx];
             } catch(e) { return null; }
@@ -1323,7 +1611,7 @@ export class MacOSPlatform {
             var idx = parseInt(parts[i]);
             if (isNaN(idx)) return null;
             try {
-              var kids = current.elements();
+              var kids = childElements(current);
               if (idx >= kids.length) return null;
               current = kids[idx];
             } catch(e) { return null; }
@@ -1408,7 +1696,7 @@ export class MacOSPlatform {
             bestScore = score;
           }
           try {
-            var kids = elem.elements();
+            var kids = childElements(elem);
             for (var i = 0; i < kids.length; i++) visit(kids[i], depth + 1);
           } catch(e) {}
         }
@@ -1456,7 +1744,9 @@ export class MacOSPlatform {
             ], { encoding: "utf-8", timeout: 15000 }).trim();
             const result = JSON.parse(out);
             if (!result.success) {
-                throw new Error(result.error || `set_value failed for element ${elementId}`);
+                throw result.error
+                    ? new Error(result.error)
+                    : new ElementNotFoundError(elementId);
             }
             const currentCached = this.elementCache.get(elementId);
             if (currentCached) {
@@ -1464,13 +1754,7 @@ export class MacOSPlatform {
             }
         }
         catch (error) {
-            if (error.message && error.message.includes("set_value failed"))
-                throw error;
-            if (String(error.message || error).includes("not allowed") ||
-                String(error.message || error).includes("permission")) {
-                throw new Error("Accessibility permission required: grant System Events access in System Preferences > Privacy & Accessibility");
-            }
-            throw new Error(`set_value failed: ${error.message || error}`);
+            rethrowElementActionError(error, "set_value", elementId);
         }
     }
 }