ucu-mcp 0.4.0 → 0.4.3

package/dist/src/platform/macos/element.js

@@ -0,0 +1,212 @@
+import { execFileSync } from "node:child_process";
+import { ElementNotFoundError } from "../../util/errors.js";
+import { rethrowElementActionError } from "./helpers.js";
+import { jxaElementActionHelpers } from "../jxa-helpers.js";
+function prepareCache(elementId) {
+    this.evictExpiredCacheEntries();
+    const cached = this.elementCache.get(elementId);
+    if (cached && this.isCacheEntryExpired(cached)) {
+        this.elementCache.delete(elementId);
+    }
+    return this.elementCache.get(elementId) ?? null;
+}
+export async function clickElement(elementId, app) {
+    const elementIdLiteral = JSON.stringify(elementId);
+    const effectiveApp = app || this.activeTarget?.appName;
+    const appLiteral = JSON.stringify(effectiveApp || "");
+    const cachedDescriptor = prepareCache.call(this, elementId);
+    const cachedJson = JSON.stringify(cachedDescriptor);
+    const jxaScript = `
+    var se = Application('System Events');
+    var _result = null;
+    ${jxaElementActionHelpers()}
+    var elemPath = ${elementIdLiteral};
+    var appName = ${appLiteral};
+    var cached = ${cachedJson};
+
+    var elem = resolveElementInApp(elemPath, appName) || resolveElementByFullPath(elemPath);
+    if (elem && !descriptorMatches(elem)) {
+      elem = refetchEquivalent() || elem;
+    }
+    if (!elem) {
+      elem = refetchEquivalent();
+    }
+
+    if (!elem) {
+      _result = {success: false, error: "Element not found: " + elemPath};
+    } else {
+      try {
+        elem.actions.AXPress.perform();
+        _result = {success: true};
+      } catch(e) {
+        try {
+          var pos = elem.position();
+          var sz = elem.size();
+          var cx = pos[0] + sz[0] / 2;
+          var cy = pos[1] + sz[1] / 2;
+          ObjC.import('CoreGraphics');
+          var src = $.CGEventSourceCreate($.kCGEventSourceStateHIDSystemState);
+          var pt = $.CGPointMake(cx, cy);
+          var down = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseDown, pt, $.kCGMouseButtonLeft);
+          $.CGEventPost($.kCGHIDEventTap, down);
+          var up = $.CGEventCreateMouseEvent(src, $.kCGEventLeftMouseUp, pt, $.kCGMouseButtonLeft);
+          $.CGEventPost($.kCGHIDEventTap, up);
+          _result = {success: true};
+        } catch(e2) {
+          _result = {success: false, error: "Could not click element: " + String(e2.message || e2)};
+        }
+      }
+    }
+    JSON.stringify(_result);
+  `;
+    try {
+        const out = execFileSync("osascript", [
+            "-l", "JavaScript",
+            "-e", jxaScript,
+        ], { encoding: "utf-8", timeout: 15000 }).trim();
+        const result = JSON.parse(out);
+        if (!result.success) {
+            throw result.error
+                ? new Error(result.error)
+                : new ElementNotFoundError(elementId);
+        }
+    }
+    catch (error) {
+        rethrowElementActionError(error, "click_element", elementId);
+    }
+}
+export async function typeInElement(elementId, text, app, clearFirst) {
+    const textLiteral = JSON.stringify(text);
+    const effectiveApp = app || this.activeTarget?.appName;
+    const appLiteral = JSON.stringify(effectiveApp || "");
+    const elementIdLiteral = JSON.stringify(elementId);
+    const cachedDescriptor = prepareCache.call(this, elementId);
+    const cachedJson = JSON.stringify(cachedDescriptor);
+    const jxaScript = `
+    var se = Application('System Events');
+    var _result = null;
+    ${jxaElementActionHelpers()}
+    var elemPath = ${elementIdLiteral};
+    var appName = ${appLiteral};
+    var textToType = ${textLiteral};
+    var shouldClear = ${clearFirst ? "true" : "false"};
+    var cached = ${cachedJson};
+
+    var elem = resolveElementInApp(elemPath, appName) || resolveElementByFullPath(elemPath);
+    if (elem && !descriptorMatches(elem)) {
+      elem = refetchEquivalent() || elem;
+    }
+    if (!elem) {
+      elem = refetchEquivalent();
+    }
+
+    if (!elem) {
+      _result = {success: false, error: "Element not found: " + elemPath};
+    } else {
+      try {
+        elem.focused = true;
+      } catch(e) {}
+
+      if (shouldClear) {
+        try {
+          elem.value = "";
+        } catch(e) {
+          try {
+            se.keystroke("a", {command: true});
+            se.keyDown("delete");
+            se.keyUp("delete");
+          } catch(e2) {}
+        }
+      }
+
+      var didSet = false;
+      try {
+        elem.value = textToType;
+        didSet = true;
+      } catch(e) {}
+
+      if (!didSet) {
+        try {
+          se.keystroke(textToType);
+          _result = {success: true};
+        } catch(e) {
+          _result = {success: false, error: "Could not type into element: " + String(e.message || e)};
+        }
+      } else {
+        _result = {success: true};
+      }
+    }
+    JSON.stringify(_result);
+  `;
+    try {
+        const out = execFileSync("osascript", [
+            "-l", "JavaScript",
+            "-e", jxaScript,
+        ], { encoding: "utf-8", timeout: 15000 }).trim();
+        const result = JSON.parse(out);
+        if (!result.success) {
+            throw result.error
+                ? new Error(result.error)
+                : new ElementNotFoundError(elementId);
+        }
+    }
+    catch (error) {
+        rethrowElementActionError(error, "type_in_element", elementId);
+    }
+}
+export async function setElementValue(elementId, value, app) {
+    const effectiveApp = app || this.activeTarget?.appName;
+    const valueLiteral = JSON.stringify(value);
+    const appLiteral = JSON.stringify(effectiveApp || "");
+    const elementIdLiteral = JSON.stringify(elementId);
+    const cachedDescriptor = prepareCache.call(this, elementId);
+    const cachedJson = JSON.stringify(cachedDescriptor);
+    const jxaScript = `
+    var se = Application('System Events');
+    var _result = null;
+    ${jxaElementActionHelpers()}
+    var elemPath = ${elementIdLiteral};
+    var appName = ${appLiteral};
+    var valueToSet = ${valueLiteral};
+    var cached = ${cachedJson};
+
+    var elem = resolveElementInApp(elemPath, appName) || resolveElementByFullPath(elemPath);
+    if (elem && !descriptorMatches(elem)) {
+      elem = refetchEquivalent() || elem;
+    }
+    if (!elem) {
+      elem = refetchEquivalent();
+    }
+
+    if (!elem) {
+      _result = {success: false, error: "Element not found: " + elemPath};
+    } else {
+      try {
+        elem.value = valueToSet;
+        _result = {success: true};
+      } catch(e) {
+        _result = {success: false, error: "Could not set AX value: " + String(e.message || e)};
+      }
+    }
+    JSON.stringify(_result);
+  `;
+    try {
+        const out = execFileSync("osascript", [
+            "-l", "JavaScript",
+            "-e", jxaScript,
+        ], { encoding: "utf-8", timeout: 15000 }).trim();
+        const result = JSON.parse(out);
+        if (!result.success) {
+            throw result.error
+                ? new Error(result.error)
+                : new ElementNotFoundError(elementId);
+        }
+        const currentCached = this.elementCache.get(elementId);
+        if (currentCached) {
+            this.elementCache.set(elementId, { ...currentCached, value, cachedAt: Date.now() });
+        }
+    }
+    catch (error) {
+        rethrowElementActionError(error, "set_value", elementId);
+    }
+}

package/dist/src/platform/macos/focus.d.ts

@@ -0,0 +1,3 @@
+import type { MacOSPlatform } from "./base.js";
+export declare function saveFocus(this: MacOSPlatform): Promise<void>;
+export declare function restoreFocus(this: MacOSPlatform): Promise<void>;

package/dist/src/platform/macos/focus.js

@@ -0,0 +1,33 @@
+import { execFileSync } from "node:child_process";
+export async function saveFocus() {
+    try {
+        const apps = await this.listApps();
+        const front = apps.find((a) => a.isFrontmost);
+        if (front) {
+            const windows = await this.listWindows();
+            const win = windows.find((w) => w.processName === front.name && w.isOnScreen);
+            this.savedFocus = {
+                appName: front.name,
+                windowTitle: win?.title ?? "",
+            };
+        }
+    }
+    catch {
+        this.savedFocus = undefined;
+    }
+}
+export async function restoreFocus() {
+    if (!this.savedFocus)
+        return;
+    try {
+        const { appName } = this.savedFocus;
+        const appNameLiteral = JSON.stringify(appName);
+        execFileSync("osascript", [
+            "-e", `tell application ${appNameLiteral} to activate`,
+        ], { timeout: 5000 });
+    }
+    catch {
+        // Best effort — don't fail the action if restore fails
+    }
+    this.savedFocus = undefined;
+}

package/dist/src/platform/macos/helpers.d.ts

@@ -0,0 +1,35 @@
+export declare function errorMessage(error: unknown): string;
+export declare function isAccessibilityPermissionError(error: unknown): boolean;
+export declare function rethrowCaptureError(error: unknown, operation: string): never;
+export declare function rethrowAccessibilityError(error: unknown, operation: string): never;
+export declare function rethrowElementActionError(error: unknown, operation: string, elementId: string): never;
+export declare function rethrowInputError(error: unknown, operation: string): never;
+export declare function normalizeAppName(name: string): string;
+export declare function appNameMatches(processName: string, requestedApp: string): boolean;
+export declare function selectWindowForApp(windows: import("../base.js").WindowInfo[], requestedApp: string): import("../base.js").WindowInfo | undefined;
+export interface MacOSPlatformOptions {
+    /**
+     * Override native helper resolution.
+     * - Map of folder name to absolute binary path to inject a specific helper.
+     * - Set a value to null to skip that helper (force JXA fallback).
+     * Used by tests to control native helper behavior without filesystem tricks.
+     */
+    nativeHelperPaths?: Record<string, string | null>;
+}
+export interface CachedElementDescriptor {
+    elementId: string;
+    appName: string;
+    role: string;
+    name: string;
+    value?: string;
+    description?: string;
+    subrole?: string;
+    identifier?: string;
+    bounds?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    cachedAt: number;
+}

package/dist/src/platform/macos/helpers.js

@@ -0,0 +1,54 @@
+import { UcuError, CaptureError, PermissionError, PlatformError, InputSynthesisError, ElementNotFoundError } from "../../util/errors.js";
+export function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+export function isAccessibilityPermissionError(error) {
+    return /not allowed|permission|assistive|accessibility/i.test(errorMessage(error));
+}
+export function rethrowCaptureError(error, operation) {
+    if (error instanceof UcuError)
+        throw error;
+    throw new CaptureError(`${operation} failed: ${errorMessage(error)}`);
+}
+export function rethrowAccessibilityError(error, operation) {
+    if (error instanceof UcuError)
+        throw error;
+    if (isAccessibilityPermissionError(error)) {
+        throw new PermissionError("accessibility", "darwin");
+    }
+    throw new PlatformError(`${operation} failed: ${errorMessage(error)}`);
+}
+export function rethrowElementActionError(error, operation, elementId) {
+    if (error instanceof UcuError)
+        throw error;
+    if (isAccessibilityPermissionError(error)) {
+        throw new PermissionError("accessibility", "darwin");
+    }
+    if (/element not found/i.test(errorMessage(error))) {
+        throw new ElementNotFoundError(elementId);
+    }
+    throw new PlatformError(`${operation} failed: ${errorMessage(error)}`);
+}
+export function rethrowInputError(error, operation) {
+    if (error instanceof UcuError)
+        throw error;
+    throw new InputSynthesisError(`${operation} failed: ${errorMessage(error)}`);
+}
+export function normalizeAppName(name) {
+    return name.trim().toLowerCase();
+}
+export function appNameMatches(processName, requestedApp) {
+    const process = normalizeAppName(processName);
+    const requested = normalizeAppName(requestedApp);
+    if (!process || !requested)
+        return false;
+    return process === requested ||
+        process.startsWith(`${requested} `) ||
+        process.startsWith(`${requested}-`) ||
+        process.includes(` ${requested} `);
+}
+export function selectWindowForApp(windows, requestedApp) {
+    const requested = normalizeAppName(requestedApp);
+    return windows.find((window) => normalizeAppName(window.processName) === requested) ??
+        windows.find((window) => appNameMatches(window.processName, requestedApp));
+}

package/dist/src/platform/macos/index.d.ts

@@ -0,0 +1,2 @@
+export { MacOSPlatform } from "./base.js";
+export type { MacOSPlatformOptions } from "./base.js";

package/dist/src/platform/macos/index.js

@@ -0,0 +1 @@
+export { MacOSPlatform } from "./base.js";

package/dist/src/platform/macos/input.d.ts

@@ -0,0 +1,9 @@
+import type { MacOSPlatform } from "./base.js";
+import type { CursorPosition } from "../base.js";
+export declare function click(this: MacOSPlatform, x: number, y: number, button?: "left" | "right" | "middle", doubleClick?: boolean): Promise<void>;
+export declare function move(this: MacOSPlatform, x: number, y: number): Promise<void>;
+export declare function drag(this: MacOSPlatform, startX: number, startY: number, endX: number, endY: number, button?: "left" | "right" | "middle", duration?: number): Promise<void>;
+export declare function scroll(this: MacOSPlatform, x: number, y: number, deltaX: number, deltaY: number): Promise<void>;
+export declare function getCursorPosition(this: MacOSPlatform): CursorPosition;
+export declare function type(this: MacOSPlatform, text: string, delay?: number): Promise<void>;
+export declare function key(this: MacOSPlatform, keys: string[]): Promise<void>;

package/dist/src/platform/macos/input.js

@@ -0,0 +1,62 @@
+import { execFileSync } from "node:child_process";
+import { click as inputClick, doubleClick as inputDoubleClick, move as inputMove, drag as inputDrag, scroll as inputScroll, typeText, pressShortcut } from "../../utils/input.js";
+import { PlatformError } from "../../util/errors.js";
+import { rethrowInputError, errorMessage } from "./helpers.js";
+export async function click(x, y, button, doubleClick) {
+    try {
+        if (doubleClick) {
+            await inputDoubleClick(x, y, button);
+        }
+        else {
+            await inputClick(x, y, button);
+        }
+    }
+    catch (error) {
+        rethrowInputError(error, doubleClick ? "double_click" : "click");
+    }
+}
+export async function move(x, y) {
+    try {
+        await inputMove(x, y);
+    }
+    catch (error) {
+        rethrowInputError(error, "move");
+    }
+}
+export async function drag(startX, startY, endX, endY, button, duration) {
+    try {
+        await inputDrag(startX, startY, endX, endY, button, duration);
+    }
+    catch (error) {
+        rethrowInputError(error, "drag");
+    }
+}
+export async function scroll(x, y, deltaX, deltaY) {
+    try {
+        await inputScroll(x, y, deltaX, deltaY);
+    }
+    catch (error) {
+        rethrowInputError(error, "scroll");
+    }
+}
+export function getCursorPosition() {
+    try {
+        const out = execFileSync("osascript", [
+            "-l", "JavaScript",
+            "-e",
+            `ObjC.import('AppKit');
+      var pt = $.NSEvent.mouseLocation;
+      JSON.stringify({x:Math.round(pt.x),y:Math.round($.NSScreen.mainScreen.frame.size.height - pt.y)});`,
+        ], { encoding: "utf-8", timeout: 5000 }).trim();
+        return JSON.parse(out);
+    }
+    catch (error) {
+        throw new PlatformError(`get_cursor_position failed: ${errorMessage(error)}`);
+    }
+}
+export async function type(text, delay) {
+    await typeText(text, delay);
+}
+export async function key(keys) {
+    await pressShortcut(keys);
+}

package/dist/src/platform/macos/screen.d.ts

@@ -0,0 +1,7 @@
+import type { MacOSPlatform } from "./base.js";
+import type { ScreenRegion, ScreenSize, ScreenshotOptions, OcrResult } from "../base.js";
+export declare function screenshot(this: MacOSPlatform, _display?: number, region?: ScreenRegion, options?: ScreenshotOptions): Promise<Buffer>;
+export declare function screenshotWindow(this: MacOSPlatform, windowId: string, options?: ScreenshotOptions): Promise<Buffer>;
+export declare function getScreenSize(this: MacOSPlatform, display?: number): ScreenSize;
+export declare function isScreenLocked(this: MacOSPlatform): boolean;
+export declare function ocr(this: MacOSPlatform, display?: number, region?: ScreenRegion): Promise<OcrResult>;

package/dist/src/platform/macos/screen.js

@@ -0,0 +1,197 @@
+import { execFileSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { captureFullScreen, captureRegion } from "../../utils/screenshot.js";
+import { WindowNotFoundError, CaptureError } from "../../util/errors.js";
+import { logger } from "../../util/logger.js";
+import { rethrowCaptureError, errorMessage } from "./helpers.js";
+export async function screenshot(_display, region, options) {
+    try {
+        const base64 = region
+            ? await captureRegion(region.x, region.y, region.width, region.height, options)
+            : await captureFullScreen(options);
+        return Buffer.from(base64, "base64");
+    }
+    catch (error) {
+        rethrowCaptureError(error, region ? "capture region" : "capture full screen");
+    }
+}
+export async function screenshotWindow(windowId, options) {
+    const win = (await this.listWindows(true)).find((w) => w.id === windowId);
+    if (!win) {
+        throw new WindowNotFoundError(windowId);
+    }
+    return this.screenshot(undefined, win.bounds, options);
+}
+export function getScreenSize(display) {
+    try {
+        const idx = display ?? 0;
+        const out = execFileSync("osascript", [
+            "-l", "JavaScript",
+            "-e",
+            `ObjC.import('AppKit');
+      var screens = $.NSScreen.screens;
+      var idx = ${idx};
+      if (idx < 0 || idx >= screens.count) idx = 0;
+      var screen = $(screens).objectAtIndex(idx);
+      var frame = screen.frame;
+      var scaleFactor = screen.backingScaleFactor;
+      JSON.stringify({width:Math.round(frame.size.width),height:Math.round(frame.size.height),scaleFactor:scaleFactor})`,
+        ], { encoding: "utf-8", timeout: 5000 }).trim();
+        return JSON.parse(out);
+    }
+    catch (error) {
+        logger.warn("getScreenSize failed, using fallback", { error: errorMessage(error) });
+        return { width: 1920, height: 1080, scaleFactor: 2, estimated: true };
+    }
+}
+export function isScreenLocked() {
+    try {
+        const out = execFileSync("/usr/sbin/ioreg", ["-n", "Root", "-d1"], {
+            encoding: "utf-8",
+            timeout: 5000,
+        });
+        return /"IOConsoleLocked"\s*=\s*Yes/.test(out);
+    }
+    catch {
+        // Fail-closed: if we can't determine lock state, assume locked
+        logger.warn("isScreenLocked check failed, assuming locked");
+        return true;
+    }
+}
+export async function ocr(display, region) {
+    const buf = await this.screenshot(display, region);
+    const { writeFile, unlink } = await import("node:fs/promises");
+    const { join } = await import("node:path");
+    const { tmpdir } = await import("node:os");
+    const tmpPath = join(tmpdir(), `ucu-ocr-${randomUUID()}.png`);
+    await writeFile(tmpPath, buf);
+    try {
+        const screenSize = this.getScreenSize(display);
+        const scaleFactor = screenSize.scaleFactor ?? 2;
+        const nativeResult = await ocrNative(tmpPath, scaleFactor, region);
+        if (nativeResult)
+            return nativeResult;
+        return await ocrJxa(tmpPath, screenSize, scaleFactor, region, buf);
+    }
+    finally {
+        await unlink(tmpPath).catch(() => { });
+    }
+}
+async function ocrNative(tmpPath, scaleFactor, region) {
+    const screenDirname = dirname(fileURLToPath(import.meta.url));
+    const candidates = [
+        join(screenDirname, "..", "..", "..", "native", "ocr", "ocr-helper"),
+        join(screenDirname, "..", "..", "native", "ocr", "ocr-helper"),
+        join(process.cwd(), "native", "ocr", "ocr-helper"),
+    ];
+    let binaryPath;
+    for (const p of candidates) {
+        if (existsSync(p)) {
+            binaryPath = p;
+            break;
+        }
+    }
+    if (!binaryPath)
+        return null;
+    try {
+        const input = JSON.stringify({ imagePath: tmpPath });
+        const out = execFileSync(binaryPath, [], {
+            input,
+            encoding: "utf-8",
+            timeout: 30000,
+        }).trim();
+        const parsed = JSON.parse(out);
+        if (parsed.error)
+            return null;
+        const elements = parsed.elements.map((el) => ({
+            text: el.text,
+            x: Math.round(el.x / scaleFactor) + (region ? region.x : 0),
+            y: Math.round(el.y / scaleFactor) + (region ? region.y : 0),
+            width: Math.round(el.width / scaleFactor),
+            height: Math.round(el.height / scaleFactor),
+            confidence: el.confidence,
+        }));
+        return { elements, fullText: parsed.fullText };
+    }
+    catch {
+        return null;
+    }
+}
+async function ocrJxa(tmpPath, screenSize, scaleFactor, region, buf) {
+    const pathLiteral = JSON.stringify(tmpPath);
+    const jxaScript = `
+    function run() {
+      ObjC.import('Vision');
+      ObjC.import('AppKit');
+      ObjC.import('Foundation');
+      var app = Application.currentApplication();
+      app.includeStandardAdditions = true;
+      var path = ${pathLiteral};
+      var url = $.NSURL.fileURLWithPath(path);
+      var image = $.NSImage.alloc.initWithContentsOfURL(url);
+      if (!image || !image.isValid) {
+        return JSON.stringify({error: "Failed to load screenshot image", elements: [], fullText: ""});
+      }
+      var cgImage = image.CGImageForProposedRectContextHints(null, null, null);
+      if (!cgImage) {
+        return JSON.stringify({error: "Failed to get CGImage from screenshot", elements: [], fullText: ""});
+      }
+      var request = $.VNRecognizeTextRequest.alloc.init;
+      request.recognitionLevel = $.VNRequestTextRecognitionLevelAccurate;
+      request.usesLanguageCorrection = true;
+      var handler = $.VNImageRequestHandler.alloc.initWithCGImageOptions(cgImage, null);
+      var performError = $();
+      var success = handler.performRequestsError([request], performError);
+      if (!success) {
+        return JSON.stringify({error: "OCR request failed", elements: [], fullText: ""});
+      }
+      var results = request.results;
+      var elements = [];
+      var fullTextParts = [];
+      var imgWidth = cgImage.width;
+      var imgHeight = cgImage.height;
+      for (var i = 0; i < results.count; i++) {
+        var obs = $(results).objectAtIndex(i);
+        var candidates = obs.topCandidates(1);
+        if (candidates && candidates.count > 0) {
+          var candidate = $(candidates).objectAtIndex(0);
+          var text = candidate.string.toString();
+          var confidence = candidate.confidence;
+          var bbox = obs.boundingBox;
+          var bx = bbox.origin.x * imgWidth;
+          var by = (1 - bbox.origin.y - bbox.size.height) * imgHeight;
+          var bw = bbox.size.width * imgWidth;
+          var bh = bbox.size.height * imgHeight;
+          elements.push({text:text,x:Math.round(bx),y:Math.round(by),width:Math.round(bw),height:Math.round(bh),confidence:confidence});
+          fullTextParts.push(text);
+        }
+      }
+      return JSON.stringify({elements:elements,fullText:fullTextParts.join("\\n"),error:null});
+    }
+    run();
+  `;
+    const out = execFileSync("osascript", ["-l", "JavaScript", "-e", jxaScript], { encoding: "utf-8", timeout: 30000 }).trim();
+    const parsed = JSON.parse(out);
+    if (parsed.error) {
+        const hint = parsed.error === "Failed to load screenshot image"
+            ? " (the screenshot file is empty or unreadable — Screen Recording permission is most likely missing; run `doctor` and grant Screen Recording to the host terminal, then retry)"
+            : parsed.error === "Failed to get CGImage from screenshot"
+                ? " (the screenshot could not be decoded — likely an empty capture; check Screen Recording permission)"
+                : "";
+        throw new CaptureError(`ocr failed: ${parsed.error}${hint}`);
+    }
+    const imgWidth = buf.readUInt32BE(16);
+    const scaleFactorX = screenSize.width / (region ? region.width : (imgWidth / scaleFactor));
+    const elements = parsed.elements.map((el) => ({
+        text: el.text,
+        x: Math.round(el.x / scaleFactor) + (region ? region.x : 0),
+        y: Math.round(el.y / scaleFactor) + (region ? region.y : 0),
+        width: Math.round(el.width / scaleFactor),
+        height: Math.round(el.height / scaleFactor),
+        confidence: el.confidence,
+    }));
+    return { elements, fullText: parsed.fullText };
+}

package/dist/src/platform/macos/window.d.ts

@@ -0,0 +1,6 @@
+import type { MacOSPlatform } from "./base.js";
+import type { AppInfo, AppTarget, WindowInfo, BrowserContext } from "../base.js";
+export declare function listApps(this: MacOSPlatform): Promise<AppInfo[]>;
+export declare function focusApp(this: MacOSPlatform, app: string): Promise<AppTarget>;
+export declare function getActiveBrowserContext(this: MacOSPlatform, app?: string): Promise<BrowserContext | undefined>;
+export declare function listWindows(this: MacOSPlatform, _includeMinimized?: boolean): Promise<WindowInfo[]>;