ucu-mcp 0.4.0 → 0.4.2

package/dist/src/mcp/tools.js

@@ -37,11 +37,15 @@ const captureAfterFields = {
     captureMaxWidth: z.number().default(1280).describe("Maximum width for the post-action screenshot"),
     captureFormat: z.enum(["png", "jpeg"]).default("jpeg").describe("Format for the post-action screenshot"),
 };
-// Exported so unit tests can pin the schema constraint directly instead
-// of going through the McpServer wrapper (which `handler()` calls
-// bypass). (Herschel review Major: 0.3.5's value='' test was a
-// tautology because it re-created a local zod schema instead of
-// asserting against this one.)
+/**
+ * Exported so unit tests can pin the schema constraint directly instead
+ * of going through the McpServer wrapper (which `handler()` calls
+ * bypass). (Herschel review Major: 0.3.5's value='' test was a
+ * tautology because it re-created a local zod schema instead of
+ * asserting against this one.)
+ *
+ * @internal Not part of the public API — may change without a semver bump.
+ */
 export const findElementInputSchema = {
     text: z.string().optional().describe("Text to search"),
     role: z.string().optional().describe("AX role"),
@@ -63,6 +67,41 @@ async function resolvePoint(x, y, windowId) {
         throw new WindowNotFoundError(windowId);
     return { x: win.bounds.x + x, y: win.bounds.y + y };
 }
+/**
+ * Build safety context (window title + URL) for the current active target.
+ *
+ * The SafetyGuard's window-skip and URL blocklist checks are dead code unless
+ * tool handlers pass `windowTitle` / `url` in the `params` object. This helper
+ * resolves both from the active target so each action tool can spread the
+ * result into its `withSafety` call with minimal overhead.
+ */
+async function getSafetyContext(windowId) {
+    const target = activeTargetContext;
+    const effectiveWindowId = windowId ?? target?.windowId;
+    let windowTitle;
+    if (effectiveWindowId) {
+        try {
+            const windows = await getPlatform().listWindows();
+            const win = windows.find(w => w.id === effectiveWindowId);
+            windowTitle = win?.title;
+        }
+        catch { /* best effort */ }
+    }
+    if (!windowTitle && target?.title) {
+        windowTitle = target.title;
+    }
+    let url;
+    const platform = getPlatform();
+    if (platform.getActiveBrowserContext) {
+        try {
+            const appName = target?.appName;
+            const ctx = await platform.getActiveBrowserContext(appName);
+            url = ctx?.url;
+        }
+        catch { /* best effort */ }
+    }
+    return { windowTitle, url };
+}
 function jsonText(value) {
     return { type: "text", text: JSON.stringify(value, null, 2) };
 }
@@ -97,7 +136,7 @@ function errorDetails(error) {
     // Some platform errors carry an inline `hint` field (added by macos.ts focusApp
     // for the Electron AX case, etc.). Surface it under `hint` so the model can
     // see remediation without parsing the message string.
-    const inlineHint = err.hint;
+    const inlineHint = err instanceof UcuError ? err.hint : undefined;
     const details = {
         name: err.name,
         code,
@@ -105,7 +144,7 @@ function errorDetails(error) {
         message: err.message,
         recovery: recoveryHint(code),
     };
-    if (typeof inlineHint === "string" && inlineHint.length > 0) {
+    if (inlineHint) {
         details.hint = inlineHint;
     }
     return details;
@@ -304,7 +343,6 @@ export function registerTools(server) {
         if (windows.length === 0) {
             let accessibility = "unknown";
             try {
-                const { checkPermission } = await import("../safety/permissions.js");
                 const { granted } = await checkPermission("accessibility");
                 accessibility = granted ? "granted" : "denied";
             }
@@ -347,7 +385,8 @@ export function registerTools(server) {
         ...captureAfterFields,
     }, async (params) => {
         const pt = await resolvePoint(params.x, params.y, params.windowId);
-        await withSafety({ action: "click", params: { x: pt.x, y: pt.y }, requiresAccessibility: true, execute: () => getPlatform().click(pt.x, pt.y, params.button) });
+        const safetyCtx = await getSafetyContext(params.windowId);
+        await withSafety({ action: "click", params: { x: pt.x, y: pt.y, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().click(pt.x, pt.y, params.button) });
         return actionResponse("click", { clicked: true, x: pt.x, y: pt.y }, { x: pt.x, y: pt.y, windowId: params.windowId }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("click");
@@ -358,7 +397,8 @@ export function registerTools(server) {
         ...captureAfterFields,
     }, async (params) => {
         const pt = await resolvePoint(params.x, params.y, params.windowId);
-        await withSafety({ action: "click", params: { x: pt.x, y: pt.y, doubleClick: true }, requiresAccessibility: true, execute: () => getPlatform().click(pt.x, pt.y, params.button, true) });
+        const safetyCtx = await getSafetyContext(params.windowId);
+        await withSafety({ action: "click", params: { x: pt.x, y: pt.y, doubleClick: true, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().click(pt.x, pt.y, params.button, true) });
         return actionResponse("double_click", { doubleClicked: true, x: pt.x, y: pt.y }, { x: pt.x, y: pt.y, windowId: params.windowId }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("double_click");
@@ -369,7 +409,8 @@ export function registerTools(server) {
     }, async (params) => {
         if (params.windowId)
             throw new UnsupportedParameterError("windowId-targeted keyboard typing is not implemented");
-        await withSafety({ action: "type_text", params: { text: params.text }, requiresAccessibility: true, execute: () => getPlatform().type(params.text, params.delay) });
+        const safetyCtx = await getSafetyContext();
+        await withSafety({ action: "type_text", params: { text: params.text, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().type(params.text, params.delay) });
         return actionResponse("type_text", { typed: true, charCount: params.text.length }, {}, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("type_text");
@@ -388,7 +429,8 @@ export function registerTools(server) {
         ];
         if (keys.length === 0)
             throw new UnsupportedParameterError("press_key requires at least one key");
-        await withSafety({ action: "press_key", params: { keys }, requiresAccessibility: true, execute: () => getPlatform().key(keys) });
+        const safetyCtx = await getSafetyContext();
+        await withSafety({ action: "press_key", params: { keys, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().key(keys) });
         return actionResponse("press_key", { pressed: true, keys }, {}, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("press_key");
@@ -400,7 +442,8 @@ export function registerTools(server) {
     }, async (params) => {
         const pt = await resolvePoint(params.x, params.y, params.windowId);
         const deltaX = params.deltaX ?? 0;
-        await withSafety({ action: "scroll", params: { x: pt.x, y: pt.y }, requiresAccessibility: true, execute: () => getPlatform().scroll(pt.x, pt.y, deltaX, params.deltaY) });
+        const safetyCtx = await getSafetyContext(params.windowId);
+        await withSafety({ action: "scroll", params: { x: pt.x, y: pt.y, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().scroll(pt.x, pt.y, deltaX, params.deltaY) });
         return actionResponse("scroll", { scrolled: true, x: pt.x, y: pt.y }, { x: pt.x, y: pt.y, windowId: params.windowId }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("scroll");
@@ -414,7 +457,8 @@ export function registerTools(server) {
     }, async (params) => {
         const start = await resolvePoint(params.startX, params.startY, params.windowId);
         const end = await resolvePoint(params.endX, params.endY, params.windowId);
-        await withSafety({ action: "drag", params: { startX: start.x, startY: start.y, endX: end.x, endY: end.y }, requiresAccessibility: true, execute: () => getPlatform().drag(start.x, start.y, end.x, end.y, params.button, params.duration) });
+        const safetyCtx = await getSafetyContext(params.windowId);
+        await withSafety({ action: "drag", params: { startX: start.x, startY: start.y, endX: end.x, endY: end.y, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().drag(start.x, start.y, end.x, end.y, params.button, params.duration) });
         return actionResponse("drag", { dragged: true, startX: start.x, startY: start.y, endX: end.x, endY: end.y }, { startX: start.x, startY: start.y, endX: end.x, endY: end.y, windowId: params.windowId }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("drag");
@@ -479,9 +523,9 @@ export function registerTools(server) {
             const ocr = resolveHelperPath(["native", "ocr", "ocr-helper"]);
             const windowlist = resolveHelperPath(["native", "windowlist", "windowlist-helper"]);
             nativeHelpers = {
-                cgevent: { ok: cgevent.path !== null, path: cgevent.path, tried: cgevent.tried.slice(0, 3) },
-                ocr: { ok: ocr.path !== null, path: ocr.path, tried: ocr.tried.slice(0, 3) },
-                windowlist: { ok: windowlist.path !== null, path: windowlist.path, tried: windowlist.tried.slice(0, 3) },
+                cgevent: { ok: cgevent.path !== null, path: cgevent.path, tried: cgevent.tried },
+                ocr: { ok: ocr.path !== null, path: ocr.path, tried: ocr.tried },
+                windowlist: { ok: windowlist.path !== null, path: windowlist.path, tried: windowlist.tried },
             };
         }
         let readiness = "ready";
@@ -658,22 +702,40 @@ export function registerTools(server) {
         ...captureAfterFields,
     }, async (params) => {
         const pt = await resolvePoint(params.x, params.y, params.windowId);
-        await withSafety({ action: "move", params: { x: pt.x, y: pt.y }, requiresAccessibility: true, execute: () => getPlatform().move(pt.x, pt.y) });
+        const safetyCtx = await getSafetyContext(params.windowId);
+        await withSafety({ action: "move", params: { x: pt.x, y: pt.y, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().move(pt.x, pt.y) });
         return actionResponse("move", { moved: true, x: pt.x, y: pt.y }, { x: pt.x, y: pt.y, windowId: params.windowId }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("move");
     registerTool("find_element", "Find accessibility elements by text, role, or value. Supports value/index/near selectors.", findElementInputSchema, async (params) => {
         const effectiveApp = params.app || getActiveTarget()?.appName;
-        const response = await withSafety({ action: "find_element", params: {}, requiresAccessibility: true,
+        const safetyCtx = await getSafetyContext(params.windowId);
+        const response = await withSafety({ action: "find_element", params: { ...safetyCtx }, requiresAccessibility: true,
             execute: () => getPlatform().findElement({ text: params.text, role: params.role, app: effectiveApp, depth: params.depth, includeBounds: params.includeBounds, maxResults: params.maxResults, textMode: params.textMode, visibleOnly: params.visibleOnly, value: params.value, index: params.index, near: params.near }) });
-        return { content: [{ type: "text", text: JSON.stringify({ results: response.results, metrics: response.metrics }, null, 2) }] };
+        const payload = { results: response.results, metrics: response.metrics };
+        // When find_element returns 0 results AND scannedCount is 0 for a specific
+        // app, the AX tree is empty — the most common cause is Electron / Chromium
+        // apps whose AX tree is not exposed to System Events. Attach a pixel-level
+        // fallback hint so the model can proceed via screenshot + ocr + click
+        // instead of retrying find_element forever.
+        if (response.results.length === 0 && effectiveApp && response.metrics.scannedCount === 0) {
+            payload.hint =
+                `${effectiveApp} returned 0 AX elements (scannedCount=0, meaning the AX tree is empty). ` +
+                    "This is typical for Electron/Chromium apps whose AX tree is not exposed to System Events. " +
+                    "Pixel-level workaround: call screenshot to capture the screen, then ocr to locate " +
+                    "the target UI text and get its bounding box coordinates, then click(x, y) at those " +
+                    "screen coordinates. Alternatively, use type_text or press_key for keyboard-based " +
+                    "interaction, or modify the app's config file or database directly.";
+        }
+        return { content: [{ type: "text", text: JSON.stringify(payload, null, 2) }] };
     });
     registry.register("find_element");
     registerTool("click_element", "Click an accessibility element by its ID", {
         elementId: z.string().describe("AX element identifier"), app: z.string().optional().describe("Target app"), ...captureAfterFields,
     }, async (params) => {
         const effectiveApp = params.app || getActiveTarget()?.appName;
-        await withSafety({ action: "click_element", params: {}, requiresAccessibility: true, execute: () => getPlatform().clickElement(params.elementId, effectiveApp) });
+        const safetyCtx = await getSafetyContext();
+        await withSafety({ action: "click_element", params: { ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().clickElement(params.elementId, effectiveApp) });
         return actionResponse("click_element", { clicked: true, elementId: params.elementId }, { elementId: params.elementId, app: effectiveApp }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("click_element");
@@ -681,7 +743,8 @@ export function registerTools(server) {
         elementId: z.string().describe("AX element identifier"), value: z.string().describe("Value to set"), app: z.string().optional().describe("Target app"), ...captureAfterFields,
     }, async (params) => {
         const effectiveApp = params.app || getActiveTarget()?.appName;
-        await withSafety({ action: "set_value", params: { value: params.value }, requiresAccessibility: true, execute: () => getPlatform().setElementValue(params.elementId, params.value, effectiveApp) });
+        const safetyCtx = await getSafetyContext();
+        await withSafety({ action: "set_value", params: { value: params.value, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().setElementValue(params.elementId, params.value, effectiveApp) });
         return actionResponse("set_value", { setValue: true, elementId: params.elementId }, { elementId: params.elementId, app: effectiveApp }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("set_value");
@@ -690,7 +753,8 @@ export function registerTools(server) {
         app: z.string().optional().describe("Target app"), clearFirst: z.boolean().optional().describe("Clear existing text before typing"), ...captureAfterFields,
     }, async (params) => {
         const effectiveApp = params.app || getActiveTarget()?.appName;
-        await withSafety({ action: "type_in_element", params: { text: params.text }, requiresAccessibility: true, execute: () => getPlatform().typeInElement(params.elementId, params.text, effectiveApp, params.clearFirst) });
+        const safetyCtx = await getSafetyContext();
+        await withSafety({ action: "type_in_element", params: { text: params.text, ...safetyCtx }, requiresAccessibility: true, execute: () => getPlatform().typeInElement(params.elementId, params.text, effectiveApp, params.clearFirst) });
         return actionResponse("type_in_element", { typed: true, elementId: params.elementId, charCount: params.text.length }, { elementId: params.elementId, app: effectiveApp }, params.captureAfter, params.captureFormat, params.captureMaxWidth);
     });
     registry.register("type_in_element");

package/dist/src/platform/base.d.ts

@@ -8,6 +8,7 @@ export interface ScreenSize {
     width: number;
     height: number;
     scaleFactor?: number;
+    estimated?: boolean;
 }
 export interface ScreenshotOptions {
     format?: "png" | "jpeg";
@@ -100,6 +101,8 @@ export interface FindElementResult {
     role: string;
     name: string;
     value?: string;
+    subrole?: string;
+    identifier?: string;
     bounds?: {
         x: number;
         y: number;

package/dist/src/platform/jxa-helpers.d.ts

@@ -0,0 +1,11 @@
+export declare function jxaChildElements(): string;
+export declare function jxaResolveElementByFullPath(): string;
+export declare function jxaResolveElementInApp(): string;
+export declare function jxaElemString(): string;
+export declare function jxaGetBounds(): string;
+export declare function jxaIsVisible(): string;
+export declare function jxaDescriptorMatches(): string;
+export declare function jxaScoreEquivalent(): string;
+export declare function jxaRefetchEquivalent(): string;
+/** Common helper set used by clickElement, typeInElement, setElementValue */
+export declare function jxaElementActionHelpers(): string;

package/dist/src/platform/jxa-helpers.js

@@ -0,0 +1,206 @@
+// JXA helper function templates for macOS accessibility scripts.
+// Each function returns a JXA code string that can be embedded in osascript calls.
+export function jxaChildElements() {
+    return `
+      function childElements(elem) {
+        try { return elem.uiElements(); } catch(e1) {
+          try { return elem.elements(); } catch(e2) { return []; }
+        }
+      }`;
+}
+export function jxaResolveElementByFullPath() {
+    return String.raw `
+      function resolveElementByFullPath(path) {
+        var parts = path.split('/');
+        if (parts.length < 2) return null;
+
+        var procName = parts[0];
+        var winPart = parts[1];
+        var winIdx = 0;
+        var match = winPart.match(/^win(\d+)$/);
+        if (match) {
+          winIdx = parseInt(match[1]);
+        }
+
+        try {
+          var proc = se.processes[procName]();
+          var wins = proc.windows();
+          if (winIdx >= wins.length) return null;
+          var current = wins[winIdx];
+
+          for (var i = 2; i < parts.length; i++) {
+            var idx = parseInt(parts[i]);
+            if (isNaN(idx)) return null;
+            try {
+              var kids = childElements(current);
+              if (idx >= kids.length) return null;
+              current = kids[idx];
+            } catch(e) { return null; }
+          }
+          return current;
+        } catch(e) { return null; }
+      }`;
+}
+export function jxaResolveElementInApp() {
+    return String.raw `
+      function resolveElementInApp(path, targetApp) {
+        if (!targetApp) return null;
+        var parts = path.split('/');
+        var start = parts[0] === targetApp ? 1 : 0;
+        var winPart = parts[start] || 'win0';
+        var winIdx = 0;
+        var match = winPart.match(/^win(\d+)$/);
+        if (match) winIdx = parseInt(match[1]);
+
+        try {
+          var proc = se.processes[targetApp]();
+          var wins = proc.windows();
+          if (winIdx >= wins.length) return null;
+          var current = wins[winIdx];
+          for (var i = start + 1; i < parts.length; i++) {
+            var idx = parseInt(parts[i]);
+            if (isNaN(idx)) return null;
+            try {
+              var kids = childElements(current);
+              if (idx >= kids.length) return null;
+              current = kids[idx];
+            } catch(e) { return null; }
+          }
+          return current;
+        } catch(e) { return null; }
+      }`;
+}
+export function jxaElemString() {
+    return `
+      function elemString(elem, getter) {
+        try {
+          var value = getter(elem);
+          return value === undefined || value === null ? '' : String(value);
+        } catch(e) {
+          return '';
+        }
+      }`;
+}
+export function jxaGetBounds() {
+    return `
+      function getBounds(elem) {
+        try {
+          var pos = elem.position();
+          var sz = elem.size();
+          return {x: pos[0] || 0, y: pos[1] || 0, width: sz[0] || 0, height: sz[1] || 0};
+        } catch(e) {
+          return {x: 0, y: 0, width: 0, height: 0};
+        }
+      }`;
+}
+export function jxaIsVisible() {
+    return `
+      function isVisible(elem) {
+        try {
+          var pos = elem.position();
+          var sz = elem.size();
+          if (!pos || !sz) return false;
+          return sz[0] > 0 && sz[1] > 0 && pos[0] > -10000 && pos[1] > -10000;
+        } catch(e) {
+          return false;
+        }
+      }`;
+}
+export function jxaDescriptorMatches() {
+    return `
+      function descriptorMatches(elem) {
+        if (!cached) return true;
+        var role = elemString(elem, function(e) { return e.role(); });
+        var name = elemString(elem, function(e) { return e.name(); });
+        var desc = elemString(elem, function(e) { return e.description(); });
+        var value = elemString(elem, function(e) { return e.value(); });
+        if (cached.role && role && role !== cached.role) return false;
+        if (cached.name && name && name !== cached.name) return false;
+        if (cached.value && value && value !== cached.value) return false;
+        if (cached.description && desc && desc !== cached.description) return false;
+        return true;
+      }`;
+}
+export function jxaScoreEquivalent() {
+    return `
+      function scoreEquivalent(elem) {
+        if (!cached) return -1;
+        var score = 0;
+        var role = elemString(elem, function(e) { return e.role(); });
+        var name = elemString(elem, function(e) { return e.name(); });
+        var desc = elemString(elem, function(e) { return e.description(); });
+        var value = elemString(elem, function(e) { return e.value(); });
+        var subrole = elemString(elem, function(e) { return e.subrole(); });
+        var identifier = elemString(elem, function(e) { return e.identifier(); });
+        if (cached.role && role === cached.role) score += 4;
+        if (cached.name && name === cached.name) score += 4;
+        if (cached.value && value === cached.value) score += 3;
+        if (cached.description && desc === cached.description) score += 2;
+        if (cached.subrole && subrole === cached.subrole) score += 2;
+        if (cached.identifier && identifier === cached.identifier) score += 3;
+        var b = getBounds(elem);
+        if (cached.bounds) {
+          var cx = b.x + b.width / 2;
+          var cy = b.y + b.height / 2;
+          var ocx = cached.bounds.x + cached.bounds.width / 2;
+          var ocy = cached.bounds.y + cached.bounds.height / 2;
+          var distance = Math.sqrt(Math.pow(cx - ocx, 2) + Math.pow(cy - ocy, 2));
+          if (distance < 8) score += 4;
+          else if (distance < 40) score += 2;
+          else if (distance < 120) score += 1;
+        }
+        return score;
+      }`;
+}
+export function jxaRefetchEquivalent() {
+    return `
+      function refetchEquivalent() {
+        if (!cached) return null;
+        var targetApp = appName || cached.appName || '';
+        var best = null;
+        var bestScore = 0;
+        var visited = [0];
+        function visit(elem, depth) {
+          if (visited[0] > 350 || depth > 10) return;
+          visited[0]++;
+          var score = scoreEquivalent(elem);
+          if (score > bestScore) {
+            best = elem;
+            bestScore = score;
+          }
+          try {
+            var kids = childElements(elem);
+            for (var i = 0; i < kids.length; i++) visit(kids[i], depth + 1);
+          } catch(e) {}
+        }
+        try {
+          if (targetApp) {
+            var proc = se.processes[targetApp]();
+            var wins = proc.windows();
+            for (var w = 0; w < wins.length; w++) visit(wins[w], 0);
+          } else {
+            var procs = se.processes();
+            for (var p = 0; p < procs.length; p++) {
+              try {
+                var wins2 = procs[p].windows();
+                for (var w2 = 0; w2 < wins2.length; w2++) visit(wins2[w2], 0);
+              } catch(e2) {}
+            }
+          }
+        } catch(e) {}
+        return bestScore >= 6 ? best : null;
+      }`;
+}
+/** Common helper set used by clickElement, typeInElement, setElementValue */
+export function jxaElementActionHelpers() {
+    return [
+        jxaChildElements(),
+        jxaResolveElementByFullPath(),
+        jxaResolveElementInApp(),
+        jxaElemString(),
+        jxaGetBounds(),
+        jxaDescriptorMatches(),
+        jxaScoreEquivalent(),
+        jxaRefetchEquivalent(),
+    ].join("\n");
+}

package/dist/src/platform/macos/ax-tree.d.ts

@@ -0,0 +1,4 @@
+import type { MacOSPlatform } from "./base.js";
+import type { WindowState, FindElementOptions, FindElementResponse } from "../base.js";
+export declare function getWindowState(this: MacOSPlatform, windowId?: string, depth?: number, includeBounds?: boolean): Promise<WindowState>;
+export declare function findElement(this: MacOSPlatform, options: FindElementOptions): Promise<FindElementResponse>;