visionclaw 0.1.0

package/dist/tools/computer-use.js

@@ -0,0 +1,311 @@
+/**
+ * Computer use MCP tools for desktop interaction.
+ * Each tool: screenshot -> resolve coordinates (if needed) -> execute -> result screenshot.
+ */
+import { z } from "zod";
+import { tool } from "@anthropic-ai/claude-agent-sdk";
+import { captureScreen, getDisplaySize, click, doubleClick, rightClick, drag, typeText, pressKey, scroll, } from "./desktop-executor.js";
+import { resolveCoordinates } from "./coordinate-resolver.js";
+import { logger } from "../logger.js";
+/** Check if we're on macOS; computer_use tools require darwin + cliclick. */
+function requireDarwin() {
+    if (process.platform !== "darwin") {
+        return {
+            content: [{ type: "text", text: "computer_use tools require macOS" }],
+            isError: true,
+        };
+    }
+    return null;
+}
+async function getApiKey() {
+    const { loadConfig } = await import("../config/index.js");
+    try {
+        const config = loadConfig();
+        return config.anthropicApiKey;
+    }
+    catch {
+        return process.env.ANTHROPIC_API_KEY ?? null;
+    }
+}
+async function resolveAndExecute(instruction, apiKey, fn) {
+    const screenshot = await captureScreen();
+    const size = getDisplaySize();
+    const resolved = await resolveCoordinates(screenshot, instruction, size.width, size.height, apiKey);
+    if (!resolved) {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `Could not resolve coordinates for: ${instruction}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+    await fn(resolved.x, resolved.y);
+    // wait for 1 second
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    const resultScreenshot = await captureScreen();
+    return {
+        content: [
+            { type: "text", text: `Executed at (${resolved.x}, ${resolved.y})` },
+            {
+                type: "image",
+                data: resultScreenshot,
+                mimeType: "image/png",
+            },
+        ],
+    };
+}
+export const computerUseClickTool = tool("computer_use_click", "Click on a UI element described in natural language (e.g. 'the Chrome icon', 'the submit button'). Resolves target via AI, then performs left/right/double-click.", {
+    target: z.string().describe("Natural language description of the element to click"),
+    button: z.enum(["left", "right", "double"]).optional().default("left"),
+}, async (args) => {
+    const err = requireDarwin();
+    if (err)
+        return err;
+    const apiKey = await getApiKey();
+    if (!apiKey) {
+        return {
+            content: [{ type: "text", text: "ANTHROPIC_API_KEY not configured" }],
+            isError: true,
+        };
+    }
+    const instruction = args.button === "right"
+        ? `Right-click on: ${args.target}`
+        : args.button === "double"
+            ? `Double-click on: ${args.target}`
+            : `Click on: ${args.target}`;
+    const fn = args.button === "right"
+        ? (x, y) => rightClick(x, y)
+        : args.button === "double"
+            ? (x, y) => doubleClick(x, y)
+            : (x, y) => click(x, y);
+    try {
+        return await resolveAndExecute(instruction, apiKey, fn);
+    }
+    catch (e) {
+        logger.err("computer_use_click failed", { error: String(e) });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `computer_use_click failed: ${e instanceof Error ? e.message : String(e)}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+});
+export const computerUseScrollTool = tool("computer_use_scroll", "Scroll at a target location in a direction. Resolves the scroll target via AI, then scrolls.", {
+    target: z.string().describe("Natural language description of where to scroll"),
+    direction: z.enum(["up", "down", "left", "right"]),
+    amount: z.number().optional().default(3).describe("Scroll amount (default 3)"),
+}, async (args) => {
+    const err = requireDarwin();
+    if (err)
+        return err;
+    const apiKey = await getApiKey();
+    if (!apiKey) {
+        return {
+            content: [{ type: "text", text: "ANTHROPIC_API_KEY not configured" }],
+            isError: true,
+        };
+    }
+    const instruction = `Move cursor to the center of: ${args.target} (for scrolling)`;
+    const amount = args.amount;
+    try {
+        return await resolveAndExecute(instruction, apiKey, async (x, y) => {
+            await scroll(x, y, args.direction, amount);
+        });
+    }
+    catch (e) {
+        logger.err("computer_use_scroll failed", { error: String(e) });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `computer_use_scroll failed: ${e instanceof Error ? e.message : String(e)}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+});
+export const computerUseDragTool = tool("computer_use_drag", "Drag from one element to another. Resolves both 'from' and 'to' via AI, then performs drag.", {
+    from: z.string().describe("Natural language description of drag start"),
+    to: z.string().describe("Natural language description of drag end"),
+}, async (args) => {
+    const err = requireDarwin();
+    if (err)
+        return err;
+    const apiKey = await getApiKey();
+    if (!apiKey) {
+        return {
+            content: [{ type: "text", text: "ANTHROPIC_API_KEY not configured" }],
+            isError: true,
+        };
+    }
+    const screenshot = await captureScreen();
+    const size = getDisplaySize();
+    const fromResolved = await resolveCoordinates(screenshot, `The start point for dragging: ${args.from}`, size.width, size.height, apiKey);
+    if (!fromResolved) {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `Could not resolve 'from' coordinates: ${args.from}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+    const toScreenshot = await captureScreen();
+    const toResolved = await resolveCoordinates(toScreenshot, `The end point for dragging: ${args.to}`, size.width, size.height, apiKey);
+    if (!toResolved) {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `Could not resolve 'to' coordinates: ${args.to}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+    try {
+        await drag(fromResolved.x, fromResolved.y, toResolved.x, toResolved.y);
+        const resultScreenshot = await captureScreen();
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `Dragged from (${fromResolved.x}, ${fromResolved.y}) to (${toResolved.x}, ${toResolved.y})`,
+                },
+                {
+                    type: "image",
+                    data: resultScreenshot,
+                    mimeType: "image/png",
+                },
+            ],
+        };
+    }
+    catch (e) {
+        logger.err("computer_use_drag failed", { error: String(e) });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `computer_use_drag failed: ${e instanceof Error ? e.message : String(e)}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+});
+export const computerUseTypeTool = tool("computer_use_type", "Type text into the focused field. Optionally clear the field first (replace) and/or press Enter after.", {
+    text: z.string().describe("Text to type"),
+    replace: z.boolean().optional().default(false).describe("If true, select-all and replace existing text first"),
+    press_enter: z.boolean().optional().default(false).describe("If true, press Enter after typing"),
+}, async (args) => {
+    const err = requireDarwin();
+    if (err)
+        return err;
+    try {
+        if (args.replace) {
+            await pressKey("cmd+a");
+        }
+        await typeText(args.text);
+        if (args.press_enter) {
+            await pressKey("enter");
+        }
+        const resultScreenshot = await captureScreen();
+        return {
+            content: [
+                { type: "text", text: `Typed text${args.press_enter ? " and pressed Enter" : ""}` },
+                {
+                    type: "image",
+                    data: resultScreenshot,
+                    mimeType: "image/png",
+                },
+            ],
+        };
+    }
+    catch (e) {
+        logger.err("computer_use_type failed", { error: String(e) });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `computer_use_type failed: ${e instanceof Error ? e.message : String(e)}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+});
+export const computerUseKeyTool = tool("computer_use_key", "Press a key or key combination (e.g. 'enter', 'escape', 'cmd+s', 'ctrl+shift+t').", {
+    key: z
+        .string()
+        .describe("Key or combo: 'enter', 'escape', 'cmd+s', 'ctrl+shift+t', etc."),
+}, async (args) => {
+    const err = requireDarwin();
+    if (err)
+        return err;
+    try {
+        await pressKey(args.key);
+        const resultScreenshot = await captureScreen();
+        return {
+            content: [
+                { type: "text", text: `Pressed: ${args.key}` },
+                {
+                    type: "image",
+                    data: resultScreenshot,
+                    mimeType: "image/png",
+                },
+            ],
+        };
+    }
+    catch (e) {
+        logger.err("computer_use_key failed", { error: String(e) });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `computer_use_key failed: ${e instanceof Error ? e.message : String(e)}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+});
+export const computerUseScreenshotTool = tool("computer_use_screenshot", "Take a screenshot of the desktop. Returns the image.", {}, async () => {
+    const err = requireDarwin();
+    if (err)
+        return err;
+    try {
+        const base64 = await captureScreen();
+        return {
+            content: [
+                {
+                    type: "image",
+                    data: base64,
+                    mimeType: "image/png",
+                },
+            ],
+        };
+    }
+    catch (e) {
+        logger.err("computer_use_screenshot failed", { error: String(e) });
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `computer_use_screenshot failed: ${e instanceof Error ? e.message : String(e)}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+});
+//# sourceMappingURL=computer-use.js.map

package/dist/tools/computer-use.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"computer-use.js","sourceRoot":"","sources":["../../src/tools/computer-use.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAEtD,OAAO,EACL,aAAa,EACb,cAAc,EACd,KAAK,EACL,WAAW,EACX,UAAU,EACV,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,MAAM,GACP,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,6EAA6E;AAC7E,SAAS,aAAa;IACpB,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kCAAkC,EAAE,CAAC;YACrE,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,SAAS;IACtB,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC,eAAe,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,WAAmB,EACnB,MAAc,EACd,EAA2C;IAE3C,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CACvC,UAAU,EACV,WAAW,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,MAAM,CACP,CAAC;IACF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,sCAAsC,WAAW,EAAE;iBAC1D;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjC,oBAAoB;IACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,MAAM,aAAa,EAAE,CAAC;IAC/C,OAAO;QACL,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,QAAQ,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,EAAE;YACpE;gBACE,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,gBAAgB;gBACtB,QAAQ,EAAE,WAAW;aACuB;SAC/C;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,CACtC,oBAAoB,EACpB,mKAAmK,EACnK;IACE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;IACnF,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;CACvE,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kCAAkC,EAAE,CAAC;YACrE,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GACf,IAAI,CAAC,MAAM,KAAK,OAAO;QACrB,CAAC,CAAC,mBAAmB,IAAI,CAAC,MAAM,EAAE;QAClC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,oBAAoB,IAAI,CAAC,MAAM,EAAE;YACnC,CAAC,CAAC,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;IAEnC,MAAM,EAAE,GACN,IAAI,CAAC,MAAM,KAAK,OAAO;QACrB,CAAC,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9C,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,8BAA8B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBACjF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CACvC,qBAAqB,EACrB,8FAA8F,EAC9F;IACE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iDAAiD,CAAC;IAC9E,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,2BAA2B,CAAC;CAC/E,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kCAAkC,EAAE,CAAC;YACrE,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,iCAAiC,IAAI,CAAC,MAAM,kBAAkB,CAAC;IACnF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAE3B,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAC5B,WAAW,EACX,MAAM,EACN,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,+BAA+B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAClF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,6FAA6F,EAC7F;IACE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;IACvE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;CACpE,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kCAAkC,EAAE,CAAC;YACrE,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAE9B,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAC3C,UAAU,EACV,iCAAiC,IAAI,CAAC,IAAI,EAAE,EAC5C,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,MAAM,CACP,CAAC;IACF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,yCAAyC,IAAI,CAAC,IAAI,EAAE;iBAC3D;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,aAAa,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,kBAAkB,CACzC,YAAY,EACZ,+BAA+B,IAAI,CAAC,EAAE,EAAE,EACxC,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,MAAM,CACP,CAAC;IACF,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,uCAAuC,IAAI,CAAC,EAAE,EAAE;iBACvD;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;QACvE,MAAM,gBAAgB,GAAG,MAAM,aAAa,EAAE,CAAC;QAC/C,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,iBAAiB,YAAY,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,SAAS,UAAU,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,GAAG;iBAClG;gBACD;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAChF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,wGAAwG,EACxG;IACE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAC9G,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,mCAAmC,CAAC;CACjG,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,gBAAgB,GAAG,MAAM,aAAa,EAAE,CAAC;QAC/C,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE;gBACnF;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAChF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CACpC,kBAAkB,EAClB,mFAAmF,EACnF;IACE,GAAG,EAAE,CAAC;SACH,MAAM,EAAE;SACR,QAAQ,CACP,gEAAgE,CACjE;CACJ,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,gBAAgB,GAAG,MAAM,aAAa,EAAE,CAAC;QAC/C,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,IAAI,CAAC,GAAG,EAAE,EAAE;gBAC9C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,4BAA4B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAC/E;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAC3C,yBAAyB,EACzB,sDAAsD,EACtD,EAAE,EACF,KAAK,IAA6B,EAAE;IAClC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,gCAAgC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnE,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,mCAAmC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBACtF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC"}

package/dist/tools/coordinate-resolver.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Resolves natural language targets to pixel coordinates using
+ * Anthropic's computer_20251124 tool via the Messages API.
+ *
+ * Sends screenshot + instruction to Claude; Claude returns a tool_use
+ * with action and coordinate. We extract coordinates only (no execution).
+ *
+ * IMPORTANT: Claude's computer-use tool has display dimension constraints.
+ * We must scale down the display dimensions we send, and then unscale
+ * the returned coordinates back to actual display space.
+ * Reference: https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool
+ */
+export interface ResolvedAction {
+    action: string;
+    x: number;
+    y: number;
+}
+/**
+ * Resolve a natural language target to pixel coordinates.
+ * Uses Messages API with computer_20251124 tool; returns the first tool_use result.
+ *
+ * The screenshot is resized to fit Claude's constraints before sending.
+ * Returned coordinates are unscaled back to actual display space.
+ */
+export declare function resolveCoordinates(screenshotBase64: string, instruction: string, displayWidth: number, displayHeight: number, apiKey: string): Promise<ResolvedAction | null>;
+//# sourceMappingURL=coordinate-resolver.d.ts.map

package/dist/tools/coordinate-resolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinate-resolver.d.ts","sourceRoot":"","sources":["../../src/tools/coordinate-resolver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAaH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX;AAyED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAyFhC"}

package/dist/tools/coordinate-resolver.js

@@ -0,0 +1,157 @@
+/**
+ * Resolves natural language targets to pixel coordinates using
+ * Anthropic's computer_20251124 tool via the Messages API.
+ *
+ * Sends screenshot + instruction to Claude; Claude returns a tool_use
+ * with action and coordinate. We extract coordinates only (no execution).
+ *
+ * IMPORTANT: Claude's computer-use tool has display dimension constraints.
+ * We must scale down the display dimensions we send, and then unscale
+ * the returned coordinates back to actual display space.
+ * Reference: https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool
+ */
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import Anthropic from "@anthropic-ai/sdk";
+import { logger } from "../logger.js";
+import { CLAUDE_MODEL } from "../agent/session.js";
+const execAsync = promisify(exec);
+// ── Claude computer-use dimension constraints ──────────────────────────
+const CLAUDE_MAX_LONG_EDGE_PX = 1568;
+const CLAUDE_MAX_PIXELS = 1_150_000;
+/**
+ * Compute how much to scale the display dimensions to fit Claude's constraints.
+ * Returns a factor <= 1.0.
+ */
+function getScaleFactor(width, height) {
+    const longEdge = Math.max(width, height);
+    const totalPixels = width * height;
+    const longEdgeScale = CLAUDE_MAX_LONG_EDGE_PX / longEdge;
+    const totalPixelsScale = Math.sqrt(CLAUDE_MAX_PIXELS / totalPixels);
+    return Math.min(1.0, longEdgeScale, totalPixelsScale);
+}
+/**
+ * Get the scaled display dimensions and scale factor to send to Claude.
+ */
+function getToolDisplaySize(displayWidth, displayHeight) {
+    const scale = getScaleFactor(displayWidth, displayHeight);
+    const width = Math.max(1, Math.floor(displayWidth * scale));
+    const height = Math.max(1, Math.floor(displayHeight * scale));
+    return { width, height, scale };
+}
+function clamp(n, min, max) {
+    return Math.max(min, Math.min(max, n));
+}
+/**
+ * Resize a base64 PNG screenshot to target dimensions.
+ * Uses sips on macOS, ImageMagick on Linux.
+ * Returns the resized image as base64.
+ */
+async function resizeScreenshot(base64, targetWidth, targetHeight) {
+    const tmpIn = path.join(os.tmpdir(), `visionclaw-resize-in-${Date.now()}.png`);
+    const tmpOut = path.join(os.tmpdir(), `visionclaw-resize-out-${Date.now()}.png`);
+    try {
+        fs.writeFileSync(tmpIn, Buffer.from(base64, "base64"));
+        if (process.platform === "darwin") {
+            await execAsync(`sips -z ${targetHeight} ${targetWidth} "${tmpIn}" --out "${tmpOut}"`);
+        }
+        else {
+            await execAsync(`convert "${tmpIn}" -resize ${targetWidth}x${targetHeight}! "${tmpOut}"`);
+        }
+        const resized = fs.readFileSync(tmpOut);
+        return resized.toString("base64");
+    }
+    finally {
+        try {
+            fs.unlinkSync(tmpIn);
+        }
+        catch { /* noop */ }
+        try {
+            fs.unlinkSync(tmpOut);
+        }
+        catch { /* noop */ }
+    }
+}
+// ── Resolver ───────────────────────────────────────────────────────────
+/**
+ * Resolve a natural language target to pixel coordinates.
+ * Uses Messages API with computer_20251124 tool; returns the first tool_use result.
+ *
+ * The screenshot is resized to fit Claude's constraints before sending.
+ * Returned coordinates are unscaled back to actual display space.
+ */
+export async function resolveCoordinates(screenshotBase64, instruction, displayWidth, displayHeight, apiKey) {
+    const toolDisplay = getToolDisplaySize(displayWidth, displayHeight);
+    logger.info(`Resolving coordinates: "${instruction}" (display ${displayWidth}x${displayHeight}, tool ${toolDisplay.width}x${toolDisplay.height}, scale=${toolDisplay.scale.toFixed(3)})`, { instruction, displayWidth, displayHeight, toolWidth: toolDisplay.width, toolHeight: toolDisplay.height, scale: toolDisplay.scale });
+    // Resize screenshot to match the tool display dimensions
+    let imageBase64 = screenshotBase64;
+    if (toolDisplay.scale < 1.0) {
+        try {
+            imageBase64 = await resizeScreenshot(screenshotBase64, toolDisplay.width, toolDisplay.height);
+            logger.debug(`Screenshot resized to ${toolDisplay.width}x${toolDisplay.height} for coordinate resolution`);
+        }
+        catch (e) {
+            logger.warn(`Failed to resize screenshot, sending original: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    const client = new Anthropic({ apiKey });
+    const response = await client.beta.messages.create({
+        model: CLAUDE_MODEL,
+        max_tokens: 1024,
+        betas: ["computer-use-2025-11-24"],
+        tools: [
+            {
+                type: "computer_20251124",
+                name: "computer",
+                display_width_px: toolDisplay.width,
+                display_height_px: toolDisplay.height,
+                display_number: 1,
+            },
+        ],
+        system: "You are a helpful assistant that can resolve natural language targets to pixel coordinates using the computer tool. Call the tool directly, no questions or explanations.",
+        tool_choice: { type: "tool", name: "computer" },
+        messages: [
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "image",
+                        source: {
+                            type: "base64",
+                            media_type: "image/png",
+                            data: imageBase64,
+                        },
+                    },
+                    {
+                        type: "text",
+                        text: instruction,
+                    },
+                ],
+            },
+        ],
+    });
+    for (const block of response.content) {
+        if (block.type === "tool_use" && block.name === "computer") {
+            const input = block.input;
+            const coord = input.coordinate;
+            const action = input.action ?? "left_click";
+            if (Array.isArray(coord) && coord.length >= 2) {
+                const rawX = coord[0];
+                const rawY = coord[1];
+                if (typeof rawX === "number" && typeof rawY === "number" && Number.isFinite(rawX) && Number.isFinite(rawY)) {
+                    // Unscale: Claude returned coords in tool display space → convert to actual display space
+                    const x = clamp(Math.round(rawX / toolDisplay.scale), 0, displayWidth);
+                    const y = clamp(Math.round(rawY / toolDisplay.scale), 0, displayHeight);
+                    logger.info(`Resolved coordinates: action=${action} raw=(${rawX},${rawY}) unscaled=(${x},${y}) scale=${toolDisplay.scale.toFixed(3)}`, { instruction, action, rawX, rawY, x, y, scale: toolDisplay.scale });
+                    return { action, x, y };
+                }
+            }
+        }
+    }
+    logger.info(`Could not resolve coordinates: "${instruction}"`, { instruction });
+    return null;
+}
+//# sourceMappingURL=coordinate-resolver.js.map

package/dist/tools/coordinate-resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinate-resolver.js","sourceRoot":"","sources":["../../src/tools/coordinate-resolver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;AAQlC,0EAA0E;AAE1E,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,iBAAiB,GAAG,SAAS,CAAC;AAEpC;;;GAGG;AACH,SAAS,cAAc,CAAC,KAAa,EAAE,MAAc;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;IAEnC,MAAM,aAAa,GAAG,uBAAuB,GAAG,QAAQ,CAAC;IACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,CAAC;IAEpE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,EAAE,gBAAgB,CAAC,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,YAAoB,EACpB,aAAqB;IAErB,MAAM,KAAK,GAAG,cAAc,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC;IAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,KAAK,CAAC,CAAS,EAAE,GAAW,EAAE,GAAW;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,gBAAgB,CAC7B,MAAc,EACd,WAAmB,EACnB,YAAoB;IAEpB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,wBAAwB,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,yBAAyB,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACjF,IAAI,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvD,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,SAAS,CACb,WAAW,YAAY,IAAI,WAAW,KAAK,KAAK,YAAY,MAAM,GAAG,CACtE,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,CACb,YAAY,KAAK,aAAa,WAAW,IAAI,YAAY,MAAM,MAAM,GAAG,CACzE,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC;YAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,0EAA0E;AAE1E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,gBAAwB,EACxB,WAAmB,EACnB,YAAoB,EACpB,aAAqB,EACrB,MAAc;IAEd,MAAM,WAAW,GAAG,kBAAkB,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAEpE,MAAM,CAAC,IAAI,CACT,2BAA2B,WAAW,cAAc,YAAY,IAAI,aAAa,UAAU,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,MAAM,WAAW,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAC5K,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CACrI,CAAC;IAEF,yDAAyD;IACzD,IAAI,WAAW,GAAG,gBAAgB,CAAC;IACnC,IAAI,WAAW,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,gBAAgB,CAClC,gBAAgB,EAChB,WAAW,CAAC,KAAK,EACjB,WAAW,CAAC,MAAM,CACnB,CAAC;YACF,MAAM,CAAC,KAAK,CACV,yBAAyB,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,MAAM,4BAA4B,CAC7F,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CACT,kDAAkD,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAEzC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACjD,KAAK,EAAE,YAAY;QACnB,UAAU,EAAE,IAAI;QAChB,KAAK,EAAE,CAAC,yBAAyB,CAAC;QAClC,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,mBAAmB;gBACzB,IAAI,EAAE,UAAU;gBAChB,gBAAgB,EAAE,WAAW,CAAC,KAAK;gBACnC,iBAAiB,EAAE,WAAW,CAAC,MAAM;gBACrC,cAAc,EAAE,CAAC;aAClB;SACF;QACD,MAAM,EAAE,2KAA2K;QACnL,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE;QAC/C,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,UAAU,EAAE,WAAW;4BACvB,IAAI,EAAE,WAAW;yBAClB;qBACF;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,WAAW;qBAClB;iBACF;aACF;SACF;KACF,CAAC,CAAC;IAEH,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAgC,CAAC;YACrD,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC;YAC/B,MAAM,MAAM,GAAI,KAAK,CAAC,MAA6B,IAAI,YAAY,CAAC;YAEpE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAY,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,IAAI,GAAY,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3G,0FAA0F;oBAC1F,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;oBACvE,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC;oBACxE,MAAM,CAAC,IAAI,CACT,gCAAgC,MAAM,SAAS,IAAI,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,WAAW,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EACzH,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CACpE,CAAC;oBACF,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,mCAAmC,WAAW,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAChF,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/tools/desktop-executor.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Low-level macOS desktop executor using cliclick, osascript, and screencapture.
+ * Used by computer_use MCP tools to perform mouse, keyboard, and screenshot actions.
+ */
+import { takeScreenshot } from "./screenshot.js";
+/** Capture the screen. Returns base64 PNG. Re-exports takeScreenshot for consistency. */
+export declare const captureScreen: typeof takeScreenshot;
+/**
+ * Initialize display size cache. Call once at agent startup.
+ */
+export declare function initDisplaySize(): Promise<void>;
+/** Return cached display size. Use after initDisplaySize() has been called. */
+export declare function getDisplaySize(): {
+    width: number;
+    height: number;
+};
+/**
+ * Left click at (x, y).
+ */
+export declare function click(x: number, y: number): Promise<void>;
+/**
+ * Double click at (x, y).
+ */
+export declare function doubleClick(x: number, y: number): Promise<void>;
+/**
+ * Right click at (x, y).
+ */
+export declare function rightClick(x: number, y: number): Promise<void>;
+/**
+ * Move cursor to (x, y).
+ */
+export declare function moveTo(x: number, y: number): Promise<void>;
+/**
+ * Drag from (x1, y1) to (x2, y2).
+ */
+export declare function drag(x1: number, y1: number, x2: number, y2: number): Promise<void>;
+/**
+ * Type text (use cliclick t: for plain text, or osascript for special chars).
+ */
+export declare function typeText(text: string): Promise<void>;
+/**
+ * Press a key or key combination (e.g. "enter", "escape", "cmd+s", "ctrl+shift+t").
+ * Uses cliclick: kd:mods for modifiers, kp: for special keys, t: for character keys.
+ */
+export declare function pressKey(key: string): Promise<void>;
+/**
+ * Scroll at (x, y) in direction. Amount is number of scroll steps (default 3).
+ * cliclick does not support scroll; we use osascript with key code 116/121 (page up/down)
+ * after moving the mouse to the target. This works in many apps (browsers, text editors).
+ */
+export declare function scroll(x: number, y: number, direction: "up" | "down" | "left" | "right", amount?: number): Promise<void>;
+//# sourceMappingURL=desktop-executor.d.ts.map

package/dist/tools/desktop-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"desktop-executor.d.ts","sourceRoot":"","sources":["../../src/tools/desktop-executor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAKjD,yFAAyF;AACzF,eAAO,MAAM,aAAa,uBAAiB,CAAC;AAK5C;;GAEG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAErD;AAED,+EAA+E;AAC/E,wBAAgB,cAAc,IAAI;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAElE;AAsFD;;GAEG;AACH,wBAAsB,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/D;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;AAED;;GAEG;AACH,wBAAsB,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhE;AAED;;GAEG;AACH,wBAAsB,IAAI,CACxB,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC,CAIf;AAED;;GAEG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI1D;AA4BD;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BzD;AAED;;;;GAIG;AACH,wBAAsB,MAAM,CAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,EAC3C,MAAM,SAAI,GACT,OAAO,CAAC,IAAI,CAAC,CASf"}