visionclaw 0.1.187-beta.8 → 0.1.187-dev.refactor-computer-use-direct-coordinates.1

package/dist/tools/computer-use.d.ts

@@ -1,10 +1,48 @@
 /**
  * Computer use MCP tools for desktop interaction.
- * Each tool: screenshot -> resolve coordinates (if needed) -> execute -> result screenshot.
+ * Each coordinate-based tool: scale screenshot coords → display coords → execute → result screenshot.
+ *
+ * The model sees resized screenshots and outputs pixel positions in that
+ * image space. The execution layer scales from screenshot coords to actual
+ * display resolution, mirroring the android-use pattern.
  */
 import { z } from "zod";
+/**
+ * Compute the expected screenshot image dimensions after vision resize.
+ * The screenshot pipeline resizes the raw capture to fit Claude's vision
+ * constraints. This computes those dimensions deterministically from the
+ * display resolution so we can map coordinates without reading the image.
+ */
+export declare function getScreenshotSize(displaySize: {
+    width: number;
+    height: number;
+}): {
+    width: number;
+    height: number;
+};
+/**
+ * Convert pixel coordinates in screenshot space to absolute display coordinates.
+ * The model outputs coordinates based on the resized screenshot it sees;
+ * we scale them to the actual display resolution for input events.
+ */
+export declare function screenshotToDisplay(point: {
+    x: number;
+    y: number;
+}, screenshotSize: {
+    width: number;
+    height: number;
+}, displaySize: {
+    width: number;
+    height: number;
+}): {
+    x: number;
+    y: number;
+};
 export declare const computerUseClickTool: import("@anthropic-ai/claude-agent-sdk").SdkMcpToolDefinition<{
-    target: z.ZodString;
+    point: z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, z.core.$strip>;
     button: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
         left: "left";
         right: "right";
@@ -12,10 +50,16 @@ export declare const computerUseClickTool: import("@anthropic-ai/claude-agent-sd
     }>>>;
 }>;
 export declare const computerUseMoveTool: import("@anthropic-ai/claude-agent-sdk").SdkMcpToolDefinition<{
-    target: z.ZodString;
+    point: z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, z.core.$strip>;
 }>;
 export declare const computerUseScrollTool: import("@anthropic-ai/claude-agent-sdk").SdkMcpToolDefinition<{
-    target: z.ZodString;
+    point: z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, z.core.$strip>;
     direction: z.ZodEnum<{
         left: "left";
         right: "right";
@@ -25,8 +69,14 @@ export declare const computerUseScrollTool: import("@anthropic-ai/claude-agent-s
     amount: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
 }>;
 export declare const computerUseDragTool: import("@anthropic-ai/claude-agent-sdk").SdkMcpToolDefinition<{
-    from: z.ZodString;
-    to: z.ZodString;
+    startPoint: z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, z.core.$strip>;
+    endPoint: z.ZodObject<{
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+    }, z.core.$strip>;
 }>;
 export declare const computerUseTypeTool: import("@anthropic-ai/claude-agent-sdk").SdkMcpToolDefinition<{
     text: z.ZodString;

package/dist/tools/computer-use.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../../src/tools/computer-use.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAsHxB,eAAO,MAAM,oBAAoB;;;;;;;EAgDhC,CAAC;AAEF,eAAO,MAAM,mBAAmB;;EAyC/B,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;;;;;;EA4CjC,CAAC;AAEF,eAAO,MAAM,mBAAmB;;;EAgG/B,CAAC;AAEF,eAAO,MAAM,mBAAmB;;;;EA+C/B,CAAC;AAEF,eAAO,MAAM,kBAAkB;;EA2C9B,CAAC;AAEF,eAAO,MAAM,yBAAyB,mEAsCrC,CAAC"}
+{"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../../src/tools/computer-use.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AA8CxB;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAOpC;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,EAC/B,cAAc,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACjD,WAAW,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAK1B;AA6DD,eAAO,MAAM,oBAAoB;;;;;;;;;;EA4BhC,CAAC;AAEF,eAAO,MAAM,mBAAmB;;;;;EAqB/B,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;;;;;;;;;EAuBjC,CAAC;AAEF,eAAO,MAAM,mBAAmB;;;;;;;;;EAyB/B,CAAC;AAEF,eAAO,MAAM,mBAAmB;;;;EAuB/B,CAAC;AAEF,eAAO,MAAM,kBAAkB;;EAiB9B,CAAC;AAEF,eAAO,MAAM,yBAAyB,mEAwCrC,CAAC"}

package/dist/tools/computer-use.js

@@ -1,24 +1,19 @@
 /**
  * Computer use MCP tools for desktop interaction.
- * Each tool: screenshot -> resolve coordinates (if needed) -> execute -> result screenshot.
+ * Each coordinate-based tool: scale screenshot coords → display coords → execute → result screenshot.
+ *
+ * The model sees resized screenshots and outputs pixel positions in that
+ * image space. The execution layer scales from screenshot coords to actual
+ * display resolution, mirroring the android-use pattern.
  */
 import { z } from "zod";
 import { tool } from "@anthropic-ai/claude-agent-sdk";
 import { captureScreen, getDisplaySize, isDisplaySizeInitialized, click, doubleClick, rightClick, moveTo, drag, typeText, pressKey, scroll, } from "./desktop-executor-factory.js";
-import { resolveCoordinates } from "./coordinate-resolver.js";
-import { takeScreenshot } from "./screenshot.js";
+import { takeScreenshot, getVisionScaleFactor } from "./screenshot.js";
 import { logger } from "../logger.js";
 /**
  * Gate every computer_use entry point on (a) a supported platform and
  * (b) a successfully-detected display size.
- *
- * The display-size check is load-bearing: every coordinate the tools
- * compute — click targets, drag endpoints, vision-resolver scaling —
- * depends on the real display dimensions. With a wrong or guessed size
- * the agent silently clicks the wrong pixels, which is worse than
- * refusing the call. When display-size detection fails at startup,
- * `initDisplaySize()` leaves the cache empty; every computer_use tool
- * then short-circuits with this error instead of running.
  */
 function requireDesktop() {
     if (process.platform !== "darwin" && process.platform !== "win32") {
@@ -40,249 +35,148 @@ function requireDesktop() {
     }
     return null;
 }
-let cachedConfig = null;
-async function getConfig() {
-    if (cachedConfig)
-        return cachedConfig;
-    const { loadConfig } = await import("../config/index.js");
-    try {
-        cachedConfig = loadConfig();
-        return cachedConfig;
-    }
-    catch {
-        if (process.env.ANTHROPIC_API_KEY) {
-            cachedConfig = {
-                model: "claude-sonnet-4-6",
-                provider: "anthropic",
-                anthropicApiKey: process.env.ANTHROPIC_API_KEY,
-            };
-            return cachedConfig;
-        }
-        return null;
-    }
+// ── Coordinate helpers (mirror android-use pattern) ─────────────────
+/**
+ * Compute the expected screenshot image dimensions after vision resize.
+ * The screenshot pipeline resizes the raw capture to fit Claude's vision
+ * constraints. This computes those dimensions deterministically from the
+ * display resolution so we can map coordinates without reading the image.
+ */
+export function getScreenshotSize(displaySize) {
+    const scale = getVisionScaleFactor(displaySize.width, displaySize.height);
+    if (scale >= 1.0)
+        return { ...displaySize };
+    return {
+        width: Math.max(1, Math.round(displaySize.width * scale)),
+        height: Math.max(1, Math.round(displaySize.height * scale)),
+    };
 }
-async function resolveAndExecute(instruction, config, fn) {
-    const { base64: screenshot, filePath: screenshotPath } = await captureScreen();
-    const size = getDisplaySize();
-    const resolved = await resolveCoordinates(screenshot, instruction, size.width, size.height, config, screenshotPath);
-    if (!resolved) {
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `Could not resolve coordinates for: ${instruction}. Please try again with a clearer instruction.`,
-                },
-            ],
-            isError: true,
-        };
-    }
-    await fn(resolved.x, resolved.y);
-    // wait for 1 second
-    await new Promise((resolve) => setTimeout(resolve, 1000));
+/**
+ * Convert pixel coordinates in screenshot space to absolute display coordinates.
+ * The model outputs coordinates based on the resized screenshot it sees;
+ * we scale them to the actual display resolution for input events.
+ */
+export function screenshotToDisplay(point, screenshotSize, displaySize) {
+    return {
+        x: Math.round((point.x / screenshotSize.width) * displaySize.width),
+        y: Math.round((point.y / screenshotSize.height) * displaySize.height),
+    };
+}
+/** Zod schema for a pixel coordinate point in the screenshot image. */
+const pointSchema = z.object({
+    x: z
+        .number()
+        .min(0)
+        .describe("X pixel coordinate in the screenshot image (0 = left edge)"),
+    y: z
+        .number()
+        .min(0)
+        .describe("Y pixel coordinate in the screenshot image (0 = top edge)"),
+});
+/**
+ * Capture a post-action screenshot and return a standard result.
+ */
+async function actionResult(text, delayMs = 1000) {
+    await new Promise((resolve) => setTimeout(resolve, delayMs));
     const { base64: resultScreenshot } = await captureScreen();
     logger.screenshot(resultScreenshot);
     return {
         content: [
-            { type: "text", text: `Executed at (${resolved.x}, ${resolved.y})` },
+            { type: "text", text },
             {
                 type: "image",
                 data: resultScreenshot,
                 mimeType: "image/png",
-            }
+            },
         ],
     };
 }
-export const computerUseClickTool = tool("computer_use_click", "Click on a UI element described in natural language (e.g. 'the Chrome icon', 'the submit button'). Resolves target via AI, then performs left/right/double-click.", {
-    target: z.string().describe("Natural language description of the element to click"),
+/** Wrap a tool handler with desktop check and error handling. */
+function withDesktopCheck(toolName, fn) {
+    return (async () => {
+        const err = requireDesktop();
+        if (err)
+            return err;
+        try {
+            return await fn();
+        }
+        catch (e) {
+            logger.err(`${toolName} failed`, { error: String(e) });
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `${toolName} failed: ${e instanceof Error ? e.message : String(e)}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    })();
+}
+export const computerUseClickTool = tool("computer_use_click", "Click at a point on the desktop screen. Coordinates are pixel positions in the screenshot image returned by computer_use_screenshot. Supports left-click, right-click, and double-click.", {
+    point: pointSchema.describe("Pixel coordinates in the screenshot image to click"),
     button: z.enum(["left", "right", "double"]).optional().default("left"),
 }, async (args) => {
-    const err = requireDesktop();
-    if (err)
-        return err;
-    const cfg = await getConfig();
-    if (!cfg) {
-        return {
-            content: [{ type: "text", text: "Claude API credentials not configured" }],
-            isError: true,
-        };
-    }
-    const instruction = args.button === "right"
-        ? `Right-click on: ${args.target}`
-        : args.button === "double"
-            ? `Double-click on: ${args.target}`
-            : `Click on: ${args.target}`;
-    const fn = args.button === "right"
-        ? (x, y) => rightClick(x, y)
-        : args.button === "double"
-            ? (x, y) => doubleClick(x, y)
-            : (x, y) => click(x, y);
-    try {
-        return await resolveAndExecute(instruction, cfg, fn);
-    }
-    catch (e) {
-        logger.err("computer_use_click failed", { error: String(e) });
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `computer_use_click failed: ${e instanceof Error ? e.message : String(e)}`,
-                },
-            ],
-            isError: true,
-        };
-    }
+    return withDesktopCheck("computer_use_click", async () => {
+        const displaySize = getDisplaySize();
+        const ssSize = getScreenshotSize(displaySize);
+        const abs = screenshotToDisplay(args.point, ssSize, displaySize);
+        if (args.button === "right") {
+            await rightClick(abs.x, abs.y);
+        }
+        else if (args.button === "double") {
+            await doubleClick(abs.x, abs.y);
+        }
+        else {
+            await click(abs.x, abs.y);
+        }
+        return actionResult(`${args.button === "double" ? "Double-clicked" : args.button === "right" ? "Right-clicked" : "Clicked"} at screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) → display (${abs.x}, ${abs.y})`);
+    });
 });
-export const computerUseMoveTool = tool("computer_use_move", "Move the mouse cursor to a UI element described in natural language (e.g. 'the search bar', 'the file menu'). Resolves target via AI, then moves cursor without clicking.", {
-    target: z.string().describe("Natural language description of the element to move the cursor to"),
+export const computerUseMoveTool = tool("computer_use_move", "Move the mouse cursor to a point on the desktop screen. Coordinates are pixel positions in the screenshot image.", {
+    point: pointSchema.describe("Pixel coordinates in the screenshot image to move the cursor to"),
 }, async (args) => {
-    const err = requireDesktop();
-    if (err)
-        return err;
-    const cfg = await getConfig();
-    if (!cfg) {
-        return {
-            content: [{ type: "text", text: "Claude API credentials not configured" }],
-            isError: true,
-        };
-    }
-    const instruction = `Move cursor to the center of: ${args.target}`;
-    try {
-        return await resolveAndExecute(instruction, cfg, async (x, y) => {
-            await moveTo(x, y);
-        });
-    }
-    catch (e) {
-        logger.err("computer_use_move failed", { error: String(e) });
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `computer_use_move failed: ${e instanceof Error ? e.message : String(e)}`,
-                },
-            ],
-            isError: true,
-        };
-    }
+    return withDesktopCheck("computer_use_move", async () => {
+        const displaySize = getDisplaySize();
+        const ssSize = getScreenshotSize(displaySize);
+        const abs = screenshotToDisplay(args.point, ssSize, displaySize);
+        await moveTo(abs.x, abs.y);
+        return actionResult(`Moved cursor to screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) → display (${abs.x}, ${abs.y})`);
+    });
 });
-export const computerUseScrollTool = tool("computer_use_scroll", "Scroll at a target location in a direction. Resolves the scroll target via AI, then scrolls.", {
-    target: z.string().describe("Natural language description of where to scroll"),
+export const computerUseScrollTool = tool("computer_use_scroll", "Scroll at a point on the desktop screen in a direction. Coordinates are pixel positions in the screenshot image.", {
+    point: pointSchema.describe("Pixel coordinates in the screenshot image for scroll origin"),
     direction: z.enum(["up", "down", "left", "right"]),
     amount: z.number().optional().default(3).describe("Scroll amount (default 3)"),
 }, async (args) => {
-    const err = requireDesktop();
-    if (err)
-        return err;
-    const cfg = await getConfig();
-    if (!cfg) {
-        return {
-            content: [{ type: "text", text: "Claude API credentials not configured" }],
-            isError: true,
-        };
-    }
-    const instruction = `Move cursor to the center of: ${args.target} (for scrolling)`;
-    const amount = args.amount;
-    try {
-        return await resolveAndExecute(instruction, cfg, async (x, y) => {
-            await scroll(x, y, args.direction, amount);
-        });
-    }
-    catch (e) {
-        logger.err("computer_use_scroll failed", { error: String(e) });
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `computer_use_scroll failed: ${e instanceof Error ? e.message : String(e)}`,
-                },
-            ],
-            isError: true,
-        };
-    }
+    return withDesktopCheck("computer_use_scroll", async () => {
+        const displaySize = getDisplaySize();
+        const ssSize = getScreenshotSize(displaySize);
+        const abs = screenshotToDisplay(args.point, ssSize, displaySize);
+        await scroll(abs.x, abs.y, args.direction, args.amount);
+        return actionResult(`Scrolled ${args.direction} at screenshot (${Math.round(args.point.x)}, ${Math.round(args.point.y)}) → display (${abs.x}, ${abs.y})`);
+    });
 });
-export const computerUseDragTool = tool("computer_use_drag", "Drag from one element to another. Resolves both 'from' and 'to' via AI, then performs drag.", {
-    from: z.string().describe("Natural language description of drag start"),
-    to: z.string().describe("Natural language description of drag end"),
+export const computerUseDragTool = tool("computer_use_drag", "Drag from one point to another on the desktop screen. Coordinates are pixel positions in the screenshot image.", {
+    startPoint: pointSchema.describe("Pixel coordinates in the screenshot image (drag source)"),
+    endPoint: pointSchema.describe("Pixel coordinates in the screenshot image (drop target)"),
 }, async (args) => {
-    const err = requireDesktop();
-    if (err)
-        return err;
-    const cfg = await getConfig();
-    if (!cfg) {
-        return {
-            content: [{ type: "text", text: "Claude API credentials not configured" }],
-            isError: true,
-        };
-    }
-    const { base64: screenshot, filePath: screenshotPath } = await captureScreen();
-    const size = getDisplaySize();
-    const [fromResolved, toResolved] = await Promise.all([
-        resolveCoordinates(screenshot, `Move cursor to the center of: ${args.from} (for dragging)`, size.width, size.height, cfg, screenshotPath),
-        resolveCoordinates(screenshot, `Move cursor to the center of: ${args.to} (for dragging)`, size.width, size.height, cfg, screenshotPath),
-    ]);
-    if (!fromResolved) {
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `Could not resolve 'from' coordinates: ${args.from}. Please try again with a clearer instruction.`,
-                },
-            ],
-            isError: true,
-        };
-    }
-    if (!toResolved) {
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `Could not resolve 'to' coordinates: ${args.to}. Please try again with a clearer instruction.`,
-                },
-            ],
-            isError: true,
-        };
-    }
-    try {
-        await drag(fromResolved.x, fromResolved.y, toResolved.x, toResolved.y);
-        // wait for 1 second
-        await new Promise((resolve) => setTimeout(resolve, 1000));
-        const { base64: resultScreenshot } = await captureScreen();
-        logger.screenshot(resultScreenshot);
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `Dragged from (${fromResolved.x}, ${fromResolved.y}) to (${toResolved.x}, ${toResolved.y})`,
-                },
-                {
-                    type: "image",
-                    data: resultScreenshot,
-                    mimeType: "image/png",
-                },
-            ],
-        };
-    }
-    catch (e) {
-        logger.err("computer_use_drag failed", { error: String(e) });
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `computer_use_drag failed: ${e instanceof Error ? e.message : String(e)}`,
-                },
-            ],
-            isError: true,
-        };
-    }
+    return withDesktopCheck("computer_use_drag", async () => {
+        const displaySize = getDisplaySize();
+        const ssSize = getScreenshotSize(displaySize);
+        const start = screenshotToDisplay(args.startPoint, ssSize, displaySize);
+        const end = screenshotToDisplay(args.endPoint, ssSize, displaySize);
+        await drag(start.x, start.y, end.x, end.y);
+        return actionResult(`Dragged from screenshot (${Math.round(args.startPoint.x)}, ${Math.round(args.startPoint.y)}) to (${Math.round(args.endPoint.x)}, ${Math.round(args.endPoint.y)}) → display (${start.x}, ${start.y}) to (${end.x}, ${end.y})`);
+    });
 });
 export const computerUseTypeTool = tool("computer_use_type", "Type text into the focused field. Optionally clear the field first (replace) and/or press Enter after.", {
     text: z.string().describe("Text to type"),
     replace: z.boolean().optional().default(false).describe("If true, select-all and replace existing text first"),
     press_enter: z.boolean().optional().default(false).describe("If true, press Enter after typing"),
 }, async (args) => {
-    const err = requireDesktop();
-    if (err)
-        return err;
-    try {
+    return withDesktopCheck("computer_use_type", async () => {
         if (args.replace) {
             await pressKey(process.platform === "win32" ? "ctrl+a" : "cmd+a");
         }
@@ -290,84 +184,33 @@ export const computerUseTypeTool = tool("computer_use_type", "Type text into the
         if (args.press_enter) {
             await pressKey("enter");
         }
-        // wait for 1 second
-        await new Promise((resolve) => setTimeout(resolve, 1000));
-        const { base64: resultScreenshot } = await captureScreen();
-        logger.screenshot(resultScreenshot);
-        return {
-            content: [
-                { type: "text", text: `Typed text${args.press_enter ? " and pressed Enter" : ""}` },
-                {
-                    type: "image",
-                    data: resultScreenshot,
-                    mimeType: "image/png",
-                },
-            ],
-        };
-    }
-    catch (e) {
-        logger.err("computer_use_type failed", { error: String(e) });
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `computer_use_type failed: ${e instanceof Error ? e.message : String(e)}`,
-                },
-            ],
-            isError: true,
-        };
-    }
+        return actionResult(`Typed text${args.press_enter ? " and pressed Enter" : ""}`);
+    });
 });
 export const computerUseKeyTool = tool("computer_use_key", "Press a key or key combination (e.g. 'enter', 'escape', 'cmd+s', 'ctrl+shift+t').", {
     key: z
         .string()
         .describe("Key or combo: 'enter', 'escape', 'cmd+s', 'ctrl+shift+t', etc."),
 }, async (args) => {
-    const err = requireDesktop();
-    if (err)
-        return err;
-    try {
+    return withDesktopCheck("computer_use_key", async () => {
         await pressKey(args.key);
-        // wait for 1 second
-        await new Promise((resolve) => setTimeout(resolve, 1000));
-        const { base64: resultScreenshot } = await captureScreen();
-        logger.screenshot(resultScreenshot);
-        return {
-            content: [
-                { type: "text", text: `Pressed: ${args.key}` },
-                {
-                    type: "image",
-                    data: resultScreenshot,
-                    mimeType: "image/png",
-                },
-            ],
-        };
-    }
-    catch (e) {
-        logger.err("computer_use_key failed", { error: String(e) });
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `computer_use_key failed: ${e instanceof Error ? e.message : String(e)}`,
-                },
-            ],
-            isError: true,
-        };
-    }
+        return actionResult(`Pressed: ${args.key}`);
+    });
 });
-export const computerUseScreenshotTool = tool("computer_use_screenshot", "Capture a screenshot of the current desktop screen. Returns the image as base64 PNG and the file path where it is saved.", {}, async (_args) => {
+export const computerUseScreenshotTool = tool("computer_use_screenshot", "Capture a screenshot of the current desktop screen. Returns the image as base64 PNG. Use the pixel coordinates from this image when calling click, move, scroll, or drag tools.", {}, async (_args) => {
     const err = requireDesktop();
     if (err)
         return err;
     try {
+        const displaySize = getDisplaySize();
+        const ssSize = getScreenshotSize(displaySize);
         const { base64, filePath } = await takeScreenshot();
         logger.screenshot(base64);
         return {
             content: [
                 {
                     type: "text",
-                    text: `Screenshot saved to ${filePath}`,
+                    text: `Screenshot saved to ${filePath}\nScreenshot image size: ${ssSize.width}x${ssSize.height}px | Display resolution: ${displaySize.width}x${displaySize.height}px\nUse pixel coordinates from this image when calling click/move/scroll/drag tools.`,
                 },
                 {
                     type: "image",

package/dist/tools/computer-use.js.map

@@ -1 +1 @@
-{"version":3,"file":"computer-use.js","sourceRoot":"","sources":["../../src/tools/computer-use.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAEtD,OAAO,EACL,aAAa,EACb,cAAc,EACd,wBAAwB,EACxB,KAAK,EACL,WAAW,EACX,UAAU,EACV,MAAM,EACN,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,MAAM,GACP,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC;;;;;;;;;;;GAWG;AACH,SAAS,cAAc;IACrB,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAClE,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,6CAA6C,EAAE,CAAC;YAChF,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,wBAAwB,EAAE,EAAE,CAAC;QAChC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,yRAAyR;iBAChS;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,IAAI,YAAY,GAA4B,IAAI,CAAC;AAEjD,KAAK,UAAU,SAAS;IACtB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC1D,IAAI,CAAC;QACH,YAAY,GAAG,UAAU,EAAE,CAAC;QAC5B,OAAO,YAAY,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YAClC,YAAY,GAAG;gBACb,KAAK,EAAE,mBAAmB;gBAC1B,QAAQ,EAAE,WAAW;gBACrB,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;aAC3B,CAAC;YACtB,OAAO,YAAY,CAAC;QACtB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,WAAmB,EACnB,MAAwB,EACxB,EAA2C;IAE3C,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;IAC/E,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CACvC,UAAU,EACV,WAAW,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,MAAM,EACN,cAAc,CACf,CAAC;IACF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,sCAAsC,WAAW,gDAAgD;iBACxG;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjC,oBAAoB;IACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;IAC3D,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;IACpC,OAAO;QACL,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,QAAQ,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,EAAE;YACpE;gBACE,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,gBAAgB;gBACtB,QAAQ,EAAE,WAAW;aACtB;SACF;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,CACtC,oBAAoB,EACpB,mKAAmK,EACnK;IACE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;IACnF,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;CACvE,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,GAAG,GAAG,MAAM,SAAS,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uCAAuC,EAAE,CAAC;YAC1E,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GACf,IAAI,CAAC,MAAM,KAAK,OAAO;QACrB,CAAC,CAAC,mBAAmB,IAAI,CAAC,MAAM,EAAE;QAClC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,oBAAoB,IAAI,CAAC,MAAM,EAAE;YACnC,CAAC,CAAC,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;IAEnC,MAAM,EAAE,GACN,IAAI,CAAC,MAAM,KAAK,OAAO;QACrB,CAAC,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9C,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAAC,WAAW,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,8BAA8B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBACjF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,2KAA2K,EAC3K;IACE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mEAAmE,CAAC;CACjG,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,GAAG,GAAG,MAAM,SAAS,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uCAAuC,EAAE,CAAC;YAC1E,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,iCAAiC,IAAI,CAAC,MAAM,EAAE,CAAC;IAEnE,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAC5B,WAAW,EACX,GAAG,EACH,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACrB,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAChF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CACvC,qBAAqB,EACrB,8FAA8F,EAC9F;IACE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iDAAiD,CAAC;IAC9E,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,2BAA2B,CAAC;CAC/E,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,GAAG,GAAG,MAAM,SAAS,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uCAAuC,EAAE,CAAC;YAC1E,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,iCAAiC,IAAI,CAAC,MAAM,kBAAkB,CAAC;IACnF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAE3B,IAAI,CAAC;QACH,OAAO,MAAM,iBAAiB,CAC5B,WAAW,EACX,GAAG,EACH,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,+BAA+B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAClF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,6FAA6F,EAC7F;IACE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;IACvE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;CACpE,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,GAAG,GAAG,MAAM,SAAS,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uCAAuC,EAAE,CAAC;YAC1E,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;IAC/E,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAE9B,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACnD,kBAAkB,CAChB,UAAU,EACV,iCAAiC,IAAI,CAAC,IAAI,iBAAiB,EAC3D,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,GAAG,EACH,cAAc,CACf;QACD,kBAAkB,CAChB,UAAU,EACV,iCAAiC,IAAI,CAAC,EAAE,iBAAiB,EACzD,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,GAAG,EACH,cAAc,CACf;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,yCAAyC,IAAI,CAAC,IAAI,gDAAgD;iBACzG;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,uCAAuC,IAAI,CAAC,EAAE,gDAAgD;iBACrG;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;QACvE,oBAAoB;QACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC1D,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;QAC3D,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACpC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,iBAAiB,YAAY,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,SAAS,UAAU,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,GAAG;iBAClG;gBACD;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAChF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,wGAAwG,EACxG;IACE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAC9G,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,mCAAmC,CAAC;CACjG,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;QACD,oBAAoB;QACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC1D,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;QAC3D,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACpC,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE;gBACnF;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAChF;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CACpC,kBAAkB,EAClB,mFAAmF,EACnF;IACE,GAAG,EAAE,CAAC;SACH,MAAM,EAAE;SACR,QAAQ,CACP,gEAAgE,CACjE;CACJ,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,oBAAoB;QACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC1D,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;QAC3D,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACpC,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,IAAI,CAAC,GAAG,EAAE,EAAE;gBAC9C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,4BAA4B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;iBAC/E;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAC3C,yBAAyB,EACzB,0HAA0H,EAC1H,EAAE,EACF,KAAK,EAAE,KAAK,EAA2B,EAAE;IACvC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,EAAE,CAAC;QAEpD,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE1B,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,uBAAuB,QAAQ,EAAE;iBACxC;gBACD;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;iBAC/E;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC"}
+{"version":3,"file":"computer-use.js","sourceRoot":"","sources":["../../src/tools/computer-use.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAEtD,OAAO,EACL,aAAa,EACb,cAAc,EACd,wBAAwB,EACxB,KAAK,EACL,WAAW,EACX,UAAU,EACV,MAAM,EACN,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,MAAM,GACP,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC;;;GAGG;AACH,SAAS,cAAc;IACrB,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAClE,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,6CAA6C,EAAE,CAAC;YAChF,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,wBAAwB,EAAE,EAAE,CAAC;QAChC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,yRAAyR;iBAChS;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uEAAuE;AAEvE;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,WAGjC;IACC,MAAM,KAAK,GAAG,oBAAoB,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1E,IAAI,KAAK,IAAI,GAAG;QAAE,OAAO,EAAE,GAAG,WAAW,EAAE,CAAC;IAC5C,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;QACzD,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;KAC5D,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,KAA+B,EAC/B,cAAiD,EACjD,WAA8C;IAE9C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC;QACnE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC;KACtE,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,CAAC,EAAE,CAAC;SACD,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CAAC,4DAA4D,CAAC;IACzE,CAAC,EAAE,CAAC;SACD,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CAAC,2DAA2D,CAAC;CACzE,CAAC,CAAC;AAEH;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,IAAY,EACZ,OAAO,GAAG,IAAI;IAEd,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7D,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,aAAa,EAAE,CAAC;IAC3D,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;IACpC,OAAO;QACL,OAAO,EAAE;YACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;YACtB;gBACE,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,gBAAgB;gBACtB,QAAQ,EAAE,WAAW;aACuB;SAC/C;KACF,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,gBAAgB,CACvB,QAAgB,EAChB,EAAiC;IAEjC,OAAO,CAAC,KAAK,IAAI,EAAE;QACjB,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;QAC7B,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACvD,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,GAAG,QAAQ,YAAY,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;qBAC1E;iBACF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;AACP,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,CACtC,oBAAoB,EACpB,0LAA0L,EAC1L;IACE,KAAK,EAAE,WAAW,CAAC,QAAQ,CACzB,oDAAoD,CACrD;IACD,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;CACvE,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,OAAO,gBAAgB,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;YAC5B,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,YAAY,CACjB,GAAG,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS,mBAAmB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CACjN,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,kHAAkH,EAClH;IACE,KAAK,EAAE,WAAW,CAAC,QAAQ,CACzB,iEAAiE,CAClE;CACF,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,OAAO,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAEjE,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAE3B,OAAO,YAAY,CACjB,+BAA+B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CACvH,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CACvC,qBAAqB,EACrB,kHAAkH,EAClH;IACE,KAAK,EAAE,WAAW,CAAC,QAAQ,CACzB,6DAA6D,CAC9D;IACD,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,2BAA2B,CAAC;CAC/E,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,OAAO,gBAAgB,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAEjE,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAExD,OAAO,YAAY,CACjB,YAAY,IAAI,CAAC,SAAS,mBAAmB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CACrI,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,gHAAgH,EAChH;IACE,UAAU,EAAE,WAAW,CAAC,QAAQ,CAC9B,yDAAyD,CAC1D;IACD,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAC5B,yDAAyD,CAC1D;CACF,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,OAAO,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QACxE,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAEpE,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAE3C,OAAO,YAAY,CACjB,4BAA4B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAC9N,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CACrC,mBAAmB,EACnB,wGAAwG,EACxG;IACE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAC9G,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,mCAAmC,CAAC;CACjG,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,OAAO,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,IAAI,EAAE;QACtD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;QAED,OAAO,YAAY,CACjB,aAAa,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5D,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CACpC,kBAAkB,EAClB,mFAAmF,EACnF;IACE,GAAG,EAAE,CAAC;SACH,MAAM,EAAE;SACR,QAAQ,CACP,gEAAgE,CACjE;CACJ,EACD,KAAK,EAAE,IAAI,EAA2B,EAAE;IACtC,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEzB,OAAO,YAAY,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CACF,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAC3C,yBAAyB,EACzB,iLAAiL,EACjL,EAAE,EACF,KAAK,EAAE,KAAK,EAA2B,EAAE;IACvC,MAAM,GAAG,GAAG,cAAc,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,EAAE,CAAC;QAEpD,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE1B,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,uBAAuB,QAAQ,4BAA4B,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,4BAA4B,WAAW,CAAC,KAAK,IAAI,WAAW,CAAC,MAAM,sFAAsF;iBACxP;gBACD;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,WAAW;iBACuB;aAC/C;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;iBAC/E;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC"}