visionclaw 0.1.187-beta.9 → 0.1.187-dev.refactor-computer-use-direct-coordinates.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "visionclaw",
-  "version": "0.1.187-beta.9",
+  "version": "0.1.187-dev.refactor-computer-use-direct-coordinates.1",
   "description": "A personal assistant agent that runs on your desktop, receives commands from messaging channels, and executes tasks autonomously.",
   "type": "module",
   "main": "dist/index.js",

package/dist/tools/coordinate-resolver.d.ts

@@ -1,30 +0,0 @@
-/**
- * Resolves natural language targets to pixel coordinates using
- * Anthropic's computer_20251124 tool via the Messages API.
- *
- * Sends screenshot + instruction to Claude; Claude returns a tool_use
- * with action and coordinate. We extract coordinates only (no execution).
- *
- * IMPORTANT: The input screenshot is assumed to be already resized to fit
- * Claude's vision constraints (done by takeScreenshot). We read the actual
- * image dimensions, tell the computer-use tool those are the display size,
- * and unscale returned coordinates back to the real display space.
- * Reference: https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool
- */
-import type { VisionClawConfig } from "../config/types.js";
-export interface ResolvedAction {
-    action: string;
-    x: number;
-    y: number;
-}
-/**
- * Resolve a natural language target to pixel coordinates.
- * Uses Messages API with computer_20251124 tool; returns the first tool_use result.
- *
- * The input screenshot is expected to already be resized to fit Claude's
- * vision constraints (by takeScreenshot). We read its actual pixel dimensions,
- * use those as display_width_px / display_height_px, and unscale the returned
- * coordinates back to the real display space using imageWidth/displayWidth ratio.
- */
-export declare function resolveCoordinates(screenshotBase64: string, instruction: string, displayWidth: number, displayHeight: number, config: VisionClawConfig, screenshotFilePath?: string): Promise<ResolvedAction | null>;
-//# sourceMappingURL=coordinate-resolver.d.ts.map

package/dist/tools/coordinate-resolver.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"coordinate-resolver.d.ts","sourceRoot":"","sources":["../../src/tools/coordinate-resolver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG3D,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX;AAQD;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CACtC,gBAAgB,EAAE,MAAM,EACxB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,MAAM,EAAE,gBAAgB,EACxB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAyFhC"}

package/dist/tools/coordinate-resolver.js

@@ -1,104 +0,0 @@
-/**
- * Resolves natural language targets to pixel coordinates using
- * Anthropic's computer_20251124 tool via the Messages API.
- *
- * Sends screenshot + instruction to Claude; Claude returns a tool_use
- * with action and coordinate. We extract coordinates only (no execution).
- *
- * IMPORTANT: The input screenshot is assumed to be already resized to fit
- * Claude's vision constraints (done by takeScreenshot). We read the actual
- * image dimensions, tell the computer-use tool those are the display size,
- * and unscale returned coordinates back to the real display space.
- * Reference: https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool
- */
-import { logger } from "../logger.js";
-import { getImageDimensions, getImageDimensionsFromFile } from "./screenshot.js";
-import { createClient, getModelId } from "../agent/providers/client-factory.js";
-function clamp(n, min, max) {
-    return Math.max(min, Math.min(max, n));
-}
-// ── Resolver ───────────────────────────────────────────────────────────
-/**
- * Resolve a natural language target to pixel coordinates.
- * Uses Messages API with computer_20251124 tool; returns the first tool_use result.
- *
- * The input screenshot is expected to already be resized to fit Claude's
- * vision constraints (by takeScreenshot). We read its actual pixel dimensions,
- * use those as display_width_px / display_height_px, and unscale the returned
- * coordinates back to the real display space using imageWidth/displayWidth ratio.
- */
-export async function resolveCoordinates(screenshotBase64, instruction, displayWidth, displayHeight, config, screenshotFilePath) {
-    // Read actual pixel dimensions of the (already resized) screenshot.
-    // Prefer file path to avoid base64 decode + temp file round-trip.
-    const dims = screenshotFilePath
-        ? await getImageDimensionsFromFile(screenshotFilePath)
-        : await getImageDimensions(Buffer.from(screenshotBase64, "base64"));
-    if (!dims) {
-        logger.warn("Could not read screenshot dimensions, aborting coordinate resolution");
-        return null;
-    }
-    const imgW = dims.width;
-    const imgH = dims.height;
-    // Scale factors to convert from image space back to actual display space
-    const scaleX = displayWidth / imgW;
-    const scaleY = displayHeight / imgH;
-    logger.info(`Resolving coordinates: "${instruction}" (display ${displayWidth}x${displayHeight}, image ${imgW}x${imgH}, scaleX=${scaleX.toFixed(3)}, scaleY=${scaleY.toFixed(3)})`, { instruction, displayWidth, displayHeight, imgW, imgH, scaleX, scaleY });
-    const client = createClient(config);
-    const response = await client.beta.messages.create({
-        model: getModelId(config),
-        max_tokens: 1024,
-        betas: ["computer-use-2025-11-24"],
-        tools: [
-            {
-                type: "computer_20251124",
-                name: "computer",
-                display_width_px: imgW,
-                display_height_px: imgH,
-                display_number: 1,
-            },
-        ],
-        system: "You are a helpful assistant that can resolve natural language targets to pixel coordinates using the computer tool. Call the tool directly, no questions or explanations.",
-        tool_choice: { type: "tool", name: "computer" },
-        messages: [
-            {
-                role: "user",
-                content: [
-                    {
-                        type: "image",
-                        source: {
-                            type: "base64",
-                            media_type: "image/png",
-                            data: screenshotBase64,
-                        },
-                    },
-                    {
-                        type: "text",
-                        text: instruction,
-                    },
-                ],
-            },
-        ],
-    });
-    for (const block of response.content) {
-        if (block.type === "tool_use" && block.name === "computer") {
-            const input = block.input;
-            const coord = input.coordinate;
-            const action = input.action ?? "left_click";
-            logger.debug(`Coordinate resolver raw response: action=${action} coordinate=${JSON.stringify(coord)}`, { instruction, action, coordinate: coord, fullInput: input });
-            if (Array.isArray(coord) && coord.length >= 2) {
-                const rawX = coord[0];
-                const rawY = coord[1];
-                if (typeof rawX === "number" && typeof rawY === "number" && Number.isFinite(rawX) && Number.isFinite(rawY)) {
-                    // Unscale: Claude returned coords in image space → convert to actual display space
-                    const x = clamp(Math.round(rawX * scaleX), 0, displayWidth);
-                    const y = clamp(Math.round(rawY * scaleY), 0, displayHeight);
-                    logger.info(`Resolved coordinates: action=${action} raw=(${rawX},${rawY}) unscaled=(${x},${y}) scaleX=${scaleX.toFixed(3)} scaleY=${scaleY.toFixed(3)}`, { instruction, action, rawX, rawY, x, y, scaleX, scaleY });
-                    return { action, x, y };
-                }
-            }
-        }
-    }
-    logger.info(`Could not resolve coordinates: "${instruction}"`, { instruction });
-    return null;
-}
-//# sourceMappingURL=coordinate-resolver.js.map

package/dist/tools/coordinate-resolver.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"coordinate-resolver.js","sourceRoot":"","sources":["../../src/tools/coordinate-resolver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAEjF,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAQhF,SAAS,KAAK,CAAC,CAAS,EAAE,GAAW,EAAE,GAAW;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,0EAA0E;AAE1E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,gBAAwB,EACxB,WAAmB,EACnB,YAAoB,EACpB,aAAqB,EACrB,MAAwB,EACxB,kBAA2B;IAE3B,oEAAoE;IACpE,kEAAkE;IAClE,MAAM,IAAI,GAAG,kBAAkB;QAC7B,CAAC,CAAC,MAAM,0BAA0B,CAAC,kBAAkB,CAAC;QACtD,CAAC,CAAC,MAAM,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC,CAAC;IACtE,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;QACpF,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;IACxB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;IAEzB,yEAAyE;IACzE,MAAM,MAAM,GAAG,YAAY,GAAG,IAAI,CAAC;IACnC,MAAM,MAAM,GAAG,aAAa,GAAG,IAAI,CAAC;IAEpC,MAAM,CAAC,IAAI,CACT,2BAA2B,WAAW,cAAc,YAAY,IAAI,aAAa,WAAW,IAAI,IAAI,IAAI,YAAY,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EACrK,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CACzE,CAAC;IAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAEpC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACjD,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC;QACzB,UAAU,EAAE,IAAI;QAChB,KAAK,EAAE,CAAC,yBAAyB,CAAC;QAClC,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,mBAAmB;gBACzB,IAAI,EAAE,UAAU;gBAChB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,IAAI;gBACvB,cAAc,EAAE,CAAC;aAClB;SACF;QACD,MAAM,EAAE,2KAA2K;QACnL,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE;QAC/C,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,UAAU,EAAE,WAAW;4BACvB,IAAI,EAAE,gBAAgB;yBACvB;qBACF;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,WAAW;qBAClB;iBACF;aACF;SACF;KACF,CAAC,CAAC;IAEH,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAgC,CAAC;YACrD,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC;YAC/B,MAAM,MAAM,GAAI,KAAK,CAAC,MAA6B,IAAI,YAAY,CAAC;YAEpE,MAAM,CAAC,KAAK,CACV,4CAA4C,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EACxF,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAC7D,CAAC;YAEF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAY,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,IAAI,GAAY,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3G,mFAAmF;oBACnF,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;oBAC5D,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC;oBAC7D,MAAM,CAAC,IAAI,CACT,gCAAgC,MAAM,SAAS,IAAI,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAC3I,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAC1D,CAAC;oBACF,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,mCAAmC,WAAW,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAChF,OAAO,IAAI,CAAC;AACd,CAAC"}