screenhand 0.3.8 → 0.3.9

package/dist/mcp-desktop.js

@@ -4219,6 +4219,27 @@ server.tool("click_with_fallback", "Click a target by text using the canonical f
                     }
                     throw new Error("Target not found via OCR");
                 }
+                case "window_buffer": {
+                    // Last resort: capture GPU window buffer (works even when window is hidden),
+                    // OCR it, find target text, translate window-relative to screen-absolute coords
+                    const wbWindowId = await resolveWindowId(targetPid);
+                    if (!wbWindowId)
+                        throw new Error("No window found for window_buffer capture");
+                    const wbShot = await bridge.call("cg.captureWindow", { windowId: wbWindowId });
+                    const wbMatches = await bridge.call("vision.findText", { imagePath: wbShot.path, searchText: target });
+                    const wbMatch = Array.isArray(wbMatches) ? wbMatches[0] : null;
+                    if (!wbMatch?.bounds)
+                        throw new Error("Target not found via window buffer OCR");
+                    // Translate window-relative coords to screen-absolute
+                    const allWins = await bridge.call("app.windows");
+                    const winInfo = allWins.find((w) => w.windowId === wbWindowId);
+                    const winX = winInfo?.bounds?.x ?? 0;
+                    const winY = winInfo?.bounds?.y ?? 0;
+                    const absX = winX + wbMatch.bounds.x + wbMatch.bounds.width / 2;
+                    const absY = winY + wbMatch.bounds.y + wbMatch.bounds.height / 2;
+                    await bridge.call("cg.mouseClick", { x: absX, y: absY });
+                    return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: `${target} at (${Math.round(absX)},${Math.round(absY)}) [window_buffer]` };
+                }
             }
             throw new Error(`Unknown method: ${method}`);
         }
@@ -4553,6 +4574,22 @@ server.tool("read_with_fallback", "Read text content from the screen or a specif
                     const ocr = await bridge.call("vision.ocr", { imagePath: shot.path });
                     return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: ocr.text?.slice(0, 4000) ?? "" };
                 }
+                case "window_buffer": {
+                    // GPU window buffer capture — reads content even when window is behind other apps
+                    const rbWindowId = await resolveWindowId(targetPid);
+                    if (!rbWindowId)
+                        throw new Error("No window found for window_buffer read");
+                    const rbShot = await bridge.call("cg.captureWindow", { windowId: rbWindowId });
+                    if (target) {
+                        const rbMatches = await bridge.call("vision.findText", { imagePath: rbShot.path, searchText: target });
+                        const rbMatch = Array.isArray(rbMatches) ? rbMatches[0] : null;
+                        if (!rbMatch)
+                            throw new Error("Text not found via window buffer OCR");
+                        return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: rbMatch.text };
+                    }
+                    const rbOcr = await bridge.call("vision.ocr", { imagePath: rbShot.path });
+                    return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: rbOcr.text?.slice(0, 4000) ?? "" };
+                }
             }
             throw new Error(`Method ${method} does not support read`);
         }
@@ -4668,6 +4705,29 @@ server.tool("locate_with_fallback", "Find an element's position on screen using
                     const b = match.bounds;
                     return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: `${target} at (${b.x},${b.y} ${b.width}x${b.height})` };
                 }
+                case "window_buffer": {
+                    // GPU window buffer capture + OCR — works even when window is hidden
+                    const lbWindowId = await resolveWindowId(targetPid);
+                    if (!lbWindowId)
+                        throw new Error("No window found for window_buffer locate");
+                    const lbShot = await bridge.call("cg.captureWindow", { windowId: lbWindowId });
+                    const lbMatches = await bridge.call("vision.findText", { imagePath: lbShot.path, searchText: target });
+                    const lbMatch = Array.isArray(lbMatches) ? lbMatches[0] : null;
+                    if (!lbMatch?.bounds)
+                        throw new Error("Target not found via window buffer OCR");
+                    // Translate window-relative to screen-absolute bounds
+                    const lbWins = await bridge.call("app.windows");
+                    const lbWinInfo = lbWins.find((w) => w.windowId === lbWindowId);
+                    const lbOffX = lbWinInfo?.bounds?.x ?? 0;
+                    const lbOffY = lbWinInfo?.bounds?.y ?? 0;
+                    const lbBounds = {
+                        x: lbOffX + lbMatch.bounds.x,
+                        y: lbOffY + lbMatch.bounds.y,
+                        width: lbMatch.bounds.width,
+                        height: lbMatch.bounds.height,
+                    };
+                    return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: `${target} at (${lbBounds.x},${lbBounds.y} ${lbBounds.width}x${lbBounds.height}) [window_buffer]` };
+                }
             }
             throw new Error(`Method ${method} does not support locate`);
         }

package/dist/src/learning/engine.js

@@ -126,6 +126,26 @@ export class LearningEngine {
     rankSensors(bundleId) {
         return this.sensors.rank(bundleId);
     }
+    /**
+     * Detect whether an app is "vision-only" — AX can't see its content,
+     * so window buffer capture + OCR is the only viable perception source.
+     * Returns true when AX has failed enough times with a low score and
+     * at least one other source (vision/ocr) has succeeded.
+     */
+    isVisionOnlyApp(bundleId) {
+        const ranked = this.sensors.rank(bundleId);
+        if (ranked.length < 2)
+            return false;
+        const ax = ranked.find(r => r.sourceType === "ax");
+        const vision = ranked.find(r => r.sourceType === "vision" || r.sourceType === "ocr");
+        // AX score near zero + vision/ocr has some success
+        if (ax && ax.score < 0.15 && vision && vision.score > 0.3)
+            return true;
+        // No AX entry at all but vision works
+        if (!ax && vision && vision.score > 0.3)
+            return true;
+        return false;
+    }
     /**
      * Query verified UI patterns for a given app, optionally filtered by tool.
      */

package/dist/src/perception/coordinator.js

@@ -768,13 +768,18 @@ export class PerceptionCoordinator extends EventEmitter {
         // Safe CLI mode is already enabled via setSafeCLI() in start().
         // This allows vision/OCR for canvas-heavy apps like Canva in Chrome.
         // Skip vision if learning engine shows it consistently fails for this app,
-        // but retry every 20th cycle to re-evaluate (apps may gain windows later)
+        // but retry every 20th cycle to re-evaluate (apps may gain windows later).
+        // Exception: vision-only apps (AX blind) — vision/OCR is their ONLY perception
+        // source, so never skip it. Window buffer capture works even when window is hidden.
         if (this.learningEngine && this.activeAppContext) {
-            const ranked = this.learningEngine.rankSensors(this.activeAppContext.bundleId);
-            const visionRank = ranked.find(r => r.sourceType === "vision");
-            if (visionRank && visionRank.score < 0.1 && ranked.length >= 2 && this.stats.slowCycles % 20 !== 0) {
-                this.stats.slowCycles++;
-                return; // Vision consistently fails for this app — skip (retry every 20th cycle)
+            const isVisionOnly = this.learningEngine.isVisionOnlyApp(this.activeAppContext.bundleId);
+            if (!isVisionOnly) {
+                const ranked = this.learningEngine.rankSensors(this.activeAppContext.bundleId);
+                const visionRank = ranked.find(r => r.sourceType === "vision");
+                if (visionRank && visionRank.score < 0.1 && ranked.length >= 2 && this.stats.slowCycles % 20 !== 0) {
+                    this.stats.slowCycles++;
+                    return; // Vision consistently fails for this app — skip (retry every 20th cycle)
+                }
             }
         }
         const timestamp = new Date().toISOString();
@@ -860,14 +865,24 @@ export class PerceptionCoordinator extends EventEmitter {
                     },
                 });
             }
-            // Record vision sensor outcome
+            // Record vision sensor outcome — also record as window_buffer for vision-only apps
+            // so the fallback chain knows this source works for element location
             if (this.learningEngine && this.activeAppContext) {
+                const latencyMs = Date.now() - new Date(timestamp).getTime();
                 this.learningEngine.recordSensorOutcome({
                     bundleId: this.activeAppContext.bundleId,
                     sourceType: "vision",
                     success: !!diffEvent,
-                    latencyMs: Date.now() - new Date(timestamp).getTime(),
+                    latencyMs,
                 });
+                if (this.learningEngine.isVisionOnlyApp(this.activeAppContext.bundleId) && ocrEvent) {
+                    this.learningEngine.recordSensorOutcome({
+                        bundleId: this.activeAppContext.bundleId,
+                        sourceType: "window_buffer",
+                        success: true,
+                        latencyMs,
+                    });
+                }
             }
         }
         catch {

package/dist/src/perception/types.js

@@ -21,6 +21,7 @@ export const DEFAULT_PERCEPTION_CONFIG = {
     enableAX: true,
     enableCDP: true,
     enableVision: true,
+    enableWindowBuffer: true,
     maxROIsPerCycle: 3,
     skipCaptureLock: false,
 };

package/dist/src/runtime/execution-contract.js

@@ -23,7 +23,7 @@
  */
 // ── 1. Fallback Chain ──────────────────────────────────────────────────
 /** Ordered list of execution methods, from fastest/most reliable to slowest/least reliable */
-const EXECUTION_METHODS = ["ax", "cdp", "ocr", "coordinates"];
+const EXECUTION_METHODS = ["ax", "cdp", "ocr", "window_buffer", "coordinates"];
 const METHOD_CAPABILITIES = {
     ax: {
         method: "ax",
@@ -61,6 +61,18 @@ const METHOD_CAPABILITIES = {
         requiresBridge: true,
         requiresCDP: false,
     },
+    window_buffer: {
+        method: "window_buffer",
+        canClick: true,
+        canType: false,
+        canRead: true,
+        canLocate: true,
+        canSelect: false,
+        canScroll: false,
+        avgLatencyMs: 350,
+        requiresBridge: true,
+        requiresCDP: false,
+    },
     coordinates: {
         method: "coordinates",
         canClick: true,
@@ -90,6 +102,7 @@ const SENSOR_TO_METHOD = {
     chrome: "cdp",
     ocr: "ocr",
     vision: "ocr",
+    window_buffer: "window_buffer",
     coordinates: "coordinates",
 };
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screenhand",
-  "version": "0.3.8",
+  "version": "0.3.9",
   "mcpName": "io.github.manushi4/screenhand",
   "description": "Give AI eyes and hands on your desktop. ScreenHand is an open-source MCP server that lets Claude and other AI agents see your screen, click buttons, type text, and control any app on macOS and Windows.",
   "homepage": "https://screenhand.com",