screenhand 0.5.2 → 0.5.3

package/dist/mcp-desktop.js

@@ -72,6 +72,7 @@ import { MenuScanner } from "./src/ingestion/menu-scanner.js";
 import { DocParser } from "./src/ingestion/doc-parser.js";
 import { TutorialExtractor } from "./src/ingestion/tutorial-extractor.js";
 import { extractFeaturesFromHTML } from "./src/ingestion/feature-extractor.js";
+import { quickScan, llmEnrich, buildVisualMeta, isSensitiveApp } from "./src/state/visual-mapper.js";
 import { CoverageAuditor } from "./src/ingestion/coverage-auditor.js";
 import { ReferenceMerger } from "./src/ingestion/reference-merger.js";
 import { PlaybookPublisher } from "./src/community/publisher.js";
@@ -6882,6 +6883,188 @@ server.tool("discover_features", "Extract features from an app's official websit
     _playbookStoreForContext.reload();
     return { content: [{ type: "text", text: lines.join("\n") }] };
 });
+// ── Visual App Mapping (Phase 3) ─────────────────────────────────
+server.tool("map_app", "Visually map an app's UI by taking a screenshot, running OCR to identify interactive elements and zones with coordinates. Makes subsequent tool calls faster and more accurate. Runs quick scan (~500ms) inline, optional LLM enrichment in background if ANTHROPIC_API_KEY is set.", {
+    bundleId: z.string().describe("macOS bundle ID (e.g. com.apple.Notes)"),
+    appName: z.string().describe("Human-readable app name"),
+    force: z.boolean().optional().describe("Re-map even if a recent map exists (default: false)"),
+    depth: z.enum(["quick", "full"]).optional().describe("'quick' = OCR only (~500ms). 'full' = OCR + LLM enrichment (~15s). Default: quick"),
+}, async ({ bundleId, appName, force, depth }) => {
+    if (isSensitiveApp(bundleId)) {
+        return { content: [{ type: "text", text: `Blocked: ${bundleId} is a sensitive app (password manager, banking, etc.). Visual mapping is not allowed for privacy reasons.` }] };
+    }
+    // Check if already mapped (unless force)
+    if (!force) {
+        const existingMeta = appMap.getVisualMeta(bundleId);
+        if (existingMeta && !appMap.isVisualMapStale(bundleId)) {
+            return { content: [{ type: "text", text: `Visual map for ${appName} already exists (${existingMeta.screensMapped.length} screens, confidence: ${existingMeta.confidence.toFixed(2)}). Use force: true to re-map.` }] };
+        }
+    }
+    await ensureBridge();
+    // Get focused app PID
+    const apps = await bridge.call("app.list", {});
+    const matchedApp = apps?.find((a) => a.bundleId === bundleId);
+    if (!matchedApp?.pid) {
+        return { content: [{ type: "text", text: `App ${bundleId} is not running. Launch it first with focus("${bundleId}") or launch("${bundleId}").` }] };
+    }
+    const pid = matchedApp.pid;
+    // Get window bounds
+    let windowTitle = "";
+    let windowBounds;
+    try {
+        const wins = await bridge.call("window.list", {});
+        const appWins = wins?.filter((w) => w.pid === pid);
+        const mainWin = appWins?.find((w) => w.focused || w.frontmost || w.isMain) ?? appWins?.[0];
+        if (mainWin) {
+            windowTitle = mainWin.title ?? "";
+            windowBounds = mainWin.bounds ?? mainWin;
+        }
+    }
+    catch { /* use defaults */ }
+    // Phase A: Quick scan (OCR)
+    const scanResult = await quickScan(bridge, pid, windowBounds);
+    if (!scanResult) {
+        return { content: [{ type: "text", text: `Failed to capture screenshot of ${appName}. Make sure the app window is visible.` }] };
+    }
+    // Get app version for staleness tracking
+    let appVersion = "unknown";
+    try {
+        const infoResult = await bridge.call("app.info", { bundleId });
+        appVersion = infoResult?.version ?? infoResult?.shortVersion ?? "unknown";
+    }
+    catch { /* use default */ }
+    // Get display scale factor
+    let scaleFactor = 2;
+    try {
+        const screenInfo = await bridge.call("screen.info", {});
+        scaleFactor = screenInfo?.scaleFactor ?? 2;
+    }
+    catch { /* default to Retina */ }
+    const meta = buildVisualMeta(scanResult.hash, scanResult.captureSize, windowTitle, appVersion, scanResult.scan.confidence, scaleFactor);
+    // Populate into AppMap
+    const { added, updated } = appMap.populateFromVisualScan(bundleId, appName, scanResult.scan, meta);
+    const lines = [
+        `Visual map for ${appName} (${bundleId}):`,
+        `  Zones identified: ${scanResult.scan.zones.length}`,
+        `  Elements mapped: ${scanResult.scan.elements.length} (${added} new, ${updated} updated)`,
+        `  Map confidence: ${scanResult.scan.confidence.toFixed(2)}`,
+        `  App version: ${appVersion}`,
+    ];
+    if (scanResult.scan.zones.length > 0) {
+        lines.push("  Zones:");
+        for (const z of scanResult.scan.zones) {
+            const elCount = scanResult.scan.elements.filter(e => e.zone === z.label).length;
+            lines.push(`    ${z.label} (${z.type}): ${elCount} elements`);
+        }
+    }
+    // Phase B: LLM enrichment (background, if depth=full and API key exists)
+    if ((depth === "full") && process.env.ANTHROPIC_API_KEY) {
+        lines.push("  LLM enrichment: starting in background...");
+        // Fire and forget — don't block the response
+        (async () => {
+            try {
+                // Get screenshot as file, then read as base64 for LLM
+                const screenshotShot = await bridge.call("cg.captureScreen", {});
+                if (!screenshotShot?.path)
+                    return;
+                const fs = await import("node:fs");
+                const screenshotBase64 = fs.readFileSync(screenshotShot.path).toString("base64");
+                const screenshotData = { base64: screenshotBase64 };
+                // Get AX tree for cross-reference
+                let axTree = "";
+                try {
+                    const tree = await bridge.call("ax.tree", { pid, depth: 3 });
+                    axTree = JSON.stringify(tree, null, 1).slice(0, 3000);
+                }
+                catch { /* proceed without AX */ }
+                const enrichment = await llmEnrich(screenshotData.base64, axTree, appName, bundleId, windowTitle, scanResult.captureSize);
+                if (enrichment) {
+                    // Merge LLM results into AppMap (LLM confidence capped at 0.5)
+                    const llmScan = {
+                        zones: enrichment.zones,
+                        elements: enrichment.elements.map(e => ({
+                            ...e,
+                            confidence: Math.min(e.confidence, 0.5), // Cap — LLM is hypothesis
+                        })),
+                        confidence: Math.min(enrichment.confidence, 0.6),
+                    };
+                    appMap.populateFromVisualScan(bundleId, appName, llmScan, {
+                        ...meta,
+                        confidence: Math.min(enrichment.confidence, 0.6),
+                    });
+                    process.stderr.write(`[visual-mapper] LLM enrichment complete for ${appName}: ${enrichment.elements.length} elements, ${enrichment.zones.length} zones\n`);
+                }
+            }
+            catch (err) {
+                process.stderr.write(`[visual-mapper] Background enrichment failed: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        })().catch(() => { });
+    }
+    else if (depth === "full" && !process.env.ANTHROPIC_API_KEY) {
+        lines.push("  LLM enrichment: skipped (no ANTHROPIC_API_KEY set)");
+    }
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
+originalTool("map_status", "Check the health of an app's visual map. Shows zones, element counts, confidence, staleness, and failure rates. Useful for debugging click failures.", {
+    bundleId: z.string().describe("macOS bundle ID"),
+}, async ({ bundleId }) => {
+    const meta = appMap.getVisualMeta(bundleId);
+    const data = appMap.getLoaded(bundleId) ?? appMap.load(bundleId);
+    if (!meta) {
+        return { content: [{ type: "text", text: `No visual map exists for ${bundleId}. Run map_app to create one.` }] };
+    }
+    const isStale = appMap.isVisualMapStale(bundleId);
+    const ageMs = Date.now() - new Date(meta.lastScannedAt).getTime();
+    const ageHours = Math.round(ageMs / 3_600_000);
+    const lines = [
+        `Visual Map Status: ${bundleId}`,
+        `  Last scanned: ${meta.lastScannedAt} (${ageHours}h ago)`,
+        `  App version: ${meta.appVersion}`,
+        `  Confidence: ${meta.confidence.toFixed(2)}`,
+        `  Staleness: ${isStale ? "STALE — consider re-mapping" : "fresh"}`,
+        `  Screens mapped: ${meta.screensMapped.join(", ") || "(none)"}`,
+        `  Scale factor: ${meta.scaleFactor}x`,
+        `  Capture size: ${meta.captureSize.w}x${meta.captureSize.h}`,
+    ];
+    // Count visual-scan elements
+    if (data) {
+        let visualElements = 0;
+        let axElements = 0;
+        let totalValidations = 0;
+        let totalMismatches = 0;
+        for (const zone of Object.values(data.zones)) {
+            for (const el of zone.elements) {
+                if (el.labelSource === "ocr" || el.labelSource === "llm") {
+                    visualElements++;
+                    totalValidations += el.validationCount ?? 0;
+                    totalMismatches += el.mismatchCount ?? 0;
+                }
+                else if (el.labelSource === "ax" || el.labelSource === "manual") {
+                    axElements++;
+                }
+            }
+        }
+        lines.push(`  Visual-scan elements: ${visualElements}`);
+        lines.push(`  AX-confirmed elements: ${axElements}`);
+        if (totalValidations + totalMismatches > 0) {
+            const matchRate = totalValidations / (totalValidations + totalMismatches);
+            lines.push(`  Position match rate: ${(matchRate * 100).toFixed(1)}% (${totalValidations} matches, ${totalMismatches} mismatches)`);
+        }
+        // Zone breakdown
+        const zoneKeys = Object.keys(data.zones);
+        if (zoneKeys.length > 0) {
+            lines.push("  Zones:");
+            for (const key of zoneKeys.slice(0, 15)) {
+                const zone = data.zones[key];
+                lines.push(`    ${key} (${zone.type}): ${zone.elements.length} elements`);
+            }
+            if (zoneKeys.length > 15) {
+                lines.push(`    ... and ${zoneKeys.length - 15} more`);
+            }
+        }
+    }
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
 server.tool("coverage_report", "Check what ScreenHand knows about an app: shortcuts, selectors, flows, playbooks, error patterns, and stability %. Useful before complex workflows to decide strategy: learn first (if empty), go fast (if high coverage), or use fallback tools (if error patterns exist). Optional for quick actions.", {
     bundleId: z.string().describe("macOS bundle ID (e.g. com.blackmagic-design.DaVinciResolveLite)"),
     appName: z.string().describe("Human-readable app name"),

package/dist/src/context-tracker.js

@@ -241,6 +241,32 @@ export class ContextTracker {
                 : map.masteryLevel.toUpperCase();
             hints.push(`🗺 AppMap [${ratingDisplay}]: ${zones} zones, ${verifiedPaths}/${totalPaths} verified paths`);
         }
+        // Visual map hint: suggest map_app if no visual data exists
+        if (hints.length < 3 && this.context.appMapData) {
+            const visualMeta = this.context.appMapData.visualMeta;
+            if (!visualMeta) {
+                hints.push("📸 No visual map — run map_app(bundleId, appName) for faster, more accurate element targeting");
+            }
+            else {
+                // Check for element with visual position near the target
+                const target = extractTarget(params);
+                if (target) {
+                    const targetLower = target.toLowerCase();
+                    for (const zone of Object.values(this.context.appMapData.zones)) {
+                        for (const el of zone.elements) {
+                            if (el.label.toLowerCase().includes(targetLower) &&
+                                el.relativeX >= 0 && el.relativeY >= 0 &&
+                                (el.visualConfidence ?? 0) >= 0.5) {
+                                hints.push(`📍 Visual map: "${el.label}" at (${el.relativeX.toFixed(2)}, ${el.relativeY.toFixed(2)}) [${el.labelSource ?? "unknown"} source, confidence ${(el.visualConfidence ?? 0).toFixed(1)}]`);
+                                break;
+                            }
+                        }
+                        if (hints.length >= 3)
+                            break;
+                    }
+                }
+            }
+        }
         // Playbook-dependent hints below
         if (!this.context.playbook)
             return hints;

package/dist/src/perception/coordinator.js

@@ -1034,6 +1034,38 @@ export class PerceptionCoordinator extends EventEmitter {
                 process.stderr.write(`[perception] recordElementOutcome failed: ${e instanceof Error ? e.message : String(e)}\n`);
             }
         }
+        // 4. Visual map validation: cross-check AX positions against visual map
+        // Uses proximity-based matching (not label matching) so OCR text
+        // that doesn't exactly match AX labels can still be validated.
+        if (this.appMap.getVisualMeta(bundleId) && focusedWin.bounds?.value) {
+            const winBounds = focusedWin.bounds.value;
+            let validated = 0;
+            for (const [, ctrl] of focusedWin.controls) {
+                if (validated >= 15)
+                    break; // Cap per cycle
+                const label = ctrl.label?.value;
+                if (!label || label.length < 2)
+                    continue;
+                const pos = ctrl.position;
+                if (!pos || typeof pos.x !== "number" || typeof pos.y !== "number")
+                    continue;
+                // Convert absolute to relative
+                const relX = winBounds.width > 0 ? (pos.x - winBounds.x) / winBounds.width : -1;
+                const relY = winBounds.height > 0 ? (pos.y - winBounds.y) / winBounds.height : -1;
+                if (relX < 0 || relX > 1 || relY < 0 || relY > 1)
+                    continue;
+                try {
+                    // Try exact label match first, fall back to proximity match
+                    const exactMatch = this.appMap.validateElementPosition(bundleId, label, relX, relY);
+                    if (exactMatch === null) {
+                        // No label match — try proximity: find nearest visual-scan element
+                        this.appMap.validateNearestElement(bundleId, label, relX, relY);
+                    }
+                    validated++;
+                }
+                catch { /* non-fatal */ }
+            }
+        }
     }
     /**
      * Wire #9: Record element visibility from OCR detections.

package/dist/src/state/app-map.js

@@ -2303,6 +2303,212 @@ export class AppMap {
             return [];
         }
     }
+    // ── Visual Mapping (Phase 3) ──────────────────────────────────────
+    /**
+     * Get visual mapping metadata for an app.
+     */
+    getVisualMeta(bundleId) {
+        const data = this.ensureLoaded(bundleId);
+        return data?.visualMeta ?? null;
+    }
+    /**
+     * Populate the app map from a visual scan result.
+     * Fills in -1,-1 coordinates for known elements and adds newly discovered ones.
+     * Does NOT overwrite elements that already have valid positions from AX/interaction data.
+     */
+    populateFromVisualScan(bundleId, appName, scan, meta) {
+        let data = this.ensureLoaded(bundleId);
+        if (!data) {
+            data = this.createEmpty(bundleId, appName);
+            this.save(data);
+        }
+        let added = 0;
+        let updated = 0;
+        // Purge all OCR/LLM elements to make room for fresh scan results.
+        // AX-confirmed and manual elements are kept — they're authoritative.
+        for (const zone of Object.values(data.zones)) {
+            zone.elements = zone.elements.filter(el => {
+                if (el.labelSource === "ax" || el.labelSource === "manual")
+                    return true;
+                // Keep elements with no labelSource that have valid AX-style positions
+                // (these come from recordElementOutcome, i.e. real interactions)
+                if (!el.labelSource && el.relativeX != null && el.relativeX >= 0)
+                    return true;
+                return false;
+            });
+        }
+        this.save(data);
+        // Populate zones from scan
+        for (const zone of scan.zones) {
+            const zoneKey = zone.label.toLowerCase().replace(/\s+/g, "_");
+            if (!data.zones[zoneKey]) {
+                this.addZone(bundleId, zoneKey, {
+                    relativePosition: zone.bounds,
+                    type: zone.type,
+                    elements: [],
+                    verified: false,
+                    lastSeen: new Date().toISOString(),
+                });
+            }
+        }
+        // Populate elements from scan
+        for (const el of scan.elements) {
+            // Check if element already exists with a valid position
+            const existing = this.findElement(bundleId, el.label);
+            if (existing) {
+                const hasValidPosition = existing.element.relativeX >= 0 && existing.element.relativeY >= 0;
+                const hasAXSource = existing.element.labelSource === "ax" || existing.element.labelSource === "manual";
+                // Don't overwrite AX-confirmed positions — they're more reliable
+                if (hasValidPosition && hasAXSource)
+                    continue;
+                // Update position if current is unknown (-1) or from lower-confidence source
+                if (!hasValidPosition || (existing.element.labelSource === "llm" && el.confidence > (existing.element.visualConfidence ?? 0))) {
+                    this.updateElementPosition(bundleId, existing.zone, el.label, el.x, el.y);
+                    // Reload after update
+                    const refreshed = this.findElement(bundleId, el.label);
+                    if (refreshed) {
+                        refreshed.element.labelSource = el.confidence >= 0.7 ? "ocr" : "llm";
+                        refreshed.element.visualConfidence = el.confidence;
+                        refreshed.element.validationCount = 0;
+                        refreshed.element.mismatchCount = 0;
+                    }
+                    updated++;
+                }
+                continue;
+            }
+            // New element — find best matching zone or use "auto_discovered"
+            const targetZone = el.zone ? el.zone.toLowerCase().replace(/\s+/g, "_") : "auto_discovered";
+            const zoneKey = data.zones[targetZone] ? targetZone : "auto_discovered";
+            const newElement = {
+                label: el.label,
+                relativeX: el.x,
+                relativeY: el.y,
+                anchor: "top-left",
+                ocrBackup: el.label,
+                successCount: 0,
+                failCount: 0,
+                lastInteracted: new Date().toISOString(),
+                sessionsSinceUse: 0,
+                labelSource: el.confidence >= 0.7 ? "ocr" : "llm",
+                visualConfidence: el.confidence,
+                validationCount: 0,
+                mismatchCount: 0,
+            };
+            this.addElement(bundleId, zoneKey, newElement);
+            added++;
+        }
+        // Store visual metadata
+        data = this.ensureLoaded(bundleId);
+        if (data) {
+            data.visualMeta = meta;
+            this.dirty.add(bundleId);
+            this.scheduleSave();
+        }
+        return { added, updated };
+    }
+    /**
+     * Validate a live AX element position against the visual map.
+     * Returns true if positions match (within tolerance), false if mismatch.
+     * Updates validationCount/mismatchCount on the stored element.
+     */
+    validateElementPosition(bundleId, label, liveX, liveY, tolerance = 0.05) {
+        const found = this.findElement(bundleId, label);
+        if (!found)
+            return null;
+        const el = found.element;
+        // Only validate elements that have visual-scan positions
+        if (el.relativeX < 0 || el.relativeY < 0)
+            return null;
+        if (!el.labelSource || el.labelSource === "ax" || el.labelSource === "manual")
+            return null;
+        const dx = Math.abs(el.relativeX - liveX);
+        const dy = Math.abs(el.relativeY - liveY);
+        const matches = dx <= tolerance && dy <= tolerance;
+        if (matches) {
+            el.validationCount = (el.validationCount ?? 0) + 1;
+            // After 3 validations, promote confidence
+            if ((el.validationCount ?? 0) >= 3 && (el.visualConfidence ?? 0) < 0.9) {
+                el.visualConfidence = 0.9;
+                el.labelSource = "ax"; // Promoted — now AX-confirmed
+            }
+        }
+        else {
+            el.mismatchCount = (el.mismatchCount ?? 0) + 1;
+            // After 3 mismatches, demote confidence
+            const total = (el.validationCount ?? 0) + (el.mismatchCount ?? 0);
+            if (total >= 10 && (el.mismatchCount ?? 0) / total > 0.3) {
+                el.visualConfidence = 0.3;
+            }
+        }
+        this.dirty.add(bundleId);
+        this.scheduleSave();
+        return matches;
+    }
+    /**
+     * Proximity-based validation: find the nearest visual-scan element to a live AX position.
+     * Unlike validateElementPosition (label match), this matches by position alone.
+     * If a visual-scan element is within tolerance of the AX position, validate it.
+     * Also updates the element's label to the AX label if it was OCR-derived.
+     */
+    validateNearestElement(bundleId, axLabel, liveX, liveY, tolerance = 0.05) {
+        const data = this.ensureLoaded(bundleId);
+        if (!data)
+            return false;
+        let bestEl = null;
+        let bestDist = Infinity;
+        for (const zone of Object.values(data.zones)) {
+            for (const el of zone.elements) {
+                // Only validate visual-scan elements (ocr/llm source)
+                if (el.relativeX < 0 || el.relativeY < 0)
+                    continue;
+                if (!el.labelSource || el.labelSource === "ax" || el.labelSource === "manual")
+                    continue;
+                const dx = Math.abs(el.relativeX - liveX);
+                const dy = Math.abs(el.relativeY - liveY);
+                const dist = Math.sqrt(dx * dx + dy * dy);
+                if (dx <= tolerance && dy <= tolerance && dist < bestDist) {
+                    bestDist = dist;
+                    bestEl = el;
+                }
+            }
+        }
+        if (!bestEl)
+            return false;
+        bestEl.validationCount = (bestEl.validationCount ?? 0) + 1;
+        // After 3 proximity validations, promote to AX-confirmed
+        if ((bestEl.validationCount ?? 0) >= 3 && (bestEl.visualConfidence ?? 0) < 0.9) {
+            bestEl.visualConfidence = 0.9;
+            bestEl.labelSource = "ax";
+            // Update label to the authoritative AX label
+            if (axLabel && axLabel.length >= 2) {
+                bestEl.label = axLabel;
+                bestEl.ocrBackup = axLabel;
+            }
+        }
+        this.dirty.add(bundleId);
+        this.scheduleSave();
+        return true;
+    }
+    /**
+     * Check if an app's visual map is stale based on app version change.
+     */
+    isVisualMapStale(bundleId, currentAppVersion) {
+        const meta = this.getVisualMeta(bundleId);
+        if (!meta)
+            return true; // No map = stale
+        // Version mismatch = stale
+        if (currentAppVersion && meta.appVersion !== currentAppVersion)
+            return true;
+        // Age-based: older than 7 days = stale
+        const ageMs = Date.now() - new Date(meta.lastScannedAt).getTime();
+        const ageDays = ageMs / 86_400_000;
+        if (ageDays > 7)
+            return true;
+        // Confidence too low = stale
+        if (meta.confidence < 0.3)
+            return true;
+        return false;
+    }
     // ── Internals ─────────────────────────────────────────────────────
     ensureLoaded(bundleId) {
         return this.cache.get(bundleId) ?? this.load(bundleId);