screenhand 0.5.0 → 0.5.3

package/dist/src/state/app-map.js

@@ -339,7 +339,9 @@ export class AppMap {
             fs.mkdirSync(this.config.mapsDir, { recursive: true });
             writeFileAtomicSync(this.ladderFilePath(bundleId), JSON.stringify(data, null, 2));
         }
-        catch { /* non-critical */ }
+        catch (e) {
+            process.stderr.write(`[app-map] saveGeneratedLadder failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     // ── Create ────────────────────────────────────────────────────────
     createEmpty(bundleId, appName, version = "unknown") {
@@ -1591,7 +1593,9 @@ export class AppMap {
                 writeFileAtomicSync(this.filePath(data.app), JSON.stringify(data, null, 2) + "\n");
                 this.dirty.delete(data.app); // Only remove the one we just wrote
             }
-            catch { /* non-fatal — will be picked up by next debounced save */ }
+            catch (e) {
+                process.stderr.write(`[app-map] urgent mastery save failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
     }
     refreshMastery(bundleId) {
@@ -2299,6 +2303,212 @@ export class AppMap {
             return [];
         }
     }
+    // ── Visual Mapping (Phase 3) ──────────────────────────────────────
+    /**
+     * Get visual mapping metadata for an app.
+     */
+    getVisualMeta(bundleId) {
+        const data = this.ensureLoaded(bundleId);
+        return data?.visualMeta ?? null;
+    }
+    /**
+     * Populate the app map from a visual scan result.
+     * Fills in -1,-1 coordinates for known elements and adds newly discovered ones.
+     * Does NOT overwrite elements that already have valid positions from AX/interaction data.
+     */
+    populateFromVisualScan(bundleId, appName, scan, meta) {
+        let data = this.ensureLoaded(bundleId);
+        if (!data) {
+            data = this.createEmpty(bundleId, appName);
+            this.save(data);
+        }
+        let added = 0;
+        let updated = 0;
+        // Purge all OCR/LLM elements to make room for fresh scan results.
+        // AX-confirmed and manual elements are kept — they're authoritative.
+        for (const zone of Object.values(data.zones)) {
+            zone.elements = zone.elements.filter(el => {
+                if (el.labelSource === "ax" || el.labelSource === "manual")
+                    return true;
+                // Keep elements with no labelSource that have valid AX-style positions
+                // (these come from recordElementOutcome, i.e. real interactions)
+                if (!el.labelSource && el.relativeX != null && el.relativeX >= 0)
+                    return true;
+                return false;
+            });
+        }
+        this.save(data);
+        // Populate zones from scan
+        for (const zone of scan.zones) {
+            const zoneKey = zone.label.toLowerCase().replace(/\s+/g, "_");
+            if (!data.zones[zoneKey]) {
+                this.addZone(bundleId, zoneKey, {
+                    relativePosition: zone.bounds,
+                    type: zone.type,
+                    elements: [],
+                    verified: false,
+                    lastSeen: new Date().toISOString(),
+                });
+            }
+        }
+        // Populate elements from scan
+        for (const el of scan.elements) {
+            // Check if element already exists with a valid position
+            const existing = this.findElement(bundleId, el.label);
+            if (existing) {
+                const hasValidPosition = existing.element.relativeX >= 0 && existing.element.relativeY >= 0;
+                const hasAXSource = existing.element.labelSource === "ax" || existing.element.labelSource === "manual";
+                // Don't overwrite AX-confirmed positions — they're more reliable
+                if (hasValidPosition && hasAXSource)
+                    continue;
+                // Update position if current is unknown (-1) or from lower-confidence source
+                if (!hasValidPosition || (existing.element.labelSource === "llm" && el.confidence > (existing.element.visualConfidence ?? 0))) {
+                    this.updateElementPosition(bundleId, existing.zone, el.label, el.x, el.y);
+                    // Reload after update
+                    const refreshed = this.findElement(bundleId, el.label);
+                    if (refreshed) {
+                        refreshed.element.labelSource = el.confidence >= 0.7 ? "ocr" : "llm";
+                        refreshed.element.visualConfidence = el.confidence;
+                        refreshed.element.validationCount = 0;
+                        refreshed.element.mismatchCount = 0;
+                    }
+                    updated++;
+                }
+                continue;
+            }
+            // New element — find best matching zone or use "auto_discovered"
+            const targetZone = el.zone ? el.zone.toLowerCase().replace(/\s+/g, "_") : "auto_discovered";
+            const zoneKey = data.zones[targetZone] ? targetZone : "auto_discovered";
+            const newElement = {
+                label: el.label,
+                relativeX: el.x,
+                relativeY: el.y,
+                anchor: "top-left",
+                ocrBackup: el.label,
+                successCount: 0,
+                failCount: 0,
+                lastInteracted: new Date().toISOString(),
+                sessionsSinceUse: 0,
+                labelSource: el.confidence >= 0.7 ? "ocr" : "llm",
+                visualConfidence: el.confidence,
+                validationCount: 0,
+                mismatchCount: 0,
+            };
+            this.addElement(bundleId, zoneKey, newElement);
+            added++;
+        }
+        // Store visual metadata
+        data = this.ensureLoaded(bundleId);
+        if (data) {
+            data.visualMeta = meta;
+            this.dirty.add(bundleId);
+            this.scheduleSave();
+        }
+        return { added, updated };
+    }
+    /**
+     * Validate a live AX element position against the visual map.
+     * Returns true if positions match (within tolerance), false if mismatch.
+     * Updates validationCount/mismatchCount on the stored element.
+     */
+    validateElementPosition(bundleId, label, liveX, liveY, tolerance = 0.05) {
+        const found = this.findElement(bundleId, label);
+        if (!found)
+            return null;
+        const el = found.element;
+        // Only validate elements that have visual-scan positions
+        if (el.relativeX < 0 || el.relativeY < 0)
+            return null;
+        if (!el.labelSource || el.labelSource === "ax" || el.labelSource === "manual")
+            return null;
+        const dx = Math.abs(el.relativeX - liveX);
+        const dy = Math.abs(el.relativeY - liveY);
+        const matches = dx <= tolerance && dy <= tolerance;
+        if (matches) {
+            el.validationCount = (el.validationCount ?? 0) + 1;
+            // After 3 validations, promote confidence
+            if ((el.validationCount ?? 0) >= 3 && (el.visualConfidence ?? 0) < 0.9) {
+                el.visualConfidence = 0.9;
+                el.labelSource = "ax"; // Promoted — now AX-confirmed
+            }
+        }
+        else {
+            el.mismatchCount = (el.mismatchCount ?? 0) + 1;
+            // After 3 mismatches, demote confidence
+            const total = (el.validationCount ?? 0) + (el.mismatchCount ?? 0);
+            if (total >= 10 && (el.mismatchCount ?? 0) / total > 0.3) {
+                el.visualConfidence = 0.3;
+            }
+        }
+        this.dirty.add(bundleId);
+        this.scheduleSave();
+        return matches;
+    }
+    /**
+     * Proximity-based validation: find the nearest visual-scan element to a live AX position.
+     * Unlike validateElementPosition (label match), this matches by position alone.
+     * If a visual-scan element is within tolerance of the AX position, validate it.
+     * Also updates the element's label to the AX label if it was OCR-derived.
+     */
+    validateNearestElement(bundleId, axLabel, liveX, liveY, tolerance = 0.05) {
+        const data = this.ensureLoaded(bundleId);
+        if (!data)
+            return false;
+        let bestEl = null;
+        let bestDist = Infinity;
+        for (const zone of Object.values(data.zones)) {
+            for (const el of zone.elements) {
+                // Only validate visual-scan elements (ocr/llm source)
+                if (el.relativeX < 0 || el.relativeY < 0)
+                    continue;
+                if (!el.labelSource || el.labelSource === "ax" || el.labelSource === "manual")
+                    continue;
+                const dx = Math.abs(el.relativeX - liveX);
+                const dy = Math.abs(el.relativeY - liveY);
+                const dist = Math.sqrt(dx * dx + dy * dy);
+                if (dx <= tolerance && dy <= tolerance && dist < bestDist) {
+                    bestDist = dist;
+                    bestEl = el;
+                }
+            }
+        }
+        if (!bestEl)
+            return false;
+        bestEl.validationCount = (bestEl.validationCount ?? 0) + 1;
+        // After 3 proximity validations, promote to AX-confirmed
+        if ((bestEl.validationCount ?? 0) >= 3 && (bestEl.visualConfidence ?? 0) < 0.9) {
+            bestEl.visualConfidence = 0.9;
+            bestEl.labelSource = "ax";
+            // Update label to the authoritative AX label
+            if (axLabel && axLabel.length >= 2) {
+                bestEl.label = axLabel;
+                bestEl.ocrBackup = axLabel;
+            }
+        }
+        this.dirty.add(bundleId);
+        this.scheduleSave();
+        return true;
+    }
+    /**
+     * Check if an app's visual map is stale based on app version change.
+     */
+    isVisualMapStale(bundleId, currentAppVersion) {
+        const meta = this.getVisualMeta(bundleId);
+        if (!meta)
+            return true; // No map = stale
+        // Version mismatch = stale
+        if (currentAppVersion && meta.appVersion !== currentAppVersion)
+            return true;
+        // Age-based: older than 7 days = stale
+        const ageMs = Date.now() - new Date(meta.lastScannedAt).getTime();
+        const ageDays = ageMs / 86_400_000;
+        if (ageDays > 7)
+            return true;
+        // Confidence too low = stale
+        if (meta.confidence < 0.3)
+            return true;
+        return false;
+    }
     // ── Internals ─────────────────────────────────────────────────────
     ensureLoaded(bundleId) {
         return this.cache.get(bundleId) ?? this.load(bundleId);

package/dist/src/state/state-watcher.js

@@ -0,0 +1,144 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class StateWatcher {
+    worldModel;
+    execute;
+    rules = new Map();
+    interval = null;
+    pollMs;
+    constructor(worldModel, execute, pollMs = 2_000) {
+        this.worldModel = worldModel;
+        this.execute = execute;
+        this.pollMs = pollMs;
+    }
+    static MAX_RULES = 50;
+    /**
+     * Register a watch rule. Returns the rule ID for later removal.
+     */
+    register(rule) {
+        if (this.rules.size >= StateWatcher.MAX_RULES && !this.rules.has(rule.id)) {
+            throw new Error(`Maximum watch rules (${StateWatcher.MAX_RULES}) reached. Remove existing rules first.`);
+        }
+        this.rules.set(rule.id, {
+            rule,
+            fireCount: 0,
+            lastFiredAt: 0,
+        });
+        return rule.id;
+    }
+    /** Register a convenience rule: fire action when a control with matching title appears. */
+    watchForElement(id, elementTitle, action, bundleId) {
+        const titleLower = elementTitle.toLowerCase();
+        return this.register({
+            id,
+            description: `Watch for element "${elementTitle}"`,
+            condition: (state) => {
+                for (const win of state.windows.values()) {
+                    for (const ctrl of win.controls.values()) {
+                        if (ctrl.label?.value?.toLowerCase().includes(titleLower)) {
+                            return true;
+                        }
+                    }
+                }
+                return false;
+            },
+            action,
+            maxFires: 1,
+            cooldownMs: 10_000,
+            ...(bundleId ? { bundleId } : {}),
+        });
+    }
+    /** Register: fire action when a dialog appears with matching text. */
+    watchForDialog(id, titlePattern, action) {
+        return this.register({
+            id,
+            description: `Watch for dialog matching ${titlePattern}`,
+            condition: (state) => state.activeDialogs.some((d) => titlePattern.test(d.title ?? "")),
+            action,
+            maxFires: 0, // unlimited — dialogs can recur
+            cooldownMs: 5_000,
+        });
+    }
+    /** Remove a watch rule by ID. */
+    unregister(id) {
+        return this.rules.delete(id);
+    }
+    /** Remove all rules. */
+    clear() {
+        this.rules.clear();
+    }
+    /** Get all registered rules. */
+    getRules() {
+        return [...this.rules.values()].map((rs) => ({
+            id: rs.rule.id,
+            description: rs.rule.description,
+            fireCount: rs.fireCount,
+        }));
+    }
+    /** Start the polling loop. */
+    start() {
+        if (this.interval)
+            return;
+        this.interval = setInterval(() => {
+            void this.tick();
+        }, this.pollMs);
+    }
+    /** Stop the polling loop. */
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+    }
+    get isRunning() {
+        return this.interval !== null;
+    }
+    async tick() {
+        const state = this.worldModel.getState();
+        const now = Date.now();
+        const focusedBundleId = state.focusedApp?.bundleId;
+        for (const [id, rs] of this.rules) {
+            // Max fires check
+            if (rs.rule.maxFires > 0 && rs.fireCount >= rs.rule.maxFires)
+                continue;
+            // Cooldown check
+            if (now - rs.lastFiredAt < rs.rule.cooldownMs)
+                continue;
+            // BundleId filter
+            if (rs.rule.bundleId && rs.rule.bundleId !== focusedBundleId)
+                continue;
+            try {
+                if (rs.rule.condition(state)) {
+                    rs.fireCount++;
+                    rs.lastFiredAt = now;
+                    process.stderr.write(`[state-watcher] Rule "${id}" fired (${rs.fireCount}x): ${rs.rule.description}\n`);
+                    // Fire and forget — don't block the poll loop
+                    this.execute(rs.rule.action.tool, rs.rule.action.params).catch((err) => {
+                        process.stderr.write(`[state-watcher] Rule "${id}" action failed: ${err instanceof Error ? err.message : String(err)}\n`);
+                    });
+                    // Remove exhausted rules
+                    if (rs.rule.maxFires > 0 && rs.fireCount >= rs.rule.maxFires) {
+                        this.rules.delete(id);
+                    }
+                }
+            }
+            catch (err) {
+                process.stderr.write(`[state-watcher] Rule "${id}" condition threw: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+    }
+}