screenhand 0.3.2 → 0.3.4

package/dist/src/learning/sensor-policy.js

@@ -97,6 +97,15 @@ export class SensorPolicy {
         });
         return qualified.length > 0 ? qualified[0].sourceType : null;
     }
+    /**
+     * Seed a single entry if no real data exists for that key.
+     * Used for cold-start bootstrap from AppMap ReadySignals / UIArchitecture.
+     */
+    seedEntry(entry) {
+        if (this.entries.has(entry.key))
+            return; // already have real data
+        this.entries.set(entry.key, { ...entry });
+    }
     clear() {
         this.entries.clear();
     }

package/dist/src/learning/timing-model.js

@@ -163,6 +163,68 @@ export class TimingModel {
         // Clear all cached distributions
         this.distributions.clear();
     }
+    /**
+     * Wire #14: Seed timing data from AppMap's TimingProfiles.
+     * Converts each profile to a synthetic TimingSample and loads it,
+     * but only for tool×bundleId keys that don't already have real samples.
+     */
+    seedFromTimingProfiles(profiles, bundleId) {
+        if (!profiles.length)
+            return;
+        // Bug #5 fix: aggregate profiles by tool type, computing weighted average
+        // Bug #6 fix: map page_load → browser_dom (a LOCATE_TOOL), add locate_with_fallback
+        const toolMap = {
+            page_load: "browser_dom",
+            element_response: "click",
+            animation: "wait_for_state",
+            data_fetch: "browser_wait",
+        };
+        // Group profiles by target tool
+        const grouped = new Map();
+        for (const profile of profiles) {
+            // Guard: skip profiles with zero/negative sample count to prevent NaN (0/0)
+            if (profile.sampleCount <= 0 || !Number.isFinite(profile.avgMs) || profile.avgMs <= 0)
+                continue;
+            const tool = toolMap[profile.type] ?? "browser_wait";
+            const key = `${tool}::${bundleId}`;
+            // Skip if we already have real samples for this key
+            if (this.samples.has(key))
+                continue;
+            const existing = grouped.get(tool);
+            if (existing) {
+                existing.totalWeightedMs += profile.avgMs * profile.sampleCount;
+                existing.totalSamples += profile.sampleCount;
+                if (profile.lastMeasured > existing.lastMeasured) {
+                    existing.lastMeasured = profile.lastMeasured;
+                }
+            }
+            else {
+                grouped.set(tool, {
+                    totalWeightedMs: profile.avgMs * profile.sampleCount,
+                    totalSamples: profile.sampleCount,
+                    lastMeasured: profile.lastMeasured,
+                });
+            }
+        }
+        // Create synthetic samples from aggregated data
+        const synthetics = [];
+        for (const [tool, agg] of grouped) {
+            const avgMs = agg.totalWeightedMs / agg.totalSamples;
+            const count = Math.min(agg.totalSamples, 5);
+            for (let i = 0; i < count; i++) {
+                synthetics.push({
+                    tool,
+                    bundleId,
+                    durationMs: avgMs,
+                    success: true,
+                    timestamp: agg.lastMeasured,
+                });
+            }
+        }
+        if (synthetics.length > 0) {
+            this.loadSamples(synthetics);
+        }
+    }
     /**
      * Compute budget for a category of tools by taking the max p95
      * across all tools in that category for the given app.

package/dist/src/memory/research.js

@@ -81,7 +81,13 @@ async function tryClaudeAPI(apiKey, query) {
         const text = data.content?.[0]?.text;
         return text && text.length > 10 ? text.trim() : null;
     }
-    catch {
+    catch (e) {
+        // Sanitize error to prevent API key leakage in logs/stack traces
+        const msg = e instanceof Error ? e.message : "unknown";
+        const sanitized = msg.replace(/x-api-key[^"]*"/gi, 'x-api-key: [REDACTED]"');
+        if (sanitized !== msg) {
+            throw new Error("API call failed: " + sanitized);
+        }
         return null;
     }
 }

package/dist/src/memory/store.js

@@ -104,22 +104,29 @@ export class MemoryStore {
     // ── file locking ──────────────────────────────
     acquireLock() {
         try {
-            // Check for stale lock (PID no longer running)
-            if (fs.existsSync(this.lockPath)) {
+            // Try to create lock atomically first (avoids TOCTOU race between exists-check and write)
+            try {
+                fs.writeFileSync(this.lockPath, String(process.pid), { flag: "wx" });
+            }
+            catch {
+                // Lock file exists — check if it's stale (PID no longer running)
                 const lockContent = fs.readFileSync(this.lockPath, "utf-8").trim();
                 const lockPid = parseInt(lockContent, 10);
                 if (lockPid && !this.isProcessRunning(lockPid)) {
-                    // Stale lock — remove it
+                    // Stale lock — remove and retry with wx
                     fs.unlinkSync(this.lockPath);
+                    fs.writeFileSync(this.lockPath, String(process.pid), { flag: "wx" });
+                }
+                else {
+                    throw new Error("Lock held by active process");
                 }
             }
-            // Write our PID
-            fs.writeFileSync(this.lockPath, String(process.pid), { flag: "wx" });
             this.hasLock = true;
         }
-        catch {
+        catch (err) {
             // Another instance holds the lock — we still work but skip writes
             // to avoid corruption. Reads are from our own cache (stale but safe).
+            console.error(`[MemoryStore] Lock acquisition failed — writes disabled: ${err instanceof Error ? err.message : err}`);
             this.hasLock = false;
         }
     }
@@ -263,7 +270,11 @@ export class MemoryStore {
             return;
         this.rotateActionsIfNeeded();
         // S75 Option C: Redact PII before persisting to disk (not in live responses)
-        const redacted = { ...entry, result: entry.result ? redactPII(entry.result) : entry.result };
+        const TYPING_TOOLS = new Set(["type_text", "browser_type", "browser_fill_form", "type_with_fallback"]);
+        const redactedParams = TYPING_TOOLS.has(entry.tool) && entry.params && "text" in entry.params
+            ? { ...entry.params, text: "[REDACTED]" }
+            : entry.params;
+        const redacted = { ...entry, params: redactedParams, result: entry.result ? redactPII(entry.result) : entry.result };
         this.pendingActionWrites.push(JSON.stringify(redacted) + "\n");
         // Schedule batch flush (debounced 100ms)
         if (!this.flushTimer) {

package/dist/src/perception/coordinator.js

@@ -57,6 +57,7 @@ export class PerceptionCoordinator extends EventEmitter {
     running = false;
     learningEngine = null;
     appMap = null;
+    contextTracker = null;
     browserEnricher = null;
     fusionPipeline = new FusionPipeline();
     // In-flight guards to prevent timer pileup when async cycles exceed their interval
@@ -71,6 +72,11 @@ export class PerceptionCoordinator extends EventEmitter {
     static IDLE_THRESHOLD_MS = 3_000;
     lastToolCallAt = Date.now();
     idle = false;
+    // Wire #9: L3→L7 — perception auto-updates AppMap
+    lastPerceptionTitle = null;
+    lastPerceptionDialogCount = 0;
+    reportedControlLabels = new Set();
+    static MAP_UPDATE_INTERVAL = 5; // every 5th medium cycle
     constructor(worldModel, axSource, cdpSource, visionSource, config) {
         super();
         this.worldModel = worldModel;
@@ -93,6 +99,12 @@ export class PerceptionCoordinator extends EventEmitter {
     setAppMap(map) {
         this.appMap = map;
     }
+    /**
+     * Wire F10: Inject context tracker for per-app perception config from references.
+     */
+    setContextTracker(tracker) {
+        this.contextTracker = tracker;
+    }
     /**
      * Set a browser enricher callback for non-CDP browsers (Safari).
      * Called during medium cycle to fetch URL/title/tabs via AppleScript.
@@ -101,6 +113,67 @@ export class PerceptionCoordinator extends EventEmitter {
     setBrowserEnricher(fn) {
         this.browserEnricher = fn;
     }
+    /**
+     * Wire #16: Adjust perception intervals based on app timing characteristics.
+     * Clamps: fast 50-500ms, medium 100-1000ms, slow 500-5000ms.
+     * Only takes effect if perception is running (restarts timers with new intervals).
+     */
+    adjustIntervals(overrides) {
+        if (!this.running) {
+            // Not running — just update config, timers will use it on next start()
+            const clamp = (val, min, max) => Math.max(min, Math.min(max, val));
+            if (overrides.fastIntervalMs != null) {
+                this.config = { ...this.config, fastIntervalMs: clamp(overrides.fastIntervalMs, 50, 500) };
+            }
+            if (overrides.mediumIntervalMs != null) {
+                this.config = { ...this.config, mediumIntervalMs: clamp(overrides.mediumIntervalMs, 100, 1000) };
+            }
+            if (overrides.slowIntervalMs != null) {
+                this.config = { ...this.config, slowIntervalMs: clamp(overrides.slowIntervalMs, 500, 5000) };
+            }
+            return;
+        }
+        const clamp = (val, min, max) => Math.max(min, Math.min(max, val));
+        if (overrides.fastIntervalMs != null) {
+            this.config = { ...this.config, fastIntervalMs: clamp(overrides.fastIntervalMs, 50, 500) };
+        }
+        if (overrides.mediumIntervalMs != null) {
+            this.config = { ...this.config, mediumIntervalMs: clamp(overrides.mediumIntervalMs, 100, 1000) };
+        }
+        if (overrides.slowIntervalMs != null) {
+            this.config = { ...this.config, slowIntervalMs: clamp(overrides.slowIntervalMs, 500, 5000) };
+        }
+        // Restart timers — only reached when this.running is true
+        if (this.running) {
+            if (this.fastTimer) {
+                clearInterval(this.fastTimer);
+                this.fastTimer = setInterval(() => {
+                    if (this.fastInFlight)
+                        return;
+                    this.fastInFlight = true;
+                    void this.fastCycle().catch(() => { }).finally(() => { this.fastInFlight = false; });
+                }, this.config.fastIntervalMs);
+            }
+            if (this.mediumTimer) {
+                clearInterval(this.mediumTimer);
+                this.mediumTimer = setInterval(() => {
+                    if (this.mediumInFlight)
+                        return;
+                    this.mediumInFlight = true;
+                    void this.mediumCycle().catch(() => { }).finally(() => { this.mediumInFlight = false; });
+                }, this.config.mediumIntervalMs);
+            }
+            if (this.slowTimer) {
+                clearInterval(this.slowTimer);
+                this.slowTimer = setInterval(() => {
+                    if (this.slowInFlight)
+                        return;
+                    this.slowInFlight = true;
+                    void this.slowCycle().catch(() => { }).finally(() => { this.slowInFlight = false; });
+                }, this.config.slowIntervalMs);
+            }
+        }
+    }
     /**
      * Notify that a tool call is happening — resets idle timer and starts stream if needed.
      * Call this from the intelligence wrapper PRE-CALL.
@@ -253,6 +326,10 @@ export class PerceptionCoordinator extends EventEmitter {
         this.fastInFlight = false;
         this.mediumInFlight = false;
         this.slowInFlight = false;
+        // Wire #9: reset AppMap tracking state on stop
+        this.lastPerceptionTitle = null;
+        this.lastPerceptionDialogCount = 0;
+        this.reportedControlLabels.clear();
         this.emit("stopped");
     }
     /**
@@ -288,6 +365,38 @@ export class PerceptionCoordinator extends EventEmitter {
         await this.stop();
         this.visionSource?.reset();
         this.cdpSource?.reset();
+        // Wire #16: reset intervals to defaults, then adjust based on AppMap timing data
+        this.config = {
+            ...this.config,
+            fastIntervalMs: DEFAULT_PERCEPTION_CONFIG.fastIntervalMs,
+            mediumIntervalMs: DEFAULT_PERCEPTION_CONFIG.mediumIntervalMs,
+            slowIntervalMs: DEFAULT_PERCEPTION_CONFIG.slowIntervalMs,
+        };
+        if (this.appMap && appContext.bundleId) {
+            const profiles = this.appMap.getTimingProfile(appContext.bundleId);
+            if (profiles.length > 0) {
+                const elementProfiles = profiles.filter((p) => p.type === "element_response");
+                // Require 3+ element_response profiles for reliable interval adaptation
+                if (elementProfiles.length >= 3) {
+                    const avgResponse = elementProfiles.reduce((sum, p) => sum + p.avgMs, 0) / elementProfiles.length;
+                    if (avgResponse > 1500) {
+                        // Slow app — increase slow interval, relax medium
+                        this.adjustIntervals({ slowIntervalMs: 2000, mediumIntervalMs: 800 });
+                    }
+                    else if (avgResponse < 300) {
+                        // Fast app — tighten slow interval for quicker visual updates
+                        this.adjustIntervals({ slowIntervalMs: 500 });
+                    }
+                }
+            }
+        }
+        // Wire F10: Apply per-app perception config from reference/playbook (L2→L3)
+        if (this.contextTracker) {
+            const refConfig = this.contextTracker.getPerceptionConfig();
+            if (refConfig) {
+                this.adjustIntervals(refConfig);
+            }
+        }
         await this.start(appContext, cdpClient);
     }
     /**
@@ -432,6 +541,10 @@ export class PerceptionCoordinator extends EventEmitter {
             catch { /* best-effort */ }
         }
         this.stats.mediumCycles++;
+        // Wire #9: L3→L7 — auto-update AppMap from perception (every 5th cycle, skip cycle 1)
+        if (this.stats.mediumCycles > 1 && this.stats.mediumCycles % PerceptionCoordinator.MAP_UPDATE_INTERVAL === 0) {
+            this.updateAppMapFromPerception();
+        }
         this.stats.lastMediumAt = timestamp;
     }
     /**
@@ -670,13 +783,17 @@ export class PerceptionCoordinator extends EventEmitter {
             this.stats.slowCycles++;
             return; // Observer daemon is capturing — skip this cycle
         }
+        let didWork = false;
         try {
             // Screenshot diff — optimized single-capture pipeline
             const windowId = this.activeWindowId ?? 0;
             if (windowId === 0)
                 return; // Vision needs a real window ID for screenshot
+            didWork = true;
             const SLOW_CYCLE_TIMEOUT_MS = 25_000;
-            const { diffEvent, ocrEvent, yoloElements } = await withTimeout(this.visionSource.captureAndDiffOptimized(windowId, this.config.maxROIsPerCycle), SLOW_CYCLE_TIMEOUT_MS, "captureAndDiffOptimized");
+            // Wire #10: L7→L3 — pass zone ROIs for targeted OCR instead of full-screen fallback
+            const zoneROIs = this.getZoneROIs();
+            const { diffEvent, ocrEvent, yoloElements } = await withTimeout(this.visionSource.captureAndDiffOptimized(windowId, this.config.maxROIsPerCycle, zoneROIs.length > 0 ? zoneROIs : undefined), SLOW_CYCLE_TIMEOUT_MS, "captureAndDiffOptimized");
             if (diffEvent) {
                 this.stats.visionDiffs++;
                 this.stats.lastVisionAt = new Date().toISOString();
@@ -707,6 +824,10 @@ export class PerceptionCoordinator extends EventEmitter {
                         this.appMap.save(mapData);
                     }
                 }
+                // Wire #9: record element visibility from OCR (every 3rd slow cycle, skip first)
+                if (this.stats.slowCycles > 0 && this.stats.slowCycles % 3 === 0 && ocrEvent.data.type === "vision_ocr") {
+                    this.recordVisibilityFromOCR(ocrEvent.data.regions);
+                }
                 this.emit("perception", ocrEvent);
             }
             // Fuse YOLO element detections with OCR text regions
@@ -761,11 +882,190 @@ export class PerceptionCoordinator extends EventEmitter {
             }
         }
         finally {
-            // Always increment stats, even on early return (windowId=0) or error
-            this.stats.slowCycles++;
-            this.stats.lastSlowAt = timestamp;
+            if (didWork) {
+                this.stats.slowCycles++;
+                this.stats.lastSlowAt = timestamp;
+            }
             if (!this.config.skipCaptureLock)
                 releaseCaptureLock();
         }
     }
+    // ── Wire #9: L3→L7 — auto-update AppMap from perception ──
+    /**
+     * Wire #9: Compare current world model state against tracked previous state
+     * and auto-record changes to AppMap. Called every MAP_UPDATE_INTERVAL medium cycles.
+     *
+     * Records:
+     * - Page transitions (window title changes)
+     * - Dialog state changes (open/closed)
+     * - New element discovery from AX/CDP controls
+     */
+    updateAppMapFromPerception() {
+        if (!this.appMap || !this.activeAppContext)
+            return;
+        const bundleId = this.activeAppContext.bundleId;
+        const state = this.worldModel.getState();
+        const focusedWin = state.focusedWindowId !== null
+            ? state.windows.get(state.focusedWindowId) ?? null
+            : null;
+        if (!focusedWin)
+            return;
+        const currentTitle = focusedWin.title?.value ?? null;
+        // 1. Page transition detection — window title changed between cycles
+        if (this.lastPerceptionTitle !== null &&
+            currentTitle !== null &&
+            this.lastPerceptionTitle !== currentTitle) {
+            try {
+                const fromTitle = this.lastPerceptionTitle.length > 80
+                    ? this.lastPerceptionTitle.slice(0, 80).trim() : this.lastPerceptionTitle;
+                const toTitle = currentTitle.length > 80
+                    ? currentTitle.slice(0, 80).trim() : currentTitle;
+                this.appMap.recordPageTransition(bundleId, fromTitle, toTitle, "perception_detected");
+            }
+            catch { /* best-effort — limits, PII filter, etc. */ }
+        }
+        this.lastPerceptionTitle = currentTitle;
+        // 2. Dialog state change detection
+        const currentDialogCount = state.activeDialogs.length;
+        if (currentDialogCount !== this.lastPerceptionDialogCount) {
+            const from = this.lastPerceptionDialogCount > 0 ? "open" : "closed";
+            const to = currentDialogCount > 0 ? "open" : "closed";
+            if (from !== to) {
+                try {
+                    this.appMap.recordStateChange(bundleId, "dialog_state", from, to, "perception_detected");
+                }
+                catch { /* best-effort */ }
+            }
+            this.lastPerceptionDialogCount = currentDialogCount;
+        }
+        // 3. New element discovery from controls (max 10 per cycle to avoid flooding)
+        // Truncate pageContext to avoid zone key bloat from dynamic window titles
+        const rawPageContext = currentTitle ?? "unknown";
+        const pageContext = rawPageContext.length > 80 ? rawPageContext.slice(0, 80).trim() : rawPageContext;
+        let added = 0;
+        // Cap reportedControlLabels — evict oldest entries instead of full clear
+        // to prevent re-flooding AppMap with already-known elements
+        if (this.reportedControlLabels.size > 5000) {
+            const iter = this.reportedControlLabels.values();
+            for (let i = 0; i < 1000; i++)
+                iter.next();
+            const keep = new Set();
+            for (const val of iter)
+                keep.add(val);
+            this.reportedControlLabels.clear();
+            for (const val of keep)
+                this.reportedControlLabels.add(val);
+        }
+        for (const [, ctrl] of focusedWin.controls) {
+            if (added >= 10)
+                break;
+            const label = ctrl.label?.value;
+            // Skip empty, too-short, too-long, or already-reported labels
+            if (!label || label.length < 2 || label.length > 60)
+                continue;
+            if (this.reportedControlLabels.has(label))
+                continue;
+            this.reportedControlLabels.add(label);
+            try {
+                // Use recordElementOutcome with zoneKey "auto" — auto-creates zones
+                // unlike addElement which silently fails when zone doesn't exist
+                this.appMap.recordElementOutcome(bundleId, "auto", label, true, pageContext);
+                added++;
+            }
+            catch { /* best-effort — zone limits, PII filter, etc. */ }
+        }
+    }
+    /**
+     * Wire #9: Record element visibility from OCR detections.
+     * Called in slow cycle after OCR produces text regions.
+     */
+    recordVisibilityFromOCR(regions) {
+        if (!this.appMap || !this.activeAppContext)
+            return;
+        const bundleId = this.activeAppContext.bundleId;
+        const state = this.worldModel.getState();
+        const focusedWin = state.focusedWindowId !== null
+            ? state.windows.get(state.focusedWindowId) ?? null
+            : null;
+        const rawPageContext = focusedWin?.title?.value ?? "unknown";
+        const pageContext = rawPageContext.length > 80 ? rawPageContext.slice(0, 80).trim() : rawPageContext;
+        // Only record visibility for OCR text that matches known AX controls —
+        // raw OCR picks up body text, logos, dates etc. that aren't UI elements
+        const knownLabels = new Set();
+        if (focusedWin) {
+            for (const [, ctrl] of focusedWin.controls) {
+                const label = ctrl.label?.value;
+                if (label && label.length >= 2)
+                    knownLabels.add(label.toLowerCase());
+            }
+        }
+        let recorded = 0;
+        for (const region of regions) {
+            if (recorded >= 20)
+                break;
+            const text = region.text.trim();
+            if (text.length < 2 || text.length > 60)
+                continue;
+            // Cross-reference: case-insensitive match against known AX control labels
+            if (!knownLabels.has(text.toLowerCase()))
+                continue;
+            try {
+                this.appMap.recordElementVisibility(bundleId, text, pageContext, true);
+                recorded++;
+            }
+            catch { /* best-effort */ }
+        }
+    }
+    // ── Wire #10: L7→L3 — zone ROI → targeted OCR ──
+    /**
+     * Wire #10: Convert AppMap zone positions to pixel ROIs for targeted OCR.
+     * Returns ROIs based on known zones, prioritized by zone type.
+     */
+    getZoneROIs() {
+        if (!this.appMap || !this.activeAppContext)
+            return [];
+        const state = this.worldModel.getState();
+        const focusedWin = state.focusedWindowId !== null
+            ? state.windows.get(state.focusedWindowId) ?? null
+            : null;
+        if (!focusedWin)
+            return [];
+        const winBounds = focusedWin.bounds?.value;
+        if (!winBounds || winBounds.width <= 0 || winBounds.height <= 0)
+            return [];
+        const mapData = this.appMap.load(this.activeAppContext.bundleId);
+        if (!mapData)
+            return [];
+        const rois = [];
+        for (const [, zone] of Object.entries(mapData.zones)) {
+            const rp = zone.relativePosition;
+            // Skip zones with zero/invalid dimensions
+            if (!rp || rp.width <= 0 || rp.height <= 0)
+                continue;
+            // Clamp relative positions to [0, 1] to prevent out-of-bounds ROIs
+            // from corrupted or stale map data
+            const left = Math.max(0, Math.min(1, rp.left));
+            const top = Math.max(0, Math.min(1, rp.top));
+            const clampedWidth = Math.min(Math.max(0, rp.width), 1.0 - left);
+            const clampedHeight = Math.min(Math.max(0, rp.height), 1.0 - top);
+            if (clampedWidth <= 0 || clampedHeight <= 0)
+                continue;
+            // ROI coordinates are relative to the window capture image (0,0 = window top-left),
+            // not screen-absolute. Zone positions are 0-1 relative to the window.
+            const pixelW = Math.max(1, Math.round(clampedWidth * winBounds.width));
+            const pixelH = Math.max(1, Math.round(clampedHeight * winBounds.height));
+            // Skip full-window ROIs from auto_discovered zones (defeats targeted OCR purpose)
+            if (clampedWidth >= 0.9 && clampedHeight >= 0.9)
+                continue;
+            rois.push({
+                x: Math.round(left * winBounds.width),
+                y: Math.round(top * winBounds.height),
+                width: pixelW,
+                height: pixelH,
+                reason: "known_zone",
+            });
+        }
+        // Cap at 5 zone ROIs to stay within budget
+        return rois.slice(0, 5);
+    }
 }

package/dist/src/perception/manager.js

@@ -37,6 +37,7 @@ export class PerceptionManager extends EventEmitter {
     lastCdpClient = null;
     pendingLearningEngine = null;
     pendingAppMap = null;
+    pendingContextTracker = null;
     constructor(worldModel, config) {
         super();
         this.worldModel = worldModel;
@@ -62,6 +63,15 @@ export class PerceptionManager extends EventEmitter {
             this.coordinator.setAppMap(map);
         }
     }
+    /**
+     * Wire F10: Inject context tracker for per-app perception config.
+     */
+    setContextTracker(tracker) {
+        this.pendingContextTracker = tracker;
+        if (this.coordinator) {
+            this.coordinator.setContextTracker(tracker);
+        }
+    }
     /**
      * Create perception sources from the bridge. Called once after ensureBridge().
      */
@@ -80,6 +90,9 @@ export class PerceptionManager extends EventEmitter {
         if (this.pendingAppMap) {
             this.coordinator.setAppMap(this.pendingAppMap);
         }
+        if (this.pendingContextTracker) {
+            this.coordinator.setContextTracker(this.pendingContextTracker);
+        }
         this.coordinator.on("perception", (event) => {
             this.handleReactiveEvent(event);
         });

package/dist/src/perception/vision-source.js

@@ -180,7 +180,7 @@ export class VisionSource {
      * Performance: unchanged ~113ms, changed ~175ms (vs ~370ms before Phase 2).
      * Phase 1 (FAST OCR) + Phase 2 (region OCR) combined.
      */
-    async captureAndDiffOptimized(windowId, maxROIs = 3) {
+    async captureAndDiffOptimized(windowId, maxROIs = 3, priorityROIs) {
         const start = Date.now();
         // 1. Capture: stream frame (~0ms) or one-shot (~112ms)
         const capture = await this.captureToFileOrStream(windowId);
@@ -208,12 +208,22 @@ export class VisionSource {
             // 4. Run OCR and YOLO in parallel on the same captured frame
             const mergedRegions = FrameDiffer.mergeRegions(changedRegions, maxROIs, 64, capture.width, capture.height);
             // OCR (region-based or full)
-            const ocrPromise = (mergedRegions.length > 0 && mergedRegions.length <= maxROIs)
+            // Wire #10: When too many changed regions, use zone ROIs (priorityROIs) for
+            // targeted OCR instead of expensive full-screen OCR fallback.
+            const useRegionOCR = mergedRegions.length > 0 && mergedRegions.length <= maxROIs;
+            const useZoneOCR = !useRegionOCR && mergedRegions.length > maxROIs && priorityROIs && priorityROIs.length > 0;
+            const ocrTargets = useRegionOCR ? mergedRegions : useZoneOCR ? priorityROIs : null;
+            const ocrPromise = ocrTargets
                 ? (async () => {
                     const regionResults = [];
-                    for (const roi of mergedRegions) {
+                    for (const roi of ocrTargets) {
                         const regionEvent = await this.ocrRegion(windowId, roi);
                         if (regionEvent?.data.type === "vision_ocr" && regionEvent.data.regions) {
+                            // Re-anchor OCR bounds from ROI-relative to window-relative coordinates
+                            for (const region of regionEvent.data.regions) {
+                                region.bounds.x += roi.x;
+                                region.bounds.y += roi.y;
+                            }
                             regionResults.push(...regionEvent.data.regions);
                         }
                     }
@@ -223,7 +233,7 @@ export class VisionSource {
                         timestamp: new Date().toISOString(),
                         data: {
                             type: "vision_ocr",
-                            roi: mergedRegions[0] ?? { x: 0, y: 0, width: 0, height: 0, reason: "changed_pixels" },
+                            roi: ocrTargets[0] ?? { x: 0, y: 0, width: 0, height: 0, reason: "changed_pixels" },
                             text: fullText,
                             regions: regionResults,
                             latencyMs: Date.now() - start - captureMs,