stelo 1.0.1

package/dist/vision.js

@@ -0,0 +1,1197 @@
+"use strict";
+// ============================================================================
+// Stelo — Vision & Change Detection Module
+// ============================================================================
+// Advanced screen analysis for desktop automation. Provides visual grounding,
+// change detection, action verification, and state tracking primitives.
+// ============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgentVision = exports.vision = void 0;
+exports.createAgentVision = createAgentVision;
+const native = require('../index.js');
+// ── Vision Module ───────────────────────────────────────────────────────────
+/**
+ * Vision and change detection utilities for automation workflows.
+ *
+ * These primitives enable visual grounding, action verification, and
+ * state tracking - essential building blocks for robust automation flows.
+ *
+ * @example
+ * ```typescript
+ * import { vision, screen } from 'stelo';
+ *
+ * // Take a reference screenshot
+ * const before = vision.captureReference();
+ *
+ * // Perform some action
+ * await mouse.click();
+ *
+ * // Verify the screen changed
+ * const diff = vision.diff(before);
+ * console.log(`${diff.changePercentage}% of screen changed`);
+ *
+ * // Wait for UI to stabilize after action
+ * await vision.waitForStable({ stabilityThreshold: 0.5 });
+ *
+ * // Analyze screen as a grid for vision model
+ * const grid = vision.analyzeGrid(16, 9);
+ * const textCells = grid.cells.filter(c => c.likelyText);
+ * ```
+ */
+exports.vision = {
+    /**
+     * Capture a reference screenshot for later comparison.
+     * Use this before triggering an action to verify it had an effect.
+     *
+     * @param region - Optional region to capture (full screen if not specified)
+     * @returns Reference object to pass to diff()
+     *
+     * @example
+     * ```typescript
+     * const before = vision.captureReference();
+     * await mouse.click();
+     * const diff = vision.diff(before);
+     * ```
+     */
+    captureReference(region) {
+        const cap = region
+            ? native.screenCapture(region.x, region.y, region.width, region.height)
+            : native.screenCapture();
+        return {
+            data: cap.data,
+            width: cap.width,
+            height: cap.height,
+            x: region?.x ?? 0,
+            y: region?.y ?? 0,
+        };
+    },
+    /**
+     * Compare current screen to a reference capture.
+     * Returns detailed diff statistics including change percentage and bounds.
+     *
+     * @param reference - Reference from captureReference()
+     * @param options - Diff options (tolerance, sample rate)
+     * @returns Diff result with change statistics
+     *
+     * @example
+     * ```typescript
+     * const diff = vision.diff(before, { tolerance: 15 });
+     * if (diff.changePercentage > 1) {
+     *   console.log('Screen changed!', diff.changedBounds);
+     * }
+     * ```
+     */
+    diff(reference, options) {
+        const result = native.visionDiff(reference.data, reference.width, reference.height, reference.x, reference.y, reference.width, reference.height, options?.tolerance, options?.sampleRate);
+        return {
+            changePercentage: result.changePercentage ?? result.change_percentage,
+            changedPixelCount: result.changedPixelCount ?? result.changed_pixel_count,
+            totalPixelCount: result.totalPixelCount ?? result.total_pixel_count,
+            changedBounds: (result.hasChanges ?? result.has_changes)
+                ? {
+                    x: result.changedX ?? result.changed_x,
+                    y: result.changedY ?? result.changed_y,
+                    width: result.changedWidth ?? result.changed_width,
+                    height: result.changedHeight ?? result.changed_height,
+                }
+                : undefined,
+            averageDiff: result.averageDiff ?? result.average_diff,
+            maxDiff: result.maxDiff ?? result.max_diff,
+            hasChanges: result.hasChanges ?? result.has_changes,
+        };
+    },
+    /**
+     * Analyze screen as a grid of cells.
+     * Each cell includes statistics useful for vision model region selection.
+     *
+     * This enables efficient visual grounding - instead of sending the entire
+     * screen to a vision model, you can identify regions of interest first.
+     *
+     * @param cols - Number of columns in grid
+     * @param rows - Number of rows in grid
+     * @param region - Optional region to analyze (full screen if not specified)
+     * @returns Grid analysis with cell statistics
+     *
+     * @example
+     * ```typescript
+     * // Analyze screen as 16x9 grid
+     * const grid = vision.analyzeGrid(16, 9);
+     *
+     * // Find cells likely containing text
+     * const textCells = grid.cells.filter(c => c.likelyText);
+     *
+     * // Get center of cell [3, 2]
+     * const center = vision.gridCellCenter(grid, 3, 2);
+     * await mouse.click(center.x, center.y);
+     * ```
+     */
+    analyzeGrid(cols, rows, region) {
+        const result = native.visionAnalyzeGrid(cols, rows, region?.x, region?.y, region?.width, region?.height);
+        const cells = result.cells.map((c) => ({
+            gridX: c.gridX ?? c.grid_x,
+            gridY: c.gridY ?? c.grid_y,
+            screenX: c.screenX ?? c.screen_x,
+            screenY: c.screenY ?? c.screen_y,
+            width: c.width,
+            height: c.height,
+            avgColor: { r: c.avgR ?? c.avg_r, g: c.avgG ?? c.avg_g, b: c.avgB ?? c.avg_b },
+            variance: c.variance,
+            likelyText: c.likelyText ?? c.likely_text,
+            likelyUI: c.likelyUi ?? c.likely_ui,
+        }));
+        const activeCells = [];
+        const activeCellsX = result.activeCellsX ?? result.active_cells_x;
+        const activeCellsY = result.activeCellsY ?? result.active_cells_y;
+        for (let i = 0; i < activeCellsX.length; i++) {
+            activeCells.push({
+                gridX: activeCellsX[i],
+                gridY: activeCellsY[i],
+            });
+        }
+        return {
+            cols: result.cols,
+            rows: result.rows,
+            cellWidth: result.cellWidth ?? result.cell_width,
+            cellHeight: result.cellHeight ?? result.cell_height,
+            cells,
+            activeCells,
+        };
+    },
+    /**
+     * Get the screen center point of a grid cell.
+     *
+     * @param grid - Grid analysis from analyzeGrid()
+     * @param gridX - Cell column index
+     * @param gridY - Cell row index
+     * @returns Screen coordinates of cell center, or undefined if out of bounds
+     */
+    gridCellCenter(grid, gridX, gridY) {
+        const cell = grid.cells.find((c) => c.gridX === gridX && c.gridY === gridY);
+        if (!cell)
+            return undefined;
+        return {
+            x: cell.screenX + Math.floor(cell.width / 2),
+            y: cell.screenY + Math.floor(cell.height / 2),
+        };
+    },
+    /**
+     * Wait until the screen changes beyond a threshold.
+     * Useful for detecting when an action triggers a visual response.
+     *
+     * @param thresholdPercent - Minimum change percentage to trigger (0-100)
+     * @param timeoutMs - Maximum time to wait
+     * @param options - Wait options (region, poll interval)
+     * @returns true if change detected, false if timed out
+     *
+     * @example
+     * ```typescript
+     * // Click and wait for visual feedback
+     * mouse.click();
+     * const changed = await vision.waitForChange(0.5, 3000);
+     * if (!changed) console.log('Button may not have responded');
+     * ```
+     */
+    async waitForChange(thresholdPercent, timeoutMs, options) {
+        return native.visionWaitForChange(thresholdPercent, timeoutMs, options?.pollIntervalMs, options?.region?.x, options?.region?.y, options?.region?.width, options?.region?.height);
+    },
+    /**
+     * Wait until the screen stabilizes (stops changing).
+     * Essential for waiting for animations, loading spinners, or transitions.
+     *
+     * @param stabilityThreshold - Maximum change % to consider "stable" (0-100)
+     * @param stableDurationMs - How long screen must remain stable
+     * @param timeoutMs - Maximum time to wait
+     * @param options - Wait options (region, poll interval)
+     * @returns true if stabilized, false if timed out
+     *
+     * @example
+     * ```typescript
+     * // Click a button and wait for animation to complete
+     * mouse.click();
+     * await vision.waitForStable(0.1, 200, 5000);
+     * // Screen is now stable - safe to read or continue
+     * ```
+     */
+    async waitForStable(stabilityThreshold, stableDurationMs, timeoutMs, options) {
+        return native.visionWaitForStable(stabilityThreshold, stableDurationMs, timeoutMs, options?.pollIntervalMs, options?.region?.x, options?.region?.y, options?.region?.width, options?.region?.height);
+    },
+    /**
+     * Compute a perceptual hash of a screen region.
+     * Two visually similar images will have hashes with low Hamming distance.
+     *
+     * Use this for fast "has the screen changed significantly?" checks
+     * without doing full pixel comparison.
+     *
+     * @param region - Optional region to hash (full screen if not specified)
+     * @returns 64-bit perceptual hash
+     *
+     * @example
+     * ```typescript
+     * const hash1 = vision.perceptualHash();
+     * await performSomeAction();
+     * const hash2 = vision.perceptualHash();
+     * const distance = vision.hashDistance(hash1, hash2);
+     * if (distance < 5) console.log('Screen looks similar');
+     * ```
+     */
+    perceptualHash(region) {
+        return native.visionPerceptualHash(region?.x, region?.y, region?.width, region?.height);
+    },
+    /**
+     * Compute Hamming distance between two perceptual hashes.
+     * Lower distance = more visually similar. 0 = identical.
+     *
+     * Rules of thumb:
+     * - 0-5: Very similar (minor changes)
+     * - 5-10: Moderately similar
+     * - 10-20: Significant differences
+     * - 20+: Completely different
+     *
+     * @param hash1 - First perceptual hash
+     * @param hash2 - Second perceptual hash
+     * @returns Hamming distance (0-64)
+     */
+    hashDistance(hash1, hash2) {
+        return native.visionHashDistance(hash1, hash2);
+    },
+    /**
+     * Find all pixels matching a color within a region.
+     * Returns all matching points up to a maximum count.
+     *
+     * @param color - Color to search for (RGB)
+     * @param tolerance - Color distance tolerance (0-441)
+     * @param options - Search options
+     * @returns Array of matching screen coordinates
+     *
+     * @example
+     * ```typescript
+     * // Find all red pixels
+     * const redPixels = vision.findAllColors(
+     *   { r: 255, g: 0, b: 0 },
+     *   30,
+     *   { maxResults: 100 }
+     * );
+     * ```
+     */
+    findAllColors(color, tolerance, options) {
+        const result = native.visionFindAllColors(color.r, color.g, color.b, tolerance, options?.maxResults, options?.region?.x, options?.region?.y, options?.region?.width, options?.region?.height);
+        return result.map((p) => ({ x: p.x, y: p.y }));
+    },
+    /**
+     * Find clusters of similar colors (potential UI elements).
+     * Clusters are groups of nearby pixels with similar colors.
+     *
+     * @param color - Color to search for (RGB)
+     * @param tolerance - Color distance tolerance (0-441)
+     * @param minClusterSize - Minimum pixels to form a cluster
+     * @param region - Optional region to search (full screen if not specified)
+     * @returns Array of bounding rectangles for each cluster
+     *
+     * @example
+     * ```typescript
+     * // Find blue button-like regions
+     * const clusters = vision.findColorClusters(
+     *   { r: 0, g: 120, b: 215 }, // Windows blue
+     *   40,
+     *   50 // At least 50 pixels
+     * );
+     * if (clusters.length > 0) {
+     *   // Click center of first cluster
+     *   const btn = clusters[0];
+     *   mouse.click(btn.x + btn.width / 2, btn.y + btn.height / 2);
+     * }
+     * ```
+     */
+    findColorClusters(color, tolerance, minClusterSize, region) {
+        const result = native.visionFindColorClusters(color.r, color.g, color.b, tolerance, minClusterSize, region?.x, region?.y, region?.width, region?.height);
+        return result.map((r) => ({
+            x: r.x,
+            y: r.y,
+            width: r.width,
+            height: r.height,
+        }));
+    },
+    /**
+     * Verify an action caused a visual change.
+     * High-level primitive that captures before/after and compares.
+     *
+     * @param action - Async action to execute and verify
+     * @param minChangePercent - Minimum change to consider verified
+     * @param timeoutMs - Maximum time to wait for change
+     * @param region - Optional region to monitor
+     * @returns Verification result with diff statistics
+     *
+     * @example
+     * ```typescript
+     * const result = await vision.verifyAction(
+     *   async () => { mouse.click(); },
+     *   0.5, // At least 0.5% change
+     *   2000
+     * );
+     * if (!result.verified) {
+     *   // Click didn't cause visual change - might need to retry
+     * }
+     * ```
+     */
+    async verifyAction(action, minChangePercent, timeoutMs, region) {
+        const before = this.captureReference(region);
+        const startTime = Date.now();
+        await action();
+        // Poll for change
+        const pollInterval = 50;
+        const deadline = startTime + timeoutMs;
+        while (Date.now() < deadline) {
+            const diff = this.diff(before);
+            if (diff.changePercentage >= minChangePercent) {
+                return {
+                    verified: true,
+                    diff,
+                    durationMs: Date.now() - startTime,
+                };
+            }
+            await new Promise((resolve) => setTimeout(resolve, pollInterval));
+        }
+        // Final check
+        const finalDiff = this.diff(before);
+        return {
+            verified: finalDiff.changePercentage >= minChangePercent,
+            diff: finalDiff,
+            durationMs: Date.now() - startTime,
+        };
+    },
+    /**
+     * Take a reference, perform action, wait for stability.
+     * Combines action execution with waiting for the UI to settle.
+     *
+     * @param action - Action to execute
+     * @param options - Wait and verification options
+     * @returns true if action completed and screen stabilized
+     *
+     * @example
+     * ```typescript
+     * // Click and wait for any animation to complete
+     * await vision.doAndWaitStable(async () => {
+     *   await mouse.click(100, 200);
+     * });
+     * // Screen is now stable
+     * ```
+     */
+    async doAndWaitStable(action, options) {
+        const { stabilityThreshold = 0.1, stableDurationMs = 150, timeoutMs = 5000, region } = options ?? {};
+        await action();
+        return this.waitForStable(stabilityThreshold, stableDurationMs, timeoutMs, { region });
+    },
+};
+// ─── Core Fingerprinting Functions ──────────────────────────────────────────
+function computeFingerprint(data, width, height, startX, startY, cellW, cellH) {
+    const brightnessHist = new Float64Array(16);
+    const hueHist = new Float64Array(8);
+    let sumBright = 0, sumBrightSq = 0;
+    let edges = 0;
+    let minBright = 255, maxBright = 0;
+    let pixelCount = 0;
+    const step = 2;
+    for (let py = startY; py < startY + cellH && py < height; py += step) {
+        for (let px = startX; px < startX + cellW && px < width; px += step) {
+            const offset = (py * width + px) * 4;
+            const r = data[offset];
+            const g = data[offset + 1];
+            const b = data[offset + 2];
+            const bright = (r * 299 + g * 587 + b * 114) / 1000;
+            sumBright += bright;
+            sumBrightSq += bright * bright;
+            pixelCount++;
+            if (bright < minBright)
+                minBright = bright;
+            if (bright > maxBright)
+                maxBright = bright;
+            const binIdx = Math.min(15, Math.floor(bright / 16));
+            brightnessHist[binIdx]++;
+            const maxC = Math.max(r, g, b);
+            const minC = Math.min(r, g, b);
+            if (maxC - minC > 20) {
+                let hue = 0;
+                if (maxC === r)
+                    hue = ((g - b) / (maxC - minC)) % 6;
+                else if (maxC === g)
+                    hue = (b - r) / (maxC - minC) + 2;
+                else
+                    hue = (r - g) / (maxC - minC) + 4;
+                if (hue < 0)
+                    hue += 6;
+                const hueBin = Math.min(7, Math.floor(hue / 6 * 8));
+                hueHist[hueBin]++;
+            }
+            if (px + step < startX + cellW && px + step < width) {
+                const rOff = (py * width + px + step) * 4;
+                const rBright = (data[rOff] * 299 + data[rOff + 1] * 587 + data[rOff + 2] * 114) / 1000;
+                if (Math.abs(bright - rBright) > 25)
+                    edges++;
+            }
+        }
+    }
+    if (pixelCount > 0) {
+        for (let i = 0; i < 16; i++)
+            brightnessHist[i] /= pixelCount;
+        let hueTotal = 0;
+        for (let i = 0; i < 8; i++)
+            hueTotal += hueHist[i];
+        if (hueTotal > 0) {
+            for (let i = 0; i < 8; i++)
+                hueHist[i] /= hueTotal;
+        }
+    }
+    if (pixelCount === 0)
+        pixelCount = 1;
+    const avgBright = sumBright / pixelCount;
+    const variance = Math.max(0, sumBrightSq / pixelCount - avgBright * avgBright);
+    const miniW = Math.floor(cellW / 4) || 1;
+    const miniH = Math.floor(cellH / 4) || 1;
+    let phash = 0;
+    for (let mr = 0; mr < 4; mr++) {
+        for (let mc = 0; mc < 4; mc++) {
+            const sx = startX + mc * miniW;
+            const sy = startY + mr * miniH;
+            if (sx < width && sy < height) {
+                const off = (sy * width + sx) * 4;
+                const b = (data[off] * 299 + data[off + 1] * 587 + data[off + 2] * 114) / 1000;
+                if (b > avgBright)
+                    phash |= (1 << (mr * 4 + mc));
+            }
+        }
+    }
+    return {
+        brightnessHist, hueHist,
+        edgeDensity: edges / Math.max(1, pixelCount),
+        avgBrightness: avgBright,
+        contrast: maxBright - minBright,
+        variance,
+        aspect: cellW / Math.max(1, cellH),
+        phash,
+    };
+}
+function fingerprintSimilarity(a, b) {
+    let histSim = 0;
+    for (let i = 0; i < 16; i++)
+        histSim += Math.min(a.brightnessHist[i], b.brightnessHist[i]);
+    let hueSim = 0;
+    let aHueSum = 0, bHueSum = 0;
+    for (let i = 0; i < 8; i++) {
+        hueSim += Math.min(a.hueHist[i], b.hueHist[i]);
+        aHueSum += a.hueHist[i];
+        bHueSum += b.hueHist[i];
+    }
+    // If both lack color, hue is irrelevant — treat as identical
+    if (aHueSum < 0.001 && bHueSum < 0.001)
+        hueSim = 1;
+    const edgeSim = 1 - Math.abs(a.edgeDensity - b.edgeDensity);
+    const contrastSim = 1 - Math.abs(a.contrast - b.contrast) / 255;
+    const brightSim = 1 - Math.abs(a.avgBrightness - b.avgBrightness) / 255;
+    let xor = a.phash ^ b.phash;
+    let hamming = 0;
+    while (xor) {
+        hamming += xor & 1;
+        xor >>= 1;
+    }
+    const phashSim = 1 - hamming / 16;
+    return (histSim * 0.25 +
+        hueSim * 0.10 +
+        edgeSim * 0.15 +
+        contrastSim * 0.15 +
+        brightSim * 0.10 +
+        phashSim * 0.25);
+}
+// ─── Helper to create LocatedElement ────────────────────────────────────────
+function makeLocatedElement(x, y, w, h, confidence, similarity, fp, label, supportCount = 1) {
+    return {
+        x, y, w, h, confidence, similarity, label, supportCount, fingerprint: fp,
+        click(button) {
+            const cx = x + Math.floor(w / 2);
+            const cy = y + Math.floor(h / 2);
+            native.mouseClickAt(cx, cy, button ?? 'left', false);
+        },
+        doubleClick() {
+            native.mouseMove(x + Math.floor(w / 2), y + Math.floor(h / 2));
+            native.mouseDoubleClick('left');
+        },
+        rightClick() {
+            native.mouseClickAt(x + Math.floor(w / 2), y + Math.floor(h / 2), 'right', false);
+        },
+        moveTo(options) {
+            const cx = x + Math.floor(w / 2);
+            const cy = y + Math.floor(h / 2);
+            if (options?.smooth) {
+                native.mouseMoveSmooth(cx, cy, options.duration ?? 300, 'easeInOut');
+            }
+            else {
+                native.mouseMove(cx, cy);
+            }
+        },
+        type(text) {
+            const cx = x + Math.floor(w / 2);
+            const cy = y + Math.floor(h / 2);
+            native.mouseClickAt(cx, cy, 'left', false);
+            native.keyboardType(text);
+        },
+        isStillPresent(threshold) {
+            const cap = native.screenCapture();
+            const nowFp = computeFingerprint(cap.data, cap.width, cap.height, x, y, w, h);
+            return fingerprintSimilarity(fp, nowFp) >= (threshold ?? 0.7);
+        },
+    };
+}
+// ─── AgentVision Class ──────────────────────────────────────────────────────
+class AgentVision {
+    cols;
+    rows;
+    maxMemories;
+    defaultRegionSize;
+    matchThreshold;
+    searchRadius;
+    appContext;
+    memories = [];
+    sequenceCounter = 0;
+    recentActions = [];
+    temporalPatterns = new Map();
+    // Grid cache
+    gridFingerprints = [];
+    gridDirty = true;
+    lastGridTime = 0;
+    lastCapture = null;
+    constructor(config = {}) {
+        this.cols = config.cols ?? 32;
+        this.rows = config.rows ?? 18;
+        this.maxMemories = config.maxMemories ?? 2000;
+        this.defaultRegionSize = config.defaultRegionSize ?? { w: 100, h: 60 };
+        this.matchThreshold = config.matchThreshold ?? 0.65;
+        this.searchRadius = config.searchRadius ?? 400;
+        this.appContext = config.appContext;
+    }
+    // ═══ FINGERPRINT — Capture visual identity of any region ══════════════
+    /**
+     * Fingerprint a screen region. Returns a compact visual descriptor
+     * that can be compared against other fingerprints for similarity.
+     *
+     * @example
+     * ```typescript
+     * const fp = agentVision.fingerprint(100, 200, 150, 40);
+     * // Later: check if the same element is somewhere else
+     * const match = agentVision.locate(fp);
+     * ```
+     */
+    fingerprint(x, y, w, h) {
+        const cap = native.screenCapture();
+        return computeFingerprint(cap.data, cap.width, cap.height, x, y, w, h);
+    }
+    /**
+     * Compute similarity between two fingerprints (0-1, 1 = identical).
+     */
+    similarity(a, b) {
+        return fingerprintSimilarity(a, b);
+    }
+    // ═══ REMEMBER — Teach the agent about screen elements ═════════════════
+    /**
+     * Remember an element at a position with a label.
+     * The agent fingerprints the region and stores it for later recognition.
+     *
+     * @example
+     * ```typescript
+     * // Agent clicked Save, now remembers what it looks like
+     * agentVision.remember('save-button', 350, 15, { w: 80, h: 30 });
+     *
+     * // Later, find it again even if it moved
+     * const saveBtn = agentVision.find('save-button');
+     * if (saveBtn) saveBtn.click();
+     * ```
+     */
+    remember(label, x, y, regionSize, action = 'click') {
+        const size = regionSize ?? this.defaultRegionSize;
+        const rx = Math.max(0, x - Math.floor(size.w / 2));
+        const ry = Math.max(0, y - Math.floor(size.h / 2));
+        const cap = native.screenCapture();
+        const fp = computeFingerprint(cap.data, cap.width, cap.height, rx, ry, size.w, size.h);
+        const memory = {
+            fingerprint: fp,
+            action,
+            label,
+            position: { x, y },
+            regionSize: size,
+            timestamp: Date.now(),
+            appContext: this.appContext,
+            sequenceIndex: this.sequenceCounter++,
+        };
+        this.memories.push(memory);
+        this.evictOldMemories();
+        this.gridDirty = true;
+        // Track temporal patterns
+        this.trackAction(action, label, x, y);
+        return memory;
+    }
+    /**
+     * Remember after performing a click — records the action AND verifies it.
+     * Captures before/after fingerprints and records whether the click did anything.
+     *
+     * @example
+     * ```typescript
+     * const result = await agentVision.rememberClick('file-menu', 44, 12);
+     * console.log(`Click ${result.success ? 'worked' : 'had no effect'}`);
+     * ```
+     */
+    async rememberClick(label, x, y, regionSize) {
+        const size = regionSize ?? this.defaultRegionSize;
+        const rx = Math.max(0, x - Math.floor(size.w / 2));
+        const ry = Math.max(0, y - Math.floor(size.h / 2));
+        // Capture before
+        const capBefore = native.screenCapture();
+        const fp = computeFingerprint(capBefore.data, capBefore.width, capBefore.height, rx, ry, size.w, size.h);
+        // Perform click
+        native.mouseClickAt(x, y, 'left', false);
+        // Wait a bit for UI response
+        await new Promise(r => setTimeout(r, 400));
+        // Capture after and measure change
+        const capAfter = native.screenCapture();
+        const diff = native.visionDiff(capBefore.data, capBefore.width, capBefore.height, undefined, undefined, undefined, undefined, 10, 2);
+        const changePercent = diff.changePercentage ?? diff.change_percentage ?? 0;
+        const success = changePercent > 0.3;
+        const memory = {
+            fingerprint: fp,
+            action: 'click',
+            label,
+            position: { x, y },
+            regionSize: size,
+            timestamp: Date.now(),
+            outcome: { screenChangePercent: changePercent, responseTimeMs: 400 },
+            success,
+            appContext: this.appContext,
+            sequenceIndex: this.sequenceCounter++,
+        };
+        this.memories.push(memory);
+        this.evictOldMemories();
+        this.gridDirty = true;
+        this.trackAction('click', label, x, y);
+        return { ...memory, success };
+    }
+    // ═══ FIND — Locate remembered elements on screen ══════════════════════
+    /**
+     * Find a previously remembered element on the current screen.
+     * Uses visual fingerprint matching — works even if the element moved.
+     *
+     * @returns The best match, or null if nothing above threshold
+     *
+     * @example
+     * ```typescript
+     * const saveBtn = agentVision.find('save-button');
+     * if (saveBtn) {
+     *   saveBtn.click();
+     * } else {
+     *   console.log('Save button not visible');
+     * }
+     * ```
+     */
+    find(label) {
+        const results = this.findAll(label, 1);
+        return results.length > 0 ? results[0] : null;
+    }
+    /**
+     * Find all instances of a remembered element on screen.
+     *
+     * @example
+     * ```typescript
+     * // Find all things that look like "close-button"
+     * const closeButtons = agentVision.findAll('close-button');
+     * console.log(`Found ${closeButtons.length} close buttons`);
+     * ```
+     */
+    findAll(label, maxResults = 20) {
+        const labeledMemories = this.memories.filter(m => m.label === label && (!this.appContext || m.appContext === this.appContext));
+        if (labeledMemories.length === 0)
+            return [];
+        this.refreshGrid();
+        const cap = this.lastCapture;
+        const cellW = Math.floor(cap.width / this.cols);
+        const cellH = Math.floor(cap.height / this.rows);
+        const results = [];
+        for (let row = 0; row < this.rows; row++) {
+            for (let col = 0; col < this.cols; col++) {
+                const idx = row * this.cols + col;
+                const fp = this.gridFingerprints[idx];
+                if (!fp)
+                    continue;
+                let bestSim = 0;
+                for (const mem of labeledMemories) {
+                    const sim = fingerprintSimilarity(fp, mem.fingerprint);
+                    if (sim > bestSim)
+                        bestSim = sim;
+                }
+                if (bestSim < this.matchThreshold)
+                    continue;
+                const x = col * cellW;
+                const y = row * cellH;
+                results.push(makeLocatedElement(x, y, cellW, cellH, bestSim, bestSim, fp, label, labeledMemories.length));
+            }
+        }
+        results.sort((a, b) => b.confidence - a.confidence);
+        return results.slice(0, maxResults);
+    }
+    // ═══ LOCATE — Find element by fingerprint anywhere on screen ══════════
+    /**
+     * Locate a specific visual fingerprint on the current screen.
+     * Does a focused scan: first near the expected position, then widens.
+     *
+     * @example
+     * ```typescript
+     * const fp = agentVision.fingerprint(100, 200, 80, 30);
+     * // ... some time later, UI may have reorganized ...
+     * const found = agentVision.locate(fp, { near: { x: 100, y: 200 } });
+     * if (found) found.click();
+     * ```
+     */
+    locate(target, options) {
+        const threshold = options?.threshold ?? this.matchThreshold;
+        const radius = options?.searchRadius ?? this.searchRadius;
+        const cap = native.screenCapture();
+        // Determine region size from fingerprint aspect ratio
+        const h = this.defaultRegionSize.h;
+        const w = Math.round(h * target.aspect);
+        let bestSim = 0;
+        let bestX = 0;
+        let bestY = 0;
+        const stepX = Math.max(8, Math.floor(w * 0.4));
+        const stepY = Math.max(8, Math.floor(h * 0.4));
+        // If we have a hint, search near it first
+        if (options?.near) {
+            const sX = Math.max(0, options.near.x - radius);
+            const eX = Math.min(cap.width - w, options.near.x + radius);
+            const sY = Math.max(0, options.near.y - radius);
+            const eY = Math.min(cap.height - h, options.near.y + radius);
+            for (let sy = sY; sy < eY; sy += stepY) {
+                for (let sx = sX; sx < eX; sx += stepX) {
+                    const candidate = computeFingerprint(cap.data, cap.width, cap.height, sx, sy, w, h);
+                    const sim = fingerprintSimilarity(target, candidate);
+                    if (sim > bestSim) {
+                        bestSim = sim;
+                        bestX = sx;
+                        bestY = sy;
+                    }
+                }
+            }
+            if (bestSim >= threshold) {
+                return makeLocatedElement(bestX, bestY, w, h, bestSim, bestSim, target);
+            }
+        }
+        // Full screen scan with grid
+        this.refreshGrid();
+        const cellW = Math.floor(cap.width / this.cols);
+        const cellH = Math.floor(cap.height / this.rows);
+        for (let row = 0; row < this.rows; row++) {
+            for (let col = 0; col < this.cols; col++) {
+                const idx = row * this.cols + col;
+                const fp = this.gridFingerprints[idx];
+                if (!fp)
+                    continue;
+                const sim = fingerprintSimilarity(target, fp);
+                if (sim > bestSim) {
+                    bestSim = sim;
+                    bestX = col * cellW;
+                    bestY = row * cellH;
+                }
+            }
+        }
+        if (bestSim >= threshold) {
+            return makeLocatedElement(bestX, bestY, cellW, cellH, bestSim, bestSim, target);
+        }
+        return null;
+    }
+    // ═══ FIND BY TEXT — Combine OCR with visual memory ════════════════════
+    /**
+     * Find an element by text using OCR, then fingerprint it for future recognition.
+     * First time: uses OCR. After that: can find it visually even without OCR.
+     *
+     * @example
+     * ```typescript
+     * // First call uses OCR to find "Save" text
+     * const save = agentVision.findByText('Save');
+     * if (save) save.click();
+     * // Now agentVision remembers what "Save" looks like visually
+     * ```
+     */
+    findByText(text, options) {
+        const shouldRemember = options?.remember !== false;
+        // First try: recall from memory (fast, no OCR needed)
+        const recalled = this.find(`text:${text}`);
+        if (recalled && recalled.isStillPresent()) {
+            return recalled;
+        }
+        // Second try: OCR
+        try {
+            const found = native.ocrFindText(text);
+            if (found) {
+                const x = found.x;
+                const y = found.y;
+                const w = found.width || this.defaultRegionSize.w;
+                const h = found.height || this.defaultRegionSize.h;
+                const cap = native.screenCapture();
+                const fp = computeFingerprint(cap.data, cap.width, cap.height, x, y, w, h);
+                if (shouldRemember) {
+                    this.remember(`text:${text}`, x + Math.floor(w / 2), y + Math.floor(h / 2), { w, h });
+                }
+                return makeLocatedElement(x, y, w, h, found.confidence ?? 1.0, 1.0, fp, `text:${text}`);
+            }
+        }
+        catch { /* OCR unavailable */ }
+        return null;
+    }
+    // ═══ VERIFY — Check if actions had an effect ══════════════════════════
+    /**
+     * Click an element and verify the screen changed.
+     * Returns the located element with success/failure status.
+     *
+     * @example
+     * ```typescript
+     * const result = await agentVision.clickAndVerify('submit-button');
+     * if (!result.verified) {
+     *   // Button didn't respond — try again or escalate
+     * }
+     * ```
+     */
+    async clickAndVerify(label, options) {
+        const el = this.find(label);
+        if (!el)
+            return { element: null, verified: false, changePercent: 0 };
+        const before = exports.vision.captureReference();
+        el.click();
+        await new Promise(r => setTimeout(r, options?.timeout ?? 500));
+        const diff = exports.vision.diff(before);
+        const minChange = options?.minChange ?? 0.3;
+        const verified = diff.changePercentage >= minChange;
+        // Update memory with outcome
+        const mem = [...this.memories].reverse().find(m => m.label === label);
+        if (mem) {
+            mem.outcome = { screenChangePercent: diff.changePercentage, responseTimeMs: options?.timeout ?? 500 };
+            mem.success = verified;
+        }
+        return { element: el, verified, changePercent: diff.changePercentage };
+    }
+    /**
+     * Check if a specific screen region visually changed since a fingerprint was taken.
+     *
+     * @example
+     * ```typescript
+     * const before = agentVision.fingerprint(100, 200, 80, 30);
+     * await agent.doSomething();
+     * const changed = agentVision.hasChanged(before, 100, 200, 80, 30);
+     * ```
+     */
+    hasChanged(previousFingerprint, x, y, w, h, changeThreshold = 0.15) {
+        const current = this.fingerprint(x, y, w, h);
+        const sim = fingerprintSimilarity(previousFingerprint, current);
+        return sim < (1 - changeThreshold);
+    }
+    // ═══ PREDICT — Temporal pattern learning & prediction ═════════════════
+    /**
+     * Predict what will happen next based on learned temporal patterns.
+     *
+     * @example
+     * ```typescript
+     * // After clicking "File" many times, the agent learns:
+     * // "After clicking file-menu, a dropdown appears"
+     * const next = agentVision.predictNext();
+     * if (next) {
+     *   console.log(`Expected: ${next.nextAction} in ${next.expectedDelayMs}ms`);
+     * }
+     * ```
+     */
+    predictNext(filterLabel) {
+        if (this.recentActions.length === 0)
+            return null;
+        const last = this.recentActions[this.recentActions.length - 1];
+        const prefix = `${last.action}:${last.label || ''}→`;
+        let bestKey = null;
+        let bestPattern = null;
+        let bestCount = 0;
+        const entries = Array.from(this.temporalPatterns.entries());
+        for (let i = 0; i < entries.length; i++) {
+            const key = entries[i][0];
+            const pattern = entries[i][1];
+            if (!key.startsWith(prefix))
+                continue;
+            if (filterLabel && !key.includes(filterLabel))
+                continue;
+            if (pattern.count > bestCount) {
+                bestKey = key;
+                bestPattern = pattern;
+                bestCount = pattern.count;
+            }
+        }
+        if (!bestKey || !bestPattern)
+            return null;
+        const nextPart = bestKey.split('→')[1] || '';
+        const parts = nextPart.split(':');
+        return {
+            nextAction: parts[0] || nextPart,
+            label: parts[1] || undefined,
+            confidence: Math.min(1, bestPattern.count / 5),
+            expectedDelayMs: bestPattern.avgDelayMs,
+            predictedRegion: bestPattern.predictedRegion,
+        };
+    }
+    /**
+     * Wait for a predicted event to happen.
+     * Uses temporal patterns to know WHEN and WHERE to look.
+     *
+     * @example
+     * ```typescript
+     * // Click File menu
+     * agentVision.find('file-menu')?.click();
+     * // Wait for the dropdown the agent learned usually appears
+     * const appeared = await agentVision.waitForPredicted('dropdown', 3000);
+     * ```
+     */
+    async waitForPredicted(label, timeoutMs = 5000) {
+        const prediction = this.predictNext(label);
+        const pollMs = 100;
+        const deadline = Date.now() + timeoutMs;
+        // Take baseline for change detection
+        const baseline = exports.vision.captureReference(prediction?.predictedRegion
+            ? { x: prediction.predictedRegion.x, y: prediction.predictedRegion.y, width: prediction.predictedRegion.w, height: prediction.predictedRegion.h }
+            : undefined);
+        while (Date.now() < deadline) {
+            // Check if something changed in the predicted region
+            const diff = exports.vision.diff(baseline);
+            if (diff.hasChanges && diff.changePercentage > 0.5) {
+                // Something appeared — try to locate the expected element
+                if (label) {
+                    const found = this.find(label);
+                    if (found)
+                        return found;
+                }
+                // Or just return the changed region
+                if (diff.changedBounds) {
+                    const b = diff.changedBounds;
+                    const cap = native.screenCapture();
+                    const fp = computeFingerprint(cap.data, cap.width, cap.height, b.x, b.y, b.width, b.height);
+                    return makeLocatedElement(b.x, b.y, b.width, b.height, 0.8, 0.8, fp, label);
+                }
+            }
+            await new Promise(r => setTimeout(r, pollMs));
+        }
+        return null;
+    }
+    /**
+     * Get all learned temporal patterns.
+     */
+    getPatterns() {
+        const result = [];
+        const entries = Array.from(this.temporalPatterns.entries());
+        for (let i = 0; i < entries.length; i++) {
+            result.push({ pattern: entries[i][0], count: entries[i][1].count, avgDelayMs: entries[i][1].avgDelayMs });
+        }
+        return result.sort((a, b) => b.count - a.count);
+    }
+    // ═══ SCAN — Full screen understanding ═════════════════════════════════
+    /**
+     * Scan the full screen and return all regions that match ANY remembered element.
+     * Gives the agent a complete understanding of "what's on screen that I recognize."
+     *
+     * @example
+     * ```typescript
+     * const recognized = agentVision.scan();
+     * for (const el of recognized) {
+     *   console.log(`Found "${el.label}" at (${el.x},${el.y}) confidence=${el.confidence}`);
+     * }
+     * ```
+     */
+    scan(threshold) {
+        const minSim = threshold ?? this.matchThreshold;
+        this.refreshGrid();
+        const cap = this.lastCapture;
+        const cellW = Math.floor(cap.width / this.cols);
+        const cellH = Math.floor(cap.height / this.rows);
+        // Group memories by label
+        const labelGroups = new Map();
+        for (const mem of this.memories) {
+            if (!mem.label)
+                continue;
+            const group = labelGroups.get(mem.label) ?? [];
+            group.push(mem);
+            labelGroups.set(mem.label, group);
+        }
+        const results = [];
+        for (let row = 0; row < this.rows; row++) {
+            for (let col = 0; col < this.cols; col++) {
+                const idx = row * this.cols + col;
+                const fp = this.gridFingerprints[idx];
+                if (!fp)
+                    continue;
+                let bestLabel;
+                let bestSim = 0;
+                let bestCount = 0;
+                const labelEntries = Array.from(labelGroups.entries());
+                for (let li = 0; li < labelEntries.length; li++) {
+                    const label = labelEntries[li][0];
+                    const mems = labelEntries[li][1];
+                    for (const mem of mems) {
+                        const sim = fingerprintSimilarity(fp, mem.fingerprint);
+                        if (sim > bestSim) {
+                            bestSim = sim;
+                            bestLabel = label;
+                            bestCount = mems.length;
+                        }
+                    }
+                }
+                if (bestSim < minSim || !bestLabel)
+                    continue;
+                results.push(makeLocatedElement(col * cellW, row * cellH, cellW, cellH, bestSim, bestSim, fp, bestLabel, bestCount));
+            }
+        }
+        // Deduplicate: keep only the best match per label per rough area
+        const deduped = [];
+        const seen = new Set();
+        results.sort((a, b) => b.confidence - a.confidence);
+        for (const el of results) {
+            const key = `${el.label}:${Math.floor(el.x / 200)}:${Math.floor(el.y / 200)}`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                deduped.push(el);
+            }
+        }
+        return deduped;
+    }
+    // ═══ PERSISTENCE — Save/load memory across sessions ═══════════════════
+    /**
+     * Export all visual memories as a JSON string.
+     *
+     * @example
+     * ```typescript
+     * const data = agentVision.save();
+     * fs.writeFileSync('agent-memory.json', data);
+     * ```
+     */
+    save() {
+        return JSON.stringify({
+            version: 1,
+            appContext: this.appContext,
+            memories: this.memories.map(m => ({
+                ...m,
+                fingerprint: {
+                    ...m.fingerprint,
+                    brightnessHist: Array.from(m.fingerprint.brightnessHist),
+                    hueHist: Array.from(m.fingerprint.hueHist),
+                },
+            })),
+            temporalPatterns: Array.from(this.temporalPatterns.entries()),
+        });
+    }
+    /**
+     * Load visual memories from a JSON string.
+     *
+     * @example
+     * ```typescript
+     * const data = fs.readFileSync('agent-memory.json', 'utf8');
+     * agentVision.load(data);
+     * // Agent now remembers everything from last session
+     * ```
+     */
+    load(json) {
+        const data = JSON.parse(json);
+        if (data.memories) {
+            for (const m of data.memories) {
+                m.fingerprint.brightnessHist = new Float64Array(m.fingerprint.brightnessHist);
+                m.fingerprint.hueHist = new Float64Array(m.fingerprint.hueHist);
+                this.memories.push(m);
+            }
+        }
+        if (data.temporalPatterns) {
+            for (const [key, val] of data.temporalPatterns) {
+                this.temporalPatterns.set(key, val);
+            }
+        }
+    }
+    // ═══ STATE ════════════════════════════════════════════════════════════
+    /** Number of stored memories. */
+    get memoryCount() { return this.memories.length; }
+    /** All unique labels the agent has learned. */
+    get knownLabels() {
+        const labels = new Set();
+        for (const m of this.memories) {
+            if (m.label)
+                labels.add(m.label);
+        }
+        return Array.from(labels);
+    }
+    /** Clear all memories. */
+    reset() {
+        this.memories = [];
+        this.temporalPatterns.clear();
+        this.recentActions = [];
+        this.sequenceCounter = 0;
+        this.gridDirty = true;
+    }
+    /** Set the app context (filters memories by app). */
+    setContext(appContext) {
+        this.appContext = appContext;
+        this.gridDirty = true;
+    }
+    // ═══ INTERNALS ════════════════════════════════════════════════════════
+    refreshGrid() {
+        const now = Date.now();
+        if (!this.gridDirty && now - this.lastGridTime < 200)
+            return;
+        const cap = native.screenCapture();
+        this.lastCapture = cap;
+        const cellW = Math.floor(cap.width / this.cols);
+        const cellH = Math.floor(cap.height / this.rows);
+        this.gridFingerprints = [];
+        for (let row = 0; row < this.rows; row++) {
+            for (let col = 0; col < this.cols; col++) {
+                this.gridFingerprints.push(computeFingerprint(cap.data, cap.width, cap.height, col * cellW, row * cellH, cellW, cellH));
+            }
+        }
+        this.gridDirty = false;
+        this.lastGridTime = now;
+    }
+    trackAction(action, label, x, y) {
+        this.recentActions.push({ action, label, time: Date.now(), pos: { x, y } });
+        // Keep only last 30s
+        const cutoff = Date.now() - 30000;
+        this.recentActions = this.recentActions.filter(a => a.time > cutoff);
+        // Learn temporal patterns from consecutive actions
+        for (let i = 1; i < this.recentActions.length; i++) {
+            const prev = this.recentActions[i - 1];
+            const curr = this.recentActions[i];
+            const key = `${prev.action}:${prev.label || ''}→${curr.action}:${curr.label || ''}`;
+            const delay = curr.time - prev.time;
+            const existing = this.temporalPatterns.get(key);
+            if (existing) {
+                existing.avgDelayMs = (existing.avgDelayMs * existing.count + delay) / (existing.count + 1);
+                existing.count++;
+                existing.predictedRegion = { x: curr.pos.x - 50, y: curr.pos.y - 50, w: 100, h: 100 };
+            }
+            else {
+                this.temporalPatterns.set(key, {
+                    avgDelayMs: delay,
+                    count: 1,
+                    predictedRegion: { x: curr.pos.x - 50, y: curr.pos.y - 50, w: 100, h: 100 },
+                });
+            }
+        }
+    }
+    evictOldMemories() {
+        while (this.memories.length > this.maxMemories) {
+            // Remove oldest non-labeled memory first, then oldest overall
+            const unlabeledIdx = this.memories.findIndex(m => !m.label);
+            if (unlabeledIdx >= 0) {
+                this.memories.splice(unlabeledIdx, 1);
+            }
+            else {
+                this.memories.shift();
+            }
+        }
+    }
+}
+exports.AgentVision = AgentVision;
+/** Create a new AgentVision instance. */
+function createAgentVision(config) {
+    return new AgentVision(config);
+}
+//# sourceMappingURL=vision.js.map