stelo 1.0.1

package/dist/vision.d.ts

@@ -0,0 +1,719 @@
+/**
+ * Result of comparing two screen captures
+ */
+export interface ScreenDiff {
+    /** Percentage of pixels that changed (0.0 - 100.0) */
+    changePercentage: number;
+    /** Total number of pixels that changed */
+    changedPixelCount: number;
+    /** Total number of pixels compared */
+    totalPixelCount: number;
+    /** Bounding box of changed region (if any) */
+    changedBounds?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    /** Average color difference across all pixels */
+    averageDiff: number;
+    /** Maximum single-pixel color difference */
+    maxDiff: number;
+    /** True if any change was detected */
+    hasChanges: boolean;
+}
+/**
+ * Grid cell analysis result
+ */
+export interface GridCell {
+    /** Cell position in grid (0-indexed) */
+    gridX: number;
+    gridY: number;
+    /** Screen coordinates of cell */
+    screenX: number;
+    screenY: number;
+    /** Cell dimensions */
+    width: number;
+    height: number;
+    /** Average color of the cell */
+    avgColor: {
+        r: number;
+        g: number;
+        b: number;
+    };
+    /** Color variance (higher = more complex content) */
+    variance: number;
+    /** Whether this cell likely contains text (high contrast) */
+    likelyText: boolean;
+    /** Whether this cell likely contains UI elements */
+    likelyUI: boolean;
+}
+/**
+ * Grid analysis result
+ */
+export interface GridAnalysis {
+    /** Grid dimensions */
+    cols: number;
+    rows: number;
+    /** Cell size in pixels */
+    cellWidth: number;
+    cellHeight: number;
+    /** All grid cells */
+    cells: GridCell[];
+    /** Cells with high activity (likely UI elements) */
+    activeCells: Array<{
+        gridX: number;
+        gridY: number;
+    }>;
+}
+/**
+ * Region on screen
+ */
+export interface Region {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+/**
+ * Reference capture for diff operations
+ */
+export interface CaptureReference {
+    data: Buffer;
+    width: number;
+    height: number;
+    x: number;
+    y: number;
+}
+/**
+ * Options for diff operations
+ */
+export interface DiffOptions {
+    /** Minimum color distance to consider a difference (0-441). Default: 10 */
+    tolerance?: number;
+    /** Only check every Nth pixel (1 = all). Higher = faster but less accurate */
+    sampleRate?: number;
+}
+/**
+ * Options for waiting operations
+ */
+export interface WaitOptions {
+    /** Region to monitor (full screen if not specified) */
+    region?: Region;
+    /** Polling interval in ms. Default: 50 */
+    pollIntervalMs?: number;
+}
+/**
+ * Vision and change detection utilities for automation workflows.
+ *
+ * These primitives enable visual grounding, action verification, and
+ * state tracking - essential building blocks for robust automation flows.
+ *
+ * @example
+ * ```typescript
+ * import { vision, screen } from 'stelo';
+ *
+ * // Take a reference screenshot
+ * const before = vision.captureReference();
+ *
+ * // Perform some action
+ * await mouse.click();
+ *
+ * // Verify the screen changed
+ * const diff = vision.diff(before);
+ * console.log(`${diff.changePercentage}% of screen changed`);
+ *
+ * // Wait for UI to stabilize after action
+ * await vision.waitForStable({ stabilityThreshold: 0.5 });
+ *
+ * // Analyze screen as a grid for vision model
+ * const grid = vision.analyzeGrid(16, 9);
+ * const textCells = grid.cells.filter(c => c.likelyText);
+ * ```
+ */
+export declare const vision: {
+    /**
+     * Capture a reference screenshot for later comparison.
+     * Use this before triggering an action to verify it had an effect.
+     *
+     * @param region - Optional region to capture (full screen if not specified)
+     * @returns Reference object to pass to diff()
+     *
+     * @example
+     * ```typescript
+     * const before = vision.captureReference();
+     * await mouse.click();
+     * const diff = vision.diff(before);
+     * ```
+     */
+    captureReference(region?: Region): CaptureReference;
+    /**
+     * Compare current screen to a reference capture.
+     * Returns detailed diff statistics including change percentage and bounds.
+     *
+     * @param reference - Reference from captureReference()
+     * @param options - Diff options (tolerance, sample rate)
+     * @returns Diff result with change statistics
+     *
+     * @example
+     * ```typescript
+     * const diff = vision.diff(before, { tolerance: 15 });
+     * if (diff.changePercentage > 1) {
+     *   console.log('Screen changed!', diff.changedBounds);
+     * }
+     * ```
+     */
+    diff(reference: CaptureReference, options?: DiffOptions): ScreenDiff;
+    /**
+     * Analyze screen as a grid of cells.
+     * Each cell includes statistics useful for vision model region selection.
+     *
+     * This enables efficient visual grounding - instead of sending the entire
+     * screen to a vision model, you can identify regions of interest first.
+     *
+     * @param cols - Number of columns in grid
+     * @param rows - Number of rows in grid
+     * @param region - Optional region to analyze (full screen if not specified)
+     * @returns Grid analysis with cell statistics
+     *
+     * @example
+     * ```typescript
+     * // Analyze screen as 16x9 grid
+     * const grid = vision.analyzeGrid(16, 9);
+     *
+     * // Find cells likely containing text
+     * const textCells = grid.cells.filter(c => c.likelyText);
+     *
+     * // Get center of cell [3, 2]
+     * const center = vision.gridCellCenter(grid, 3, 2);
+     * await mouse.click(center.x, center.y);
+     * ```
+     */
+    analyzeGrid(cols: number, rows: number, region?: Region): GridAnalysis;
+    /**
+     * Get the screen center point of a grid cell.
+     *
+     * @param grid - Grid analysis from analyzeGrid()
+     * @param gridX - Cell column index
+     * @param gridY - Cell row index
+     * @returns Screen coordinates of cell center, or undefined if out of bounds
+     */
+    gridCellCenter(grid: GridAnalysis, gridX: number, gridY: number): {
+        x: number;
+        y: number;
+    } | undefined;
+    /**
+     * Wait until the screen changes beyond a threshold.
+     * Useful for detecting when an action triggers a visual response.
+     *
+     * @param thresholdPercent - Minimum change percentage to trigger (0-100)
+     * @param timeoutMs - Maximum time to wait
+     * @param options - Wait options (region, poll interval)
+     * @returns true if change detected, false if timed out
+     *
+     * @example
+     * ```typescript
+     * // Click and wait for visual feedback
+     * mouse.click();
+     * const changed = await vision.waitForChange(0.5, 3000);
+     * if (!changed) console.log('Button may not have responded');
+     * ```
+     */
+    waitForChange(thresholdPercent: number, timeoutMs: number, options?: WaitOptions): Promise<boolean>;
+    /**
+     * Wait until the screen stabilizes (stops changing).
+     * Essential for waiting for animations, loading spinners, or transitions.
+     *
+     * @param stabilityThreshold - Maximum change % to consider "stable" (0-100)
+     * @param stableDurationMs - How long screen must remain stable
+     * @param timeoutMs - Maximum time to wait
+     * @param options - Wait options (region, poll interval)
+     * @returns true if stabilized, false if timed out
+     *
+     * @example
+     * ```typescript
+     * // Click a button and wait for animation to complete
+     * mouse.click();
+     * await vision.waitForStable(0.1, 200, 5000);
+     * // Screen is now stable - safe to read or continue
+     * ```
+     */
+    waitForStable(stabilityThreshold: number, stableDurationMs: number, timeoutMs: number, options?: WaitOptions): Promise<boolean>;
+    /**
+     * Compute a perceptual hash of a screen region.
+     * Two visually similar images will have hashes with low Hamming distance.
+     *
+     * Use this for fast "has the screen changed significantly?" checks
+     * without doing full pixel comparison.
+     *
+     * @param region - Optional region to hash (full screen if not specified)
+     * @returns 64-bit perceptual hash
+     *
+     * @example
+     * ```typescript
+     * const hash1 = vision.perceptualHash();
+     * await performSomeAction();
+     * const hash2 = vision.perceptualHash();
+     * const distance = vision.hashDistance(hash1, hash2);
+     * if (distance < 5) console.log('Screen looks similar');
+     * ```
+     */
+    perceptualHash(region?: Region): number;
+    /**
+     * Compute Hamming distance between two perceptual hashes.
+     * Lower distance = more visually similar. 0 = identical.
+     *
+     * Rules of thumb:
+     * - 0-5: Very similar (minor changes)
+     * - 5-10: Moderately similar
+     * - 10-20: Significant differences
+     * - 20+: Completely different
+     *
+     * @param hash1 - First perceptual hash
+     * @param hash2 - Second perceptual hash
+     * @returns Hamming distance (0-64)
+     */
+    hashDistance(hash1: number, hash2: number): number;
+    /**
+     * Find all pixels matching a color within a region.
+     * Returns all matching points up to a maximum count.
+     *
+     * @param color - Color to search for (RGB)
+     * @param tolerance - Color distance tolerance (0-441)
+     * @param options - Search options
+     * @returns Array of matching screen coordinates
+     *
+     * @example
+     * ```typescript
+     * // Find all red pixels
+     * const redPixels = vision.findAllColors(
+     *   { r: 255, g: 0, b: 0 },
+     *   30,
+     *   { maxResults: 100 }
+     * );
+     * ```
+     */
+    findAllColors(color: {
+        r: number;
+        g: number;
+        b: number;
+    }, tolerance: number, options?: {
+        maxResults?: number;
+        region?: Region;
+    }): Array<{
+        x: number;
+        y: number;
+    }>;
+    /**
+     * Find clusters of similar colors (potential UI elements).
+     * Clusters are groups of nearby pixels with similar colors.
+     *
+     * @param color - Color to search for (RGB)
+     * @param tolerance - Color distance tolerance (0-441)
+     * @param minClusterSize - Minimum pixels to form a cluster
+     * @param region - Optional region to search (full screen if not specified)
+     * @returns Array of bounding rectangles for each cluster
+     *
+     * @example
+     * ```typescript
+     * // Find blue button-like regions
+     * const clusters = vision.findColorClusters(
+     *   { r: 0, g: 120, b: 215 }, // Windows blue
+     *   40,
+     *   50 // At least 50 pixels
+     * );
+     * if (clusters.length > 0) {
+     *   // Click center of first cluster
+     *   const btn = clusters[0];
+     *   mouse.click(btn.x + btn.width / 2, btn.y + btn.height / 2);
+     * }
+     * ```
+     */
+    findColorClusters(color: {
+        r: number;
+        g: number;
+        b: number;
+    }, tolerance: number, minClusterSize: number, region?: Region): Region[];
+    /**
+     * Verify an action caused a visual change.
+     * High-level primitive that captures before/after and compares.
+     *
+     * @param action - Async action to execute and verify
+     * @param minChangePercent - Minimum change to consider verified
+     * @param timeoutMs - Maximum time to wait for change
+     * @param region - Optional region to monitor
+     * @returns Verification result with diff statistics
+     *
+     * @example
+     * ```typescript
+     * const result = await vision.verifyAction(
+     *   async () => { mouse.click(); },
+     *   0.5, // At least 0.5% change
+     *   2000
+     * );
+     * if (!result.verified) {
+     *   // Click didn't cause visual change - might need to retry
+     * }
+     * ```
+     */
+    verifyAction(action: () => Promise<void> | void, minChangePercent: number, timeoutMs: number, region?: Region): Promise<{
+        verified: boolean;
+        diff: ScreenDiff;
+        durationMs: number;
+    }>;
+    /**
+     * Take a reference, perform action, wait for stability.
+     * Combines action execution with waiting for the UI to settle.
+     *
+     * @param action - Action to execute
+     * @param options - Wait and verification options
+     * @returns true if action completed and screen stabilized
+     *
+     * @example
+     * ```typescript
+     * // Click and wait for any animation to complete
+     * await vision.doAndWaitStable(async () => {
+     *   await mouse.click(100, 200);
+     * });
+     * // Screen is now stable
+     * ```
+     */
+    doAndWaitStable(action: () => Promise<void> | void, options?: {
+        stabilityThreshold?: number;
+        stableDurationMs?: number;
+        timeoutMs?: number;
+        region?: Region;
+    }): Promise<boolean>;
+};
+/** Compact visual descriptor for a screen region. */
+export interface VisualFingerprint {
+    brightnessHist: Float64Array;
+    hueHist: Float64Array;
+    edgeDensity: number;
+    avgBrightness: number;
+    contrast: number;
+    variance: number;
+    aspect: number;
+    phash: number;
+}
+/** A remembered interaction with a screen element. */
+export interface VisualMemory {
+    fingerprint: VisualFingerprint;
+    action: string;
+    label?: string;
+    position: {
+        x: number;
+        y: number;
+    };
+    regionSize: {
+        w: number;
+        h: number;
+    };
+    timestamp: number;
+    outcome?: {
+        screenChangePercent: number;
+        responseTimeMs: number;
+    };
+    success?: boolean;
+    appContext?: string;
+    sequenceIndex: number;
+}
+/** A located element on screen. */
+export interface LocatedElement {
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+    confidence: number;
+    similarity: number;
+    label?: string;
+    supportCount: number;
+    fingerprint: VisualFingerprint;
+    click(button?: string): void;
+    doubleClick(): void;
+    rightClick(): void;
+    moveTo(options?: {
+        smooth?: boolean;
+        duration?: number;
+    }): void;
+    type(text: string): void;
+    /** Verify this element is still visually present */
+    isStillPresent(threshold?: number): boolean;
+}
+/** Temporal prediction result. */
+export interface TemporalPrediction {
+    nextAction: string;
+    confidence: number;
+    expectedDelayMs: number;
+    predictedRegion?: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    label?: string;
+}
+/** Configuration for the AgentVision system. */
+export interface AgentVisionConfig {
+    /** Grid resolution for screen scanning. Default: 32x18 */
+    cols?: number;
+    rows?: number;
+    /** Maximum memories to keep. Default: 2000 */
+    maxMemories?: number;
+    /** Fingerprint region around a point. Default: 100x60 */
+    defaultRegionSize?: {
+        w: number;
+        h: number;
+    };
+    /** Minimum similarity to consider a match. Default: 0.65 */
+    matchThreshold?: number;
+    /** How far to search around original position. Default: 400px */
+    searchRadius?: number;
+    /** Application context (e.g. "notepad", "chrome"). Helps filter memories. */
+    appContext?: string;
+}
+export declare class AgentVision {
+    private cols;
+    private rows;
+    private maxMemories;
+    private defaultRegionSize;
+    private matchThreshold;
+    private searchRadius;
+    private appContext?;
+    private memories;
+    private sequenceCounter;
+    private recentActions;
+    private temporalPatterns;
+    private gridFingerprints;
+    private gridDirty;
+    private lastGridTime;
+    private lastCapture;
+    constructor(config?: AgentVisionConfig);
+    /**
+     * Fingerprint a screen region. Returns a compact visual descriptor
+     * that can be compared against other fingerprints for similarity.
+     *
+     * @example
+     * ```typescript
+     * const fp = agentVision.fingerprint(100, 200, 150, 40);
+     * // Later: check if the same element is somewhere else
+     * const match = agentVision.locate(fp);
+     * ```
+     */
+    fingerprint(x: number, y: number, w: number, h: number): VisualFingerprint;
+    /**
+     * Compute similarity between two fingerprints (0-1, 1 = identical).
+     */
+    similarity(a: VisualFingerprint, b: VisualFingerprint): number;
+    /**
+     * Remember an element at a position with a label.
+     * The agent fingerprints the region and stores it for later recognition.
+     *
+     * @example
+     * ```typescript
+     * // Agent clicked Save, now remembers what it looks like
+     * agentVision.remember('save-button', 350, 15, { w: 80, h: 30 });
+     *
+     * // Later, find it again even if it moved
+     * const saveBtn = agentVision.find('save-button');
+     * if (saveBtn) saveBtn.click();
+     * ```
+     */
+    remember(label: string, x: number, y: number, regionSize?: {
+        w: number;
+        h: number;
+    }, action?: string): VisualMemory;
+    /**
+     * Remember after performing a click — records the action AND verifies it.
+     * Captures before/after fingerprints and records whether the click did anything.
+     *
+     * @example
+     * ```typescript
+     * const result = await agentVision.rememberClick('file-menu', 44, 12);
+     * console.log(`Click ${result.success ? 'worked' : 'had no effect'}`);
+     * ```
+     */
+    rememberClick(label: string, x: number, y: number, regionSize?: {
+        w: number;
+        h: number;
+    }): Promise<VisualMemory & {
+        success: boolean;
+    }>;
+    /**
+     * Find a previously remembered element on the current screen.
+     * Uses visual fingerprint matching — works even if the element moved.
+     *
+     * @returns The best match, or null if nothing above threshold
+     *
+     * @example
+     * ```typescript
+     * const saveBtn = agentVision.find('save-button');
+     * if (saveBtn) {
+     *   saveBtn.click();
+     * } else {
+     *   console.log('Save button not visible');
+     * }
+     * ```
+     */
+    find(label: string): LocatedElement | null;
+    /**
+     * Find all instances of a remembered element on screen.
+     *
+     * @example
+     * ```typescript
+     * // Find all things that look like "close-button"
+     * const closeButtons = agentVision.findAll('close-button');
+     * console.log(`Found ${closeButtons.length} close buttons`);
+     * ```
+     */
+    findAll(label: string, maxResults?: number): LocatedElement[];
+    /**
+     * Locate a specific visual fingerprint on the current screen.
+     * Does a focused scan: first near the expected position, then widens.
+     *
+     * @example
+     * ```typescript
+     * const fp = agentVision.fingerprint(100, 200, 80, 30);
+     * // ... some time later, UI may have reorganized ...
+     * const found = agentVision.locate(fp, { near: { x: 100, y: 200 } });
+     * if (found) found.click();
+     * ```
+     */
+    locate(target: VisualFingerprint, options?: {
+        near?: {
+            x: number;
+            y: number;
+        };
+        threshold?: number;
+        searchRadius?: number;
+    }): LocatedElement | null;
+    /**
+     * Find an element by text using OCR, then fingerprint it for future recognition.
+     * First time: uses OCR. After that: can find it visually even without OCR.
+     *
+     * @example
+     * ```typescript
+     * // First call uses OCR to find "Save" text
+     * const save = agentVision.findByText('Save');
+     * if (save) save.click();
+     * // Now agentVision remembers what "Save" looks like visually
+     * ```
+     */
+    findByText(text: string, options?: {
+        remember?: boolean;
+    }): LocatedElement | null;
+    /**
+     * Click an element and verify the screen changed.
+     * Returns the located element with success/failure status.
+     *
+     * @example
+     * ```typescript
+     * const result = await agentVision.clickAndVerify('submit-button');
+     * if (!result.verified) {
+     *   // Button didn't respond — try again or escalate
+     * }
+     * ```
+     */
+    clickAndVerify(label: string, options?: {
+        timeout?: number;
+        minChange?: number;
+    }): Promise<{
+        element: LocatedElement | null;
+        verified: boolean;
+        changePercent: number;
+    }>;
+    /**
+     * Check if a specific screen region visually changed since a fingerprint was taken.
+     *
+     * @example
+     * ```typescript
+     * const before = agentVision.fingerprint(100, 200, 80, 30);
+     * await agent.doSomething();
+     * const changed = agentVision.hasChanged(before, 100, 200, 80, 30);
+     * ```
+     */
+    hasChanged(previousFingerprint: VisualFingerprint, x: number, y: number, w: number, h: number, changeThreshold?: number): boolean;
+    /**
+     * Predict what will happen next based on learned temporal patterns.
+     *
+     * @example
+     * ```typescript
+     * // After clicking "File" many times, the agent learns:
+     * // "After clicking file-menu, a dropdown appears"
+     * const next = agentVision.predictNext();
+     * if (next) {
+     *   console.log(`Expected: ${next.nextAction} in ${next.expectedDelayMs}ms`);
+     * }
+     * ```
+     */
+    predictNext(filterLabel?: string): TemporalPrediction | null;
+    /**
+     * Wait for a predicted event to happen.
+     * Uses temporal patterns to know WHEN and WHERE to look.
+     *
+     * @example
+     * ```typescript
+     * // Click File menu
+     * agentVision.find('file-menu')?.click();
+     * // Wait for the dropdown the agent learned usually appears
+     * const appeared = await agentVision.waitForPredicted('dropdown', 3000);
+     * ```
+     */
+    waitForPredicted(label?: string, timeoutMs?: number): Promise<LocatedElement | null>;
+    /**
+     * Get all learned temporal patterns.
+     */
+    getPatterns(): Array<{
+        pattern: string;
+        count: number;
+        avgDelayMs: number;
+    }>;
+    /**
+     * Scan the full screen and return all regions that match ANY remembered element.
+     * Gives the agent a complete understanding of "what's on screen that I recognize."
+     *
+     * @example
+     * ```typescript
+     * const recognized = agentVision.scan();
+     * for (const el of recognized) {
+     *   console.log(`Found "${el.label}" at (${el.x},${el.y}) confidence=${el.confidence}`);
+     * }
+     * ```
+     */
+    scan(threshold?: number): LocatedElement[];
+    /**
+     * Export all visual memories as a JSON string.
+     *
+     * @example
+     * ```typescript
+     * const data = agentVision.save();
+     * fs.writeFileSync('agent-memory.json', data);
+     * ```
+     */
+    save(): string;
+    /**
+     * Load visual memories from a JSON string.
+     *
+     * @example
+     * ```typescript
+     * const data = fs.readFileSync('agent-memory.json', 'utf8');
+     * agentVision.load(data);
+     * // Agent now remembers everything from last session
+     * ```
+     */
+    load(json: string): void;
+    /** Number of stored memories. */
+    get memoryCount(): number;
+    /** All unique labels the agent has learned. */
+    get knownLabels(): string[];
+    /** Clear all memories. */
+    reset(): void;
+    /** Set the app context (filters memories by app). */
+    setContext(appContext: string): void;
+    private refreshGrid;
+    private trackAction;
+    private evictOldMemories;
+}
+/** Create a new AgentVision instance. */
+export declare function createAgentVision(config?: AgentVisionConfig): AgentVision;
+//# sourceMappingURL=vision.d.ts.map