screenhand 0.2.0 → 0.3.1

package/dist/src/perception/frame-differ.js

@@ -0,0 +1,287 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import crypto from "node:crypto";
+import fs from "node:fs";
+/**
+ * Fast frame differencing using content hashing.
+ * Compares PNG buffers (in-memory, no disk I/O) and extracts changed regions
+ * by dividing the frame into a grid and hashing each cell.
+ */
+export class FrameDiffer {
+    lastFrameHash = null;
+    lastFrameBuffer = null;
+    lastGridHashes = null;
+    /** Grid cell size for region detection (pixels). */
+    cellSize;
+    constructor(cellSize = 128) {
+        this.cellSize = cellSize;
+    }
+    /**
+     * Hash a frame buffer. Uses MD5 for speed (not security).
+     */
+    hashBuffer(buffer) {
+        return crypto.createHash("md5").update(buffer).digest("hex");
+    }
+    /**
+     * Compare a new frame against the last.
+     * Returns whether anything changed and which regions differ.
+     *
+     * For PNG buffers, we do whole-frame hash for quick "anything changed?" check,
+     * then grid-based hashing for region extraction.
+     *
+     * IMPORTANT: PNG is a compressed format, so byte-offset slicing does NOT map
+     * to pixel coordinates. The grid-based region detection is an approximation
+     * that detects *which chunk of the compressed stream* changed, not the exact
+     * pixel region. The returned ROI coordinates are estimates — use them as hints
+     * for OCR, not as precise bounding boxes. For exact pixel-level regions, use
+     * the native bridge's `cg.captureWindowBuffer` (raw RGBA) + `vision.ocrRegion`.
+     */
+    diff(buffer, frameWidth, frameHeight) {
+        const hash = this.hashBuffer(buffer);
+        const changed = this.lastFrameHash !== null && hash !== this.lastFrameHash;
+        let changedRegions = [];
+        if (changed && this.lastGridHashes !== null) {
+            changedRegions = this.detectChangedRegions(buffer, frameWidth, frameHeight);
+        }
+        // Update grid hashes for next comparison
+        this.lastGridHashes = this.computeGridHashes(buffer, frameWidth, frameHeight);
+        this.lastFrameHash = hash;
+        this.lastFrameBuffer = buffer;
+        return { changed, hash, changedRegions };
+    }
+    /**
+     * Quick check: did anything change? (~0.1ms for hash comparison)
+     */
+    quickChanged(buffer) {
+        const hash = this.hashBuffer(buffer);
+        return this.lastFrameHash !== null && hash !== this.lastFrameHash;
+    }
+    /**
+     * Hash a file on disk directly (skips base64 round-trip).
+     */
+    hashFile(filePath) {
+        const buf = fs.readFileSync(filePath);
+        return crypto.createHash("md5").update(buf).digest("hex");
+    }
+    /**
+     * Quick change detection from a file path. Hashes file on disk,
+     * compares against last frame hash. Skips grid hashing entirely.
+     */
+    quickChangedFile(filePath) {
+        const hash = this.hashFile(filePath);
+        const changed = this.lastFrameHash !== null && hash !== this.lastFrameHash;
+        this.lastFrameHash = hash;
+        return { changed, hash };
+    }
+    /**
+     * Full diff from a file path — detects changed regions using grid hashing.
+     * More expensive than quickChangedFile (~5ms vs ~1ms) but returns ROIs
+     * that can be used for region-based OCR.
+     */
+    diffFile(filePath, frameWidth, frameHeight) {
+        const buffer = fs.readFileSync(filePath);
+        return this.diff(buffer, frameWidth, frameHeight);
+    }
+    /**
+     * Merge adjacent ROI cells into larger rectangles and pad with extra pixels
+     * to ensure OCR captures text at region boundaries. Returns at most
+     * maxRegions merged ROIs, sorted by area (largest first).
+     */
+    static mergeRegions(regions, maxRegions, padding, frameWidth, frameHeight, cellSize = 128) {
+        if (regions.length === 0)
+            return [];
+        if (regions.length === 1) {
+            return [FrameDiffer.padRegion(regions[0], padding, frameWidth, frameHeight)];
+        }
+        // Sort by position (top-left to bottom-right) for merge pass
+        const sorted = [...regions].sort((a, b) => a.y - b.y || a.x - b.x);
+        // Greedy merge: combine overlapping/adjacent regions
+        const merged = [];
+        let current = { ...sorted[0] };
+        for (let i = 1; i < sorted.length; i++) {
+            const next = sorted[i];
+            // Check if next region overlaps or is adjacent to current (within 1 cell gap)
+            const currentRight = current.x + current.width;
+            const currentBottom = current.y + current.height;
+            const nextRight = next.x + next.width;
+            const nextBottom = next.y + next.height;
+            const GAP = cellSize; // one cell-sized gap tolerance for adjacency
+            const horizontalOverlap = next.x <= currentRight + GAP && nextRight >= current.x;
+            const verticalOverlap = next.y <= currentBottom + GAP && nextBottom >= current.y;
+            if (horizontalOverlap && verticalOverlap) {
+                // Merge: expand current to encompass next
+                const newX = Math.min(current.x, next.x);
+                const newY = Math.min(current.y, next.y);
+                current = {
+                    x: newX,
+                    y: newY,
+                    width: Math.max(currentRight, nextRight) - newX,
+                    height: Math.max(currentBottom, nextBottom) - newY,
+                    reason: "changed_pixels",
+                };
+            }
+            else {
+                merged.push(current);
+                current = { ...next };
+            }
+        }
+        merged.push(current);
+        // Pad and sort by area (largest first), cap at maxRegions
+        return merged
+            .map((r) => FrameDiffer.padRegion(r, padding, frameWidth, frameHeight))
+            .sort((a, b) => b.width * b.height - a.width * a.height)
+            .slice(0, maxRegions);
+    }
+    static padRegion(roi, padding, frameWidth, frameHeight) {
+        const x = Math.max(0, roi.x - padding);
+        const y = Math.max(0, roi.y - padding);
+        return {
+            x,
+            y,
+            width: Math.min(roi.x + roi.width + padding, frameWidth) - x,
+            height: Math.min(roi.y + roi.height + padding, frameHeight) - y,
+            reason: roi.reason,
+        };
+    }
+    /** Reset state (e.g., on context switch). */
+    reset() {
+        this.lastFrameHash = null;
+        this.lastFrameBuffer = null;
+        this.lastGridHashes = null;
+    }
+    /** Get last frame hash (for external state tracking). */
+    getLastHash() {
+        return this.lastFrameHash;
+    }
+    computeGridHashes(buffer, width, height) {
+        const hashes = new Map();
+        if (width <= 0 || height <= 0 || buffer.length === 0)
+            return hashes;
+        const cols = Math.ceil(width / this.cellSize);
+        const rows = Math.ceil(height / this.cellSize);
+        const bytesPerRow = Math.ceil(buffer.length / height) || 1;
+        for (let row = 0; row < rows; row++) {
+            for (let col = 0; col < cols; col++) {
+                const key = `${col},${row}`;
+                const startByte = row * this.cellSize * bytesPerRow + col * this.cellSize;
+                const endByte = Math.min(startByte + this.cellSize * bytesPerRow, buffer.length);
+                if (startByte >= buffer.length)
+                    continue;
+                const slice = buffer.subarray(startByte, endByte);
+                hashes.set(key, crypto.createHash("md5").update(slice).digest("hex"));
+            }
+        }
+        return hashes;
+    }
+    detectChangedRegions(buffer, width, height) {
+        const currentGrid = this.computeGridHashes(buffer, width, height);
+        const regions = [];
+        for (const [key, hash] of currentGrid) {
+            const prevHash = this.lastGridHashes?.get(key);
+            if (prevHash && prevHash !== hash) {
+                const [colStr, rowStr] = key.split(",");
+                const col = Number(colStr);
+                const row = Number(rowStr);
+                regions.push({
+                    x: col * this.cellSize,
+                    y: row * this.cellSize,
+                    width: Math.min(this.cellSize, width - col * this.cellSize),
+                    height: Math.min(this.cellSize, height - row * this.cellSize),
+                    reason: "changed_pixels",
+                });
+            }
+        }
+        return regions;
+    }
+    // ── Raw RGBA pixel-accurate diffing ──
+    lastRawHash = null;
+    lastRawGridHashes = null;
+    /**
+     * Diff raw RGBA pixel data for accurate ROI detection.
+     *
+     * Unlike `diff()` which operates on compressed PNG bytes (approximate ROIs),
+     * this method works with uncompressed RGBA buffers where byte offsets map
+     * directly to pixel coordinates. Use with the native bridge's
+     * `cg.captureWindowBuffer` which returns raw RGBA data.
+     *
+     * @param rgba Raw RGBA pixel buffer (4 bytes per pixel, row-major)
+     * @param width Frame width in pixels
+     * @param height Frame height in pixels
+     */
+    diffRaw(rgba, width, height) {
+        const hash = this.hashBuffer(rgba);
+        const changed = this.lastRawHash !== null && hash !== this.lastRawHash;
+        let changedRegions = [];
+        if (changed && this.lastRawGridHashes !== null) {
+            changedRegions = this.detectRawChangedRegions(rgba, width, height);
+        }
+        this.lastRawGridHashes = this.computeRawGridHashes(rgba, width, height);
+        this.lastRawHash = hash;
+        return { changed, hash, changedRegions };
+    }
+    /**
+     * Compute grid hashes from raw RGBA data using pixel-accurate slicing.
+     * Each cell is hashed using its actual pixel rows, not byte-offset estimates.
+     */
+    computeRawGridHashes(rgba, width, height) {
+        const hashes = new Map();
+        const cols = Math.ceil(width / this.cellSize);
+        const rows = Math.ceil(height / this.cellSize);
+        const bytesPerPixel = 4; // RGBA
+        const stride = width * bytesPerPixel;
+        for (let row = 0; row < rows; row++) {
+            for (let col = 0; col < cols; col++) {
+                const key = `${col},${row}`;
+                const cellX = col * this.cellSize;
+                const cellY = row * this.cellSize;
+                const cellW = Math.min(this.cellSize, width - cellX);
+                const cellH = Math.min(this.cellSize, height - cellY);
+                // Hash the actual pixel data for this cell
+                const hasher = crypto.createHash("md5");
+                for (let y = cellY; y < cellY + cellH; y++) {
+                    const rowStart = y * stride + cellX * bytesPerPixel;
+                    const rowEnd = rowStart + cellW * bytesPerPixel;
+                    if (rowEnd <= rgba.length) {
+                        hasher.update(rgba.subarray(rowStart, rowEnd));
+                    }
+                }
+                hashes.set(key, hasher.digest("hex"));
+            }
+        }
+        return hashes;
+    }
+    detectRawChangedRegions(rgba, width, height) {
+        const currentGrid = this.computeRawGridHashes(rgba, width, height);
+        const regions = [];
+        for (const [key, hash] of currentGrid) {
+            const prevHash = this.lastRawGridHashes?.get(key);
+            if (prevHash && prevHash !== hash) {
+                const [colStr, rowStr] = key.split(",");
+                const col = Number(colStr);
+                const row = Number(rowStr);
+                regions.push({
+                    x: col * this.cellSize,
+                    y: row * this.cellSize,
+                    width: Math.min(this.cellSize, width - col * this.cellSize),
+                    height: Math.min(this.cellSize, height - row * this.cellSize),
+                    reason: "changed_pixels",
+                });
+            }
+        }
+        return regions;
+    }
+}

package/dist/src/perception/index.js

@@ -0,0 +1,22 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export { PerceptionCoordinator } from "./coordinator.js";
+export { PerceptionManager } from "./manager.js";
+export { AXSource } from "./ax-source.js";
+export { CDPSource } from "./cdp-source.js";
+export { VisionSource } from "./vision-source.js";
+export { FrameDiffer } from "./frame-differ.js";

package/dist/src/perception/manager.js

@@ -0,0 +1,199 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { EventEmitter } from "node:events";
+import { StateObserver } from "../runtime/state-observer.js";
+import { AXSource } from "./ax-source.js";
+import { CDPSource } from "./cdp-source.js";
+import { VisionSource } from "./vision-source.js";
+import { PerceptionCoordinator } from "./coordinator.js";
+import { createEmptyStats } from "./types.js";
+/**
+ * PerceptionManager — creates sources lazily when the bridge is ready,
+ * auto-starts perception on first app context, manages context switches,
+ * and emits reactive events (dialog_detected, app_switched).
+ */
+export class PerceptionManager extends EventEmitter {
+    worldModel;
+    config;
+    coordinator = null;
+    sourcesCreated = false;
+    currentContext = null;
+    currentPid = null;
+    currentBundleId = null;
+    lastCdpClient = null;
+    pendingLearningEngine = null;
+    pendingAppMap = null;
+    constructor(worldModel, config) {
+        super();
+        this.worldModel = worldModel;
+        this.config = config;
+    }
+    /**
+     * Inject the learning engine. If coordinator already exists, wires immediately.
+     * Otherwise, defers until createSources() is called.
+     */
+    setLearningEngine(engine) {
+        this.pendingLearningEngine = engine;
+        if (this.coordinator) {
+            this.coordinator.setLearningEngine(engine);
+        }
+    }
+    /**
+     * Inject the app mastery map. If coordinator already exists, wires immediately.
+     * Otherwise, defers until createSources() is called.
+     */
+    setAppMap(map) {
+        this.pendingAppMap = map;
+        if (this.coordinator) {
+            this.coordinator.setAppMap(map);
+        }
+    }
+    /**
+     * Create perception sources from the bridge. Called once after ensureBridge().
+     */
+    createSources(bridge) {
+        if (this.sourcesCreated)
+            return;
+        this.sourcesCreated = true;
+        const observer = new StateObserver(bridge);
+        const axSource = new AXSource(observer, bridge);
+        const cdpSource = new CDPSource();
+        const visionSource = new VisionSource(bridge);
+        this.coordinator = new PerceptionCoordinator(this.worldModel, axSource, cdpSource, visionSource, { enableVision: true, ...this.config });
+        if (this.pendingLearningEngine) {
+            this.coordinator.setLearningEngine(this.pendingLearningEngine);
+        }
+        if (this.pendingAppMap) {
+            this.coordinator.setAppMap(this.pendingAppMap);
+        }
+        this.coordinator.on("perception", (event) => {
+            this.handleReactiveEvent(event);
+        });
+    }
+    /**
+     * Ensure perception is started for the given app context.
+     * Idempotent — starts if not running, switches context if app changed.
+     */
+    async ensureStarted(appContext, cdpClient) {
+        if (!this.coordinator)
+            return;
+        const client = cdpClient ?? this.lastCdpClient;
+        if (!this.coordinator.isRunning) {
+            this.currentContext = appContext;
+            this.currentPid = appContext.pid;
+            this.currentBundleId = appContext.bundleId;
+            await this.coordinator.start(appContext, client);
+        }
+        else if (this.currentPid !== appContext.pid ||
+            (appContext.windowId != null && this.currentContext?.windowId !== appContext.windowId)) {
+            // Switch context when PID changes or when windowId is now available but wasn't before
+            this.currentContext = appContext;
+            this.currentPid = appContext.pid;
+            this.currentBundleId = appContext.bundleId;
+            await this.coordinator.switchContext(appContext, client);
+        }
+    }
+    /**
+     * Activate CDP source with a new client.
+     * Uses hot-inject on the running coordinator instead of stop+restart
+     * to preserve AX polling state and avoid resetting counters.
+     */
+    activateCDP(cdpClient) {
+        this.lastCdpClient = cdpClient;
+        if (this.coordinator?.isRunning) {
+            this.coordinator.activateCDP(cdpClient);
+        }
+    }
+    /**
+     * Best-effort auto-start: if perception isn't running and a focused app
+     * is known, resolve its windowId and start perception silently.
+     * Non-blocking — failures are swallowed.
+     */
+    async tryAutoStart(focusedApp, bridge) {
+        if (!this.coordinator || this.coordinator.isRunning)
+            return;
+        if (!focusedApp.pid)
+            return;
+        let windowId;
+        try {
+            const wins = await bridge.call("window.list", {});
+            const matching = wins?.filter((w) => w.pid === focusedApp.pid);
+            if (matching && matching.length > 0) {
+                const frontmost = matching.find((w) => w.focused || w.frontmost || w.isMain);
+                windowId = (frontmost ?? matching[0])?.windowId;
+            }
+        }
+        catch { /* best-effort */ }
+        const ctx = {
+            bundleId: focusedApp.bundleId,
+            appName: focusedApp.bundleId,
+            pid: focusedApp.pid,
+            windowTitle: "",
+            ...(windowId != null ? { windowId } : {}),
+        };
+        await this.ensureStarted(ctx);
+    }
+    async stop() {
+        if (this.coordinator?.isRunning) {
+            await this.coordinator.stop();
+        }
+        this.currentContext = null;
+        this.currentPid = null;
+        this.currentBundleId = null;
+    }
+    get isRunning() {
+        return this.coordinator?.isRunning ?? false;
+    }
+    getStats() {
+        return this.coordinator?.getStats() ?? createEmptyStats();
+    }
+    getFreshnessSummary() {
+        return this.coordinator?.getFreshnessSummary() ?? "Perception: not initialized";
+    }
+    getConfig() {
+        return this.coordinator?.getConfig() ?? null;
+    }
+    getCoordinator() {
+        return this.coordinator;
+    }
+    /**
+     * Notify perception that a tool call is happening — resets idle timer.
+     */
+    notifyToolCall() {
+        this.coordinator?.notifyToolCall();
+    }
+    handleReactiveEvent(event) {
+        if (event.data?.type === "ax_events" && Array.isArray(event.data.events)) {
+            for (const uiEvent of event.data.events) {
+                if (uiEvent.type === "dialog_appeared") {
+                    this.emit("dialog_detected", {
+                        title: uiEvent.windowTitle ?? "",
+                        pid: uiEvent.pid,
+                    });
+                }
+                if (uiEvent.type === "app_activated" &&
+                    uiEvent.bundleId &&
+                    uiEvent.bundleId !== this.currentBundleId) {
+                    this.emit("app_switched", {
+                        bundleId: uiEvent.bundleId,
+                        pid: uiEvent.pid,
+                    });
+                }
+            }
+        }
+    }
+}

package/dist/src/perception/types.js

@@ -0,0 +1,47 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export const DEFAULT_PERCEPTION_CONFIG = {
+    fastIntervalMs: 100,
+    mediumIntervalMs: 300,
+    slowIntervalMs: 1000,
+    enableAX: true,
+    enableCDP: true,
+    enableVision: true,
+    maxROIsPerCycle: 3,
+    skipCaptureLock: false,
+};
+export function createEmptyStats() {
+    return {
+        started: false,
+        startedAt: null,
+        fastCycles: 0,
+        mediumCycles: 0,
+        slowCycles: 0,
+        axEventsProcessed: 0,
+        axTreePolls: 0,
+        cdpMutationsProcessed: 0,
+        cdpSnapshots: 0,
+        visionDiffs: 0,
+        visionOCRs: 0,
+        lastFastAt: null,
+        lastMediumAt: null,
+        lastSlowAt: null,
+        lastAXAt: null,
+        lastCDPAt: null,
+        lastVisionAt: null,
+    };
+}