screenhand 0.1.1 → 0.2.0

package/dist/src/runtime/vision-adapter.js

@@ -0,0 +1,224 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+const POLL_INTERVAL_MS = 200;
+/**
+ * Vision-based adapter for apps with poor/no accessibility support.
+ * Uses screenshots + OCR to locate elements and CG events to interact.
+ */
+export class VisionAdapter {
+    bridge;
+    sessions = new Map();
+    sessionsByProfile = new Map();
+    constructor(bridge) {
+        this.bridge = bridge;
+    }
+    async attach(profile, reuseSessionId) {
+        const existing = this.sessionsByProfile.get(profile);
+        if (existing)
+            return existing.info;
+        await this.bridge.start();
+        const frontmost = await this.bridge.call("app.frontmost");
+        const info = {
+            sessionId: reuseSessionId ?? `vision_session_${profile}_${Date.now()}`,
+            profile,
+            createdAt: new Date().toISOString(),
+            adapterType: "vision",
+        };
+        const state = {
+            info,
+            pid: frontmost.pid,
+            bundleId: frontmost.bundleId,
+            appName: frontmost.name,
+        };
+        this.sessions.set(info.sessionId, state);
+        this.sessionsByProfile.set(profile, state);
+        return info;
+    }
+    async getAppContext(sessionId) {
+        const state = this.requireSession(sessionId);
+        return {
+            bundleId: state.bundleId,
+            appName: state.appName,
+            pid: state.pid,
+            windowTitle: state.appName,
+        };
+    }
+    async getPageMeta(sessionId) {
+        const ctx = await this.getAppContext(sessionId);
+        return {
+            url: `app://${ctx.bundleId}`,
+            title: ctx.appName,
+        };
+    }
+    async navigate(sessionId, url, _timeoutMs) {
+        if (url.startsWith("app://")) {
+            const bundleId = url.slice(6);
+            const result = await this.bridge.call("app.launch", { bundleId });
+            const state = this.requireSession(sessionId);
+            state.pid = result.pid;
+            state.bundleId = result.bundleId;
+            state.appName = result.appName;
+        }
+        return this.getPageMeta(sessionId);
+    }
+    async locate(sessionId, target, timeoutMs) {
+        const state = this.requireSession(sessionId);
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            // Take a screenshot
+            const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+            state.lastScreenshotPath = screenshotResult.path;
+            const searchText = this.getSearchText(target);
+            if (!searchText) {
+                // For coordinate targets, just return coordinates directly
+                if (target.type === "coordinates") {
+                    return {
+                        handleId: `vision_coords_${target.x}_${target.y}`,
+                        locatorUsed: "vision:coordinates",
+                        coordinates: { x: target.x, y: target.y, width: 1, height: 1 },
+                    };
+                }
+                return null;
+            }
+            // OCR the screenshot
+            const matches = await this.bridge.call("vision.findText", {
+                imagePath: screenshotResult.path,
+                searchText,
+            });
+            if (matches.length > 0) {
+                const best = matches.reduce((a, b) => (a.confidence > b.confidence ? a : b));
+                return {
+                    handleId: `vision_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+                    locatorUsed: `vision:text:${searchText}`,
+                    label: best.text,
+                    coordinates: best.bounds,
+                };
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return null;
+    }
+    async click(_sessionId, element) {
+        if (!element.coordinates) {
+            throw new Error("Vision adapter requires coordinates to click");
+        }
+        const cx = element.coordinates.x + element.coordinates.width / 2;
+        const cy = element.coordinates.y + element.coordinates.height / 2;
+        await this.bridge.call("cg.mouseClick", { x: cx, y: cy });
+    }
+    async setValue(_sessionId, element, text, clear) {
+        // Click to focus
+        await this.click(_sessionId, element);
+        await sleep(100);
+        if (clear) {
+            await this.bridge.call("cg.keyCombo", { keys: ["cmd", "a"] });
+            await sleep(50);
+        }
+        await this.bridge.call("cg.typeText", { text });
+    }
+    async getValue(_sessionId, element) {
+        // Vision can't reliably read values; return label if available
+        return element.label ?? "";
+    }
+    async waitFor(sessionId, condition, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            if (condition.type === "text_appears") {
+                const found = await this.locate(sessionId, { type: "text", value: condition.text }, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_exists") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_gone") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (!found)
+                    return true;
+            }
+            else {
+                // Unsupported condition types
+                return false;
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return false;
+    }
+    async extract(sessionId, _target, format) {
+        const state = this.requireSession(sessionId);
+        // Take a fresh screenshot and OCR it
+        const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+        state.lastScreenshotPath = screenshotResult.path;
+        const ocrResult = await this.bridge.call("vision.ocr", { imagePath: screenshotResult.path });
+        if (format === "text") {
+            return ocrResult.text;
+        }
+        if (format === "json") {
+            return ocrResult;
+        }
+        // table format
+        return {
+            headers: ["text", "confidence", "x", "y", "width", "height"],
+            rows: ocrResult.regions.map((r) => [
+                r.text,
+                r.confidence,
+                r.bounds.x,
+                r.bounds.y,
+                r.bounds.width,
+                r.bounds.height,
+            ]),
+        };
+    }
+    async screenshot(_sessionId, region) {
+        const result = await this.bridge.call("cg.captureScreen", region ? { region } : {});
+        return result.path;
+    }
+    async keyCombo(_sessionId, keys) {
+        await this.bridge.call("cg.keyCombo", { keys });
+    }
+    async elementTree(_sessionId, _maxDepth, _root) {
+        throw new Error("Vision adapter does not support elementTree — use accessibility adapter");
+    }
+    // ── Private ──
+    requireSession(sessionId) {
+        const state = this.sessions.get(sessionId);
+        if (!state)
+            throw new Error(`Session not found: ${sessionId}`);
+        return state;
+    }
+    getSearchText(target) {
+        switch (target.type) {
+            case "text":
+                return target.value;
+            case "role":
+                return target.name;
+            case "selector":
+                return target.value;
+            case "ax_attribute":
+                return target.value;
+            case "image":
+            case "coordinates":
+            case "ax_path":
+                return null;
+        }
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/src/supervisor/locks.js

@@ -0,0 +1,186 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * LeaseManager — filesystem-based session lease management.
+ *
+ * Each lease is a JSON lock file: {app}__{windowId}.lock
+ * Provides mutual exclusion so only one client controls a window at a time.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { writeFileAtomicSync } from "../util/atomic-write.js";
+const DEFAULT_LEASE_TIMEOUT_MS = 300000; // 5 min
+export class LeaseManager {
+    lockDir;
+    leaseTimeoutMs;
+    constructor(lockDir, leaseTimeoutMs = DEFAULT_LEASE_TIMEOUT_MS) {
+        this.lockDir = lockDir;
+        this.leaseTimeoutMs = leaseTimeoutMs;
+        fs.mkdirSync(this.lockDir, { recursive: true });
+    }
+    /**
+     * Claim a window for a client. Fails if another active lease exists.
+     * Returns the new SessionLease, or null if the window is already claimed.
+     */
+    claim(client, app, windowId) {
+        const existing = this.isLocked(app, windowId);
+        if (existing) {
+            return null;
+        }
+        const now = new Date();
+        const sessionId = "lease_" + Date.now().toString(36) + "_" + Math.random().toString(36).slice(2, 8);
+        const lease = {
+            sessionId,
+            client,
+            app,
+            windowId,
+            claimedAt: now.toISOString(),
+            lastHeartbeat: now.toISOString(),
+            expiresAt: new Date(now.getTime() + this.leaseTimeoutMs).toISOString(),
+        };
+        const lockFile = this.lockFilePath(app, windowId);
+        try {
+            fs.writeFileSync(lockFile, JSON.stringify(lease, null, 2), { flag: "wx" });
+        }
+        catch {
+            // File already exists (race condition) — another client claimed it
+            return null;
+        }
+        return lease;
+    }
+    /**
+     * Refresh heartbeat for an existing lease.
+     * Returns true if the lease was found and updated, false otherwise.
+     */
+    heartbeat(sessionId) {
+        const leases = this.readAllLeases();
+        for (const { lease, filePath } of leases) {
+            if (lease.sessionId === sessionId) {
+                const now = new Date();
+                lease.lastHeartbeat = now.toISOString();
+                lease.expiresAt = new Date(now.getTime() + this.leaseTimeoutMs).toISOString();
+                writeFileAtomicSync(filePath, JSON.stringify(lease, null, 2));
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Release a lease by session ID.
+     * Returns true if the lease was found and removed, false otherwise.
+     */
+    release(sessionId) {
+        const leases = this.readAllLeases();
+        for (const { lease, filePath } of leases) {
+            if (lease.sessionId === sessionId) {
+                try {
+                    fs.unlinkSync(filePath);
+                }
+                catch {
+                    // Already removed
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Get all active (non-expired) leases.
+     */
+    getActive() {
+        const now = Date.now();
+        return this.readAllLeases()
+            .filter(({ lease }) => new Date(lease.expiresAt).getTime() > now)
+            .map(({ lease }) => lease);
+    }
+    /**
+     * Clean up expired leases by deleting their lock files.
+     * Returns the number of leases expired.
+     */
+    expireStale() {
+        const now = Date.now();
+        let count = 0;
+        for (const { lease, filePath } of this.readAllLeases()) {
+            if (new Date(lease.expiresAt).getTime() <= now) {
+                try {
+                    fs.unlinkSync(filePath);
+                    count++;
+                }
+                catch {
+                    // Already removed
+                }
+            }
+        }
+        return count;
+    }
+    /**
+     * Check if a window is claimed by an active (non-expired) lease.
+     * Returns the lease if found, null otherwise.
+     */
+    isLocked(app, windowId) {
+        const lockFile = this.lockFilePath(app, windowId);
+        try {
+            const data = fs.readFileSync(lockFile, "utf-8");
+            const lease = JSON.parse(data);
+            // Check if expired
+            if (new Date(lease.expiresAt).getTime() <= Date.now()) {
+                // Expired — clean up and report as unlocked
+                try {
+                    fs.unlinkSync(lockFile);
+                }
+                catch {
+                    // Ignore
+                }
+                return null;
+            }
+            return lease;
+        }
+        catch {
+            return null;
+        }
+    }
+    // ── Private helpers ──
+    lockFilePath(app, windowId) {
+        // Sanitize app name for filesystem safety
+        const safeApp = app.replace(/[^a-zA-Z0-9._-]/g, "_");
+        return path.join(this.lockDir, `${safeApp}__${windowId}.lock`);
+    }
+    readAllLeases() {
+        const results = [];
+        let files;
+        try {
+            files = fs.readdirSync(this.lockDir);
+        }
+        catch {
+            return results;
+        }
+        for (const file of files) {
+            if (!file.endsWith(".lock"))
+                continue;
+            const filePath = path.join(this.lockDir, file);
+            try {
+                const data = fs.readFileSync(filePath, "utf-8");
+                const lease = JSON.parse(data);
+                results.push({ lease, filePath });
+            }
+            catch {
+                // Corrupt lock file — skip
+            }
+        }
+        return results;
+    }
+}