screenhand 0.1.1 → 0.3.0

package/dist/src/runtime/accessibility-adapter.js

@@ -0,0 +1,430 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { randomUUID } from "node:crypto";
+import { toAXRole } from "./ax-role-map.js";
+const POLL_INTERVAL_MS = 100;
+export class AccessibilityAdapter {
+    bridge;
+    sessions = new Map();
+    sessionsByProfile = new Map();
+    lastPidRefresh = 0;
+    constructor(bridge) {
+        this.bridge = bridge;
+    }
+    async attach(profile, reuseSessionId) {
+        const existing = this.sessionsByProfile.get(profile);
+        if (existing)
+            return existing.info;
+        // Ensure bridge is started
+        await this.bridge.start();
+        // Check accessibility permissions
+        const perms = await this.bridge.checkPermissions();
+        if (!perms.trusted) {
+            throw new Error("Accessibility permission not granted. Go to System Settings → Privacy & Security → Accessibility and enable this app.");
+        }
+        const info = {
+            sessionId: reuseSessionId ?? `ax_session_${profile}_${Date.now()}_${randomUUID().slice(0, 8)}`,
+            profile,
+            createdAt: new Date().toISOString(),
+            adapterType: "accessibility",
+        };
+        // Default to frontmost app
+        const frontmost = await this.bridge.call("app.frontmost");
+        const state = {
+            info,
+            pid: frontmost.pid,
+            bundleId: frontmost.bundleId,
+            appName: frontmost.name,
+        };
+        this.sessions.set(info.sessionId, state);
+        this.sessionsByProfile.set(profile, state);
+        return info;
+    }
+    async getAppContext(sessionId) {
+        const state = this.requireSession(sessionId);
+        // Get window title from AX tree
+        let windowTitle = "";
+        try {
+            const tree = await this.bridge.call("ax.getElementTree", {
+                pid: state.pid,
+                maxDepth: 1,
+            });
+            const window = tree.children?.find((c) => c.role === "AXWindow");
+            windowTitle = window?.title ?? "";
+        }
+        catch {
+            // Ignore tree errors
+        }
+        return {
+            bundleId: state.bundleId,
+            appName: state.appName,
+            pid: state.pid,
+            windowTitle,
+        };
+    }
+    async getPageMeta(sessionId) {
+        const ctx = await this.getAppContext(sessionId);
+        return {
+            url: ctx.url ?? `app://${ctx.bundleId}`,
+            title: ctx.windowTitle || ctx.appName,
+        };
+    }
+    async navigate(sessionId, url, _timeoutMs) {
+        // For desktop apps, "navigate" means launching/focusing an app by bundle ID
+        const state = this.requireSession(sessionId);
+        if (url.startsWith("app://")) {
+            const bundleId = url.slice(6);
+            const result = await this.bridge.call("app.launch", { bundleId });
+            state.pid = result.pid;
+            state.bundleId = result.bundleId;
+            state.appName = result.appName;
+        }
+        return this.getPageMeta(sessionId);
+    }
+    async locate(sessionId, target, timeoutMs) {
+        const state = this.requireSession(sessionId);
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            try {
+                const params = this.buildFindParams(target, state.pid);
+                const result = await this.bridge.call("ax.findElement", params);
+                if (result) {
+                    const located = {
+                        handleId: result.handleId,
+                        locatorUsed: `ax:${target.type}`,
+                        role: result.role,
+                        label: result.title,
+                    };
+                    if (result.bounds) {
+                        located.coordinates = result.bounds;
+                    }
+                    return located;
+                }
+            }
+            catch {
+                // Element not found yet, keep polling
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return null;
+    }
+    async click(sessionId, element) {
+        const state = this.requireSession(sessionId);
+        await this.refreshPidIfNeeded(state);
+        const elementPath = this.parseElementPath(element.handleId);
+        if (elementPath) {
+            await this.bridge.call("ax.performAction", {
+                pid: state.pid,
+                elementPath,
+                action: "AXPress",
+            });
+        }
+        else if (element.coordinates) {
+            // Fallback to coordinate click
+            const cx = element.coordinates.x + element.coordinates.width / 2;
+            const cy = element.coordinates.y + element.coordinates.height / 2;
+            await this.bridge.call("cg.mouseClick", { x: cx, y: cy, targetPid: state.pid });
+        }
+        else {
+            throw new Error("Cannot click: no element path or coordinates");
+        }
+    }
+    async setValue(sessionId, element, text, clear) {
+        const state = this.requireSession(sessionId);
+        await this.refreshPidIfNeeded(state);
+        const elementPath = this.parseElementPath(element.handleId);
+        if (clear && elementPath) {
+            // Try AX value set first
+            try {
+                await this.bridge.call("ax.setElementValue", {
+                    pid: state.pid,
+                    elementPath,
+                    value: text,
+                });
+                return;
+            }
+            catch {
+                // Fallback: click, select all, type
+            }
+        }
+        // Fallback: click to focus, select all if clearing, then type
+        // Use PID-targeted events to prevent keystrokes going to wrong app
+        const targetPid = state.pid;
+        await this.click(sessionId, element);
+        await sleep(50);
+        if (clear) {
+            await this.bridge.call("cg.keyCombo", { keys: ["cmd", "a"], targetPid });
+            await sleep(50);
+        }
+        await this.bridge.call("cg.typeText", { text, targetPid });
+    }
+    async getValue(sessionId, element) {
+        const state = this.requireSession(sessionId);
+        const elementPath = this.parseElementPath(element.handleId);
+        if (!elementPath)
+            return "";
+        const result = await this.bridge.call("ax.getElementValue", {
+            pid: state.pid,
+            elementPath,
+        });
+        return result.value;
+    }
+    async waitFor(sessionId, condition, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            const met = await this.checkCondition(sessionId, condition);
+            if (met)
+                return true;
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return false;
+    }
+    async extract(sessionId, target, format) {
+        const state = this.requireSession(sessionId);
+        if (format === "text") {
+            // Get element tree and extract text content
+            const element = await this.locate(sessionId, target, 1500);
+            if (!element)
+                throw new Error("Extract target not found");
+            const result = await this.getValue(sessionId, element);
+            return result || element.label || "";
+        }
+        if (format === "json") {
+            // Return the AX tree as JSON
+            const tree = await this.bridge.call("ax.getElementTree", {
+                pid: state.pid,
+                maxDepth: 3,
+            });
+            return tree;
+        }
+        // table format: return element tree in tabular form
+        const tree = await this.bridge.call("ax.getElementTree", {
+            pid: state.pid,
+            maxDepth: 2,
+        });
+        return {
+            headers: ["role", "title", "value"],
+            rows: this.flattenTree(tree).map((n) => [n.role, n.title ?? "", n.value ?? ""]),
+        };
+    }
+    async screenshot(sessionId, region) {
+        const state = this.requireSession(sessionId);
+        // Get window list to find the window ID for the app
+        const windows = await this.bridge.call("app.windows");
+        const appWindow = windows.find((w) => w.pid === state.pid);
+        if (appWindow) {
+            const result = await this.bridge.call("cg.captureWindow", { windowId: appWindow.windowId });
+            return result.path;
+        }
+        // Fallback to screen capture with region
+        const result = await this.bridge.call("cg.captureScreen", region ? { region } : {});
+        return result.path;
+    }
+    // ── Desktop-specific methods ──
+    async launchApp(sessionId, bundleId) {
+        const state = this.requireSession(sessionId);
+        const result = await this.bridge.call("app.launch", { bundleId });
+        // Update session to track new app
+        state.pid = result.pid;
+        state.bundleId = result.bundleId;
+        state.appName = result.appName;
+        return {
+            bundleId: result.bundleId,
+            appName: result.appName,
+            pid: result.pid,
+            windowTitle: "",
+        };
+    }
+    async focusApp(sessionId, bundleId) {
+        const state = this.requireSession(sessionId);
+        await this.bridge.call("app.focus", { bundleId });
+        // Verify focus was achieved by checking frontmost app
+        let frontmost = await this.bridge.call("app.frontmost");
+        // If focus didn't take, retry once
+        if (frontmost.bundleId !== bundleId) {
+            await this.bridge.call("app.focus", { bundleId });
+            frontmost = await this.bridge.call("app.frontmost");
+        }
+        // Update state based on actual frontmost app, not optimistic assumption
+        state.pid = frontmost.pid;
+        state.bundleId = frontmost.bundleId;
+        state.appName = frontmost.name;
+        this.lastPidRefresh = Date.now();
+    }
+    async listApps(_sessionId) {
+        return this.bridge.call("app.list");
+    }
+    async listWindows(_sessionId) {
+        return this.bridge.call("app.windows");
+    }
+    async menuClick(sessionId, menuPath) {
+        const state = this.requireSession(sessionId);
+        await this.refreshPidIfNeeded(state);
+        await this.bridge.call("ax.menuClick", { pid: state.pid, menuPath });
+    }
+    async keyCombo(sessionId, keys) {
+        const state = this.requireSession(sessionId);
+        await this.bridge.call("cg.keyCombo", { keys, targetPid: state.pid });
+    }
+    async elementTree(sessionId, maxDepth, _root) {
+        const state = this.requireSession(sessionId);
+        return this.bridge.call("ax.getElementTree", {
+            pid: state.pid,
+            maxDepth: maxDepth ?? 5,
+        });
+    }
+    async drag(sessionId, from, to) {
+        if (!from.coordinates || !to.coordinates) {
+            throw new Error("Drag requires elements with coordinates");
+        }
+        const fromX = from.coordinates.x + from.coordinates.width / 2;
+        const fromY = from.coordinates.y + from.coordinates.height / 2;
+        const toX = to.coordinates.x + to.coordinates.width / 2;
+        const toY = to.coordinates.y + to.coordinates.height / 2;
+        const state = this.requireSession(sessionId);
+        await this.bridge.call("cg.mouseDrag", { fromX, fromY, toX, toY, targetPid: state.pid });
+    }
+    async scroll(sessionId, direction, amount, element) {
+        let x = 500;
+        let y = 400;
+        if (element?.coordinates) {
+            x = element.coordinates.x + element.coordinates.width / 2;
+            y = element.coordinates.y + element.coordinates.height / 2;
+        }
+        const deltaMap = {
+            up: { deltaX: 0, deltaY: -amount },
+            down: { deltaX: 0, deltaY: amount },
+            left: { deltaX: -amount, deltaY: 0 },
+            right: { deltaX: amount, deltaY: 0 },
+        };
+        const delta = deltaMap[direction];
+        const state = this.requireSession(sessionId);
+        await this.bridge.call("cg.scroll", { x, y, ...delta, targetPid: state.pid });
+    }
+    async isFrontmost() {
+        // Check if *any* session's bundleId matches the current frontmost app.
+        // Used by the executor to verify focus before acting.
+        const frontmost = await this.bridge.call("app.frontmost");
+        for (const state of this.sessions.values()) {
+            if (state.bundleId === frontmost.bundleId) {
+                return true;
+            }
+        }
+        return false;
+    }
+    // ── Private helpers ──
+    async refreshPidIfNeeded(state) {
+        if (Date.now() - this.lastPidRefresh < 500)
+            return;
+        try {
+            const frontmost = await this.bridge.call("app.frontmost");
+            if (frontmost.bundleId === state.bundleId) {
+                state.pid = frontmost.pid;
+                this.lastPidRefresh = Date.now();
+            }
+        }
+        catch {
+            // Best-effort refresh; don't break the caller
+        }
+    }
+    requireSession(sessionId) {
+        const state = this.sessions.get(sessionId);
+        if (!state)
+            throw new Error(`Session not found: ${sessionId}`);
+        return state;
+    }
+    buildFindParams(target, pid) {
+        const params = { pid };
+        switch (target.type) {
+            case "role":
+                params.role = toAXRole(target.role);
+                params.title = target.name;
+                params.exact = target.exact ?? true;
+                break;
+            case "text":
+                params.title = target.value;
+                params.exact = target.exact ?? true;
+                break;
+            case "selector":
+                // For AX, treat selector as an identifier
+                params.identifier = target.value;
+                break;
+            case "ax_path":
+                // Direct path resolution handled differently
+                params.role = target.path[target.path.length - 1];
+                break;
+            case "ax_attribute":
+                params[target.attribute] = target.value;
+                break;
+            case "coordinates":
+                // Can't find by coordinates via AX, will fallback to vision
+                throw new Error("Cannot locate by coordinates using accessibility adapter");
+            case "image":
+                throw new Error("Cannot locate by image using accessibility adapter");
+        }
+        return params;
+    }
+    parseElementPath(handleId) {
+        // Handle IDs from the bridge are formatted as "ax_0_1_2"
+        if (!handleId.startsWith("ax_"))
+            return null;
+        const parts = handleId.slice(3).split("_");
+        const indices = parts.map(Number).filter((n) => !isNaN(n));
+        return indices.length > 0 ? indices : null;
+    }
+    async checkCondition(sessionId, condition) {
+        switch (condition.type) {
+            case "element_exists": {
+                const found = await this.locate(sessionId, condition.target, 100);
+                return found !== null;
+            }
+            case "element_gone": {
+                const found = await this.locate(sessionId, condition.target, 100);
+                return found === null;
+            }
+            case "window_title_matches": {
+                const ctx = await this.getAppContext(sessionId);
+                return new RegExp(condition.regex).test(ctx.windowTitle);
+            }
+            case "text_appears": {
+                const found = await this.locate(sessionId, { type: "text", value: condition.text }, 100);
+                return found !== null;
+            }
+            case "app_idle":
+                // Simplified: always return true after a short delay
+                return true;
+            case "selector_visible":
+            case "selector_hidden":
+            case "url_matches":
+            case "spinner_disappears":
+                // Browser-specific conditions not fully supported
+                return false;
+        }
+    }
+    flattenTree(node) {
+        const result = [node];
+        if (node.children) {
+            for (const child of node.children) {
+                result.push(...this.flattenTree(child));
+            }
+        }
+        return result;
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/src/runtime/app-adapter.js

@@ -0,0 +1,64 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * Placeholder adapter that returns stubs for all methods.
+ * Used for testing or when no real adapter is configured.
+ */
+export class PlaceholderAppAdapter {
+    async attach(profile, reuseSessionId) {
+        return {
+            sessionId: reuseSessionId ?? `session_${profile}_${Date.now()}`,
+            profile,
+            createdAt: new Date().toISOString(),
+        };
+    }
+    async getAppContext(_sessionId) {
+        return {
+            bundleId: "com.placeholder",
+            appName: "Placeholder",
+            pid: 0,
+            windowTitle: "Placeholder Session",
+        };
+    }
+    async getPageMeta(_sessionId) {
+        return { url: "about:blank", title: "Placeholder Session" };
+    }
+    async navigate(_sessionId, url, _timeoutMs) {
+        return { url, title: "Placeholder Navigation" };
+    }
+    async locate(_sessionId, _target, _timeoutMs) {
+        throw new Error("App adapter not implemented: locate");
+    }
+    async click(_sessionId, _element) {
+        throw new Error("App adapter not implemented: click");
+    }
+    async setValue(_sessionId, _element, _text, _clear) {
+        throw new Error("App adapter not implemented: setValue");
+    }
+    async getValue(_sessionId, _element) {
+        throw new Error("App adapter not implemented: getValue");
+    }
+    async waitFor(_sessionId, _condition, _timeoutMs) {
+        throw new Error("App adapter not implemented: waitFor");
+    }
+    async extract(_sessionId, _target, _format) {
+        throw new Error("App adapter not implemented: extract");
+    }
+    async screenshot(_sessionId, _region) {
+        throw new Error("App adapter not implemented: screenshot");
+    }
+}