screenhand 0.1.1 → 0.2.0

package/dist/src/runtime/planning-loop.js

@@ -0,0 +1,63 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * Bidirectional planning loop that buffers UI events between LLM actions
+ * and provides state snapshots for the LLM to react to.
+ */
+export class PlanningLoop {
+    observer;
+    adapter;
+    lastActionResults = new Map();
+    constructor(observer, adapter) {
+        this.observer = observer;
+        this.adapter = adapter;
+    }
+    /** Get a state snapshot for the LLM after an action. */
+    async getStateSnapshot(sessionId) {
+        const recentEvents = this.observer.drainEvents();
+        let appContext = null;
+        try {
+            appContext = await this.adapter.getAppContext(sessionId);
+        }
+        catch {
+            // May not have an active session
+        }
+        return {
+            recentEvents,
+            appContext,
+            lastActionResult: this.lastActionResults.get(sessionId) ?? null,
+            observing: this.observer.isObserving,
+            timestamp: new Date().toISOString(),
+        };
+    }
+    /** Record the result of the last action for a session. */
+    recordActionResult(sessionId, result) {
+        this.lastActionResults.set(sessionId, result);
+    }
+    /** Start observing a process for state changes. */
+    async startObserving(sessionId, pid) {
+        await this.observer.startObserving(pid);
+    }
+    /** Stop observing a process. */
+    async stopObserving(sessionId, pid) {
+        await this.observer.stopObserving(pid);
+    }
+    /** Peek at recent events without draining. */
+    peekEvents(limit = 50) {
+        return this.observer.peekEvents(limit);
+    }
+}

package/dist/src/runtime/service.js

@@ -0,0 +1,388 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { DEFAULT_NAVIGATE_TIMEOUT_MS, DEFAULT_PROFILE, DEFAULT_WAIT_TIMEOUT_MS, } from "../config.js";
+import { Executor } from "./executor.js";
+import { LocatorCache } from "./locator-cache.js";
+import { SessionManager } from "./session-manager.js";
+export class AutomationRuntimeService {
+    adapter;
+    logger;
+    sessions;
+    executor;
+    constructor(adapter, logger, cache = new LocatorCache()) {
+        this.adapter = adapter;
+        this.logger = logger;
+        this.sessions = new SessionManager(adapter);
+        this.executor = new Executor(adapter, cache, logger);
+    }
+    async sessionStart(profile = DEFAULT_PROFILE) {
+        return this.sessions.sessionStart(profile);
+    }
+    async navigate(input) {
+        const telemetry = this.logger.start("navigate", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            const page = await this.adapter.navigate(input.sessionId, input.url, input.timeoutMs ?? DEFAULT_NAVIGATE_TIMEOUT_MS);
+            return {
+                ok: true,
+                data: page,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Navigate failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async waitFor(input) {
+        const telemetry = this.logger.start("wait_for", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            const matched = await this.adapter.waitFor(input.sessionId, input.condition, input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS);
+            return {
+                ok: true,
+                data: { matched },
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Wait failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async press(input) {
+        await this.sessions.requireSessionResilent(input.sessionId);
+        return this.executor.press(input);
+    }
+    async typeInto(input) {
+        await this.sessions.requireSessionResilent(input.sessionId);
+        return this.executor.typeInto(input);
+    }
+    async extract(input) {
+        const telemetry = this.logger.start("extract", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            const data = await this.adapter.extract(input.sessionId, input.target, input.format);
+            return {
+                ok: true,
+                data,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Extract failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async screenshot(input) {
+        const telemetry = this.logger.start("screenshot", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            const path = await this.adapter.screenshot(input.sessionId, input.region);
+            return {
+                ok: true,
+                data: { path },
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Screenshot failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    // ── Desktop-specific methods ──
+    async appLaunch(input) {
+        const telemetry = this.logger.start("app_launch", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            if (!this.adapter.launchApp) {
+                throw new Error("Adapter does not support launchApp");
+            }
+            const ctx = await this.adapter.launchApp(input.sessionId, input.bundleId);
+            return {
+                ok: true,
+                data: ctx,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "App launch failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async appFocus(input) {
+        const telemetry = this.logger.start("app_focus", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            if (!this.adapter.focusApp) {
+                throw new Error("Adapter does not support focusApp");
+            }
+            await this.adapter.focusApp(input.sessionId, input.bundleId);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "App focus failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async appList(sessionId) {
+        const telemetry = this.logger.start("app_list", sessionId);
+        try {
+            await this.sessions.requireSessionResilent(sessionId);
+            if (!this.adapter.listApps) {
+                throw new Error("Adapter does not support listApps");
+            }
+            const apps = await this.adapter.listApps(sessionId);
+            return {
+                ok: true,
+                data: apps,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "App list failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async windowList(sessionId) {
+        const telemetry = this.logger.start("window_list", sessionId);
+        try {
+            await this.sessions.requireSessionResilent(sessionId);
+            if (!this.adapter.listWindows) {
+                throw new Error("Adapter does not support listWindows");
+            }
+            const windows = await this.adapter.listWindows(sessionId);
+            return {
+                ok: true,
+                data: windows,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Window list failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async menuClick(input) {
+        const telemetry = this.logger.start("menu_click", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            if (!this.adapter.menuClick) {
+                throw new Error("Adapter does not support menuClick");
+            }
+            await this.adapter.menuClick(input.sessionId, input.menuPath);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Menu click failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async keyCombo(input) {
+        const telemetry = this.logger.start("key_combo", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            if (!this.adapter.keyCombo) {
+                throw new Error("Adapter does not support keyCombo");
+            }
+            await this.adapter.keyCombo(input.sessionId, input.keys);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Key combo failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async elementTree(input) {
+        const telemetry = this.logger.start("element_tree", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            if (!this.adapter.elementTree) {
+                throw new Error("Adapter does not support elementTree");
+            }
+            const tree = await this.adapter.elementTree(input.sessionId, input.maxDepth, input.root);
+            return {
+                ok: true,
+                data: tree,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Element tree failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async drag(input) {
+        const telemetry = this.logger.start("drag", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            if (!this.adapter.drag) {
+                throw new Error("Adapter does not support drag");
+            }
+            const fromEl = await this.adapter.locate(input.sessionId, input.from, 800);
+            const toEl = await this.adapter.locate(input.sessionId, input.to, 800);
+            if (!fromEl || !toEl) {
+                throw new Error("Could not locate drag source or destination");
+            }
+            await this.adapter.drag(input.sessionId, fromEl, toEl);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Drag failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async scroll(input) {
+        const telemetry = this.logger.start("scroll", input.sessionId);
+        try {
+            await this.sessions.requireSessionResilent(input.sessionId);
+            if (!this.adapter.scroll) {
+                throw new Error("Adapter does not support scroll");
+            }
+            let element;
+            if (input.target) {
+                const found = await this.adapter.locate(input.sessionId, input.target, 800);
+                if (found)
+                    element = found;
+            }
+            await this.adapter.scroll(input.sessionId, input.direction, input.amount ?? 3, element);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Scroll failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async observeStart(_input) {
+        const telemetry = this.logger.start("observe_start", _input.sessionId);
+        // Implemented in Phase 4 when StateObserver is available
+        return {
+            ok: true,
+            data: undefined,
+            telemetry: this.logger.finish(telemetry, "success"),
+        };
+    }
+    async observeStop(_input) {
+        const telemetry = this.logger.start("observe_stop", _input.sessionId);
+        return {
+            ok: true,
+            data: undefined,
+            telemetry: this.logger.finish(telemetry, "success"),
+        };
+    }
+    getTimeline(limit = 100) {
+        return this.logger.getRecent(limit);
+    }
+}

package/dist/src/runtime/session-manager.js

@@ -0,0 +1,60 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class SessionManager {
+    adapter;
+    sessionsByProfile = new Map();
+    sessionsById = new Map();
+    constructor(adapter) {
+        this.adapter = adapter;
+    }
+    async sessionStart(profile) {
+        const existing = this.sessionsByProfile.get(profile);
+        if (existing) {
+            return existing;
+        }
+        const created = await this.adapter.attach(profile);
+        this.sessionsByProfile.set(profile, created);
+        this.sessionsById.set(created.sessionId, created);
+        return created;
+    }
+    getSession(sessionId) {
+        return this.sessionsById.get(sessionId);
+    }
+    requireSession(sessionId) {
+        const session = this.getSession(sessionId);
+        if (!session) {
+            throw new Error(`Session not found: ${sessionId}`);
+        }
+        return session;
+    }
+    /**
+     * Like requireSession but auto-recreates expired/missing sessions.
+     * MCP servers restart between tool calls, losing in-memory state.
+     * This re-attaches transparently so the caller's sessionId stays valid.
+     */
+    async requireSessionResilent(sessionId) {
+        const existing = this.getSession(sessionId);
+        if (existing)
+            return existing;
+        const match = sessionId.match(/^(?:ax|cdp|as|vision|composite)_session_(.+)_\d+$/);
+        const profile = match?.[1] ?? "automation";
+        const created = await this.adapter.attach(profile, sessionId);
+        this.sessionsByProfile.set(profile, created);
+        this.sessionsById.set(created.sessionId, created);
+        return created;
+    }
+}

package/dist/src/runtime/state-observer.js

@@ -0,0 +1,121 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { EventEmitter } from "node:events";
+/**
+ * Wraps the native bridge's AX observer events into typed UIEvent objects.
+ * Buffers events for consumption by the planning loop.
+ */
+export class StateObserver extends EventEmitter {
+    bridge;
+    observedPids = new Set();
+    eventBuffer = [];
+    maxBufferSize;
+    constructor(bridge, maxBufferSize = 200) {
+        super();
+        this.bridge = bridge;
+        this.maxBufferSize = maxBufferSize;
+        // Listen for AX events from the bridge
+        this.bridge.on("ax-event", (raw) => {
+            const event = this.parseEvent(raw);
+            if (event) {
+                this.eventBuffer.push(event);
+                if (this.eventBuffer.length > this.maxBufferSize) {
+                    this.eventBuffer.shift();
+                }
+                this.emit("event", event);
+            }
+        });
+    }
+    async startObserving(pid, eventTypes) {
+        if (this.observedPids.has(pid))
+            return;
+        const notifications = eventTypes
+            ? this.mapEventTypesToNotifications(eventTypes)
+            : undefined;
+        await this.bridge.call("observer.start", {
+            pid,
+            notifications,
+        });
+        this.observedPids.add(pid);
+    }
+    async stopObserving(pid) {
+        if (!this.observedPids.has(pid))
+            return;
+        await this.bridge.call("observer.stop", { pid });
+        this.observedPids.delete(pid);
+    }
+    /** Get and clear the event buffer. */
+    drainEvents() {
+        const events = [...this.eventBuffer];
+        this.eventBuffer = [];
+        return events;
+    }
+    /** Get recent events without clearing. */
+    peekEvents(limit = 50) {
+        return this.eventBuffer.slice(-limit);
+    }
+    /** Clear the event buffer. */
+    clearEvents() {
+        this.eventBuffer = [];
+    }
+    get isObserving() {
+        return this.observedPids.size > 0;
+    }
+    get observedProcesses() {
+        return [...this.observedPids];
+    }
+    parseEvent(raw) {
+        const type = raw.type;
+        if (!type)
+            return null;
+        const event = {
+            type,
+            timestamp: raw.timestamp ?? new Date().toISOString(),
+            pid: raw.pid ?? 0,
+        };
+        if (typeof raw.bundleId === "string")
+            event.bundleId = raw.bundleId;
+        if (typeof raw.elementRole === "string")
+            event.elementRole = raw.elementRole;
+        if (typeof raw.elementLabel === "string")
+            event.elementLabel = raw.elementLabel;
+        if (typeof raw.oldValue === "string")
+            event.oldValue = raw.oldValue;
+        if (typeof raw.newValue === "string")
+            event.newValue = raw.newValue;
+        if (typeof raw.windowTitle === "string")
+            event.windowTitle = raw.windowTitle;
+        return event;
+    }
+    mapEventTypesToNotifications(types) {
+        const map = {
+            value_changed: "AXValueChanged",
+            focus_changed: "AXFocusedUIElementChanged",
+            window_created: "AXWindowCreated",
+            window_closed: "AXUIElementDestroyed",
+            title_changed: "AXTitleChanged",
+            menu_opened: "AXMenuOpened",
+            layout_changed: "AXLayoutChanged",
+            dialog_appeared: "AXSheetCreated",
+            app_activated: "AXApplicationActivated",
+            app_deactivated: "AXApplicationDeactivated",
+        };
+        return types
+            .map((t) => map[t])
+            .filter((n) => n !== undefined);
+    }
+}