screenhand 0.1.1 → 0.3.0

package/dist/src/runtime/locator-cache.js

@@ -0,0 +1,50 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class LocatorCache {
+    store = new Map();
+    learningEngine = null;
+    /**
+     * Inject the learning engine for fallback on cache miss.
+     * Called after both are constructed to avoid circular dependencies.
+     */
+    setLearningEngine(engine) {
+        this.learningEngine = engine;
+    }
+    get(siteKey, actionKey) {
+        // 1. Check in-memory cache first
+        const cached = this.store.get(this.key(siteKey, actionKey));
+        if (cached)
+            return cached;
+        // 2. Fallback: ask learning engine for a proven locator
+        if (this.learningEngine) {
+            const learned = this.learningEngine.recommendLocator(siteKey, actionKey);
+            if (learned) {
+                // Promote to cache for fast subsequent lookups
+                this.store.set(this.key(siteKey, actionKey), learned.locator);
+                return learned.locator;
+            }
+        }
+        return undefined;
+    }
+    set(siteKey, actionKey, locator) {
+        this.store.set(this.key(siteKey, actionKey), locator);
+    }
+    key(siteKey, actionKey) {
+        // Use length-prefixed format to avoid collision when keys contain the separator
+        return `${siteKey.length}:${siteKey}\0${actionKey}`;
+    }
+}

package/dist/src/runtime/planning-loop.js

@@ -0,0 +1,63 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * Bidirectional planning loop that buffers UI events between LLM actions
+ * and provides state snapshots for the LLM to react to.
+ */
+export class PlanningLoop {
+    observer;
+    adapter;
+    lastActionResults = new Map();
+    constructor(observer, adapter) {
+        this.observer = observer;
+        this.adapter = adapter;
+    }
+    /** Get a state snapshot for the LLM after an action. */
+    async getStateSnapshot(sessionId) {
+        const recentEvents = this.observer.drainEvents();
+        let appContext = null;
+        try {
+            appContext = await this.adapter.getAppContext(sessionId);
+        }
+        catch {
+            // May not have an active session
+        }
+        return {
+            recentEvents,
+            appContext,
+            lastActionResult: this.lastActionResults.get(sessionId) ?? null,
+            observing: this.observer.isObserving,
+            timestamp: new Date().toISOString(),
+        };
+    }
+    /** Record the result of the last action for a session. */
+    recordActionResult(sessionId, result) {
+        this.lastActionResults.set(sessionId, result);
+    }
+    /** Start observing a process for state changes. */
+    async startObserving(sessionId, pid) {
+        await this.observer.startObserving(pid);
+    }
+    /** Stop observing a process. */
+    async stopObserving(sessionId, pid) {
+        await this.observer.stopObserving(pid);
+    }
+    /** Peek at recent events without draining. */
+    peekEvents(limit = 50) {
+        return this.observer.peekEvents(limit);
+    }
+}

package/dist/src/runtime/service.js

@@ -0,0 +1,432 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { DEFAULT_NAVIGATE_TIMEOUT_MS, DEFAULT_PROFILE, DEFAULT_WAIT_TIMEOUT_MS, } from "../config.js";
+import { Executor } from "./executor.js";
+import { LocatorCache } from "./locator-cache.js";
+import { SessionManager } from "./session-manager.js";
+export class AutomationRuntimeService {
+    adapter;
+    logger;
+    sessions;
+    executor;
+    worldModel = null;
+    constructor(adapter, logger, cache = new LocatorCache()) {
+        this.adapter = adapter;
+        this.logger = logger;
+        this.sessions = new SessionManager(adapter);
+        this.executor = new Executor(adapter, cache, logger);
+    }
+    /**
+     * Inject the WorldModel so runtime actions update shared state.
+     */
+    setWorldModel(model) {
+        this.worldModel = model;
+    }
+    async sessionStart(profile = DEFAULT_PROFILE) {
+        return this.sessions.sessionStart(profile);
+    }
+    /**
+     * Ensure session exists (re-attaches if lost after MCP restart).
+     * Also reloads the world model from disk when re-attaching so world
+     * state survives across MCP server restarts.
+     */
+    async ensureSession(sessionId) {
+        const hadSession = !!this.sessions.getSession(sessionId);
+        const session = await this.sessions.requireSessionResilent(sessionId);
+        // If we had to re-attach, reload persisted world state
+        if (!hadSession && this.worldModel) {
+            this.worldModel.init(sessionId);
+        }
+        return session;
+    }
+    // L2-74 fix: Centralized URL protocol validation for all navigate paths
+    static BLOCKED_URL_PROTOCOLS = ["javascript:", "data:", "blob:", "vbscript:"];
+    async navigate(input) {
+        const telemetry = this.logger.start("navigate", input.sessionId);
+        try {
+            // L2-74 fix: Block dangerous URL protocols at the service level
+            const urlLower = input.url.trim().toLowerCase();
+            for (const proto of AutomationRuntimeService.BLOCKED_URL_PROTOCOLS) {
+                if (urlLower.startsWith(proto)) {
+                    throw new Error(`Blocked: "${proto}" URLs are not allowed for security reasons`);
+                }
+            }
+            await this.ensureSession(input.sessionId);
+            const page = await this.adapter.navigate(input.sessionId, input.url, input.timeoutMs ?? DEFAULT_NAVIGATE_TIMEOUT_MS);
+            // Feed navigation result to world model for domain state tracking
+            if (this.worldModel) {
+                const bundleId = this.worldModel.getState().focusedApp?.bundleId;
+                if (bundleId) {
+                    this.worldModel.ingestCDPSnapshot(bundleId, input.url, page.title ?? "");
+                }
+            }
+            return {
+                ok: true,
+                data: page,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Navigate failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async waitFor(input) {
+        const telemetry = this.logger.start("wait_for", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            const matched = await this.adapter.waitFor(input.sessionId, input.condition, input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS);
+            return {
+                ok: true,
+                data: { matched },
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Wait failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async press(input) {
+        await this.ensureSession(input.sessionId);
+        return this.executor.press(input);
+    }
+    async typeInto(input) {
+        await this.ensureSession(input.sessionId);
+        return this.executor.typeInto(input);
+    }
+    async extract(input) {
+        const telemetry = this.logger.start("extract", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            const data = await this.adapter.extract(input.sessionId, input.target, input.format);
+            return {
+                ok: true,
+                data,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Extract failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async screenshot(input) {
+        const telemetry = this.logger.start("screenshot", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            const path = await this.adapter.screenshot(input.sessionId, input.region);
+            return {
+                ok: true,
+                data: { path },
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Screenshot failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    // ── Desktop-specific methods ──
+    async appLaunch(input) {
+        const telemetry = this.logger.start("app_launch", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            if (!this.adapter.launchApp) {
+                throw new Error("Adapter does not support launchApp");
+            }
+            const ctx = await this.adapter.launchApp(input.sessionId, input.bundleId);
+            this.worldModel?.updateFocusedApp(ctx);
+            return {
+                ok: true,
+                data: ctx,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "App launch failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async appFocus(input) {
+        const telemetry = this.logger.start("app_focus", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            if (!this.adapter.focusApp) {
+                throw new Error("Adapter does not support focusApp");
+            }
+            await this.adapter.focusApp(input.sessionId, input.bundleId);
+            this.worldModel?.updateFocusedApp({
+                bundleId: input.bundleId,
+                appName: input.bundleId,
+                pid: 0,
+                windowTitle: "",
+            });
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "App focus failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async appList(sessionId) {
+        const telemetry = this.logger.start("app_list", sessionId);
+        try {
+            await this.ensureSession(sessionId);
+            if (!this.adapter.listApps) {
+                throw new Error("Adapter does not support listApps");
+            }
+            const apps = await this.adapter.listApps(sessionId);
+            return {
+                ok: true,
+                data: apps,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "App list failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async windowList(sessionId) {
+        const telemetry = this.logger.start("window_list", sessionId);
+        try {
+            await this.ensureSession(sessionId);
+            if (!this.adapter.listWindows) {
+                throw new Error("Adapter does not support listWindows");
+            }
+            const windows = await this.adapter.listWindows(sessionId);
+            return {
+                ok: true,
+                data: windows,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Window list failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async menuClick(input) {
+        const telemetry = this.logger.start("menu_click", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            if (!this.adapter.menuClick) {
+                throw new Error("Adapter does not support menuClick");
+            }
+            await this.adapter.menuClick(input.sessionId, input.menuPath);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Menu click failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async keyCombo(input) {
+        const telemetry = this.logger.start("key_combo", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            if (!this.adapter.keyCombo) {
+                throw new Error("Adapter does not support keyCombo");
+            }
+            await this.adapter.keyCombo(input.sessionId, input.keys);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Key combo failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async elementTree(input) {
+        const telemetry = this.logger.start("element_tree", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            if (!this.adapter.elementTree) {
+                throw new Error("Adapter does not support elementTree");
+            }
+            const tree = await this.adapter.elementTree(input.sessionId, input.maxDepth, input.root);
+            return {
+                ok: true,
+                data: tree,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Element tree failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async drag(input) {
+        const telemetry = this.logger.start("drag", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            if (!this.adapter.drag) {
+                throw new Error("Adapter does not support drag");
+            }
+            const fromEl = await this.adapter.locate(input.sessionId, input.from, 800);
+            const toEl = await this.adapter.locate(input.sessionId, input.to, 800);
+            if (!fromEl || !toEl) {
+                throw new Error("Could not locate drag source or destination");
+            }
+            await this.adapter.drag(input.sessionId, fromEl, toEl);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Drag failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async scroll(input) {
+        const telemetry = this.logger.start("scroll", input.sessionId);
+        try {
+            await this.ensureSession(input.sessionId);
+            if (!this.adapter.scroll) {
+                throw new Error("Adapter does not support scroll");
+            }
+            let element;
+            if (input.target) {
+                const found = await this.adapter.locate(input.sessionId, input.target, 800);
+                if (found)
+                    element = found;
+            }
+            await this.adapter.scroll(input.sessionId, input.direction, input.amount ?? 3, element);
+            return {
+                ok: true,
+                data: undefined,
+                telemetry: this.logger.finish(telemetry, "success"),
+            };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: {
+                    code: "ACTION_FAILED",
+                    message: error instanceof Error ? error.message : "Scroll failed",
+                },
+                telemetry: this.logger.finish(telemetry, "failed"),
+            };
+        }
+    }
+    async observeStart(_input) {
+        const telemetry = this.logger.start("observe_start", _input.sessionId);
+        // Implemented in Phase 4 when StateObserver is available
+        return {
+            ok: true,
+            data: undefined,
+            telemetry: this.logger.finish(telemetry, "success"),
+        };
+    }
+    async observeStop(_input) {
+        const telemetry = this.logger.start("observe_stop", _input.sessionId);
+        return {
+            ok: true,
+            data: undefined,
+            telemetry: this.logger.finish(telemetry, "success"),
+        };
+    }
+    getTimeline(limit = 100) {
+        return this.logger.getRecent(limit);
+    }
+}

package/dist/src/runtime/session-manager.js

@@ -0,0 +1,63 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class SessionManager {
+    adapter;
+    sessionsByProfile = new Map();
+    sessionsById = new Map();
+    constructor(adapter) {
+        this.adapter = adapter;
+    }
+    async sessionStart(profile) {
+        const existing = this.sessionsByProfile.get(profile);
+        if (existing) {
+            return existing;
+        }
+        const created = await this.adapter.attach(profile);
+        this.sessionsByProfile.set(profile, created);
+        this.sessionsById.set(created.sessionId, created);
+        return created;
+    }
+    getSession(sessionId) {
+        return this.sessionsById.get(sessionId);
+    }
+    requireSession(sessionId) {
+        const session = this.getSession(sessionId);
+        if (!session) {
+            throw new Error(`Session not found: ${sessionId}`);
+        }
+        return session;
+    }
+    /**
+     * Like requireSession but auto-recreates expired/missing sessions.
+     * MCP servers restart between tool calls, losing in-memory state.
+     * This re-attaches transparently so the caller's sessionId stays valid.
+     */
+    async requireSessionResilent(sessionId) {
+        const existing = this.getSession(sessionId);
+        if (existing)
+            return existing;
+        // Session IDs: {prefix}_session_{profile}_{timestamp}_{random8} (new)
+        // or legacy:   {prefix}_session_{profile}_{timestamp}
+        // Use greedy .+ so profiles with digits (e.g. "user_1234567890") capture fully
+        const match = sessionId.match(/^(?:ax|cdp|as|vision|composite)_session_(.+)_\d{13,}(?:_[a-f0-9]{8})?$/);
+        const profile = match?.[1] ?? "automation";
+        const created = await this.adapter.attach(profile, sessionId);
+        this.sessionsByProfile.set(profile, created);
+        this.sessionsById.set(created.sessionId, created);
+        return created;
+    }
+}