screenhand 0.2.0 → 0.3.1

package/dist/src/orchestrator/state.js

@@ -0,0 +1,68 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * Orchestrator state helpers — read/write state file, PID management.
+ */
+import fs from "node:fs";
+import crypto from "node:crypto";
+import { readJsonWithRecovery, writeFileAtomicSync } from "../util/atomic-write.js";
+import { ORCHESTRATOR_DIR, ORCHESTRATOR_STATE_FILE, ORCHESTRATOR_PID_FILE, } from "./types.js";
+/** Read current orchestrator state from disk. */
+export function readOrchestratorState() {
+    return readJsonWithRecovery(ORCHESTRATOR_STATE_FILE);
+}
+/** Write orchestrator state to disk (atomic). */
+export function writeOrchestratorState(state) {
+    fs.mkdirSync(ORCHESTRATOR_DIR, { recursive: true });
+    writeFileAtomicSync(ORCHESTRATOR_STATE_FILE, JSON.stringify(state, null, 2));
+}
+/** Get PID of running orchestrator daemon, or null. */
+export function getOrchestratorDaemonPid() {
+    try {
+        const pid = Number(fs.readFileSync(ORCHESTRATOR_PID_FILE, "utf-8").trim());
+        if (Number.isNaN(pid))
+            return null;
+        try {
+            process.kill(pid, 0);
+            return pid;
+        }
+        catch {
+            try {
+                fs.unlinkSync(ORCHESTRATOR_PID_FILE);
+            }
+            catch { /* ignore */ }
+            return null;
+        }
+    }
+    catch {
+        return null;
+    }
+}
+/** Generate a unique task ID */
+export function generateTaskId() {
+    return "task_" + Date.now().toString(36) + "_" + crypto.randomBytes(3).toString("hex");
+}
+/** Auto-detect task mode from playbook and parameters */
+export function detectTaskMode(playbookId, bundleId) {
+    // If it has a bundleId, it's at least partially native
+    if (bundleId)
+        return "native";
+    // CDP-only playbooks are web
+    // For now, default to web if no bundleId specified
+    return "web";
+}
+/** Create a new task */
+export function createTask(task, opts = {}) {
+    return {
+        id: generateTaskId(),
+        task,
+        mode: opts.mode ?? detectTaskMode(opts.playbookId, opts.bundleId),
+        ...(opts.playbookId !== undefined ? { playbookId: opts.playbookId } : {}),
+        ...(opts.bundleId !== undefined ? { bundleId: opts.bundleId } : {}),
+        ...(opts.windowId !== undefined ? { windowId: opts.windowId } : {}),
+        ...(opts.vars ? { vars: opts.vars } : {}),
+        priority: opts.priority ?? 10,
+        status: "queued",
+        createdAt: new Date().toISOString(),
+    };
+}

package/dist/src/orchestrator/types.js

@@ -0,0 +1,22 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * Orchestrator types — multi-agent task routing and coordination.
+ *
+ * Architecture:
+ *   Orchestrator daemon manages a pool of worker slots.
+ *   Web tasks (CDP-only) run in parallel — no mouse/keyboard conflict.
+ *   Native tasks (AX/keyboard) are serialized per-app via lease locks.
+ *   Mixed tasks split into web + native phases automatically.
+ */
+import os from "node:os";
+import path from "node:path";
+export const DEFAULT_ORCHESTRATOR_CONFIG = {
+    webSlots: 4,
+    nativeSlots: 1,
+    pollMs: 1000,
+};
+export const ORCHESTRATOR_DIR = path.join(os.homedir(), ".screenhand", "orchestrator");
+export const ORCHESTRATOR_STATE_FILE = path.join(ORCHESTRATOR_DIR, "state.json");
+export const ORCHESTRATOR_PID_FILE = path.join(ORCHESTRATOR_DIR, "orchestrator.pid");
+export const ORCHESTRATOR_LOG_FILE = path.join(ORCHESTRATOR_DIR, "orchestrator.log");

package/dist/src/perception/ax-source.js

@@ -0,0 +1,162 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * AX perception source — wraps StateObserver for push events
+ * and provides periodic AX tree polling for structured snapshots.
+ */
+export class AXSource {
+    observer;
+    bridge;
+    /** Rolling average of recent poll latencies for adaptive backoff */
+    recentLatencies = [];
+    static LATENCY_WINDOW = 5;
+    constructor(observer, bridge) {
+        this.observer = observer;
+        this.bridge = bridge;
+    }
+    /**
+     * FAST rate: drain buffered AX events from StateObserver.
+     * Returns a PerceptionEvent if there are events, null otherwise.
+     */
+    drainEvents() {
+        const events = this.observer.drainEvents();
+        if (events.length === 0)
+            return null;
+        return {
+            source: "ax_events",
+            rate: "fast",
+            timestamp: new Date().toISOString(),
+            data: {
+                type: "ax_events",
+                events,
+            },
+        };
+    }
+    /** Known browser bundle IDs — use lower AX tree depth to avoid blowing up on heavy web DOMs */
+    static BROWSER_BUNDLE_IDS = new Set([
+        "com.apple.Safari",
+        "com.google.Chrome",
+        "com.brave.Browser",
+        "com.microsoft.edgemac",
+        "org.mozilla.firefox",
+        "com.operasoftware.Opera",
+        "company.thebrowser.Browser", // Arc
+    ]);
+    /**
+     * MEDIUM rate: poll the full AX tree for the active window.
+     * More expensive but gives a complete structural snapshot.
+     * Uses lower depth for browsers to prevent crash on heavy web apps (Canva, Figma).
+     */
+    /**
+     * Get the adaptive max depth based on recent poll performance.
+     * If recent polls are slow or returning huge trees, reduce depth automatically.
+     */
+    getAdaptiveMaxDepth(bundleId) {
+        const isBrowser = AXSource.BROWSER_BUNDLE_IDS.has(bundleId);
+        const baseDepth = isBrowser ? 5 : 10;
+        if (this.recentLatencies.length < 2)
+            return baseDepth;
+        const avgLatency = this.recentLatencies.reduce((a, b) => a + b, 0) / this.recentLatencies.length;
+        // If average latency > 3s, reduce depth aggressively
+        if (avgLatency > 3000)
+            return Math.max(2, baseDepth - 3);
+        // If average latency > 1.5s, reduce depth by 1
+        if (avgLatency > 1500)
+            return Math.max(3, baseDepth - 1);
+        return baseDepth;
+    }
+    /**
+     * Check if AX polling should be skipped this cycle due to recent instability.
+     * Returns true if the last few polls were extremely slow (>5s average).
+     */
+    shouldSkipPoll() {
+        if (this.recentLatencies.length < 3)
+            return false;
+        const avgLatency = this.recentLatencies.reduce((a, b) => a + b, 0) / this.recentLatencies.length;
+        return avgLatency > 5000;
+    }
+    /**
+     * Get average recent poll latency (ms). Returns 0 if no data.
+     */
+    getAverageLatency() {
+        if (this.recentLatencies.length === 0)
+            return 0;
+        return this.recentLatencies.reduce((a, b) => a + b, 0) / this.recentLatencies.length;
+    }
+    async pollAXTree(pid, windowId, appContext, maxDepth) {
+        // Use adaptive depth if not explicitly provided
+        if (maxDepth === undefined) {
+            maxDepth = this.getAdaptiveMaxDepth(appContext.bundleId);
+        }
+        const start = Date.now();
+        try {
+            const tree = await this.bridge.call("ax.getElementTree", {
+                pid,
+                maxDepth,
+                windowId: windowId > 0 ? windowId : undefined,
+            });
+            const latencyMs = Date.now() - start;
+            const nodeCount = tree?._nodeCount ?? 0;
+            // Track latency for adaptive backoff
+            this.recentLatencies.push(latencyMs);
+            if (this.recentLatencies.length > AXSource.LATENCY_WINDOW) {
+                this.recentLatencies.shift();
+            }
+            if (!tree)
+                return { event: null, latencyMs, nodeCount: 0 };
+            return {
+                event: {
+                    source: "ax_tree",
+                    rate: "medium",
+                    timestamp: new Date().toISOString(),
+                    data: {
+                        type: "ax_tree",
+                        windowId,
+                        tree,
+                        appContext,
+                    },
+                },
+                latencyMs,
+                nodeCount,
+            };
+        }
+        catch {
+            const latencyMs = Date.now() - start;
+            // Track failures as high-latency for backoff purposes
+            this.recentLatencies.push(latencyMs > 1000 ? latencyMs : 5000);
+            if (this.recentLatencies.length > AXSource.LATENCY_WINDOW) {
+                this.recentLatencies.shift();
+            }
+            return { event: null, latencyMs, nodeCount: 0 };
+        }
+    }
+    /**
+     * Start observing a process for AX events.
+     */
+    async startObserving(pid) {
+        await this.observer.startObserving(pid);
+    }
+    /**
+     * Stop observing a process.
+     */
+    async stopObserving(pid) {
+        await this.observer.stopObserving(pid);
+    }
+    get isObserving() {
+        return this.observer.isObserving;
+    }
+}

package/dist/src/perception/cdp-source.js

@@ -0,0 +1,162 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * CDP perception source — tracks DOM mutations and provides snapshots
+ * via Chrome DevTools Protocol.
+ *
+ * The CDP client is injected as `any` since it comes from the `chrome-remote-interface`
+ * package which is dynamically imported in mcp-desktop.ts.
+ */
+export class CDPSource {
+    mutationBuffer = [];
+    observerInstalled = false;
+    maxBufferSize = 100;
+    /**
+     * Install a DOM mutation observer via CDP Runtime.evaluate.
+     * This injects a MutationObserver into the page that sends mutations
+     * back via CDP's Runtime.consoleAPICalled.
+     */
+    async installMutationObserver(cdpClient) {
+        if (this.observerInstalled)
+            return;
+        try {
+            // Enable DOM and Runtime domains
+            await cdpClient.DOM.enable();
+            await cdpClient.Runtime.enable();
+            // Inject MutationObserver that logs changes as JSON
+            await cdpClient.Runtime.evaluate({
+                expression: `
+          (() => {
+            if (window.__shMutationObserver) return;
+            const observer = new MutationObserver((mutations) => {
+              const summary = mutations.slice(0, 20).map(m => ({
+                type: m.type,
+                target: m.target.nodeName + (m.target.id ? '#' + m.target.id : ''),
+                attribute: m.attributeName || undefined,
+                addedNodes: m.addedNodes.length,
+                removedNodes: m.removedNodes.length,
+              }));
+              console.debug('__sh_mutations__' + JSON.stringify(summary));
+            });
+            observer.observe(document.body, {
+              childList: true, subtree: true,
+              attributes: true, attributeOldValue: true,
+            });
+            window.__shMutationObserver = observer;
+          })();
+        `,
+                returnByValue: true,
+            });
+            // Listen for console messages that contain mutation data
+            cdpClient.Runtime.consoleAPICalled((params) => {
+                if (params.type === "debug" && params.args?.length > 0) {
+                    const msg = params.args[0]?.value;
+                    if (typeof msg === "string") {
+                        this.processCDPConsoleMessage(msg);
+                    }
+                }
+            });
+            this.observerInstalled = true;
+        }
+        catch {
+            // CDP not available or page not ready
+        }
+    }
+    /**
+     * Process a CDP console message that might contain mutation data.
+     * Call this from a Runtime.consoleAPICalled handler.
+     */
+    processCDPConsoleMessage(message) {
+        if (!message.startsWith("__sh_mutations__"))
+            return;
+        try {
+            const raw = JSON.parse(message.slice("__sh_mutations__".length));
+            for (const m of raw) {
+                const entry = {
+                    selector: m.target,
+                };
+                if (m.attribute)
+                    entry.attribute = m.attribute;
+                if (m.addedNodes)
+                    entry.addedNodes = m.addedNodes;
+                if (m.removedNodes)
+                    entry.removedNodes = m.removedNodes;
+                this.mutationBuffer.push(entry);
+            }
+            // Cap buffer
+            if (this.mutationBuffer.length > this.maxBufferSize) {
+                this.mutationBuffer = this.mutationBuffer.slice(-this.maxBufferSize);
+            }
+        }
+        catch {
+            // Malformed message
+        }
+    }
+    /**
+     * FAST rate: drain buffered DOM mutations.
+     */
+    drainMutations() {
+        if (this.mutationBuffer.length === 0)
+            return null;
+        const mutations = [...this.mutationBuffer];
+        this.mutationBuffer = [];
+        return {
+            source: "cdp_mutations",
+            rate: "fast",
+            timestamp: new Date().toISOString(),
+            data: {
+                type: "cdp_mutations",
+                mutations,
+            },
+        };
+    }
+    /**
+     * MEDIUM rate: take a DOM snapshot — page URL, title, node count.
+     */
+    async pollSnapshot(cdpClient) {
+        try {
+            const result = await cdpClient.Runtime.evaluate({
+                expression: `JSON.stringify({
+          url: location.href,
+          title: document.title,
+          nodeCount: document.querySelectorAll('*').length,
+        })`,
+                returnByValue: true,
+            });
+            const data = JSON.parse(result.result.value);
+            return {
+                source: "cdp_snapshot",
+                rate: "medium",
+                timestamp: new Date().toISOString(),
+                data: {
+                    type: "cdp_snapshot",
+                    ...data,
+                },
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Reset state (e.g., on page navigation).
+     */
+    reset() {
+        this.mutationBuffer = [];
+        this.observerInstalled = false;
+    }
+}