screenhand 0.1.1 → 0.2.0

package/dist/src/native/bridge-client.js

@@ -0,0 +1,190 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { spawn } from "node:child_process";
+import { EventEmitter } from "node:events";
+import path from "node:path";
+import { createInterface } from "node:readline";
+/**
+ * Per-method timeout overrides (ms).
+ * Methods not listed here use the default 10s timeout.
+ */
+const METHOD_TIMEOUTS = {
+    "app.launch": 30_000,
+    "cg.captureScreen": 15_000,
+    "cg.captureWindow": 15_000,
+    "vision.ocr": 20_000,
+    "vision.findText": 20_000,
+};
+/**
+ * Resolves the correct native bridge binary path for the current platform.
+ */
+function defaultBinaryPath() {
+    // import.meta.dirname is Node 20+; for Node 18 derive from import.meta.url
+    const base = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
+    if (process.platform === "win32") {
+        return path.resolve(base, "../../native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe");
+    }
+    // macOS (default)
+    return path.resolve(base, "../../native/macos-bridge/.build/release/macos-bridge");
+}
+/**
+ * Platform-aware native bridge client.
+ * Spawns the correct bridge binary (macOS Swift or Windows C#) based on the OS,
+ * communicating via the same JSON-RPC-over-stdio protocol.
+ *
+ * Drop-in replacement for the original MacOSBridgeClient.
+ */
+export class BridgeClient extends EventEmitter {
+    process = null;
+    nextId = 1;
+    pending = new Map();
+    binaryPath;
+    restarting = false;
+    started = false;
+    constructor(binaryPath) {
+        super();
+        this.binaryPath = binaryPath ?? defaultBinaryPath();
+    }
+    async start() {
+        if (this.started)
+            return;
+        await this.spawn();
+        this.started = true;
+    }
+    async stop() {
+        this.started = false;
+        if (this.process) {
+            this.process.kill();
+            this.process = null;
+        }
+        // Reject all pending
+        for (const [id, pending] of this.pending) {
+            clearTimeout(pending.timer);
+            pending.reject(new Error("Bridge stopped"));
+            this.pending.delete(id);
+        }
+    }
+    async call(method, params, timeoutMs) {
+        const effectiveTimeout = timeoutMs ?? METHOD_TIMEOUTS[method] ?? 10_000;
+        if (!this.process || this.process.exitCode !== null) {
+            await this.restart();
+        }
+        const id = this.nextId++;
+        const request = { id, method };
+        if (params) {
+            request.params = params;
+        }
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pending.delete(id);
+                reject(new Error(`Bridge call "${method}" timed out after ${effectiveTimeout}ms`));
+            }, effectiveTimeout);
+            this.pending.set(id, {
+                resolve: resolve,
+                reject,
+                timer,
+            });
+            const line = JSON.stringify(request) + "\n";
+            this.process.stdin.write(line);
+        });
+    }
+    async ping() {
+        return this.call("ping");
+    }
+    async checkPermissions() {
+        return this.call("check_permissions");
+    }
+    async spawn() {
+        const child = spawn(this.binaryPath, [], {
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        child.on("error", (err) => {
+            this.emit("error", err);
+            if (this.started) {
+                this.restart().catch(() => { });
+            }
+        });
+        child.on("exit", (code) => {
+            this.emit("exit", code);
+            if (this.started && !this.restarting) {
+                this.restart().catch(() => { });
+            }
+        });
+        // Parse stdout line by line
+        const rl = createInterface({ input: child.stdout });
+        rl.on("line", (line) => {
+            this.handleLine(line);
+        });
+        // Log stderr
+        child.stderr?.on("data", (data) => {
+            this.emit("stderr", data.toString());
+        });
+        this.process = child;
+    }
+    handleLine(line) {
+        let response;
+        try {
+            response = JSON.parse(line);
+        }
+        catch {
+            return; // Ignore malformed lines
+        }
+        // Event (streaming notification from observer)
+        if (response.event) {
+            this.emit("ax-event", response.event);
+            return;
+        }
+        // Response to a pending request
+        const pending = this.pending.get(response.id);
+        if (!pending)
+            return;
+        this.pending.delete(response.id);
+        clearTimeout(pending.timer);
+        if (response.error) {
+            pending.reject(new Error(response.error.message));
+        }
+        else {
+            pending.resolve(response.result);
+        }
+    }
+    async restart() {
+        if (this.restarting)
+            return;
+        this.restarting = true;
+        // Reject all pending requests
+        for (const [id, pending] of this.pending) {
+            clearTimeout(pending.timer);
+            pending.reject(new Error("Bridge process crashed, restarting"));
+            this.pending.delete(id);
+        }
+        try {
+            if (this.process) {
+                this.process.kill();
+                this.process = null;
+            }
+            await this.spawn();
+            this.emit("restart");
+        }
+        finally {
+            this.restarting = false;
+        }
+    }
+}
+/**
+ * @deprecated Use BridgeClient instead. This alias exists for backward compatibility.
+ */
+export const MacOSBridgeClient = BridgeClient;

package/{src/native/macos-bridge-client.ts → dist/src/native/macos-bridge-client.js}

@@ -14,7 +14,6 @@
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
 /**
  * @deprecated Import from "./bridge-client.js" instead.
  * This file re-exports for backward compatibility.

package/dist/src/playbook/engine.js

@@ -0,0 +1,201 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+const DEFAULT_VERIFY_TIMEOUT = 5000;
+const STEP_DELAY_MS = 300;
+export class PlaybookEngine {
+    runtime;
+    constructor(runtime) {
+        this.runtime = runtime;
+    }
+    /**
+     * Execute a playbook against a live session.
+     * Returns result with success/failure and which step broke.
+     */
+    async run(sessionId, playbook, options = {}) {
+        const start = Date.now();
+        let stepsCompleted = 0;
+        for (let i = 0; i < playbook.steps.length; i++) {
+            const step = playbook.steps[i];
+            try {
+                const result = await this.executeStep(sessionId, step);
+                stepsCompleted++;
+                if (options.onStep) {
+                    options.onStep(i, step, result);
+                }
+                // Verify step if needed
+                if (step.verify) {
+                    const verified = await this.verifyStep(sessionId, step);
+                    if (!verified && !step.optional) {
+                        return {
+                            playbook: playbook.id,
+                            success: false,
+                            stepsCompleted,
+                            totalSteps: playbook.steps.length,
+                            failedAtStep: i,
+                            error: `Verification failed at step ${i}: ${step.description ?? step.action}`,
+                            durationMs: Date.now() - start,
+                        };
+                    }
+                }
+                // Small delay between steps for UI to settle
+                await sleep(STEP_DELAY_MS);
+            }
+            catch (err) {
+                if (step.optional) {
+                    stepsCompleted++;
+                    if (options.onStep) {
+                        options.onStep(i, step, `Skipped (optional): ${err instanceof Error ? err.message : String(err)}`);
+                    }
+                    continue;
+                }
+                return {
+                    playbook: playbook.id,
+                    success: false,
+                    stepsCompleted,
+                    totalSteps: playbook.steps.length,
+                    failedAtStep: i,
+                    error: err instanceof Error ? err.message : String(err),
+                    durationMs: Date.now() - start,
+                };
+            }
+        }
+        return {
+            playbook: playbook.id,
+            success: true,
+            stepsCompleted,
+            totalSteps: playbook.steps.length,
+            failedAtStep: -1,
+            durationMs: Date.now() - start,
+        };
+    }
+    /**
+     * Execute a single playbook step.
+     */
+    async executeStep(sessionId, step) {
+        const target = this.resolveTarget(step.target);
+        switch (step.action) {
+            case "navigate": {
+                if (!step.url)
+                    throw new Error("navigate step missing url");
+                const r = await this.runtime.navigate({ sessionId, url: step.url });
+                if (!r.ok)
+                    throw new Error(r.error.message);
+                return `Navigated to ${step.url}`;
+            }
+            case "press": {
+                if (!target)
+                    throw new Error("press step missing target");
+                const r = await this.runtime.press({ sessionId, target });
+                if (!r.ok)
+                    throw new Error(r.error.message);
+                return `Pressed ${JSON.stringify(step.target)}`;
+            }
+            case "type_into": {
+                if (!target)
+                    throw new Error("type_into step missing target");
+                if (!step.text)
+                    throw new Error("type_into step missing text");
+                const r = await this.runtime.typeInto({ sessionId, target, text: step.text });
+                if (!r.ok)
+                    throw new Error(r.error.message);
+                return `Typed "${step.text}" into ${JSON.stringify(step.target)}`;
+            }
+            case "extract": {
+                if (!target)
+                    throw new Error("extract step missing target");
+                const r = await this.runtime.extract({
+                    sessionId,
+                    target,
+                    format: step.format ?? "text",
+                });
+                if (!r.ok)
+                    throw new Error(r.error.message);
+                return `Extracted: ${JSON.stringify(r.data).slice(0, 200)}`;
+            }
+            case "key_combo": {
+                if (!step.keys || step.keys.length === 0)
+                    throw new Error("key_combo step missing keys");
+                const r = await this.runtime.keyCombo({ sessionId, keys: step.keys });
+                if (!r.ok)
+                    throw new Error(r.error.message);
+                return `Key combo: ${step.keys.join("+")}`;
+            }
+            case "scroll": {
+                const input = {
+                    sessionId,
+                    direction: step.direction ?? "down",
+                };
+                if (step.amount != null)
+                    input.amount = step.amount;
+                const r = await this.runtime.scroll(input);
+                if (!r.ok)
+                    throw new Error(r.error.message);
+                return `Scrolled ${step.direction ?? "down"}`;
+            }
+            case "wait": {
+                await sleep(step.ms ?? 1000);
+                return `Waited ${step.ms ?? 1000}ms`;
+            }
+            case "screenshot": {
+                const r = await this.runtime.screenshot({ sessionId });
+                if (!r.ok)
+                    throw new Error(r.error.message);
+                return `Screenshot taken`;
+            }
+            default:
+                throw new Error(`Unknown action: ${step.action}`);
+        }
+    }
+    /**
+     * Verify a step's postcondition via CSS selector check.
+     */
+    async verifyStep(sessionId, step) {
+        if (!step.verify)
+            return true;
+        const timeout = step.verifyTimeoutMs ?? DEFAULT_VERIFY_TIMEOUT;
+        const r = await this.runtime.waitFor({
+            sessionId,
+            condition: { type: "selector_visible", selector: step.verify },
+            timeoutMs: timeout,
+        });
+        return r.ok && r.data.matched;
+    }
+    /**
+     * Convert playbook target format to runtime Target format.
+     */
+    resolveTarget(target) {
+        if (!target)
+            return undefined;
+        if (typeof target === "string") {
+            // CSS selector if starts with common patterns, else treat as text
+            if (target.startsWith("[") || target.startsWith("#") || target.startsWith(".") || target.startsWith("css=")) {
+                return { type: "selector", value: target.replace(/^css=/, "") };
+            }
+            return { type: "text", value: target };
+        }
+        if ("selector" in target) {
+            return { type: "selector", value: target.selector };
+        }
+        if ("x" in target && "y" in target) {
+            return { type: "coordinates", x: target.x, y: target.y };
+        }
+        return undefined;
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/src/playbook/index.js

@@ -0,0 +1,20 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export { PlaybookEngine } from "./engine.js";
+export { PlaybookStore } from "./store.js";
+export { PlaybookRunner } from "./runner.js";
+export { PlaybookRecorder } from "./recorder.js";