screenhand 0.1.1 → 0.2.0

package/dist/scripts/supervisor-daemon.js

@@ -0,0 +1,272 @@
+#!/usr/bin/env npx tsx
+/**
+ * Supervisor Daemon — runs as a standalone background process.
+ *
+ * Survives Claude Code restarts. Manages session leases,
+ * detects stalls via OCR, and executes recovery actions.
+ *
+ * Usage:
+ *   npx tsx scripts/supervisor-daemon.ts
+ *   npx tsx scripts/supervisor-daemon.ts --poll 5000 --stall 300000
+ *
+ * State files:
+ *   ~/.screenhand/supervisor/state.json      — supervisor state
+ *   ~/.screenhand/supervisor/recoveries.json  — recovery queue
+ *   ~/.screenhand/supervisor/supervisor.pid   — PID of this process
+ *   ~/.screenhand/supervisor/supervisor.log   — log output
+ *   ~/.screenhand/locks/                      — session leases
+ */
+import path from "node:path";
+import fs from "node:fs";
+import os from "node:os";
+import { BridgeClient } from "../src/native/bridge-client.js";
+import { SessionSupervisor } from "../src/supervisor/supervisor.js";
+// ── Config from CLI args ──
+const args = process.argv.slice(2);
+function getArg(name, fallback) {
+    const idx = args.indexOf("--" + name);
+    if (idx === -1)
+        return fallback;
+    return args[idx + 1] ?? fallback;
+}
+const POLL_MS = Number(getArg("poll", "5000"));
+const STALL_MS = Number(getArg("stall", "300000"));
+const LEASE_TIMEOUT_MS = Number(getArg("lease-timeout", "300000"));
+const AUTO_RECOVER = getArg("no-auto-recover") === undefined;
+const DRY_RUN = args.includes("--dry-run");
+// ── Logging ──
+const STATE_DIR = path.join(os.homedir(), ".screenhand", "supervisor");
+fs.mkdirSync(STATE_DIR, { recursive: true });
+const LOG_FILE = path.join(STATE_DIR, "supervisor.log");
+const PID_FILE = path.join(STATE_DIR, "supervisor.pid");
+const logStream = fs.createWriteStream(LOG_FILE, { flags: "a" });
+let daemonized = false;
+function log(msg) {
+    const line = `[${new Date().toISOString()}] ${msg}`;
+    logStream.write(line + "\n");
+    if (!daemonized)
+        process.stderr.write(line + "\n");
+}
+// ── Bridge setup ──
+const scriptDir = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
+// When running from dist/scripts/, go up two levels to reach the real project root
+const projectRoot = scriptDir.includes("/dist/")
+    ? path.resolve(scriptDir, "../..")
+    : path.resolve(scriptDir, "..");
+const bridgePath = process.platform === "win32"
+    ? path.resolve(projectRoot, "native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe")
+    : path.resolve(projectRoot, "native/macos-bridge/.build/release/macos-bridge");
+const bridge = new BridgeClient(bridgePath);
+let bridgeReady = false;
+async function ensureBridge() {
+    if (!bridgeReady) {
+        await bridge.start();
+        bridgeReady = true;
+    }
+}
+// ── Supervisor ──
+const supervisor = new SessionSupervisor({
+    pollMs: POLL_MS,
+    stallThresholdMs: STALL_MS,
+    leaseTimeoutMs: LEASE_TIMEOUT_MS,
+    autoRecover: AUTO_RECOVER,
+});
+let stopped = false;
+// ── Recovery execution (the real control) ──
+async function executeRecovery(recovery, lease) {
+    await ensureBridge();
+    switch (recovery.type) {
+        case "nudge": {
+            // Focus the app and press Enter to nudge a stalled agent
+            log(`Nudging session ${recovery.sessionId}: focusing ${lease.app}`);
+            try {
+                await bridge.call("app.focus", { bundleId: lease.app });
+                await sleep(300);
+                await bridge.call("cg.keyCombo", { keys: ["enter"] });
+                return "Focused app and pressed Enter";
+            }
+            catch (err) {
+                return `Nudge failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
+        }
+        case "restart": {
+            // Close and relaunch the app
+            log(`Restarting session ${recovery.sessionId}: relaunching ${lease.app}`);
+            try {
+                await bridge.call("app.focus", { bundleId: lease.app });
+                await sleep(200);
+                // Cmd+Q to quit, then relaunch
+                await bridge.call("cg.keyCombo", { keys: ["cmd", "q"] });
+                await sleep(2000);
+                await bridge.call("app.launch", { bundleId: lease.app });
+                return "App relaunched";
+            }
+            catch (err) {
+                return `Restart failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
+        }
+        case "custom": {
+            // Type the custom instruction into the focused window
+            log(`Custom recovery for ${recovery.sessionId}: "${recovery.instruction.slice(0, 60)}"`);
+            try {
+                await bridge.call("app.focus", { bundleId: lease.app });
+                await sleep(300);
+                await bridge.call("cg.typeText", { text: recovery.instruction });
+                await sleep(100);
+                await bridge.call("cg.keyCombo", { keys: ["enter"] });
+                return `Typed: "${recovery.instruction.slice(0, 80)}"`;
+            }
+            catch (err) {
+                return `Custom recovery failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
+        }
+        case "escalate": {
+            // Notify via macOS notification
+            if (process.platform === "darwin") {
+                try {
+                    const { execFileSync } = await import("node:child_process");
+                    const msg = recovery.instruction.replace(/"/g, '\\"');
+                    execFileSync("osascript", [
+                        "-e",
+                        `display notification "${msg}" with title "ScreenHand Supervisor" subtitle "Session needs help"`,
+                    ]);
+                    return "Notification sent";
+                }
+                catch {
+                    return "Escalation: notification failed, check supervisor log";
+                }
+            }
+            return "Escalation logged — requires human intervention";
+        }
+        default:
+            return `Unknown recovery type: ${recovery.type}`;
+    }
+}
+// ── OCR-based stall content capture ──
+async function captureWindowContent(lease) {
+    try {
+        await ensureBridge();
+        const shot = await bridge.call("cg.captureWindow", {
+            windowId: lease.windowId,
+        });
+        const ocr = await bridge.call("vision.ocr", {
+            imagePath: shot.path,
+        });
+        return ocr.text;
+    }
+    catch {
+        // OCR failed — try AX tree as fallback
+        try {
+            const tree = await bridge.call("ax.getElementTree", {
+                pid: lease.client.pid ?? 0,
+                maxDepth: 4,
+            });
+            return extractText(tree);
+        }
+        catch {
+            return null;
+        }
+    }
+}
+function extractText(node, depth = 0) {
+    if (depth > 6)
+        return null;
+    if (node.value && typeof node.value === "string" && node.value.length > 20) {
+        return node.value;
+    }
+    if (node.children) {
+        for (const child of node.children) {
+            const found = extractText(child, depth + 1);
+            if (found)
+                return found;
+        }
+    }
+    return null;
+}
+// ── Main poll loop ──
+async function poll() {
+    // 1. Update screen content for all active sessions (for blocker matching)
+    const state = supervisor.getState();
+    for (const lease of state.sessions) {
+        const content = await captureWindowContent(lease);
+        if (content) {
+            supervisor.setScreenContent(lease.sessionId, content);
+        }
+    }
+    // 2. Let the supervisor run its poll cycle (detect stalls, create recoveries)
+    // The supervisor's internal pollCycle is on a timer, but we also call detectStalls
+    // to feed screen content into it
+    // 3. Execute pending recoveries with actual bridge actions
+    const pending = supervisor.getRecoveries("attempted");
+    for (const recovery of pending) {
+        const lease = state.sessions.find((s) => s.sessionId === recovery.sessionId);
+        if (!lease) {
+            supervisor.updateRecovery(recovery.id, "failed", "Session no longer active");
+            continue;
+        }
+        log(`Executing recovery ${recovery.id} (type=${recovery.type})`);
+        let resultText;
+        if (DRY_RUN) {
+            resultText = `[DRY RUN] Would execute ${recovery.type}: ${recovery.instruction.slice(0, 80)}`;
+        }
+        else {
+            resultText = await executeRecovery(recovery, lease);
+        }
+        const status = resultText.toLowerCase().includes("failed") ? "failed" : "succeeded";
+        supervisor.updateRecovery(recovery.id, status, resultText);
+        log(`Recovery ${recovery.id}: ${status} — ${resultText}`);
+    }
+}
+async function main() {
+    // Enforce single daemon — abort if another is already running
+    const existingPid = supervisor.getExistingDaemonPid();
+    if (existingPid !== null && existingPid !== process.pid) {
+        const msg = `Another supervisor daemon is already running (pid=${existingPid}). Aborting.`;
+        log(msg);
+        process.stderr.write(msg + "\n");
+        process.exit(1);
+    }
+    fs.writeFileSync(PID_FILE, String(process.pid));
+    daemonized = true;
+    log(`Supervisor daemon started (pid=${process.pid})`);
+    log(`Config: poll=${POLL_MS}ms stall=${STALL_MS}ms lease-timeout=${LEASE_TIMEOUT_MS}ms auto-recover=${AUTO_RECOVER} dry-run=${DRY_RUN}`);
+    await supervisor.start();
+    log("Supervisor poll loop started");
+    // Additional poll for OCR + recovery execution
+    while (!stopped) {
+        try {
+            await poll();
+        }
+        catch (err) {
+            log(`Poll error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        await sleep(POLL_MS);
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+// ── Graceful shutdown ──
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);
+async function shutdown() {
+    if (stopped)
+        return;
+    stopped = true;
+    log("Shutting down...");
+    await supervisor.stop();
+    try {
+        fs.unlinkSync(PID_FILE);
+    }
+    catch { /* ignore */ }
+    try {
+        await bridge.stop();
+    }
+    catch { /* ignore */ }
+    logStream.end();
+    process.exit(0);
+}
+main().catch((err) => {
+    log(`Fatal: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+});

package/dist/scripts/worker-daemon.js

@@ -0,0 +1,228 @@
+#!/usr/bin/env npx tsx
+/**
+ * Worker Daemon — runs as a standalone background process.
+ *
+ * Survives MCP/client restarts. Continuously processes the job queue
+ * via JobRunner with playbook engine support.
+ *
+ * Usage:
+ *   npx tsx scripts/worker-daemon.ts
+ *   npx tsx scripts/worker-daemon.ts --poll 3000 --max-jobs 0
+ *
+ * State files:
+ *   ~/.screenhand/worker/state.json   — worker status + recent results
+ *   ~/.screenhand/worker/worker.pid   — PID of this process
+ *   ~/.screenhand/worker/worker.log   — log output
+ *   ~/.screenhand/jobs/               — job persistence
+ *   ~/.screenhand/locks/              — session leases
+ */
+import path from "node:path";
+import fs from "node:fs";
+import os from "node:os";
+import { BridgeClient } from "../src/native/bridge-client.js";
+import { SessionSupervisor, LeaseManager } from "../src/supervisor/supervisor.js";
+import { JobManager } from "../src/jobs/manager.js";
+import { JobRunner } from "../src/jobs/runner.js";
+import { PlaybookEngine } from "../src/playbook/engine.js";
+import { PlaybookStore } from "../src/playbook/store.js";
+import { AccessibilityAdapter } from "../src/runtime/accessibility-adapter.js";
+import { AutomationRuntimeService } from "../src/runtime/service.js";
+import { TimelineLogger } from "../src/logging/timeline-logger.js";
+import { MemoryService } from "../src/memory/service.js";
+import { WORKER_DIR, WORKER_PID_FILE, WORKER_LOG_FILE, getWorkerDaemonPid, writeWorkerStatus, } from "../src/jobs/worker.js";
+// ── Config from CLI args ──
+const args = process.argv.slice(2);
+function getArg(name, fallback) {
+    const idx = args.indexOf("--" + name);
+    if (idx === -1)
+        return fallback;
+    return args[idx + 1] ?? fallback;
+}
+const POLL_MS = Number(getArg("poll", "3000"));
+const MAX_JOBS = Number(getArg("max-jobs", "0")); // 0 = unlimited
+// ── Directories ──
+const JOB_DIR = path.join(os.homedir(), ".screenhand", "jobs");
+const LOCK_DIR = path.join(os.homedir(), ".screenhand", "locks");
+const PLAYBOOKS_DIR = path.join(os.homedir(), ".screenhand", "playbooks");
+const SUPERVISOR_STATE_DIR = path.join(os.homedir(), ".screenhand", "supervisor");
+fs.mkdirSync(WORKER_DIR, { recursive: true });
+fs.mkdirSync(JOB_DIR, { recursive: true });
+// ── Logging ──
+const logStream = fs.createWriteStream(WORKER_LOG_FILE, { flags: "a" });
+let daemonized = false;
+function log(msg) {
+    const line = `[${new Date().toISOString()}] ${msg}`;
+    logStream.write(line + "\n");
+    if (!daemonized)
+        process.stderr.write(line + "\n");
+}
+// ── Bridge setup ──
+const scriptDir = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
+const projectRoot = scriptDir.includes("/dist/")
+    ? path.resolve(scriptDir, "../..")
+    : path.resolve(scriptDir, "..");
+const bridgePath = process.platform === "win32"
+    ? path.resolve(projectRoot, "native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe")
+    : path.resolve(projectRoot, "native/macos-bridge/.build/release/macos-bridge");
+const bridge = new BridgeClient(bridgePath);
+let bridgeReady = false;
+async function ensureBridge() {
+    if (!bridgeReady) {
+        await bridge.start();
+        bridgeReady = true;
+    }
+}
+// ── Services ──
+const leaseManager = new LeaseManager(LOCK_DIR);
+const supervisor = new SessionSupervisor({
+    stateDir: SUPERVISOR_STATE_DIR,
+    lockDir: LOCK_DIR,
+});
+const memory = new MemoryService(os.homedir());
+const jobManager = new JobManager({ jobDir: JOB_DIR, memory, supervisor });
+jobManager.init();
+// ── State ──
+let stopped = false;
+let processing = false;
+const recentResults = [];
+const MAX_RECENT = 50;
+let jobsProcessed = 0;
+let jobsDone = 0;
+let jobsFailed = 0;
+let jobsBlocked = 0;
+let lastJobId = null;
+let lastJobState = null;
+let startedAt = null;
+function buildStatus() {
+    return {
+        pid: process.pid,
+        running: !stopped,
+        startedAt,
+        pollMs: POLL_MS,
+        maxJobs: MAX_JOBS,
+        jobsProcessed,
+        jobsDone,
+        jobsFailed,
+        jobsBlocked,
+        lastJobId,
+        lastJobState,
+        uptimeMs: startedAt ? Date.now() - new Date(startedAt).getTime() : 0,
+        recentResults: recentResults.slice(-MAX_RECENT),
+    };
+}
+function persistState() {
+    try {
+        writeWorkerStatus(buildStatus());
+    }
+    catch {
+        // Non-fatal
+    }
+}
+function recordResult(result) {
+    jobsProcessed++;
+    lastJobId = result.jobId;
+    lastJobState = result.finalState;
+    switch (result.finalState) {
+        case "done":
+            jobsDone++;
+            break;
+        case "failed":
+            jobsFailed++;
+            break;
+        case "blocked":
+        case "waiting_human":
+            jobsBlocked++;
+            break;
+    }
+    recentResults.push(result);
+    if (recentResults.length > MAX_RECENT)
+        recentResults.shift();
+    log(`Completed: ${result.jobId} → ${result.finalState} (${result.stepsCompleted}/${result.totalSteps} steps, ${result.durationMs}ms)`);
+    persistState();
+}
+// ── Main loop ──
+async function main() {
+    // Enforce single daemon
+    const existingPid = getWorkerDaemonPid();
+    if (existingPid !== null && existingPid !== process.pid) {
+        const msg = `Another worker daemon is already running (pid=${existingPid}). Aborting.`;
+        log(msg);
+        process.stderr.write(msg + "\n");
+        process.exit(1);
+    }
+    fs.writeFileSync(WORKER_PID_FILE, String(process.pid));
+    daemonized = true;
+    startedAt = new Date().toISOString();
+    log(`Worker daemon started (pid=${process.pid})`);
+    log(`Config: poll=${POLL_MS}ms max-jobs=${MAX_JOBS || "unlimited"}`);
+    // Ensure bridge is ready
+    await ensureBridge();
+    // Build playbook engine stack
+    const adapter = new AccessibilityAdapter(bridge);
+    const logger = new TimelineLogger();
+    const runtimeService = new AutomationRuntimeService(adapter, logger);
+    const playbookEngine = new PlaybookEngine(runtimeService);
+    const playbookStore = new PlaybookStore(PLAYBOOKS_DIR);
+    playbookStore.load();
+    const runner = new JobRunner(bridge, jobManager, leaseManager, supervisor, {
+        playbookEngine,
+        playbookStore,
+        runtimeService,
+        onLog: log,
+    });
+    persistState();
+    // Poll loop
+    while (!stopped) {
+        if (!processing) {
+            processing = true;
+            try {
+                const result = await runner.run();
+                if (result) {
+                    recordResult(result);
+                    // Check maxJobs limit
+                    if (MAX_JOBS > 0 && jobsProcessed >= MAX_JOBS) {
+                        log(`Reached max-jobs limit (${MAX_JOBS})`);
+                        break;
+                    }
+                    // Job found — poll again immediately
+                    continue;
+                }
+            }
+            catch (err) {
+                log(`Poll error: ${err instanceof Error ? err.message : String(err)}`);
+            }
+            finally {
+                processing = false;
+            }
+        }
+        // Queue empty or error — wait before next poll
+        await sleep(POLL_MS);
+    }
+    log(`Worker daemon exiting (${jobsProcessed} jobs: ${jobsDone} done, ${jobsFailed} failed, ${jobsBlocked} blocked)`);
+    await shutdown();
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+// ── Graceful shutdown ──
+process.on("SIGINT", () => { stopped = true; });
+process.on("SIGTERM", () => { stopped = true; });
+async function shutdown() {
+    stopped = true;
+    persistState();
+    try {
+        fs.unlinkSync(WORKER_PID_FILE);
+    }
+    catch { /* ignore */ }
+    try {
+        await bridge.stop();
+    }
+    catch { /* ignore */ }
+    logStream.end();
+    process.exit(0);
+}
+main().catch((err) => {
+    log(`Fatal: ${err instanceof Error ? err.message : String(err)}`);
+    persistState();
+    process.exit(1);
+});

package/dist/src/agent/cli.js

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { TimelineLogger } from "../logging/timeline-logger.js";
+import { AutomationRuntimeService } from "../runtime/service.js";
+import { runAgentLoop } from "./loop.js";
+const task = process.argv.slice(2).join(" ");
+if (!task) {
+    console.error("Usage: screenhand-agent <task description>");
+    console.error("Example: screenhand-agent \"Open Safari and search for MCP protocol\"");
+    process.exit(1);
+}
+async function createAdapter() {
+    const adapterType = process.env.SCREENHAND_ADAPTER ?? "accessibility";
+    switch (adapterType) {
+        case "placeholder": {
+            const { PlaceholderAppAdapter } = await import("../runtime/app-adapter.js");
+            return new PlaceholderAppAdapter();
+        }
+        case "cdp": {
+            const { CdpChromeAdapter } = await import("../runtime/cdp-chrome-adapter.js");
+            return new CdpChromeAdapter({ headless: process.env.SCREENHAND_HEADLESS === "1" });
+        }
+        case "composite": {
+            const { BridgeClient } = await import("../native/bridge-client.js");
+            const { CompositeAdapter } = await import("../runtime/composite-adapter.js");
+            return new CompositeAdapter(new BridgeClient(), { headless: process.env.SCREENHAND_HEADLESS === "1" });
+        }
+        default: {
+            const { BridgeClient } = await import("../native/bridge-client.js");
+            const { AccessibilityAdapter } = await import("../runtime/accessibility-adapter.js");
+            return new AccessibilityAdapter(new BridgeClient());
+        }
+    }
+}
+try {
+    const adapter = await createAdapter();
+    const runtime = new AutomationRuntimeService(adapter, new TimelineLogger());
+    const session = await runtime.sessionStart();
+    console.log(`\n🔄 Task: ${task}`);
+    console.log(`   Session: ${session.sessionId}`);
+    console.log(`   Model: ${process.env.SCREENHAND_MODEL ?? "claude-sonnet-4-20250514"}\n`);
+    const cliModel = process.env.SCREENHAND_MODEL;
+    const result = await runAgentLoop(runtime, session.sessionId, task, {
+        maxSteps: parseInt(process.env.SCREENHAND_MAX_STEPS ?? "50", 10),
+        ...(cliModel ? { model: cliModel } : {}),
+        onStep: (step) => {
+            const icon = step.done ? "✅" : step.action ? "→" : "⚠️";
+            console.log(`  ${icon} [${step.index}] ${step.reasoning.slice(0, 100)}`);
+            if (step.action && step.action.tool !== "done") {
+                console.log(`     ${step.action.tool}: ${JSON.stringify(step.action).slice(0, 120)}`);
+            }
+            if (step.result) {
+                console.log(`     Result: ${step.result.slice(0, 100)}`);
+            }
+            console.log(`     (${step.durationMs}ms)\n`);
+        },
+    });
+    console.log(`\n${"=".repeat(60)}`);
+    console.log(`${result.success ? "✅ SUCCESS" : "❌ INCOMPLETE"}: ${result.summary}`);
+    console.log(`Steps: ${result.steps.length} | Total: ${result.totalMs}ms`);
+    console.log(`${"=".repeat(60)}\n`);
+    process.exit(result.success ? 0 : 1);
+}
+catch (e) {
+    console.error(`Fatal: ${e instanceof Error ? e.message : String(e)}`);
+    process.exit(1);
+}