screenhand 0.1.1 → 0.3.0

package/dist/scripts/supervisor-daemon.js

@@ -0,0 +1,272 @@
+#!/usr/bin/env npx tsx
+/**
+ * Supervisor Daemon — runs as a standalone background process.
+ *
+ * Survives Claude Code restarts. Manages session leases,
+ * detects stalls via OCR, and executes recovery actions.
+ *
+ * Usage:
+ *   npx tsx scripts/supervisor-daemon.ts
+ *   npx tsx scripts/supervisor-daemon.ts --poll 5000 --stall 300000
+ *
+ * State files:
+ *   ~/.screenhand/supervisor/state.json      — supervisor state
+ *   ~/.screenhand/supervisor/recoveries.json  — recovery queue
+ *   ~/.screenhand/supervisor/supervisor.pid   — PID of this process
+ *   ~/.screenhand/supervisor/supervisor.log   — log output
+ *   ~/.screenhand/locks/                      — session leases
+ */
+import path from "node:path";
+import fs from "node:fs";
+import os from "node:os";
+import { BridgeClient } from "../src/native/bridge-client.js";
+import { SessionSupervisor } from "../src/supervisor/supervisor.js";
+// ── Config from CLI args ──
+const args = process.argv.slice(2);
+function getArg(name, fallback) {
+    const idx = args.indexOf("--" + name);
+    if (idx === -1)
+        return fallback;
+    return args[idx + 1] ?? fallback;
+}
+const POLL_MS = Number(getArg("poll", "5000"));
+const STALL_MS = Number(getArg("stall", "300000"));
+const LEASE_TIMEOUT_MS = Number(getArg("lease-timeout", "300000"));
+const AUTO_RECOVER = getArg("no-auto-recover") === undefined;
+const DRY_RUN = args.includes("--dry-run");
+// ── Logging ──
+const STATE_DIR = path.join(os.homedir(), ".screenhand", "supervisor");
+fs.mkdirSync(STATE_DIR, { recursive: true });
+const LOG_FILE = path.join(STATE_DIR, "supervisor.log");
+const PID_FILE = path.join(STATE_DIR, "supervisor.pid");
+const logStream = fs.createWriteStream(LOG_FILE, { flags: "a" });
+let daemonized = false;
+function log(msg) {
+    const line = `[${new Date().toISOString()}] ${msg}`;
+    logStream.write(line + "\n");
+    if (!daemonized)
+        process.stderr.write(line + "\n");
+}
+// ── Bridge setup ──
+const scriptDir = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
+// When running from dist/scripts/, go up two levels to reach the real project root
+const projectRoot = scriptDir.includes("/dist/")
+    ? path.resolve(scriptDir, "../..")
+    : path.resolve(scriptDir, "..");
+const bridgePath = process.platform === "win32"
+    ? path.resolve(projectRoot, "native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe")
+    : path.resolve(projectRoot, "native/macos-bridge/.build/release/macos-bridge");
+const bridge = new BridgeClient(bridgePath);
+let bridgeReady = false;
+async function ensureBridge() {
+    if (!bridgeReady) {
+        await bridge.start();
+        bridgeReady = true;
+    }
+}
+// ── Supervisor ──
+const supervisor = new SessionSupervisor({
+    pollMs: POLL_MS,
+    stallThresholdMs: STALL_MS,
+    leaseTimeoutMs: LEASE_TIMEOUT_MS,
+    autoRecover: AUTO_RECOVER,
+});
+let stopped = false;
+// ── Recovery execution (the real control) ──
+async function executeRecovery(recovery, lease) {
+    await ensureBridge();
+    switch (recovery.type) {
+        case "nudge": {
+            // Focus the app and press Enter to nudge a stalled agent
+            log(`Nudging session ${recovery.sessionId}: focusing ${lease.app}`);
+            try {
+                await bridge.call("app.focus", { bundleId: lease.app });
+                await sleep(300);
+                await bridge.call("cg.keyCombo", { keys: ["enter"] });
+                return "Focused app and pressed Enter";
+            }
+            catch (err) {
+                return `Nudge failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
+        }
+        case "restart": {
+            // Close and relaunch the app
+            log(`Restarting session ${recovery.sessionId}: relaunching ${lease.app}`);
+            try {
+                await bridge.call("app.focus", { bundleId: lease.app });
+                await sleep(200);
+                // Cmd+Q to quit, then relaunch
+                await bridge.call("cg.keyCombo", { keys: ["cmd", "q"] });
+                await sleep(2000);
+                await bridge.call("app.launch", { bundleId: lease.app });
+                return "App relaunched";
+            }
+            catch (err) {
+                return `Restart failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
+        }
+        case "custom": {
+            // Type the custom instruction into the focused window
+            log(`Custom recovery for ${recovery.sessionId}: "${recovery.instruction.slice(0, 60)}"`);
+            try {
+                await bridge.call("app.focus", { bundleId: lease.app });
+                await sleep(300);
+                await bridge.call("cg.typeText", { text: recovery.instruction });
+                await sleep(100);
+                await bridge.call("cg.keyCombo", { keys: ["enter"] });
+                return `Typed: "${recovery.instruction.slice(0, 80)}"`;
+            }
+            catch (err) {
+                return `Custom recovery failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
+        }
+        case "escalate": {
+            // Notify via macOS notification
+            if (process.platform === "darwin") {
+                try {
+                    const { execFileSync } = await import("node:child_process");
+                    const msg = recovery.instruction.replace(/"/g, '\\"');
+                    execFileSync("osascript", [
+                        "-e",
+                        `display notification "${msg}" with title "ScreenHand Supervisor" subtitle "Session needs help"`,
+                    ]);
+                    return "Notification sent";
+                }
+                catch {
+                    return "Escalation: notification failed, check supervisor log";
+                }
+            }
+            return "Escalation logged — requires human intervention";
+        }
+        default:
+            return `Unknown recovery type: ${recovery.type}`;
+    }
+}
+// ── OCR-based stall content capture ──
+async function captureWindowContent(lease) {
+    try {
+        await ensureBridge();
+        const shot = await bridge.call("cg.captureWindow", {
+            windowId: lease.windowId,
+        });
+        const ocr = await bridge.call("vision.ocr", {
+            imagePath: shot.path,
+        });
+        return ocr.text;
+    }
+    catch {
+        // OCR failed — try AX tree as fallback
+        try {
+            const tree = await bridge.call("ax.getElementTree", {
+                pid: lease.client.pid ?? 0,
+                maxDepth: 4,
+            });
+            return extractText(tree);
+        }
+        catch {
+            return null;
+        }
+    }
+}
+function extractText(node, depth = 0) {
+    if (depth > 6)
+        return null;
+    if (node.value && typeof node.value === "string" && node.value.length > 20) {
+        return node.value;
+    }
+    if (node.children) {
+        for (const child of node.children) {
+            const found = extractText(child, depth + 1);
+            if (found)
+                return found;
+        }
+    }
+    return null;
+}
+// ── Main poll loop ──
+async function poll() {
+    // 1. Update screen content for all active sessions (for blocker matching)
+    const state = supervisor.getState();
+    for (const lease of state.sessions) {
+        const content = await captureWindowContent(lease);
+        if (content) {
+            supervisor.setScreenContent(lease.sessionId, content);
+        }
+    }
+    // 2. Let the supervisor run its poll cycle (detect stalls, create recoveries)
+    // The supervisor's internal pollCycle is on a timer, but we also call detectStalls
+    // to feed screen content into it
+    // 3. Execute pending recoveries with actual bridge actions
+    const pending = supervisor.getRecoveries("attempted");
+    for (const recovery of pending) {
+        const lease = state.sessions.find((s) => s.sessionId === recovery.sessionId);
+        if (!lease) {
+            supervisor.updateRecovery(recovery.id, "failed", "Session no longer active");
+            continue;
+        }
+        log(`Executing recovery ${recovery.id} (type=${recovery.type})`);
+        let resultText;
+        if (DRY_RUN) {
+            resultText = `[DRY RUN] Would execute ${recovery.type}: ${recovery.instruction.slice(0, 80)}`;
+        }
+        else {
+            resultText = await executeRecovery(recovery, lease);
+        }
+        const status = resultText.toLowerCase().includes("failed") ? "failed" : "succeeded";
+        supervisor.updateRecovery(recovery.id, status, resultText);
+        log(`Recovery ${recovery.id}: ${status} — ${resultText}`);
+    }
+}
+async function main() {
+    // Enforce single daemon — abort if another is already running
+    const existingPid = supervisor.getExistingDaemonPid();
+    if (existingPid !== null && existingPid !== process.pid) {
+        const msg = `Another supervisor daemon is already running (pid=${existingPid}). Aborting.`;
+        log(msg);
+        process.stderr.write(msg + "\n");
+        process.exit(1);
+    }
+    fs.writeFileSync(PID_FILE, String(process.pid));
+    daemonized = true;
+    log(`Supervisor daemon started (pid=${process.pid})`);
+    log(`Config: poll=${POLL_MS}ms stall=${STALL_MS}ms lease-timeout=${LEASE_TIMEOUT_MS}ms auto-recover=${AUTO_RECOVER} dry-run=${DRY_RUN}`);
+    await supervisor.start();
+    log("Supervisor poll loop started");
+    // Additional poll for OCR + recovery execution
+    while (!stopped) {
+        try {
+            await poll();
+        }
+        catch (err) {
+            log(`Poll error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        await sleep(POLL_MS);
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+// ── Graceful shutdown ──
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);
+async function shutdown() {
+    if (stopped)
+        return;
+    stopped = true;
+    log("Shutting down...");
+    await supervisor.stop();
+    try {
+        fs.unlinkSync(PID_FILE);
+    }
+    catch { /* ignore */ }
+    try {
+        await bridge.stop();
+    }
+    catch { /* ignore */ }
+    logStream.end();
+    process.exit(0);
+}
+main().catch((err) => {
+    log(`Fatal: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+});

package/dist/scripts/threads-campaign.js

@@ -0,0 +1,208 @@
+/**
+ * ScreenHand Twitter/X 2-Hour Campaign Runner
+ *
+ * Executes the engagement cadence from threads-campaign.md
+ * Uses ScreenHand MCP tools via direct function calls
+ *
+ * Usage: tsx scripts/threads-campaign.ts
+ * With cron/loop: keeps alive for 2 hours
+ */
+import { join } from 'path';
+// Campaign config
+const CONFIG = {
+    duration: 2 * 60 * 60 * 1000, // 2 hours in ms
+    tabId: '', // filled at runtime
+    searchQueries: [
+        '"MCP server" -is:retweet',
+        '"AI agent" automation -is:retweet',
+        '"Claude Code" -is:retweet',
+        '"desktop automation" -is:retweet',
+        '"Selenium frustrating" OR "Playwright broken" -is:retweet',
+        '"browser automation" -is:retweet',
+        '"Figma to code" -is:retweet',
+    ],
+    tweets: [
+        {
+            time: 0,
+            content: `AI agents can think, code, and reason.
+
+But they can't click a button.
+
+That's the gap ScreenHand fills — an MCP server giving any AI agent native desktop control on macOS + Windows.
+
+No API keys. No cloud. No fixed scripts. Just real OS-level control. 82 tools. One protocol.`
+        },
+        {
+            time: 20,
+            content: `If your automation breaks when a button moves 10 pixels, you don't have automation. You have a time bomb.
+
+ScreenHand doesn't rely on fixed selectors. It uses fallback chains:
+
+Accessibility API → CDP → Vision → AppleScript
+
+If one fails, it adapts. It also remembers what worked — gets faster every run.`
+        },
+        {
+            time: 40,
+            content: `The AI stack everyone's sleeping on:
+
+Claude Code → thinks + codes
+Figma MCP → design tokens
+ScreenHand → opens the actual browser, verifies pixels, clicks deploy
+
+What takes 45 min manually takes 3 min with this stack. The AI has a brain. ScreenHand gives it hands.`
+        },
+        {
+            time: 60,
+            content: `Every cloud automation tool has the same problem — your credentials pass through someone else's server.
+
+ScreenHand runs locally. On your machine. No cloud relay. No API keys to leak. No third-party credential storage.
+
+AGPL licensed. You own it completely.`
+        },
+        {
+            time: 80,
+            content: `Building AI agents? Here's what nobody tells you:
+
+The hardest part isn't the reasoning. It's the last mile — actually executing actions in real software.
+
+We built ScreenHand because we hit this wall ourselves. 82 MCP tools later, our agents can control any app on the desktop.
+
+What are you building?`
+        },
+        {
+            time: 100,
+            content: `What ScreenHand can do in one MCP call:
+
+- browser_human_click — clicks like a real human, anti-detection built in
+- memory_recall — remembers patterns from previous runs
+- execution_plan — plans multi-step workflows with fallback strategies
+- export_playbook — saves learned flows for autonomous replay
+
+Not scripts. Not recordings. Adaptive execution.`
+        },
+    ],
+    replyTemplates: [
+        `This is exactly why we built ScreenHand — the execution layer is the missing piece. Agents can reason but need OS-level control to do real work. We use accessibility APIs + CDP, not screenshots.`,
+        `Interesting approach. One thing we've found — the fallback chain matters more than any single method. AX API fails? Try CDP. CDP fails? Try Vision. Resilience > precision.`,
+        `Have you tried giving your agent actual desktop control? We built an MCP server (ScreenHand) that lets Claude/GPT control any app natively. Game changer for automation.`,
+        `MCP is the right protocol for this. We built ScreenHand as an MCP server — 82 tools for desktop + browser automation. Any MCP client can drive it.`,
+        `The composability of MCP is what makes it powerful. ScreenHand (desktop control) stacks with tools like yours naturally. Brain + hands.`,
+        `Felt this. Selector-based automation is brittle. We switched to accessibility-tree-first with ScreenHand — the OS exposes UI elements semantically, not positionally.`,
+        `The maintenance tax on Selenium scripts is brutal. We built fallback chains (AX → CDP → Vision → AppleScript) because no single method is reliable enough alone.`,
+        `This is why we went API-free. No API keys = no rate limits, no platform approval needed. ScreenHand controls the actual UI like a human would.`,
+        `Claude Code + ScreenHand is a wild combo. Claude thinks and codes, ScreenHand handles everything outside the IDE — browser testing, deployment UIs, verification.`,
+        `Figma MCP gives you design tokens. But who verifies the implementation matches? ScreenHand opens the browser, screenshots the result, compares. Loop closed.`,
+    ],
+    // Rate limits
+    maxLikesPerHour: 18,
+    maxRepliesPerHour: 6,
+    maxDmsPerHour: 2,
+    maxTweetsPerHour: 3,
+};
+// Randomized delay (human-like)
+function randomDelay(minSec, maxSec) {
+    return (Math.floor(Math.random() * (maxSec - minSec + 1)) + minSec) * 1000;
+}
+function createInitialState() {
+    return {
+        startTime: Date.now(),
+        tweetsPosted: 0,
+        likesGiven: 0,
+        repliesSent: 0,
+        dmsSent: 0,
+        searchIndex: 0,
+        replyIndex: 0,
+        minutesElapsed: 0,
+    };
+}
+const SCHEDULE = [
+    { minute: 0, type: 'post', tweetIndex: 0 },
+    { minute: 3, type: 'like', count: 3 },
+    { minute: 8, type: 'reply', replyTemplate: 0 },
+    { minute: 10, type: 'reply', replyTemplate: 3 },
+    { minute: 15, type: 'like', count: 2 },
+    { minute: 20, type: 'post', tweetIndex: 1 },
+    { minute: 25, type: 'reply', replyTemplate: 5 },
+    { minute: 27, type: 'reply', replyTemplate: 7 },
+    { minute: 30, type: 'like', count: 3 },
+    { minute: 35, type: 'reply', replyTemplate: 8 },
+    { minute: 40, type: 'post', tweetIndex: 2 },
+    { minute: 45, type: 'like', count: 3 },
+    { minute: 50, type: 'reply', replyTemplate: 9 },
+    { minute: 52, type: 'reply', replyTemplate: 1 },
+    { minute: 55, type: 'search' },
+    { minute: 60, type: 'post', tweetIndex: 3 },
+    { minute: 65, type: 'like', count: 2 },
+    { minute: 70, type: 'reply', replyTemplate: 2 },
+    { minute: 72, type: 'reply', replyTemplate: 4 },
+    { minute: 75, type: 'like', count: 3 },
+    { minute: 80, type: 'post', tweetIndex: 4 },
+    { minute: 85, type: 'reply', replyTemplate: 6 },
+    { minute: 87, type: 'reply', replyTemplate: 1 },
+    { minute: 90, type: 'search' },
+    { minute: 95, type: 'like', count: 3 },
+    { minute: 100, type: 'post', tweetIndex: 5 },
+    { minute: 105, type: 'reply', replyTemplate: 3 },
+    { minute: 107, type: 'reply', replyTemplate: 8 },
+    { minute: 110, type: 'like', count: 2 },
+    { minute: 115, type: 'like', count: 3 },
+    { minute: 120, type: 'search' },
+];
+// Log action for tracking
+function logAction(action, details) {
+    const ts = new Date().toISOString();
+    const line = `[${ts}] ${action}: ${details}`;
+    console.log(line);
+    // Append to campaign log
+    const logPath = join(__dirname, '..', 'threads_campaign_log.txt');
+    const { appendFileSync } = require('fs');
+    appendFileSync(logPath, line + '\n');
+}
+// Main — this file is designed to be driven by ScreenHand MCP tools
+// When run standalone, it outputs the schedule for the current minute
+if (require.main === module) {
+    const state = createInitialState();
+    const elapsed = Math.floor((Date.now() - state.startTime) / 60000);
+    // Find next actions
+    const upcoming = SCHEDULE.filter(a => a.minute >= elapsed).slice(0, 3);
+    console.log('=== ScreenHand Campaign Runner ===');
+    console.log(`Started: ${new Date(state.startTime).toLocaleTimeString()}`);
+    console.log(`Elapsed: ${elapsed} minutes`);
+    console.log(`\nNext actions:`);
+    for (const action of upcoming) {
+        const inMinutes = action.minute - elapsed;
+        let desc = '';
+        switch (action.type) {
+            case 'post':
+                desc = `Post tweet #${(action.tweetIndex ?? 0) + 1}`;
+                break;
+            case 'like':
+                desc = `Like ${action.count} relevant tweets`;
+                break;
+            case 'reply':
+                desc = `Reply using template #${(action.replyTemplate ?? 0) + 1}`;
+                break;
+            case 'search':
+                desc = `Search for new engagement targets`;
+                break;
+            case 'dm':
+                desc = `Send warm DM`;
+                break;
+        }
+        console.log(`  ${inMinutes === 0 ? 'NOW' : `in ${inMinutes}m`}: ${desc}`);
+    }
+    console.log(`\nTweet content for next post:`);
+    const nextPost = upcoming.find(a => a.type === 'post');
+    if (nextPost?.tweetIndex !== undefined) {
+        console.log(CONFIG.tweets[nextPost.tweetIndex]?.content.substring(0, 200) + '...');
+    }
+    console.log(`\nReply template for next reply:`);
+    const nextReply = upcoming.find(a => a.type === 'reply');
+    if (nextReply?.replyTemplate !== undefined) {
+        console.log(CONFIG.replyTemplates[nextReply.replyTemplate]?.substring(0, 200) + '...');
+    }
+    console.log('\n--- Campaign will be executed via ScreenHand MCP tools ---');
+    console.log('Use /loop 5m to keep the campaign runner alive');
+}
+export { CONFIG, SCHEDULE, createInitialState, logAction, randomDelay };