screenhand 0.1.1 → 0.2.0

package/dist/runtime/vision-adapter.js

@@ -0,0 +1,208 @@
+const POLL_INTERVAL_MS = 200;
+/**
+ * Vision-based adapter for apps with poor/no accessibility support.
+ * Uses screenshots + OCR to locate elements and CG events to interact.
+ */
+export class VisionAdapter {
+    bridge;
+    sessions = new Map();
+    sessionsByProfile = new Map();
+    constructor(bridge) {
+        this.bridge = bridge;
+    }
+    async attach(profile) {
+        const existing = this.sessionsByProfile.get(profile);
+        if (existing)
+            return existing.info;
+        await this.bridge.start();
+        const frontmost = await this.bridge.call("app.frontmost");
+        const info = {
+            sessionId: `vision_session_${profile}_${Date.now()}`,
+            profile,
+            createdAt: new Date().toISOString(),
+            adapterType: "vision",
+        };
+        const state = {
+            info,
+            pid: frontmost.pid,
+            bundleId: frontmost.bundleId,
+            appName: frontmost.name,
+        };
+        this.sessions.set(info.sessionId, state);
+        this.sessionsByProfile.set(profile, state);
+        return info;
+    }
+    async getAppContext(sessionId) {
+        const state = this.requireSession(sessionId);
+        return {
+            bundleId: state.bundleId,
+            appName: state.appName,
+            pid: state.pid,
+            windowTitle: state.appName,
+        };
+    }
+    async getPageMeta(sessionId) {
+        const ctx = await this.getAppContext(sessionId);
+        return {
+            url: `app://${ctx.bundleId}`,
+            title: ctx.appName,
+        };
+    }
+    async navigate(sessionId, url, _timeoutMs) {
+        if (url.startsWith("app://")) {
+            const bundleId = url.slice(6);
+            const result = await this.bridge.call("app.launch", { bundleId });
+            const state = this.requireSession(sessionId);
+            state.pid = result.pid;
+            state.bundleId = result.bundleId;
+            state.appName = result.appName;
+        }
+        return this.getPageMeta(sessionId);
+    }
+    async locate(sessionId, target, timeoutMs) {
+        const state = this.requireSession(sessionId);
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            // Take a screenshot
+            const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+            state.lastScreenshotPath = screenshotResult.path;
+            const searchText = this.getSearchText(target);
+            if (!searchText) {
+                // For coordinate targets, just return coordinates directly
+                if (target.type === "coordinates") {
+                    return {
+                        handleId: `vision_coords_${target.x}_${target.y}`,
+                        locatorUsed: "vision:coordinates",
+                        coordinates: { x: target.x, y: target.y, width: 1, height: 1 },
+                    };
+                }
+                return null;
+            }
+            // OCR the screenshot
+            const matches = await this.bridge.call("vision.findText", {
+                imagePath: screenshotResult.path,
+                searchText,
+            });
+            if (matches.length > 0) {
+                const best = matches.reduce((a, b) => (a.confidence > b.confidence ? a : b));
+                return {
+                    handleId: `vision_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+                    locatorUsed: `vision:text:${searchText}`,
+                    label: best.text,
+                    coordinates: best.bounds,
+                };
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return null;
+    }
+    async click(_sessionId, element) {
+        if (!element.coordinates) {
+            throw new Error("Vision adapter requires coordinates to click");
+        }
+        const cx = element.coordinates.x + element.coordinates.width / 2;
+        const cy = element.coordinates.y + element.coordinates.height / 2;
+        await this.bridge.call("cg.mouseClick", { x: cx, y: cy });
+    }
+    async setValue(_sessionId, element, text, clear) {
+        // Click to focus
+        await this.click(_sessionId, element);
+        await sleep(100);
+        if (clear) {
+            await this.bridge.call("cg.keyCombo", { keys: ["cmd", "a"] });
+            await sleep(50);
+        }
+        await this.bridge.call("cg.typeText", { text });
+    }
+    async getValue(_sessionId, element) {
+        // Vision can't reliably read values; return label if available
+        return element.label ?? "";
+    }
+    async waitFor(sessionId, condition, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            if (condition.type === "text_appears") {
+                const found = await this.locate(sessionId, { type: "text", value: condition.text }, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_exists") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_gone") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (!found)
+                    return true;
+            }
+            else {
+                // Unsupported condition types
+                return false;
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return false;
+    }
+    async extract(sessionId, _target, format) {
+        const state = this.requireSession(sessionId);
+        // Take a fresh screenshot and OCR it
+        const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+        state.lastScreenshotPath = screenshotResult.path;
+        const ocrResult = await this.bridge.call("vision.ocr", { imagePath: screenshotResult.path });
+        if (format === "text") {
+            return ocrResult.text;
+        }
+        if (format === "json") {
+            return ocrResult;
+        }
+        // table format
+        return {
+            headers: ["text", "confidence", "x", "y", "width", "height"],
+            rows: ocrResult.regions.map((r) => [
+                r.text,
+                r.confidence,
+                r.bounds.x,
+                r.bounds.y,
+                r.bounds.width,
+                r.bounds.height,
+            ]),
+        };
+    }
+    async screenshot(_sessionId, region) {
+        const result = await this.bridge.call("cg.captureScreen", region ? { region } : {});
+        return result.path;
+    }
+    async keyCombo(_sessionId, keys) {
+        await this.bridge.call("cg.keyCombo", { keys });
+    }
+    async elementTree(_sessionId, _maxDepth, _root) {
+        throw new Error("Vision adapter does not support elementTree — use accessibility adapter");
+    }
+    // ── Private ──
+    requireSession(sessionId) {
+        const state = this.sessions.get(sessionId);
+        if (!state)
+            throw new Error(`Session not found: ${sessionId}`);
+        return state;
+    }
+    getSearchText(target) {
+        switch (target.type) {
+            case "text":
+                return target.value;
+            case "role":
+                return target.name;
+            case "selector":
+                return target.value;
+            case "ax_attribute":
+                return target.value;
+            case "image":
+            case "coordinates":
+            case "ax_path":
+                return null;
+        }
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/scripts/codex-monitor-daemon.js

@@ -0,0 +1,335 @@
+#!/usr/bin/env npx tsx
+/**
+ * Codex Monitor Daemon — runs as a standalone background process.
+ *
+ * Survives Claude Code restarts. Writes state to ~/.screenhand/monitor/
+ * so MCP tools can read status and enqueue tasks via the filesystem.
+ *
+ * Usage:
+ *   npx tsx scripts/codex-monitor-daemon.ts --pid 56966
+ *   npx tsx scripts/codex-monitor-daemon.ts --pid 56966 --window 5015 --poll 5000
+ *
+ * State files:
+ *   ~/.screenhand/monitor/state.json    — terminal states + status
+ *   ~/.screenhand/monitor/tasks.json    — task queue
+ *   ~/.screenhand/monitor/daemon.pid    — PID of this process
+ *   ~/.screenhand/monitor/daemon.log    — log output
+ */
+import path from "node:path";
+import fs from "node:fs";
+import os from "node:os";
+import { BridgeClient } from "../src/native/bridge-client.js";
+// ── Config from CLI args ──
+const args = process.argv.slice(2);
+function getArg(name, fallback) {
+    const idx = args.indexOf("--" + name);
+    if (idx === -1)
+        return fallback;
+    return args[idx + 1] ?? fallback;
+}
+const VSCODE_PID = Number(getArg("pid", "0"));
+const WINDOW_ID = getArg("window") ? Number(getArg("window")) : undefined;
+const POLL_MS = Number(getArg("poll", "3000"));
+const LABEL = getArg("label", "Terminal") ?? "Terminal";
+const AUTO_ASSIGN = getArg("no-auto-assign") === undefined;
+if (!VSCODE_PID) {
+    console.error("Usage: codex-monitor-daemon.ts --pid <vscode_pid> [--window <id>] [--poll <ms>]");
+    process.exit(1);
+}
+// ── State directory ──
+const STATE_DIR = path.join(os.homedir(), ".screenhand", "monitor");
+const STATE_FILE = path.join(STATE_DIR, "state.json");
+const TASKS_FILE = path.join(STATE_DIR, "tasks.json");
+const PID_FILE = path.join(STATE_DIR, "daemon.pid");
+const LOG_FILE = path.join(STATE_DIR, "daemon.log");
+fs.mkdirSync(STATE_DIR, { recursive: true });
+// ── Logging ──
+const logStream = fs.createWriteStream(LOG_FILE, { flags: "a" });
+function log(msg) {
+    const line = `[${new Date().toISOString()}] ${msg}`;
+    logStream.write(line + "\n");
+    // Also write to stderr for initial debugging
+    if (!daemonized)
+        process.stderr.write(line + "\n");
+}
+let daemonized = false;
+// ── Status detection patterns ──
+const RUNNING_PATTERNS = [
+    "thinking", "working", "generating", "analyzing", "reading",
+    "writing", "searching", "running", "executing", "...",
+    "in progress",
+];
+const IDLE_PATTERNS = [
+    "codex>", "> ", "$ ", "done", "complete", "finished",
+    "task completed", "all done", "ready", "waiting for input",
+    "what would you like", "how can i help",
+];
+const ERROR_PATTERNS = [
+    "error:", "failed", "exception", "traceback", "panic:",
+    "fatal", "cannot", "could not",
+];
+// ── Bridge setup ──
+const scriptDir = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
+const projectRoot = path.resolve(scriptDir, "..");
+const bridgePath = process.platform === "win32"
+    ? path.resolve(projectRoot, "native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe")
+    : path.resolve(projectRoot, "native/macos-bridge/.build/release/macos-bridge");
+const bridge = new BridgeClient(bridgePath);
+// ── State ──
+let resolvedWindowId = WINDOW_ID;
+const terminal = {
+    id: `term_${VSCODE_PID}_${Date.now().toString(36)}`,
+    vscodePid: VSCODE_PID,
+    ...(WINDOW_ID != null ? { windowId: WINDOW_ID } : {}),
+    label: LABEL,
+    status: "unknown",
+    lastOutput: "",
+    lastTask: null,
+    lastPollAt: new Date().toISOString(),
+    tasksCompleted: 0,
+    taskHistory: [],
+};
+let stopped = false;
+// ── Filesystem I/O ──
+function writeState() {
+    const state = {
+        pid: process.pid,
+        startedAt: new Date().toISOString(),
+        terminals: [terminal],
+        running: !stopped,
+    };
+    fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
+}
+function readTasks() {
+    try {
+        if (!fs.existsSync(TASKS_FILE))
+            return [];
+        return JSON.parse(fs.readFileSync(TASKS_FILE, "utf-8"));
+    }
+    catch {
+        return [];
+    }
+}
+function writeTasks(tasks) {
+    fs.writeFileSync(TASKS_FILE, JSON.stringify(tasks, null, 2));
+}
+// ── Detection logic ──
+function detectStatus(text) {
+    const lower = text.toLowerCase();
+    const lines = text.split("\n").filter((l) => l.trim().length > 0);
+    const lastLine = (lines[lines.length - 1] ?? "").trim().toLowerCase();
+    // Check errors in last 5 lines
+    const lastFew = lines.slice(-5).join("\n").toLowerCase();
+    for (const p of ERROR_PATTERNS) {
+        if (lastFew.includes(p))
+            return "error";
+    }
+    // Check idle (prompt on last line)
+    for (const p of IDLE_PATTERNS) {
+        if (lastLine.includes(p) || lastLine.endsWith(p.trim()))
+            return "idle";
+    }
+    // Check running
+    for (const p of RUNNING_PATTERNS) {
+        if (lower.includes(p))
+            return "running";
+    }
+    return "unknown";
+}
+// ── Terminal reading ──
+async function readTerminalContent() {
+    try {
+        // Find window if not set
+        if (!resolvedWindowId) {
+            const wins = await bridge.call("app.windows");
+            // Find VS Code windows, pick the largest one (skip title bar / small panels)
+            const vscodeWins = wins.filter((w) => w.pid === VSCODE_PID || w.bundleId === "com.microsoft.VSCode");
+            if (vscodeWins.length === 0) {
+                log("VS Code window not found");
+                return null;
+            }
+            const largest = vscodeWins.reduce((a, b) => {
+                const areaA = (a.bounds?.width ?? 0) * (a.bounds?.height ?? 0);
+                const areaB = (b.bounds?.width ?? 0) * (b.bounds?.height ?? 0);
+                return areaA >= areaB ? a : b;
+            });
+            resolvedWindowId = largest.windowId;
+            if (resolvedWindowId !== undefined) {
+                terminal.windowId = resolvedWindowId;
+            }
+            log(`Auto-detected window ${resolvedWindowId} (${largest.bounds?.width}x${largest.bounds?.height})`);
+        }
+        // Screenshot + OCR
+        const shot = await bridge.call("cg.captureWindow", {
+            windowId: resolvedWindowId,
+        });
+        const ocr = await bridge.call("vision.ocr", {
+            imagePath: shot.path,
+        });
+        return ocr.text;
+    }
+    catch (err) {
+        log(`OCR failed: ${err instanceof Error ? err.message : String(err)}`);
+        // Fallback: AX tree
+        try {
+            const tree = await bridge.call("ax.getElementTree", {
+                pid: VSCODE_PID,
+                maxDepth: 6,
+            });
+            return extractTerminalText(tree);
+        }
+        catch {
+            return null;
+        }
+    }
+}
+function extractTerminalText(node, depth = 0) {
+    if (depth > 8)
+        return null;
+    const role = (node.role || "").toLowerCase();
+    const title = (node.title || "").toLowerCase();
+    if ((role.includes("terminal") || title.includes("terminal")) &&
+        node.value) {
+        return node.value;
+    }
+    if (node.children) {
+        for (const child of node.children) {
+            const found = extractTerminalText(child, depth + 1);
+            if (found)
+                return found;
+        }
+    }
+    return null;
+}
+// ── Task assignment ──
+async function typeIntoTerminal(text) {
+    try {
+        await bridge.call("app.focus", { bundleId: "com.microsoft.VSCode" });
+        await sleep(300);
+        await bridge.call("cg.typeText", { text });
+        await sleep(100);
+        await bridge.call("cg.keyCombo", { keys: ["enter"] });
+        return true;
+    }
+    catch (err) {
+        log(`Failed to type: ${err instanceof Error ? err.message : String(err)}`);
+        return false;
+    }
+}
+async function tryAssignTask() {
+    if (!AUTO_ASSIGN || terminal.status !== "idle")
+        return;
+    const tasks = readTasks();
+    const next = tasks.find((t) => t.status === "queued" && (t.terminalId === null || t.terminalId === terminal.id));
+    if (!next)
+        return;
+    log(`Assigning task: "${next.prompt.slice(0, 60)}"`);
+    next.status = "assigned";
+    next.terminalId = terminal.id;
+    next.assignedAt = new Date().toISOString();
+    writeTasks(tasks);
+    terminal.lastTask = next.prompt;
+    const success = await typeIntoTerminal(next.prompt);
+    if (success) {
+        next.status = "running";
+        terminal.status = "running";
+        log(`Task running: ${next.id}`);
+    }
+    else {
+        next.status = "failed";
+        next.result = "Failed to type into terminal";
+        log(`Task failed to assign: ${next.id}`);
+    }
+    writeTasks(tasks);
+}
+function handleIdleTransition() {
+    // Complete current task
+    if (terminal.lastTask) {
+        terminal.taskHistory.push({
+            task: terminal.lastTask,
+            completedAt: new Date().toISOString(),
+        });
+        terminal.tasksCompleted++;
+        // Update task in queue
+        const tasks = readTasks();
+        const running = tasks.find((t) => t.status === "running" && t.terminalId === terminal.id);
+        if (running) {
+            running.status = "completed";
+            running.completedAt = new Date().toISOString();
+            running.result = terminal.lastOutput.split("\n").slice(-20).join("\n");
+            writeTasks(tasks);
+        }
+        terminal.lastTask = null;
+        log(`Task completed (${terminal.tasksCompleted} total)`);
+    }
+    // Delay then try assign
+    setTimeout(() => tryAssignTask(), 2000);
+}
+// ── Main loop ──
+async function poll() {
+    const output = await readTerminalContent();
+    if (output === null)
+        return;
+    const oldStatus = terminal.status;
+    terminal.lastOutput = output;
+    terminal.lastPollAt = new Date().toISOString();
+    const lastLines = output.split("\n").slice(-15).join("\n");
+    terminal.status = detectStatus(lastLines);
+    if (oldStatus !== terminal.status) {
+        log(`Status: ${oldStatus} -> ${terminal.status}`);
+        if (terminal.status === "idle" && (oldStatus === "running" || oldStatus === "unknown")) {
+            handleIdleTransition();
+        }
+    }
+    writeState();
+}
+async function main() {
+    // Write PID file
+    fs.writeFileSync(PID_FILE, String(process.pid));
+    // Initialize tasks file if missing
+    if (!fs.existsSync(TASKS_FILE)) {
+        writeTasks([]);
+    }
+    log(`Daemon started (pid=${process.pid})`);
+    log(`Watching VS Code pid=${VSCODE_PID} window=${WINDOW_ID ?? "auto"} poll=${POLL_MS}ms`);
+    await bridge.start();
+    log("Bridge started");
+    writeState();
+    // Poll loop
+    while (!stopped) {
+        try {
+            await poll();
+        }
+        catch (err) {
+            log(`Poll error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        await sleep(POLL_MS);
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+// ── Graceful shutdown ──
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);
+async function shutdown() {
+    if (stopped)
+        return;
+    stopped = true;
+    log("Shutting down...");
+    writeState();
+    try {
+        fs.unlinkSync(PID_FILE);
+    }
+    catch { /* ignore */ }
+    try {
+        await bridge.stop();
+    }
+    catch { /* ignore */ }
+    logStream.end();
+    process.exit(0);
+}
+main().catch((err) => {
+    log(`Fatal: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+});