screenhand 0.2.0 → 0.3.1

package/dist/monitor/codex-monitor.js

@@ -1,377 +0,0 @@
-/**
- * Codex Monitor — watches VS Code terminals for AI agent activity
- *
- * Uses the native bridge to:
- * 1. Find VS Code windows + terminal panels via accessibility tree
- * 2. Poll terminal content via OCR (screenshot + vision.ocr)
- * 3. Detect agent status (running/idle/error) via pattern matching
- * 4. Auto-assign queued tasks when a terminal goes idle
- *
- * The monitor runs as a background polling loop, similar to PlaybookRecorder.
- */
-import { TaskQueue } from "./task-queue.js";
-import { DEFAULT_MONITOR_CONFIG } from "./types.js";
-export class CodexMonitor {
-    bridge;
-    terminals = new Map();
-    pollTimer = null;
-    config;
-    running = false;
-    assignTimers = new Map();
-    queue;
-    /** Callback when a terminal status changes */
-    onStatusChange;
-    /** Callback when a task is auto-assigned */
-    onTaskAssigned;
-    /** Callback for log messages */
-    onLog;
-    constructor(bridge, config = {}) {
-        this.bridge = bridge;
-        this.config = { ...DEFAULT_MONITOR_CONFIG, ...config };
-        this.queue = new TaskQueue();
-    }
-    log(msg) {
-        if (this.onLog)
-            this.onLog(msg);
-    }
-    /**
-     * Start monitoring a VS Code terminal.
-     * Finds VS Code by PID, identifies terminal panels, begins polling.
-     */
-    async addTerminal(opts) {
-        const id = "term_" + opts.vscodePid + "_" + Date.now().toString(36);
-        const state = {
-            id,
-            vscodePid: opts.vscodePid,
-            ...(opts.windowId != null ? { windowId: opts.windowId } : {}),
-            terminalLabel: opts.label ?? "Terminal",
-            status: "unknown",
-            lastOutput: "",
-            lastTask: null,
-            startedAt: new Date().toISOString(),
-            lastPollAt: new Date().toISOString(),
-            tasksCompleted: 0,
-            taskHistory: [],
-        };
-        this.terminals.set(id, state);
-        this.log(`Added terminal ${id} (pid=${opts.vscodePid})`);
-        // Do an initial poll
-        await this.pollTerminal(state);
-        return state;
-    }
-    /** Remove a terminal from monitoring */
-    removeTerminal(terminalId) {
-        const timer = this.assignTimers.get(terminalId);
-        if (timer) {
-            clearTimeout(timer);
-            this.assignTimers.delete(terminalId);
-        }
-        return this.terminals.delete(terminalId);
-    }
-    /** Start the polling loop */
-    start() {
-        if (this.running)
-            return;
-        this.running = true;
-        this.pollTimer = setInterval(async () => {
-            if (!this.running)
-                return;
-            for (const terminal of this.terminals.values()) {
-                try {
-                    await this.pollTerminal(terminal);
-                }
-                catch (err) {
-                    this.log(`Poll error for ${terminal.id}: ${err instanceof Error ? err.message : String(err)}`);
-                }
-            }
-        }, this.config.pollIntervalMs);
-        this.log(`Monitor started (poll every ${this.config.pollIntervalMs}ms)`);
-    }
-    /** Stop the polling loop */
-    stop() {
-        this.running = false;
-        if (this.pollTimer) {
-            clearInterval(this.pollTimer);
-            this.pollTimer = null;
-        }
-        for (const timer of this.assignTimers.values()) {
-            clearTimeout(timer);
-        }
-        this.assignTimers.clear();
-        this.log("Monitor stopped");
-    }
-    /** Get all monitored terminals */
-    getTerminals() {
-        return [...this.terminals.values()];
-    }
-    /** Get a specific terminal */
-    getTerminal(id) {
-        return this.terminals.get(id);
-    }
-    get isRunning() {
-        return this.running;
-    }
-    // ── Core polling logic ──
-    async pollTerminal(terminal) {
-        terminal.lastPollAt = new Date().toISOString();
-        // Strategy: use OCR on the VS Code window to read terminal content.
-        // This is more reliable than AX tree for terminal text content.
-        const output = await this.readTerminalContent(terminal);
-        if (output === null)
-            return; // couldn't read
-        const oldStatus = terminal.status;
-        terminal.lastOutput = output;
-        // Detect status from the last few lines of output
-        const lastLines = output.split("\n").slice(-15).join("\n");
-        terminal.status = this.detectStatus(lastLines);
-        // Status transition handling
-        if (oldStatus !== terminal.status) {
-            this.log(`Terminal ${terminal.id}: ${oldStatus} -> ${terminal.status}`);
-            if (this.onStatusChange) {
-                this.onStatusChange(terminal, oldStatus);
-            }
-            // If terminal just went idle, handle task completion + auto-assign
-            if (terminal.status === "idle" && (oldStatus === "running" || oldStatus === "unknown")) {
-                this.handleTerminalIdle(terminal);
-            }
-        }
-    }
-    /**
-     * Read terminal content via screenshot + OCR of the VS Code window.
-     */
-    async readTerminalContent(terminal) {
-        try {
-            let shotPath;
-            if (terminal.windowId) {
-                // Capture specific window
-                const shot = await this.bridge.call("cg.captureWindow", {
-                    windowId: terminal.windowId,
-                });
-                shotPath = shot.path;
-            }
-            else {
-                // Try to find VS Code window
-                const wins = await this.bridge.call("app.windows");
-                const vscodeWin = wins.find((w) => w.pid === terminal.vscodePid || w.bundleId === "com.microsoft.VSCode");
-                if (!vscodeWin) {
-                    this.log(`VS Code window not found for pid=${terminal.vscodePid}`);
-                    return null;
-                }
-                terminal.windowId = vscodeWin.windowId;
-                const shot = await this.bridge.call("cg.captureWindow", {
-                    windowId: vscodeWin.windowId,
-                });
-                shotPath = shot.path;
-            }
-            // OCR the screenshot
-            const ocr = await this.bridge.call("vision.ocr", {
-                imagePath: shotPath,
-            });
-            return ocr.text;
-        }
-        catch (err) {
-            this.log(`OCR failed for ${terminal.id}: ${err instanceof Error ? err.message : String(err)}`);
-            // Fallback: try AX tree to read terminal content
-            return this.readTerminalViaAX(terminal);
-        }
-    }
-    /**
-     * Fallback: read terminal content from accessibility tree.
-     */
-    async readTerminalViaAX(terminal) {
-        try {
-            const tree = await this.bridge.call("ax.getElementTree", {
-                pid: terminal.vscodePid,
-                maxDepth: 6,
-            });
-            // Find terminal text areas in the AX tree
-            const terminalText = this.extractTerminalText(tree);
-            return terminalText;
-        }
-        catch {
-            return null;
-        }
-    }
-    /**
-     * Recursively search AX tree for terminal content.
-     * VS Code terminals usually show up as AXTextArea or AXGroup with role "terminal".
-     */
-    extractTerminalText(node, depth = 0) {
-        if (depth > 8)
-            return null;
-        const role = (node.role || "").toLowerCase();
-        const title = (node.title || "").toLowerCase();
-        const desc = (node.description || "").toLowerCase();
-        // Look for terminal-like elements
-        const isTerminal = role.includes("terminal") ||
-            title.includes("terminal") ||
-            desc.includes("terminal") ||
-            (role === "textarea" && (title.includes("terminal") || desc.includes("terminal")));
-        if (isTerminal && node.value) {
-            return node.value;
-        }
-        if (node.children) {
-            for (const child of node.children) {
-                const found = this.extractTerminalText(child, depth + 1);
-                if (found)
-                    return found;
-            }
-        }
-        return null;
-    }
-    /**
-     * Detect Codex status from terminal output text.
-     */
-    detectStatus(text) {
-        const lower = text.toLowerCase();
-        // Check error patterns first (highest priority)
-        for (const pattern of this.config.errorPatterns) {
-            if (lower.includes(pattern.toLowerCase())) {
-                // Only if it appears in the last few lines
-                const lastLines = text.split("\n").slice(-5).join("\n").toLowerCase();
-                if (lastLines.includes(pattern.toLowerCase())) {
-                    return "error";
-                }
-            }
-        }
-        // Check idle patterns (prompt waiting for input)
-        const lastLines = text.split("\n").filter((l) => l.trim().length > 0);
-        const lastLine = lastLines[lastLines.length - 1] ?? "";
-        const lastLineTrimmed = lastLine.trim();
-        for (const pattern of this.config.idlePatterns) {
-            if (lastLineTrimmed.includes(pattern) || lastLineTrimmed.endsWith(pattern.trim())) {
-                return "idle";
-            }
-        }
-        // Check running patterns
-        for (const pattern of this.config.runningPatterns) {
-            if (lower.includes(pattern.toLowerCase())) {
-                return "running";
-            }
-        }
-        // If we can read content but can't determine status
-        return "unknown";
-    }
-    /**
-     * Handle a terminal going idle — complete current task and maybe assign next.
-     */
-    handleTerminalIdle(terminal) {
-        // Record task completion if there was an active task
-        if (terminal.lastTask) {
-            const completed = {
-                task: terminal.lastTask,
-                startedAt: terminal.taskHistory.length > 0
-                    ? terminal.taskHistory[terminal.taskHistory.length - 1]?.completedAt ?? terminal.startedAt
-                    : terminal.startedAt,
-                completedAt: new Date().toISOString(),
-                output: terminal.lastOutput.split("\n").slice(-20).join("\n"),
-            };
-            terminal.taskHistory.push(completed);
-            terminal.tasksCompleted++;
-            terminal.lastTask = null;
-            // Complete the task in the queue
-            const runningTask = this.queue.all().find((t) => t.status === "running" && t.terminalId === terminal.id);
-            if (runningTask) {
-                this.queue.complete(runningTask.id, completed.output);
-            }
-            this.log(`Terminal ${terminal.id}: task completed (${terminal.tasksCompleted} total)`);
-        }
-        // Auto-assign next task if enabled
-        if (this.config.autoAssign) {
-            // Clear any existing assign timer
-            const existingTimer = this.assignTimers.get(terminal.id);
-            if (existingTimer)
-                clearTimeout(existingTimer);
-            const timer = setTimeout(() => {
-                this.assignTimers.delete(terminal.id);
-                this.tryAssignTask(terminal);
-            }, this.config.assignDelayMs);
-            this.assignTimers.set(terminal.id, timer);
-        }
-    }
-    /**
-     * Try to assign the next queued task to a terminal.
-     */
-    async tryAssignTask(terminal) {
-        // Only assign if still idle
-        if (terminal.status !== "idle")
-            return false;
-        const task = this.queue.next(terminal.id);
-        if (!task) {
-            this.log(`No queued tasks for terminal ${terminal.id}`);
-            return false;
-        }
-        this.queue.assign(task.id, terminal.id);
-        terminal.lastTask = task.prompt;
-        this.log(`Assigning task "${task.prompt.slice(0, 50)}" to terminal ${terminal.id}`);
-        if (this.onTaskAssigned) {
-            this.onTaskAssigned(terminal.id, task);
-        }
-        // Type the task into the terminal
-        try {
-            await this.typeIntoTerminal(terminal, task.prompt);
-            this.queue.markRunning(task.id);
-            terminal.status = "running";
-            return true;
-        }
-        catch (err) {
-            this.log(`Failed to type task: ${err instanceof Error ? err.message : String(err)}`);
-            this.queue.fail(task.id, String(err));
-            return false;
-        }
-    }
-    /**
-     * Type a command into the terminal by focusing VS Code and using keyboard input.
-     */
-    async typeIntoTerminal(terminal, text) {
-        // 1. Focus VS Code
-        await this.bridge.call("app.focus", { bundleId: "com.microsoft.VSCode" });
-        await sleep(300);
-        // 2. If we know the terminal label, try to focus that specific terminal pane
-        // Use accessibility to find and click the terminal
-        try {
-            await this.bridge.call("ax.findElement", {
-                pid: terminal.vscodePid,
-                title: terminal.terminalLabel,
-                exact: false,
-            });
-        }
-        catch {
-            // Terminal pane might already be focused, continue
-        }
-        // 3. Type the command
-        await this.bridge.call("cg.typeText", { text });
-        await sleep(100);
-        // 4. Press Enter to execute
-        await this.bridge.call("cg.keyCombo", { keys: ["enter"] });
-    }
-    /**
-     * Manually assign a task to a terminal (bypasses queue).
-     */
-    async assignDirect(terminalId, prompt) {
-        const terminal = this.terminals.get(terminalId);
-        if (!terminal)
-            return false;
-        terminal.lastTask = prompt;
-        try {
-            await this.typeIntoTerminal(terminal, prompt);
-            terminal.status = "running";
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    /** Update monitor config */
-    updateConfig(config) {
-        this.config = { ...this.config, ...config };
-        // Restart polling if interval changed
-        if (config.pollIntervalMs && this.running) {
-            this.stop();
-            this.start();
-        }
-    }
-}
-function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}

package/dist/monitor/task-queue.js

@@ -1,84 +0,0 @@
-/**
- * Task Queue — manages tasks to be assigned to Codex terminals
- */
-export class TaskQueue {
-    tasks = [];
-    /** Add a task to the queue */
-    enqueue(prompt, options = {}) {
-        const task = {
-            id: "task_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
-            prompt,
-            priority: options.priority ?? 10,
-            terminalId: options.terminalId ?? null,
-            status: "queued",
-            createdAt: new Date().toISOString(),
-            assignedAt: null,
-            completedAt: null,
-            result: null,
-        };
-        this.tasks.push(task);
-        this.tasks.sort((a, b) => a.priority - b.priority);
-        return task;
-    }
-    /** Get the next queued task for a specific terminal (or any terminal) */
-    next(terminalId) {
-        // First try tasks assigned to this specific terminal
-        const specific = this.tasks.find((t) => t.status === "queued" && t.terminalId === terminalId);
-        if (specific)
-            return specific;
-        // Then try unassigned tasks
-        const any = this.tasks.find((t) => t.status === "queued" && t.terminalId === null);
-        return any ?? null;
-    }
-    /** Mark a task as assigned */
-    assign(taskId, terminalId) {
-        const task = this.tasks.find((t) => t.id === taskId);
-        if (task) {
-            task.status = "assigned";
-            task.terminalId = terminalId;
-            task.assignedAt = new Date().toISOString();
-        }
-    }
-    /** Mark a task as running */
-    markRunning(taskId) {
-        const task = this.tasks.find((t) => t.id === taskId);
-        if (task)
-            task.status = "running";
-    }
-    /** Mark a task as completed */
-    complete(taskId, result) {
-        const task = this.tasks.find((t) => t.id === taskId);
-        if (task) {
-            task.status = "completed";
-            task.completedAt = new Date().toISOString();
-            task.result = result;
-        }
-    }
-    /** Mark a task as failed */
-    fail(taskId, result) {
-        const task = this.tasks.find((t) => t.id === taskId);
-        if (task) {
-            task.status = "failed";
-            task.completedAt = new Date().toISOString();
-            task.result = result;
-        }
-    }
-    /** Get all tasks */
-    all() {
-        return [...this.tasks];
-    }
-    /** Get queued tasks count */
-    queuedCount() {
-        return this.tasks.filter((t) => t.status === "queued").length;
-    }
-    /** Remove completed/failed tasks older than given ms */
-    cleanup(olderThanMs = 3600000) {
-        const cutoff = Date.now() - olderThanMs;
-        const before = this.tasks.length;
-        this.tasks = this.tasks.filter((t) => t.status === "queued" ||
-            t.status === "assigned" ||
-            t.status === "running" ||
-            (t.completedAt && new Date(t.completedAt).getTime() > cutoff));
-        return before - this.tasks.length;
-    }
-}

package/dist/monitor/types.js

@@ -1,49 +0,0 @@
-/**
- * Codex Monitor types — track AI coding agents in VS Code terminals
- */
-export const DEFAULT_MONITOR_CONFIG = {
-    pollIntervalMs: 3000,
-    runningPatterns: [
-        "Thinking",
-        "Working",
-        "Generating",
-        "Analyzing",
-        "Reading",
-        "Writing",
-        "Searching",
-        "Running",
-        "Executing",
-        "...",
-        "spinning",
-        "in progress",
-    ],
-    idlePatterns: [
-        "codex>",
-        "Codex>",
-        "> ",
-        "$ ",
-        "Done",
-        "Complete",
-        "Finished",
-        "Task completed",
-        "All done",
-        "ready",
-        "waiting for input",
-        "What would you like",
-        "How can I help",
-    ],
-    errorPatterns: [
-        "Error:",
-        "error:",
-        "FAILED",
-        "failed",
-        "Exception",
-        "Traceback",
-        "panic:",
-        "FATAL",
-        "Cannot",
-        "could not",
-    ],
-    autoAssign: true,
-    assignDelayMs: 2000,
-};

package/dist/native/bridge-client.js

@@ -1,174 +0,0 @@
-import { spawn } from "node:child_process";
-import { EventEmitter } from "node:events";
-import path from "node:path";
-import { createInterface } from "node:readline";
-/**
- * Per-method timeout overrides (ms).
- * Methods not listed here use the default 10s timeout.
- */
-const METHOD_TIMEOUTS = {
-    "app.launch": 30_000,
-    "cg.captureScreen": 15_000,
-    "cg.captureWindow": 15_000,
-    "vision.ocr": 20_000,
-    "vision.findText": 20_000,
-};
-/**
- * Resolves the correct native bridge binary path for the current platform.
- */
-function defaultBinaryPath() {
-    // import.meta.dirname is Node 20+; for Node 18 derive from import.meta.url
-    const base = import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname);
-    if (process.platform === "win32") {
-        return path.resolve(base, "../../native/windows-bridge/bin/Release/net8.0-windows/windows-bridge.exe");
-    }
-    // macOS (default)
-    return path.resolve(base, "../../native/macos-bridge/.build/release/macos-bridge");
-}
-/**
- * Platform-aware native bridge client.
- * Spawns the correct bridge binary (macOS Swift or Windows C#) based on the OS,
- * communicating via the same JSON-RPC-over-stdio protocol.
- *
- * Drop-in replacement for the original MacOSBridgeClient.
- */
-export class BridgeClient extends EventEmitter {
-    process = null;
-    nextId = 1;
-    pending = new Map();
-    binaryPath;
-    restarting = false;
-    started = false;
-    constructor(binaryPath) {
-        super();
-        this.binaryPath = binaryPath ?? defaultBinaryPath();
-    }
-    async start() {
-        if (this.started)
-            return;
-        await this.spawn();
-        this.started = true;
-    }
-    async stop() {
-        this.started = false;
-        if (this.process) {
-            this.process.kill();
-            this.process = null;
-        }
-        // Reject all pending
-        for (const [id, pending] of this.pending) {
-            clearTimeout(pending.timer);
-            pending.reject(new Error("Bridge stopped"));
-            this.pending.delete(id);
-        }
-    }
-    async call(method, params, timeoutMs) {
-        const effectiveTimeout = timeoutMs ?? METHOD_TIMEOUTS[method] ?? 10_000;
-        if (!this.process || this.process.exitCode !== null) {
-            await this.restart();
-        }
-        const id = this.nextId++;
-        const request = { id, method };
-        if (params) {
-            request.params = params;
-        }
-        return new Promise((resolve, reject) => {
-            const timer = setTimeout(() => {
-                this.pending.delete(id);
-                reject(new Error(`Bridge call "${method}" timed out after ${effectiveTimeout}ms`));
-            }, effectiveTimeout);
-            this.pending.set(id, {
-                resolve: resolve,
-                reject,
-                timer,
-            });
-            const line = JSON.stringify(request) + "\n";
-            this.process.stdin.write(line);
-        });
-    }
-    async ping() {
-        return this.call("ping");
-    }
-    async checkPermissions() {
-        return this.call("check_permissions");
-    }
-    async spawn() {
-        const child = spawn(this.binaryPath, [], {
-            stdio: ["pipe", "pipe", "pipe"],
-        });
-        child.on("error", (err) => {
-            this.emit("error", err);
-            if (this.started) {
-                this.restart().catch(() => { });
-            }
-        });
-        child.on("exit", (code) => {
-            this.emit("exit", code);
-            if (this.started && !this.restarting) {
-                this.restart().catch(() => { });
-            }
-        });
-        // Parse stdout line by line
-        const rl = createInterface({ input: child.stdout });
-        rl.on("line", (line) => {
-            this.handleLine(line);
-        });
-        // Log stderr
-        child.stderr?.on("data", (data) => {
-            this.emit("stderr", data.toString());
-        });
-        this.process = child;
-    }
-    handleLine(line) {
-        let response;
-        try {
-            response = JSON.parse(line);
-        }
-        catch {
-            return; // Ignore malformed lines
-        }
-        // Event (streaming notification from observer)
-        if (response.event) {
-            this.emit("ax-event", response.event);
-            return;
-        }
-        // Response to a pending request
-        const pending = this.pending.get(response.id);
-        if (!pending)
-            return;
-        this.pending.delete(response.id);
-        clearTimeout(pending.timer);
-        if (response.error) {
-            pending.reject(new Error(response.error.message));
-        }
-        else {
-            pending.resolve(response.result);
-        }
-    }
-    async restart() {
-        if (this.restarting)
-            return;
-        this.restarting = true;
-        // Reject all pending requests
-        for (const [id, pending] of this.pending) {
-            clearTimeout(pending.timer);
-            pending.reject(new Error("Bridge process crashed, restarting"));
-            this.pending.delete(id);
-        }
-        try {
-            if (this.process) {
-                this.process.kill();
-                this.process = null;
-            }
-            await this.spawn();
-            this.emit("restart");
-        }
-        finally {
-            this.restarting = false;
-        }
-    }
-}
-/**
- * @deprecated Use BridgeClient instead. This alias exists for backward compatibility.
- */
-export const MacOSBridgeClient = BridgeClient;

package/dist/native/macos-bridge-client.js

@@ -1,5 +0,0 @@
-/**
- * @deprecated Import from "./bridge-client.js" instead.
- * This file re-exports for backward compatibility.
- */
-export { BridgeClient, BridgeClient as MacOSBridgeClient } from "./bridge-client.js";