screenhand 0.1.1 → 0.3.0

package/dist/src/monitor/codex-monitor.js

@@ -0,0 +1,382 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { TaskQueue } from "./task-queue.js";
+import { DEFAULT_MONITOR_CONFIG } from "./types.js";
+export class CodexMonitor {
+    bridge;
+    terminals = new Map();
+    pollTimer = null;
+    config;
+    running = false;
+    assignTimers = new Map();
+    queue;
+    /** Callback when a terminal status changes */
+    onStatusChange;
+    /** Callback when a task is auto-assigned */
+    onTaskAssigned;
+    /** Callback for log messages */
+    onLog;
+    constructor(bridge, config = {}) {
+        this.bridge = bridge;
+        this.config = { ...DEFAULT_MONITOR_CONFIG, ...config };
+        this.queue = new TaskQueue();
+    }
+    log(msg) {
+        if (this.onLog)
+            this.onLog(msg);
+    }
+    /**
+     * Start monitoring a VS Code terminal.
+     * Finds VS Code by PID, identifies terminal panels, begins polling.
+     */
+    async addTerminal(opts) {
+        const id = "term_" + opts.vscodePid + "_" + Date.now().toString(36);
+        const state = {
+            id,
+            vscodePid: opts.vscodePid,
+            ...(opts.windowId != null ? { windowId: opts.windowId } : {}),
+            terminalLabel: opts.label ?? "Terminal",
+            status: "unknown",
+            lastOutput: "",
+            lastTask: null,
+            startedAt: new Date().toISOString(),
+            lastPollAt: new Date().toISOString(),
+            tasksCompleted: 0,
+            taskHistory: [],
+        };
+        this.terminals.set(id, state);
+        this.log(`Added terminal ${id} (pid=${opts.vscodePid})`);
+        // Do an initial poll
+        await this.pollTerminal(state);
+        return state;
+    }
+    /** Remove a terminal from monitoring */
+    removeTerminal(terminalId) {
+        const timer = this.assignTimers.get(terminalId);
+        if (timer) {
+            clearTimeout(timer);
+            this.assignTimers.delete(terminalId);
+        }
+        return this.terminals.delete(terminalId);
+    }
+    /** Start the polling loop */
+    start() {
+        if (this.running)
+            return;
+        this.running = true;
+        this.pollTimer = setInterval(async () => {
+            if (!this.running)
+                return;
+            for (const terminal of this.terminals.values()) {
+                try {
+                    await this.pollTerminal(terminal);
+                }
+                catch (err) {
+                    this.log(`Poll error for ${terminal.id}: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+        }, this.config.pollIntervalMs);
+        this.log(`Monitor started (poll every ${this.config.pollIntervalMs}ms)`);
+    }
+    /** Stop the polling loop */
+    stop() {
+        this.running = false;
+        if (this.pollTimer) {
+            clearInterval(this.pollTimer);
+            this.pollTimer = null;
+        }
+        for (const timer of this.assignTimers.values()) {
+            clearTimeout(timer);
+        }
+        this.assignTimers.clear();
+        this.log("Monitor stopped");
+    }
+    /** Get all monitored terminals */
+    getTerminals() {
+        return [...this.terminals.values()];
+    }
+    /** Get a specific terminal */
+    getTerminal(id) {
+        return this.terminals.get(id);
+    }
+    get isRunning() {
+        return this.running;
+    }
+    // ── Core polling logic ──
+    async pollTerminal(terminal) {
+        terminal.lastPollAt = new Date().toISOString();
+        // Strategy: use OCR on the VS Code window to read terminal content.
+        // This is more reliable than AX tree for terminal text content.
+        const output = await this.readTerminalContent(terminal);
+        if (output === null)
+            return; // couldn't read
+        const oldStatus = terminal.status;
+        terminal.lastOutput = output;
+        // Detect status from the last few lines of output
+        const lastLines = output.split("\n").slice(-15).join("\n");
+        terminal.status = this.detectStatus(lastLines);
+        // Status transition handling
+        if (oldStatus !== terminal.status) {
+            this.log(`Terminal ${terminal.id}: ${oldStatus} -> ${terminal.status}`);
+            if (this.onStatusChange) {
+                this.onStatusChange(terminal, oldStatus);
+            }
+            // If terminal just went idle, handle task completion + auto-assign
+            if (terminal.status === "idle" && (oldStatus === "running" || oldStatus === "unknown")) {
+                this.handleTerminalIdle(terminal);
+            }
+        }
+    }
+    /**
+     * Read terminal content via screenshot + OCR of the VS Code window.
+     */
+    async readTerminalContent(terminal) {
+        try {
+            let shotPath;
+            if (terminal.windowId) {
+                // Capture specific window
+                const shot = await this.bridge.call("cg.captureWindow", {
+                    windowId: terminal.windowId,
+                });
+                shotPath = shot.path;
+            }
+            else {
+                // Try to find VS Code window
+                const wins = await this.bridge.call("app.windows");
+                const vscodeWin = wins.find((w) => w.pid === terminal.vscodePid || w.bundleId === "com.microsoft.VSCode");
+                if (!vscodeWin) {
+                    this.log(`VS Code window not found for pid=${terminal.vscodePid}`);
+                    return null;
+                }
+                terminal.windowId = vscodeWin.windowId;
+                const shot = await this.bridge.call("cg.captureWindow", {
+                    windowId: vscodeWin.windowId,
+                });
+                shotPath = shot.path;
+            }
+            // OCR the screenshot
+            const ocr = await this.bridge.call("vision.ocr", {
+                imagePath: shotPath,
+            });
+            return ocr.text;
+        }
+        catch (err) {
+            this.log(`OCR failed for ${terminal.id}: ${err instanceof Error ? err.message : String(err)}`);
+            // Fallback: try AX tree to read terminal content
+            return this.readTerminalViaAX(terminal);
+        }
+    }
+    /**
+     * Fallback: read terminal content from accessibility tree.
+     */
+    async readTerminalViaAX(terminal) {
+        try {
+            const tree = await this.bridge.call("ax.getElementTree", {
+                pid: terminal.vscodePid,
+                maxDepth: 6,
+            });
+            // Find terminal text areas in the AX tree
+            const terminalText = this.extractTerminalText(tree);
+            return terminalText;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Recursively search AX tree for terminal content.
+     * VS Code terminals usually show up as AXTextArea or AXGroup with role "terminal".
+     */
+    extractTerminalText(node, depth = 0) {
+        if (depth > 8)
+            return null;
+        const role = (node.role || "").toLowerCase();
+        const title = (node.title || "").toLowerCase();
+        const desc = (node.description || "").toLowerCase();
+        // Look for terminal-like elements
+        const isTerminal = role.includes("terminal") ||
+            title.includes("terminal") ||
+            desc.includes("terminal") ||
+            (role === "textarea" && (title.includes("terminal") || desc.includes("terminal")));
+        if (isTerminal && node.value) {
+            return node.value;
+        }
+        if (node.children) {
+            for (const child of node.children) {
+                const found = this.extractTerminalText(child, depth + 1);
+                if (found)
+                    return found;
+            }
+        }
+        return null;
+    }
+    /**
+     * Detect Codex status from terminal output text.
+     */
+    detectStatus(text) {
+        const lower = text.toLowerCase();
+        // Check error patterns first (highest priority)
+        for (const pattern of this.config.errorPatterns) {
+            if (lower.includes(pattern.toLowerCase())) {
+                // Only if it appears in the last few lines
+                const lastLines = text.split("\n").slice(-5).join("\n").toLowerCase();
+                if (lastLines.includes(pattern.toLowerCase())) {
+                    return "error";
+                }
+            }
+        }
+        // Check idle patterns (prompt waiting for input)
+        const lastLines = text.split("\n").filter((l) => l.trim().length > 0);
+        const lastLine = lastLines[lastLines.length - 1] ?? "";
+        const lastLineTrimmed = lastLine.trim();
+        for (const pattern of this.config.idlePatterns) {
+            if (lastLineTrimmed.includes(pattern) || lastLineTrimmed.endsWith(pattern.trim())) {
+                return "idle";
+            }
+        }
+        // Check running patterns
+        for (const pattern of this.config.runningPatterns) {
+            if (lower.includes(pattern.toLowerCase())) {
+                return "running";
+            }
+        }
+        // If we can read content but can't determine status
+        return "unknown";
+    }
+    /**
+     * Handle a terminal going idle — complete current task and maybe assign next.
+     */
+    handleTerminalIdle(terminal) {
+        // Record task completion if there was an active task
+        if (terminal.lastTask) {
+            const completed = {
+                task: terminal.lastTask,
+                startedAt: terminal.taskHistory.length > 0
+                    ? terminal.taskHistory[terminal.taskHistory.length - 1]?.completedAt ?? terminal.startedAt
+                    : terminal.startedAt,
+                completedAt: new Date().toISOString(),
+                output: terminal.lastOutput.split("\n").slice(-20).join("\n"),
+            };
+            terminal.taskHistory.push(completed);
+            terminal.tasksCompleted++;
+            terminal.lastTask = null;
+            // Complete the task in the queue
+            const runningTask = this.queue.all().find((t) => t.status === "running" && t.terminalId === terminal.id);
+            if (runningTask) {
+                this.queue.complete(runningTask.id, completed.output);
+            }
+            this.log(`Terminal ${terminal.id}: task completed (${terminal.tasksCompleted} total)`);
+        }
+        // Auto-assign next task if enabled
+        if (this.config.autoAssign) {
+            // Clear any existing assign timer
+            const existingTimer = this.assignTimers.get(terminal.id);
+            if (existingTimer)
+                clearTimeout(existingTimer);
+            const timer = setTimeout(() => {
+                this.assignTimers.delete(terminal.id);
+                this.tryAssignTask(terminal);
+            }, this.config.assignDelayMs);
+            this.assignTimers.set(terminal.id, timer);
+        }
+    }
+    /**
+     * Try to assign the next queued task to a terminal.
+     */
+    async tryAssignTask(terminal) {
+        // Only assign if still idle
+        if (terminal.status !== "idle")
+            return false;
+        const task = this.queue.next(terminal.id);
+        if (!task) {
+            this.log(`No queued tasks for terminal ${terminal.id}`);
+            return false;
+        }
+        this.queue.assign(task.id, terminal.id);
+        terminal.lastTask = task.prompt;
+        this.log(`Assigning task "${task.prompt.slice(0, 50)}" to terminal ${terminal.id}`);
+        if (this.onTaskAssigned) {
+            this.onTaskAssigned(terminal.id, task);
+        }
+        // Type the task into the terminal
+        try {
+            await this.typeIntoTerminal(terminal, task.prompt);
+            this.queue.markRunning(task.id);
+            terminal.status = "running";
+            return true;
+        }
+        catch (err) {
+            this.log(`Failed to type task: ${err instanceof Error ? err.message : String(err)}`);
+            this.queue.fail(task.id, String(err));
+            return false;
+        }
+    }
+    /**
+     * Type a command into the terminal by focusing VS Code and using keyboard input.
+     */
+    async typeIntoTerminal(terminal, text) {
+        // 1. Focus VS Code
+        await this.bridge.call("app.focus", { bundleId: "com.microsoft.VSCode" });
+        await sleep(300);
+        // 2. If we know the terminal label, try to focus that specific terminal pane
+        // Use accessibility to find and click the terminal
+        try {
+            await this.bridge.call("ax.findElement", {
+                pid: terminal.vscodePid,
+                title: terminal.terminalLabel,
+                exact: false,
+            });
+        }
+        catch {
+            // Terminal pane might already be focused, continue
+        }
+        // 3. Type the command
+        await this.bridge.call("cg.typeText", { text });
+        await sleep(100);
+        // 4. Press Enter to execute
+        await this.bridge.call("cg.keyCombo", { keys: ["enter"] });
+    }
+    /**
+     * Manually assign a task to a terminal (bypasses queue).
+     */
+    async assignDirect(terminalId, prompt) {
+        const terminal = this.terminals.get(terminalId);
+        if (!terminal)
+            return false;
+        terminal.lastTask = prompt;
+        try {
+            await this.typeIntoTerminal(terminal, prompt);
+            terminal.status = "running";
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /** Update monitor config */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+        // Restart polling if interval changed
+        if (config.pollIntervalMs && this.running) {
+            this.stop();
+            this.start();
+        }
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/src/monitor/task-queue.js

@@ -0,0 +1,97 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class TaskQueue {
+    tasks = [];
+    /** Add a task to the queue */
+    enqueue(prompt, options = {}) {
+        const task = {
+            id: "task_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+            prompt,
+            priority: options.priority ?? 10,
+            terminalId: options.terminalId ?? null,
+            status: "queued",
+            createdAt: new Date().toISOString(),
+            assignedAt: null,
+            completedAt: null,
+            result: null,
+        };
+        this.tasks.push(task);
+        this.tasks.sort((a, b) => a.priority - b.priority);
+        return task;
+    }
+    /** Get the next queued task for a specific terminal (or any terminal) */
+    next(terminalId) {
+        // First try tasks assigned to this specific terminal
+        const specific = this.tasks.find((t) => t.status === "queued" && t.terminalId === terminalId);
+        if (specific)
+            return specific;
+        // Then try unassigned tasks
+        const any = this.tasks.find((t) => t.status === "queued" && t.terminalId === null);
+        return any ?? null;
+    }
+    /** Mark a task as assigned */
+    assign(taskId, terminalId) {
+        const task = this.tasks.find((t) => t.id === taskId);
+        if (task) {
+            task.status = "assigned";
+            task.terminalId = terminalId;
+            task.assignedAt = new Date().toISOString();
+        }
+    }
+    /** Mark a task as running */
+    markRunning(taskId) {
+        const task = this.tasks.find((t) => t.id === taskId);
+        if (task)
+            task.status = "running";
+    }
+    /** Mark a task as completed */
+    complete(taskId, result) {
+        const task = this.tasks.find((t) => t.id === taskId);
+        if (task) {
+            task.status = "completed";
+            task.completedAt = new Date().toISOString();
+            task.result = result;
+        }
+    }
+    /** Mark a task as failed */
+    fail(taskId, result) {
+        const task = this.tasks.find((t) => t.id === taskId);
+        if (task) {
+            task.status = "failed";
+            task.completedAt = new Date().toISOString();
+            task.result = result;
+        }
+    }
+    /** Get all tasks */
+    all() {
+        return [...this.tasks];
+    }
+    /** Get queued tasks count */
+    queuedCount() {
+        return this.tasks.filter((t) => t.status === "queued").length;
+    }
+    /** Remove completed/failed tasks older than given ms */
+    cleanup(olderThanMs = 3600000) {
+        const cutoff = Date.now() - olderThanMs;
+        const before = this.tasks.length;
+        this.tasks = this.tasks.filter((t) => t.status === "queued" ||
+            t.status === "assigned" ||
+            t.status === "running" ||
+            (t.completedAt && new Date(t.completedAt).getTime() > cutoff));
+        return before - this.tasks.length;
+    }
+}

package/dist/src/monitor/types.js

@@ -0,0 +1,62 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export const DEFAULT_MONITOR_CONFIG = {
+    pollIntervalMs: 3000,
+    runningPatterns: [
+        "Thinking",
+        "Working",
+        "Generating",
+        "Analyzing",
+        "Reading",
+        "Writing",
+        "Searching",
+        "Running",
+        "Executing",
+        "...",
+        "spinning",
+        "in progress",
+    ],
+    idlePatterns: [
+        "codex>",
+        "Codex>",
+        "> ",
+        "$ ",
+        "Done",
+        "Complete",
+        "Finished",
+        "Task completed",
+        "All done",
+        "ready",
+        "waiting for input",
+        "What would you like",
+        "How can I help",
+    ],
+    errorPatterns: [
+        "Error:",
+        "error:",
+        "FAILED",
+        "failed",
+        "Exception",
+        "Traceback",
+        "panic:",
+        "FATAL",
+        "Cannot",
+        "could not",
+    ],
+    autoAssign: true,
+    assignDelayMs: 2000,
+};