screenhand 0.1.0

package/dist/mcp-entry.js

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+/**
+ * ScreenHand MCP Server — stdio entry point.
+ *
+ * Usage in claude_desktop_config.json or openclaw.json:
+ *
+ *   {
+ *     "mcpServers": {
+ *       "screenhand": {
+ *         "command": "npx",
+ *         "args": ["tsx", "/path/to/screenhand/src/mcp-entry.ts"]
+ *       }
+ *     }
+ *   }
+ *
+ * Environment variables:
+ *   SCREENHAND_ADAPTER  - "accessibility" (default), "composite", "cdp", "placeholder"
+ *   SCREENHAND_HEADLESS - "1" to run browser in headless mode
+ */
+import { PlaceholderAppAdapter } from "./runtime/app-adapter.js";
+import { CdpChromeAdapter } from "./runtime/cdp-chrome-adapter.js";
+import { TimelineLogger } from "./logging/timeline-logger.js";
+import { AutomationRuntimeService } from "./runtime/service.js";
+import { startMcpStdioServer } from "./mcp/mcp-stdio-server.js";
+async function createAdapter() {
+    const adapterType = process.env.SCREENHAND_ADAPTER ?? process.env.AUTOMATOR_ADAPTER ?? "accessibility";
+    switch (adapterType) {
+        case "placeholder":
+            return new PlaceholderAppAdapter();
+        case "cdp":
+            return new CdpChromeAdapter({
+                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
+            });
+        case "composite": {
+            const { BridgeClient } = await import("./native/bridge-client.js");
+            const { CompositeAdapter } = await import("./runtime/composite-adapter.js");
+            const bridge = new BridgeClient();
+            return new CompositeAdapter(bridge, {
+                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
+            });
+        }
+        case "accessibility":
+        default: {
+            const { BridgeClient } = await import("./native/bridge-client.js");
+            const { AccessibilityAdapter } = await import("./runtime/accessibility-adapter.js");
+            const bridge = new BridgeClient();
+            return new AccessibilityAdapter(bridge);
+        }
+    }
+}
+try {
+    const adapter = await createAdapter();
+    const logger = new TimelineLogger();
+    const runtime = new AutomationRuntimeService(adapter, logger);
+    process.stderr.write("ScreenHand MCP server starting...\n");
+    await startMcpStdioServer(runtime);
+    process.stderr.write("ScreenHand MCP server connected.\n");
+}
+catch (e) {
+    process.stderr.write(`ScreenHand startup error: ${e instanceof Error ? e.message : String(e)}\n`);
+    process.exit(1);
+}

package/dist/memory/recall.js

@@ -0,0 +1,160 @@
+/**
+ * Learning Memory — Recall engine (in-memory)
+ *
+ * All searches run against cached data — no disk IO.
+ * Provides fast methods for the interceptor to call on every tool invocation.
+ */
+import { MemoryStore } from "./store.js";
+export class RecallEngine {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    /**
+     * Find strategies matching a task description (~0ms, in-memory).
+     * Strategies with high fail rates are penalized.
+     */
+    recallStrategies(query, limit = 5) {
+        const strategies = this.store.readStrategies();
+        if (strategies.length === 0)
+            return [];
+        const queryTokens = tokenize(query);
+        if (queryTokens.length === 0)
+            return [];
+        const scored = strategies.map((s) => {
+            const targetTokens = new Set([
+                ...tokenize(s.task),
+                ...s.tags,
+                ...s.steps.map((step) => step.tool),
+                ...s.steps.flatMap((step) => Object.values(step.params)
+                    .filter((v) => typeof v === "string")
+                    .flatMap(tokenize)),
+            ]);
+            let matches = 0;
+            for (const qt of queryTokens) {
+                for (const tt of targetTokens) {
+                    if (tt.includes(qt) || qt.includes(tt)) {
+                        matches++;
+                        break;
+                    }
+                }
+            }
+            const relevance = matches / queryTokens.length;
+            const ageMs = Date.now() - new Date(s.lastUsed).getTime();
+            const ageDays = ageMs / (1000 * 60 * 60 * 24);
+            const recency = Math.max(0.5, 1.0 - ageDays / 365);
+            const successBoost = 1 + Math.log2(Math.max(1, s.successCount)) * 0.1;
+            // Penalty for strategies that have failed — reduces score proportionally
+            const failCount = s.failCount ?? 0;
+            const totalAttempts = s.successCount + failCount;
+            const reliabilityPenalty = totalAttempts > 0
+                ? s.successCount / totalAttempts
+                : 1;
+            const score = relevance * recency * successBoost * reliabilityPenalty;
+            return { ...s, score };
+        });
+        return scored
+            .filter((s) => s.score > 0)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    /**
+     * O(1) exact match by tool sequence fingerprint.
+     * Returns the strategy if found and it has a positive reliability score.
+     */
+    recallByFingerprint(tools) {
+        const fp = MemoryStore.makeFingerprint(tools);
+        const strategy = this.store.lookupByFingerprint(fp);
+        if (!strategy)
+            return null;
+        // Skip strategies that fail more than they succeed
+        const failCount = strategy.failCount ?? 0;
+        if (failCount > strategy.successCount)
+            return null;
+        return strategy;
+    }
+    /**
+     * Quick error lookup for a tool — used by interceptor on every call (~0ms).
+     * Returns the most relevant error pattern or null.
+     */
+    quickErrorCheck(tool) {
+        const errors = this.store.readErrors();
+        let best = null;
+        for (const e of errors) {
+            if (e.tool === tool && e.resolution) {
+                if (!best || e.occurrences > best.occurrences)
+                    best = e;
+            }
+        }
+        return best;
+    }
+    /**
+     * Quick strategy hint for a tool sequence — used by interceptor.
+     * Tries fingerprint prefix match first (O(1)), then falls back to scan.
+     * Skips unreliable strategies (failCount > successCount).
+     */
+    quickStrategyHint(recentTools) {
+        if (recentTools.length === 0)
+            return null;
+        const strategies = this.store.readStrategies();
+        for (const s of strategies) {
+            if (s.steps.length <= recentTools.length)
+                continue;
+            // Skip unreliable strategies
+            const failCount = s.failCount ?? 0;
+            if (failCount > s.successCount)
+                continue;
+            const strategyToolPrefix = s.steps.slice(0, recentTools.length).map((st) => st.tool);
+            const matches = recentTools.every((t, i) => t === strategyToolPrefix[i]);
+            if (matches) {
+                return {
+                    strategy: s,
+                    nextStep: s.steps[recentTools.length],
+                    fingerprint: s.fingerprint ?? MemoryStore.makeFingerprint(s.steps.map((st) => st.tool)),
+                };
+            }
+        }
+        return null;
+    }
+    /** Find error patterns for a specific tool or all tools */
+    recallErrors(tool, params) {
+        const errors = this.store.readErrors();
+        if (!tool)
+            return errors;
+        let filtered = errors.filter((e) => e.tool === tool);
+        if (params && filtered.length > 1) {
+            const paramStr = JSON.stringify(params).toLowerCase();
+            filtered.sort((a, b) => {
+                const aScore = stringSimilarity(paramStr, JSON.stringify(a.params).toLowerCase());
+                const bScore = stringSimilarity(paramStr, JSON.stringify(b.params).toLowerCase());
+                return bScore - aScore;
+            });
+        }
+        return filtered;
+    }
+}
+/** Tokenize a string into lowercase keywords (3+ chars) */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/[\W_]+/)
+        .filter((w) => w.length >= 3);
+}
+/** Simple string similarity: shared character bigrams / total bigrams */
+function stringSimilarity(a, b) {
+    if (a === b)
+        return 1;
+    if (a.length < 2 || b.length < 2)
+        return 0;
+    const bigramsA = new Set();
+    for (let i = 0; i < a.length - 1; i++)
+        bigramsA.add(a.slice(i, i + 2));
+    let matches = 0;
+    let total = 0;
+    for (let i = 0; i < b.length - 1; i++) {
+        total++;
+        if (bigramsA.has(b.slice(i, i + 2)))
+            matches++;
+    }
+    return total > 0 ? matches / total : 0;
+}

package/dist/memory/research.js

@@ -0,0 +1,98 @@
+/**
+ * Background web research — fire-and-forget resolution lookup.
+ *
+ * When a tool fails and no resolution exists, this searches for a fix
+ * in the background (non-blocking) and saves it for next time.
+ *
+ * Resolution paths:
+ * 1. Claude API (haiku) — if ANTHROPIC_API_KEY is set
+ * 2. DuckDuckGo instant answer — free fallback, no auth
+ */
+export function backgroundResearch(store, tool, params, errorMessage) {
+    // Fire-and-forget — never blocks, never throws
+    doResearch(store, tool, params, errorMessage).catch(() => { });
+}
+async function doResearch(store, tool, params, errorMessage) {
+    const query = `macOS automation: "${tool}" failed with "${errorMessage.slice(0, 200)}"`;
+    let resolution = null;
+    const apiKey = process.env["ANTHROPIC_API_KEY"];
+    if (apiKey) {
+        resolution = await tryClaudeAPI(apiKey, query);
+    }
+    if (!resolution) {
+        resolution = await tryDuckDuckGo(query);
+    }
+    if (!resolution)
+        return;
+    // Save resolution to error cache
+    store.appendError({
+        id: "err_research_" + Date.now().toString(36),
+        tool,
+        params,
+        error: errorMessage,
+        resolution,
+        occurrences: 1,
+        lastSeen: new Date().toISOString(),
+    });
+    // Save as a reusable strategy
+    store.appendStrategy({
+        id: "str_research_" + Date.now().toString(36),
+        task: `Fix: ${tool} — ${errorMessage.slice(0, 100)}`,
+        steps: [{ tool, params }],
+        totalDurationMs: 0,
+        successCount: 1,
+        failCount: 0,
+        lastUsed: new Date().toISOString(),
+        tags: [tool, "research", "fix"],
+        fingerprint: "",
+    });
+}
+async function tryClaudeAPI(apiKey, query) {
+    try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 10000);
+        const response = await fetch("https://api.anthropic.com/v1/messages", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-api-key": apiKey,
+                "anthropic-version": "2023-06-01",
+            },
+            body: JSON.stringify({
+                model: "claude-haiku-4-5-20251001",
+                max_tokens: 200,
+                messages: [{
+                        role: "user",
+                        content: `You are a macOS automation expert. Give a brief fix (1-2 sentences) for this error:\n\n${query}`,
+                    }],
+            }),
+            signal: controller.signal,
+        });
+        clearTimeout(timeout);
+        if (!response.ok)
+            return null;
+        const data = await response.json();
+        const text = data.content?.[0]?.text;
+        return text && text.length > 10 ? text.trim() : null;
+    }
+    catch {
+        return null;
+    }
+}
+async function tryDuckDuckGo(query) {
+    try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 5000);
+        const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1`;
+        const response = await fetch(url, { signal: controller.signal });
+        clearTimeout(timeout);
+        if (!response.ok)
+            return null;
+        const data = await response.json();
+        const text = data.AbstractText || data.Abstract || data.Answer;
+        return text && text.length > 10 ? text.trim() : null;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/memory/seeds.js

@@ -0,0 +1,89 @@
+/**
+ * Predefined seed strategies — common macOS desktop workflows.
+ * Loaded on first boot so the memory system has knowledge from day one.
+ */
+let seedCounter = 0;
+function makeFingerprint(tools) {
+    return tools.join("→");
+}
+function seed(task, steps, tags) {
+    seedCounter++;
+    return {
+        id: `seed_${String(seedCounter).padStart(3, "0")}`,
+        task,
+        steps,
+        totalDurationMs: 0,
+        successCount: 10,
+        failCount: 0,
+        lastUsed: new Date().toISOString(),
+        tags,
+        fingerprint: makeFingerprint(steps.map((s) => s.tool)),
+    };
+}
+export const SEED_STRATEGIES = [
+    // 1. Take a photo with Photo Booth
+    seed("Take a photo with Photo Booth", [
+        { tool: "launch", params: { bundleId: "com.apple.PhotoBooth" } },
+        { tool: "ui_press", params: { title: "Take Photo" } },
+    ], ["photo", "booth", "camera"]),
+    // 2. Open a URL in Chrome
+    seed("Open a URL in Chrome", [
+        { tool: "launch", params: { bundleId: "com.google.Chrome" } },
+        { tool: "browser_navigate", params: { url: "" } },
+    ], ["chrome", "browse", "url"]),
+    // 3. Save current document
+    seed("Save current document", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+s" } },
+    ], ["save", "document"]),
+    // 4. Copy from one app and paste into another
+    seed("Copy from one app and paste into another", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+c" } },
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+v" } },
+    ], ["copy", "paste"]),
+    // 5. Navigate to a folder in Finder
+    seed("Navigate to a folder in Finder", [
+        { tool: "focus", params: { bundleId: "com.apple.finder" } },
+        { tool: "key", params: { combo: "cmd+shift+g" } },
+        { tool: "type_text", params: { text: "" } },
+    ], ["finder", "folder", "navigate"]),
+    // 6. Create a new folder in Finder
+    seed("Create a new folder in Finder", [
+        { tool: "focus", params: { bundleId: "com.apple.finder" } },
+        { tool: "key", params: { combo: "cmd+shift+n" } },
+        { tool: "type_text", params: { text: "" } },
+    ], ["finder", "folder", "create"]),
+    // 7. Close the current window
+    seed("Close the current window", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+w" } },
+    ], ["close", "window"]),
+    // 8. Select all and copy
+    seed("Select all content and copy", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+a" } },
+        { tool: "key", params: { combo: "cmd+c" } },
+    ], ["select", "all", "copy"]),
+    // 9. List running apps
+    seed("List all running applications", [
+        { tool: "apps", params: {} },
+    ], ["apps", "list", "running"]),
+    // 10. Inspect app UI tree
+    seed("Inspect an app's UI element tree", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "ui_tree", params: { pid: 0 } },
+    ], ["inspect", "tree", "accessibility"]),
+    // 11. Open a new tab in Chrome and navigate
+    seed("Open a new Chrome tab and navigate to URL", [
+        { tool: "focus", params: { bundleId: "com.google.Chrome" } },
+        { tool: "key", params: { combo: "cmd+t" } },
+        { tool: "browser_navigate", params: { url: "" } },
+    ], ["chrome", "tab", "new"]),
+    // 12. Export as PDF via menu
+    seed("Export document as PDF", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "menu_click", params: { menuPath: "File/Export as PDF" } },
+    ], ["export", "pdf"]),
+];

package/dist/memory/session.js

@@ -0,0 +1,161 @@
+/**
+ * Learning Memory — Session tracking with auto-save
+ *
+ * Tracks a rolling buffer of actions within a "task session".
+ * Auto-saves strategies when a successful sequence is detected:
+ * - 3+ consecutive successes followed by a gap (>60s) or session end
+ * - Or explicit endSession() call
+ */
+import { MemoryStore } from "./store.js";
+const SESSION_GAP_MS = 60_000; // 60s gap = new session
+const MAX_BUFFER_SIZE = 100;
+const MIN_AUTO_SAVE_STEPS = 3; // Need at least 3 successful steps to auto-save
+export class SessionTracker {
+    store;
+    sessionId;
+    taskDescription = null;
+    buffer = [];
+    lastActionTime = 0;
+    constructor(store) {
+        this.store = store;
+        this.sessionId = SessionTracker.generateId();
+    }
+    static generateId() {
+        return "s_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
+    }
+    /** Start (or restart) a named task session */
+    startSession(taskDescription) {
+        // Auto-save previous session if it had successful actions
+        this.tryAutoSave();
+        this.sessionId = SessionTracker.generateId();
+        this.taskDescription = taskDescription ?? null;
+        this.buffer = [];
+        this.lastActionTime = Date.now();
+        return this.sessionId;
+    }
+    /** Get the current session ID, auto-rotating if stale */
+    getSessionId() {
+        const now = Date.now();
+        if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
+            // Session gap detected — auto-save previous sequence then start fresh
+            this.tryAutoSave();
+            this.sessionId = SessionTracker.generateId();
+            this.buffer = [];
+            this.taskDescription = null;
+        }
+        return this.sessionId;
+    }
+    /** Record an action into the current session buffer */
+    recordAction(entry) {
+        const now = Date.now();
+        if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
+            // Gap detected — auto-save then start new session
+            this.tryAutoSave();
+            this.sessionId = SessionTracker.generateId();
+            this.buffer = [];
+            this.taskDescription = null;
+        }
+        this.lastActionTime = now;
+        this.buffer.push(entry);
+        if (this.buffer.length > MAX_BUFFER_SIZE) {
+            this.buffer.shift();
+        }
+    }
+    /** End the session and save a strategy if successful */
+    endSession(success, taskDescription) {
+        const task = taskDescription ?? this.taskDescription;
+        if (!success || !task || this.buffer.length === 0) {
+            this.buffer = [];
+            return null;
+        }
+        const strategy = this.buildStrategy(task, this.buffer);
+        this.store.appendStrategy(strategy);
+        this.buffer = [];
+        return strategy;
+    }
+    /** Get the current session's action buffer */
+    getBuffer() {
+        return [...this.buffer];
+    }
+    /** Get recent tool names (for strategy hint matching) */
+    getRecentToolNames(limit = 10) {
+        return this.buffer.slice(-limit).map((a) => a.tool);
+    }
+    /** Get current task description */
+    getTaskDescription() {
+        return this.taskDescription;
+    }
+    // ── auto-save logic ────────────────────────────
+    /**
+     * Try to auto-save the current buffer as a strategy.
+     * Only saves if there are MIN_AUTO_SAVE_STEPS+ consecutive successes.
+     * Uses tool sequence as task description if no explicit one was given.
+     */
+    tryAutoSave() {
+        if (this.buffer.length < MIN_AUTO_SAVE_STEPS)
+            return;
+        // Find the longest trailing streak of successes
+        let successStreak = [];
+        for (let i = this.buffer.length - 1; i >= 0; i--) {
+            if (this.buffer[i].success) {
+                successStreak.unshift(this.buffer[i]);
+            }
+            else {
+                break;
+            }
+        }
+        if (successStreak.length < MIN_AUTO_SAVE_STEPS)
+            return;
+        // Build a task description from the tool sequence if none provided
+        const task = this.taskDescription ?? this.inferTaskDescription(successStreak);
+        const strategy = this.buildStrategy(task, successStreak);
+        this.store.appendStrategy(strategy);
+    }
+    /** Infer a task description from a sequence of actions */
+    inferTaskDescription(actions) {
+        const tools = [...new Set(actions.map((a) => a.tool))];
+        // Extract key param values (bundle IDs, titles, URLs, etc.)
+        const keyParams = [];
+        for (const a of actions) {
+            for (const [key, val] of Object.entries(a.params)) {
+                if (typeof val === "string" && val.length > 2 && val.length < 60) {
+                    if (["bundleId", "title", "url", "text", "script", "selector", "menuPath"].includes(key)) {
+                        keyParams.push(val);
+                    }
+                }
+            }
+        }
+        const paramHint = keyParams.length > 0 ? ` (${keyParams.slice(0, 3).join(", ")})` : "";
+        return `${tools.join(" → ")}${paramHint}`;
+    }
+    buildStrategy(task, actions) {
+        const steps = actions.map((a) => ({
+            tool: a.tool,
+            params: a.params,
+        }));
+        const totalDurationMs = actions.reduce((sum, a) => sum + a.durationMs, 0);
+        const tags = extractTags(task, steps);
+        const fingerprint = MemoryStore.makeFingerprint(steps.map((s) => s.tool));
+        return {
+            id: "str_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+            task,
+            steps,
+            totalDurationMs,
+            successCount: 1,
+            failCount: 0,
+            lastUsed: new Date().toISOString(),
+            tags,
+            fingerprint,
+        };
+    }
+}
+/** Extract tags from task description and tool names */
+function extractTags(task, steps) {
+    const tags = new Set();
+    const words = task.toLowerCase().split(/\W+/).filter((w) => w.length >= 3);
+    for (const w of words)
+        tags.add(w);
+    for (const s of steps)
+        tags.add(s.tool);
+    return [...tags];
+}