screenhand 0.2.0 → 0.3.1

package/dist/mcp/server.js

@@ -1,347 +0,0 @@
-export class MvpMcpServer {
-    runtime;
-    constructor(runtime) {
-        this.runtime = runtime;
-    }
-    async invoke(request) {
-        switch (request.tool) {
-            case "session_start":
-                return this.runtime.sessionStart(optionalString(request.args, "profile"));
-            case "navigate": {
-                const timeoutMs = optionalNumber(request.args, "timeoutMs");
-                const input = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                    url: requiredString(request.args, "url"),
-                };
-                if (typeof timeoutMs === "number") {
-                    input.timeoutMs = timeoutMs;
-                }
-                return this.runtime.navigate(input);
-            }
-            case "press": {
-                const verify = parseOptionalWaitCondition(request.args.verify);
-                const input = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                    target: parseTarget(request.args.target),
-                };
-                if (verify) {
-                    input.verify = verify;
-                }
-                return this.runtime.press(input);
-            }
-            case "type_into": {
-                const clear = optionalBoolean(request.args, "clear");
-                const verify = parseOptionalWaitCondition(request.args.verify);
-                const input = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                    target: parseTarget(request.args.target),
-                    text: requiredString(request.args, "text"),
-                };
-                if (typeof clear === "boolean") {
-                    input.clear = clear;
-                }
-                if (verify) {
-                    input.verify = verify;
-                }
-                return this.runtime.typeInto(input);
-            }
-            case "wait_for": {
-                const timeoutMs = optionalNumber(request.args, "timeoutMs");
-                const input = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                    condition: parseWaitCondition(request.args.condition),
-                };
-                if (typeof timeoutMs === "number") {
-                    input.timeoutMs = timeoutMs;
-                }
-                return this.runtime.waitFor(input);
-            }
-            case "extract":
-                return this.runtime.extract({
-                    sessionId: requiredString(request.args, "sessionId"),
-                    target: parseTarget(request.args.target),
-                    format: parseExtractFormat(request.args.format),
-                });
-            case "screenshot": {
-                const region = parseOptionalRegion(request.args.region);
-                const input = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                };
-                if (region) {
-                    input.region = region;
-                }
-                return this.runtime.screenshot(input);
-            }
-            // ── Desktop automation tools ──
-            case "app_launch":
-                return this.runtime.appLaunch({
-                    sessionId: requiredString(request.args, "sessionId"),
-                    bundleId: requiredString(request.args, "bundleId"),
-                });
-            case "app_focus":
-                return this.runtime.appFocus({
-                    sessionId: requiredString(request.args, "sessionId"),
-                    bundleId: requiredString(request.args, "bundleId"),
-                });
-            case "app_list":
-                return this.runtime.appList(requiredString(request.args, "sessionId"));
-            case "window_list":
-                return this.runtime.windowList(requiredString(request.args, "sessionId"));
-            case "menu_click":
-                return this.runtime.menuClick({
-                    sessionId: requiredString(request.args, "sessionId"),
-                    menuPath: requiredStringArray(request.args, "menuPath"),
-                });
-            case "key_combo":
-                return this.runtime.keyCombo({
-                    sessionId: requiredString(request.args, "sessionId"),
-                    keys: requiredStringArray(request.args, "keys"),
-                });
-            case "element_tree": {
-                const maxDepth = optionalNumber(request.args, "maxDepth");
-                const root = request.args.root ? parseTarget(request.args.root) : undefined;
-                const etInput = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                };
-                if (typeof maxDepth === "number")
-                    etInput.maxDepth = maxDepth;
-                if (root)
-                    etInput.root = root;
-                return this.runtime.elementTree(etInput);
-            }
-            case "observe_start": {
-                const events = request.args.events;
-                const osInput = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                };
-                if (Array.isArray(events)) {
-                    osInput.events = events;
-                }
-                return this.runtime.observeStart(osInput);
-            }
-            case "observe_stop":
-                return this.runtime.observeStop({
-                    sessionId: requiredString(request.args, "sessionId"),
-                });
-            case "drag":
-                return this.runtime.drag({
-                    sessionId: requiredString(request.args, "sessionId"),
-                    from: parseTarget(request.args.from),
-                    to: parseTarget(request.args.to),
-                });
-            case "scroll": {
-                const scrollTarget = request.args.target ? parseTarget(request.args.target) : undefined;
-                const scrollAmount = optionalNumber(request.args, "amount");
-                const scrollInput = {
-                    sessionId: requiredString(request.args, "sessionId"),
-                    direction: requiredString(request.args, "direction"),
-                };
-                if (scrollTarget)
-                    scrollInput.target = scrollTarget;
-                if (typeof scrollAmount === "number")
-                    scrollInput.amount = scrollAmount;
-                return this.runtime.scroll(scrollInput);
-            }
-            default:
-                throw new Error(`Unsupported tool: ${String(request.tool)}`);
-        }
-    }
-}
-function parseTarget(input) {
-    if (typeof input === "string") {
-        if (input.startsWith("css=")) {
-            return { type: "selector", value: input.slice(4) };
-        }
-        if (input.startsWith("text=")) {
-            return { type: "text", value: input.slice(5), exact: true };
-        }
-        if (input.startsWith("ax_id=")) {
-            return { type: "ax_attribute", attribute: "identifier", value: input.slice(6) };
-        }
-        return { type: "text", value: input };
-    }
-    if (!isRecord(input)) {
-        throw new Error("target must be a string or object");
-    }
-    if (typeof input.selector === "string") {
-        return { type: "selector", value: input.selector };
-    }
-    if (typeof input.text === "string") {
-        return {
-            type: "text",
-            value: input.text,
-            exact: input.exact === true,
-        };
-    }
-    if (typeof input.role === "string" && typeof input.name === "string") {
-        return {
-            type: "role",
-            role: input.role,
-            name: input.name,
-            exact: input.exact === true,
-        };
-    }
-    if (Array.isArray(input.path)) {
-        return { type: "ax_path", path: input.path };
-    }
-    if (typeof input.attribute === "string" && typeof input.value === "string") {
-        return { type: "ax_attribute", attribute: input.attribute, value: input.value };
-    }
-    if (typeof input.x === "number" && typeof input.y === "number") {
-        return { type: "coordinates", x: input.x, y: input.y };
-    }
-    if (typeof input.base64 === "string") {
-        const target = { type: "image", base64: input.base64 };
-        if (typeof input.confidence === "number") {
-            target.confidence = input.confidence;
-        }
-        return target;
-    }
-    throw new Error("target object must contain selector, text, role+name, path, attribute+value, x+y, or base64");
-}
-function parseWaitCondition(input) {
-    if (!isRecord(input) || typeof input.type !== "string") {
-        throw new Error("condition must be an object with a type");
-    }
-    switch (input.type) {
-        case "selector_visible":
-            return {
-                type: "selector_visible",
-                selector: requiredObjectString(input, "selector"),
-            };
-        case "selector_hidden":
-            return {
-                type: "selector_hidden",
-                selector: requiredObjectString(input, "selector"),
-            };
-        case "url_matches":
-            return {
-                type: "url_matches",
-                regex: requiredObjectString(input, "regex"),
-            };
-        case "text_appears":
-            return {
-                type: "text_appears",
-                text: requiredObjectString(input, "text"),
-            };
-        case "spinner_disappears":
-            return {
-                type: "spinner_disappears",
-                selector: requiredObjectString(input, "selector"),
-            };
-        case "element_exists":
-            return {
-                type: "element_exists",
-                target: parseTarget(input.target),
-            };
-        case "element_gone":
-            return {
-                type: "element_gone",
-                target: parseTarget(input.target),
-            };
-        case "window_title_matches":
-            return {
-                type: "window_title_matches",
-                regex: requiredObjectString(input, "regex"),
-            };
-        case "app_idle": {
-            const cond = {
-                type: "app_idle",
-                bundleId: requiredObjectString(input, "bundleId"),
-            };
-            if (typeof input.timeoutMs === "number") {
-                cond.timeoutMs = input.timeoutMs;
-            }
-            return cond;
-        }
-        default:
-            throw new Error(`Unsupported condition type: ${input.type}`);
-    }
-}
-function parseOptionalWaitCondition(input) {
-    if (typeof input === "undefined") {
-        return undefined;
-    }
-    return parseWaitCondition(input);
-}
-function parseExtractFormat(input) {
-    if (input === "text" || input === "table" || input === "json") {
-        return input;
-    }
-    throw new Error("format must be one of: text, table, json");
-}
-function parseOptionalRegion(input) {
-    if (typeof input === "undefined") {
-        return undefined;
-    }
-    if (!isRecord(input)) {
-        throw new Error("region must be an object");
-    }
-    return {
-        x: requiredObjectNumber(input, "x"),
-        y: requiredObjectNumber(input, "y"),
-        width: requiredObjectNumber(input, "width"),
-        height: requiredObjectNumber(input, "height"),
-    };
-}
-function requiredString(input, key) {
-    const value = input[key];
-    if (typeof value !== "string") {
-        throw new Error(`${key} must be a string`);
-    }
-    return value;
-}
-function requiredStringArray(input, key) {
-    const value = input[key];
-    if (!Array.isArray(value) || !value.every((v) => typeof v === "string")) {
-        throw new Error(`${key} must be an array of strings`);
-    }
-    return value;
-}
-function optionalString(input, key) {
-    const value = input[key];
-    if (typeof value === "undefined") {
-        return undefined;
-    }
-    if (typeof value !== "string") {
-        throw new Error(`${key} must be a string`);
-    }
-    return value;
-}
-function optionalNumber(input, key) {
-    const value = input[key];
-    if (typeof value === "undefined") {
-        return undefined;
-    }
-    if (typeof value !== "number") {
-        throw new Error(`${key} must be a number`);
-    }
-    return value;
-}
-function optionalBoolean(input, key) {
-    const value = input[key];
-    if (typeof value === "undefined") {
-        return undefined;
-    }
-    if (typeof value !== "boolean") {
-        throw new Error(`${key} must be a boolean`);
-    }
-    return value;
-}
-function requiredObjectString(input, key) {
-    const value = input[key];
-    if (typeof value !== "string") {
-        throw new Error(`${key} must be a string`);
-    }
-    return value;
-}
-function requiredObjectNumber(input, key) {
-    const value = input[key];
-    if (typeof value !== "number") {
-        throw new Error(`${key} must be a number`);
-    }
-    return value;
-}
-function isRecord(input) {
-    return typeof input === "object" && input !== null;
-}

package/dist/mcp-entry.js

@@ -1,59 +0,0 @@
-#!/usr/bin/env node
-/**
- * ScreenHand MCP Server — MODULAR entry point (alternative).
- *
- * NOTE: The primary/canonical MCP server is `mcp-desktop.ts` at the project root.
- * It has the full 40+ tool set (desktop, browser, memory, playbooks, codex monitor).
- *
- * This modular entrypoint exposes a smaller subset of tools via the runtime service
- * abstraction. It exists for adapter experimentation and future refactoring.
- *
- * For production use, prefer: npx tsx mcp-desktop.ts
- *
- * Environment variables:
- *   SCREENHAND_ADAPTER  - "accessibility" (default), "composite", "cdp", "placeholder"
- *   SCREENHAND_HEADLESS - "1" to run browser in headless mode
- */
-import { PlaceholderAppAdapter } from "./runtime/app-adapter.js";
-import { CdpChromeAdapter } from "./runtime/cdp-chrome-adapter.js";
-import { TimelineLogger } from "./logging/timeline-logger.js";
-import { AutomationRuntimeService } from "./runtime/service.js";
-import { startMcpStdioServer } from "./mcp/mcp-stdio-server.js";
-async function createAdapter() {
-    const adapterType = process.env.SCREENHAND_ADAPTER ?? process.env.AUTOMATOR_ADAPTER ?? "accessibility";
-    switch (adapterType) {
-        case "placeholder":
-            return new PlaceholderAppAdapter();
-        case "cdp":
-            return new CdpChromeAdapter({
-                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
-            });
-        case "composite": {
-            const { BridgeClient } = await import("./native/bridge-client.js");
-            const { CompositeAdapter } = await import("./runtime/composite-adapter.js");
-            const bridge = new BridgeClient();
-            return new CompositeAdapter(bridge, {
-                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
-            });
-        }
-        case "accessibility":
-        default: {
-            const { BridgeClient } = await import("./native/bridge-client.js");
-            const { AccessibilityAdapter } = await import("./runtime/accessibility-adapter.js");
-            const bridge = new BridgeClient();
-            return new AccessibilityAdapter(bridge);
-        }
-    }
-}
-try {
-    const adapter = await createAdapter();
-    const logger = new TimelineLogger();
-    const runtime = new AutomationRuntimeService(adapter, logger);
-    process.stderr.write("ScreenHand MCP server starting...\n");
-    await startMcpStdioServer(runtime);
-    process.stderr.write("ScreenHand MCP server connected.\n");
-}
-catch (e) {
-    process.stderr.write(`ScreenHand startup error: ${e instanceof Error ? e.message : String(e)}\n`);
-    process.exit(1);
-}

package/dist/memory/recall.js

@@ -1,160 +0,0 @@
-/**
- * Learning Memory — Recall engine (in-memory)
- *
- * All searches run against cached data — no disk IO.
- * Provides fast methods for the interceptor to call on every tool invocation.
- */
-import { MemoryStore } from "./store.js";
-export class RecallEngine {
-    store;
-    constructor(store) {
-        this.store = store;
-    }
-    /**
-     * Find strategies matching a task description (~0ms, in-memory).
-     * Strategies with high fail rates are penalized.
-     */
-    recallStrategies(query, limit = 5) {
-        const strategies = this.store.readStrategies();
-        if (strategies.length === 0)
-            return [];
-        const queryTokens = tokenize(query);
-        if (queryTokens.length === 0)
-            return [];
-        const scored = strategies.map((s) => {
-            const targetTokens = new Set([
-                ...tokenize(s.task),
-                ...s.tags,
-                ...s.steps.map((step) => step.tool),
-                ...s.steps.flatMap((step) => Object.values(step.params)
-                    .filter((v) => typeof v === "string")
-                    .flatMap(tokenize)),
-            ]);
-            let matches = 0;
-            for (const qt of queryTokens) {
-                for (const tt of targetTokens) {
-                    if (tt.includes(qt) || qt.includes(tt)) {
-                        matches++;
-                        break;
-                    }
-                }
-            }
-            const relevance = matches / queryTokens.length;
-            const ageMs = Date.now() - new Date(s.lastUsed).getTime();
-            const ageDays = ageMs / (1000 * 60 * 60 * 24);
-            const recency = Math.max(0.5, 1.0 - ageDays / 365);
-            const successBoost = 1 + Math.log2(Math.max(1, s.successCount)) * 0.1;
-            // Penalty for strategies that have failed — reduces score proportionally
-            const failCount = s.failCount ?? 0;
-            const totalAttempts = s.successCount + failCount;
-            const reliabilityPenalty = totalAttempts > 0
-                ? s.successCount / totalAttempts
-                : 1;
-            const score = relevance * recency * successBoost * reliabilityPenalty;
-            return { ...s, score };
-        });
-        return scored
-            .filter((s) => s.score > 0)
-            .sort((a, b) => b.score - a.score)
-            .slice(0, limit);
-    }
-    /**
-     * O(1) exact match by tool sequence fingerprint.
-     * Returns the strategy if found and it has a positive reliability score.
-     */
-    recallByFingerprint(tools) {
-        const fp = MemoryStore.makeFingerprint(tools);
-        const strategy = this.store.lookupByFingerprint(fp);
-        if (!strategy)
-            return null;
-        // Skip strategies that fail more than they succeed
-        const failCount = strategy.failCount ?? 0;
-        if (failCount > strategy.successCount)
-            return null;
-        return strategy;
-    }
-    /**
-     * Quick error lookup for a tool — used by interceptor on every call (~0ms).
-     * Returns the most relevant error pattern or null.
-     */
-    quickErrorCheck(tool) {
-        const errors = this.store.readErrors();
-        let best = null;
-        for (const e of errors) {
-            if (e.tool === tool && e.resolution) {
-                if (!best || e.occurrences > best.occurrences)
-                    best = e;
-            }
-        }
-        return best;
-    }
-    /**
-     * Quick strategy hint for a tool sequence — used by interceptor.
-     * Tries fingerprint prefix match first (O(1)), then falls back to scan.
-     * Skips unreliable strategies (failCount > successCount).
-     */
-    quickStrategyHint(recentTools) {
-        if (recentTools.length === 0)
-            return null;
-        const strategies = this.store.readStrategies();
-        for (const s of strategies) {
-            if (s.steps.length <= recentTools.length)
-                continue;
-            // Skip unreliable strategies
-            const failCount = s.failCount ?? 0;
-            if (failCount > s.successCount)
-                continue;
-            const strategyToolPrefix = s.steps.slice(0, recentTools.length).map((st) => st.tool);
-            const matches = recentTools.every((t, i) => t === strategyToolPrefix[i]);
-            if (matches) {
-                return {
-                    strategy: s,
-                    nextStep: s.steps[recentTools.length],
-                    fingerprint: s.fingerprint ?? MemoryStore.makeFingerprint(s.steps.map((st) => st.tool)),
-                };
-            }
-        }
-        return null;
-    }
-    /** Find error patterns for a specific tool or all tools */
-    recallErrors(tool, params) {
-        const errors = this.store.readErrors();
-        if (!tool)
-            return errors;
-        let filtered = errors.filter((e) => e.tool === tool);
-        if (params && filtered.length > 1) {
-            const paramStr = JSON.stringify(params).toLowerCase();
-            filtered.sort((a, b) => {
-                const aScore = stringSimilarity(paramStr, JSON.stringify(a.params).toLowerCase());
-                const bScore = stringSimilarity(paramStr, JSON.stringify(b.params).toLowerCase());
-                return bScore - aScore;
-            });
-        }
-        return filtered;
-    }
-}
-/** Tokenize a string into lowercase keywords (3+ chars) */
-function tokenize(text) {
-    return text
-        .toLowerCase()
-        .split(/[\W_]+/)
-        .filter((w) => w.length >= 3);
-}
-/** Simple string similarity: shared character bigrams / total bigrams */
-function stringSimilarity(a, b) {
-    if (a === b)
-        return 1;
-    if (a.length < 2 || b.length < 2)
-        return 0;
-    const bigramsA = new Set();
-    for (let i = 0; i < a.length - 1; i++)
-        bigramsA.add(a.slice(i, i + 2));
-    let matches = 0;
-    let total = 0;
-    for (let i = 0; i < b.length - 1; i++) {
-        total++;
-        if (bigramsA.has(b.slice(i, i + 2)))
-            matches++;
-    }
-    return total > 0 ? matches / total : 0;
-}

package/dist/memory/research.js

@@ -1,98 +0,0 @@
-/**
- * Background web research — fire-and-forget resolution lookup.
- *
- * When a tool fails and no resolution exists, this searches for a fix
- * in the background (non-blocking) and saves it for next time.
- *
- * Resolution paths:
- * 1. Claude API (haiku) — if ANTHROPIC_API_KEY is set
- * 2. DuckDuckGo instant answer — free fallback, no auth
- */
-export function backgroundResearch(store, tool, params, errorMessage) {
-    // Fire-and-forget — never blocks, never throws
-    doResearch(store, tool, params, errorMessage).catch(() => { });
-}
-async function doResearch(store, tool, params, errorMessage) {
-    const query = `macOS automation: "${tool}" failed with "${errorMessage.slice(0, 200)}"`;
-    let resolution = null;
-    const apiKey = process.env["ANTHROPIC_API_KEY"];
-    if (apiKey) {
-        resolution = await tryClaudeAPI(apiKey, query);
-    }
-    if (!resolution) {
-        resolution = await tryDuckDuckGo(query);
-    }
-    if (!resolution)
-        return;
-    // Save resolution to error cache
-    store.appendError({
-        id: "err_research_" + Date.now().toString(36),
-        tool,
-        params,
-        error: errorMessage,
-        resolution,
-        occurrences: 1,
-        lastSeen: new Date().toISOString(),
-    });
-    // Save as a reusable strategy
-    store.appendStrategy({
-        id: "str_research_" + Date.now().toString(36),
-        task: `Fix: ${tool} — ${errorMessage.slice(0, 100)}`,
-        steps: [{ tool, params }],
-        totalDurationMs: 0,
-        successCount: 1,
-        failCount: 0,
-        lastUsed: new Date().toISOString(),
-        tags: [tool, "research", "fix"],
-        fingerprint: "",
-    });
-}
-async function tryClaudeAPI(apiKey, query) {
-    try {
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), 10000);
-        const response = await fetch("https://api.anthropic.com/v1/messages", {
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json",
-                "x-api-key": apiKey,
-                "anthropic-version": "2023-06-01",
-            },
-            body: JSON.stringify({
-                model: "claude-haiku-4-5-20251001",
-                max_tokens: 200,
-                messages: [{
-                        role: "user",
-                        content: `You are a macOS automation expert. Give a brief fix (1-2 sentences) for this error:\n\n${query}`,
-                    }],
-            }),
-            signal: controller.signal,
-        });
-        clearTimeout(timeout);
-        if (!response.ok)
-            return null;
-        const data = await response.json();
-        const text = data.content?.[0]?.text;
-        return text && text.length > 10 ? text.trim() : null;
-    }
-    catch {
-        return null;
-    }
-}
-async function tryDuckDuckGo(query) {
-    try {
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), 5000);
-        const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1`;
-        const response = await fetch(url, { signal: controller.signal });
-        clearTimeout(timeout);
-        if (!response.ok)
-            return null;
-        const data = await response.json();
-        const text = data.AbstractText || data.Abstract || data.Answer;
-        return text && text.length > 10 ? text.trim() : null;
-    }
-    catch {
-        return null;
-    }
-}