screenhand 0.1.1 → 0.2.0

package/dist/src/mcp/server.js

@@ -0,0 +1,363 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class MvpMcpServer {
+    runtime;
+    constructor(runtime) {
+        this.runtime = runtime;
+    }
+    async invoke(request) {
+        switch (request.tool) {
+            case "session_start":
+                return this.runtime.sessionStart(optionalString(request.args, "profile"));
+            case "navigate": {
+                const timeoutMs = optionalNumber(request.args, "timeoutMs");
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    url: requiredString(request.args, "url"),
+                };
+                if (typeof timeoutMs === "number") {
+                    input.timeoutMs = timeoutMs;
+                }
+                return this.runtime.navigate(input);
+            }
+            case "press": {
+                const verify = parseOptionalWaitCondition(request.args.verify);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                };
+                if (verify) {
+                    input.verify = verify;
+                }
+                return this.runtime.press(input);
+            }
+            case "type_into": {
+                const clear = optionalBoolean(request.args, "clear");
+                const verify = parseOptionalWaitCondition(request.args.verify);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                    text: requiredString(request.args, "text"),
+                };
+                if (typeof clear === "boolean") {
+                    input.clear = clear;
+                }
+                if (verify) {
+                    input.verify = verify;
+                }
+                return this.runtime.typeInto(input);
+            }
+            case "wait_for": {
+                const timeoutMs = optionalNumber(request.args, "timeoutMs");
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    condition: parseWaitCondition(request.args.condition),
+                };
+                if (typeof timeoutMs === "number") {
+                    input.timeoutMs = timeoutMs;
+                }
+                return this.runtime.waitFor(input);
+            }
+            case "extract":
+                return this.runtime.extract({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                    format: parseExtractFormat(request.args.format),
+                });
+            case "screenshot": {
+                const region = parseOptionalRegion(request.args.region);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (region) {
+                    input.region = region;
+                }
+                return this.runtime.screenshot(input);
+            }
+            // ── Desktop automation tools ──
+            case "app_launch":
+                return this.runtime.appLaunch({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    bundleId: requiredString(request.args, "bundleId"),
+                });
+            case "app_focus":
+                return this.runtime.appFocus({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    bundleId: requiredString(request.args, "bundleId"),
+                });
+            case "app_list":
+                return this.runtime.appList(requiredString(request.args, "sessionId"));
+            case "window_list":
+                return this.runtime.windowList(requiredString(request.args, "sessionId"));
+            case "menu_click":
+                return this.runtime.menuClick({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    menuPath: requiredStringArray(request.args, "menuPath"),
+                });
+            case "key_combo":
+                return this.runtime.keyCombo({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    keys: requiredStringArray(request.args, "keys"),
+                });
+            case "element_tree": {
+                const maxDepth = optionalNumber(request.args, "maxDepth");
+                const root = request.args.root ? parseTarget(request.args.root) : undefined;
+                const etInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (typeof maxDepth === "number")
+                    etInput.maxDepth = maxDepth;
+                if (root)
+                    etInput.root = root;
+                return this.runtime.elementTree(etInput);
+            }
+            case "observe_start": {
+                const events = request.args.events;
+                const osInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (Array.isArray(events)) {
+                    osInput.events = events;
+                }
+                return this.runtime.observeStart(osInput);
+            }
+            case "observe_stop":
+                return this.runtime.observeStop({
+                    sessionId: requiredString(request.args, "sessionId"),
+                });
+            case "drag":
+                return this.runtime.drag({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    from: parseTarget(request.args.from),
+                    to: parseTarget(request.args.to),
+                });
+            case "scroll": {
+                const scrollTarget = request.args.target ? parseTarget(request.args.target) : undefined;
+                const scrollAmount = optionalNumber(request.args, "amount");
+                const scrollInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    direction: requiredString(request.args, "direction"),
+                };
+                if (scrollTarget)
+                    scrollInput.target = scrollTarget;
+                if (typeof scrollAmount === "number")
+                    scrollInput.amount = scrollAmount;
+                return this.runtime.scroll(scrollInput);
+            }
+            default:
+                throw new Error(`Unsupported tool: ${String(request.tool)}`);
+        }
+    }
+}
+function parseTarget(input) {
+    if (typeof input === "string") {
+        if (input.startsWith("css=")) {
+            return { type: "selector", value: input.slice(4) };
+        }
+        if (input.startsWith("text=")) {
+            return { type: "text", value: input.slice(5), exact: true };
+        }
+        if (input.startsWith("ax_id=")) {
+            return { type: "ax_attribute", attribute: "identifier", value: input.slice(6) };
+        }
+        return { type: "text", value: input };
+    }
+    if (!isRecord(input)) {
+        throw new Error("target must be a string or object");
+    }
+    if (typeof input.selector === "string") {
+        return { type: "selector", value: input.selector };
+    }
+    if (typeof input.text === "string") {
+        return {
+            type: "text",
+            value: input.text,
+            exact: input.exact === true,
+        };
+    }
+    if (typeof input.role === "string" && typeof input.name === "string") {
+        return {
+            type: "role",
+            role: input.role,
+            name: input.name,
+            exact: input.exact === true,
+        };
+    }
+    if (Array.isArray(input.path)) {
+        return { type: "ax_path", path: input.path };
+    }
+    if (typeof input.attribute === "string" && typeof input.value === "string") {
+        return { type: "ax_attribute", attribute: input.attribute, value: input.value };
+    }
+    if (typeof input.x === "number" && typeof input.y === "number") {
+        return { type: "coordinates", x: input.x, y: input.y };
+    }
+    if (typeof input.base64 === "string") {
+        const target = { type: "image", base64: input.base64 };
+        if (typeof input.confidence === "number") {
+            target.confidence = input.confidence;
+        }
+        return target;
+    }
+    throw new Error("target object must contain selector, text, role+name, path, attribute+value, x+y, or base64");
+}
+function parseWaitCondition(input) {
+    if (!isRecord(input) || typeof input.type !== "string") {
+        throw new Error("condition must be an object with a type");
+    }
+    switch (input.type) {
+        case "selector_visible":
+            return {
+                type: "selector_visible",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "selector_hidden":
+            return {
+                type: "selector_hidden",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "url_matches":
+            return {
+                type: "url_matches",
+                regex: requiredObjectString(input, "regex"),
+            };
+        case "text_appears":
+            return {
+                type: "text_appears",
+                text: requiredObjectString(input, "text"),
+            };
+        case "spinner_disappears":
+            return {
+                type: "spinner_disappears",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "element_exists":
+            return {
+                type: "element_exists",
+                target: parseTarget(input.target),
+            };
+        case "element_gone":
+            return {
+                type: "element_gone",
+                target: parseTarget(input.target),
+            };
+        case "window_title_matches":
+            return {
+                type: "window_title_matches",
+                regex: requiredObjectString(input, "regex"),
+            };
+        case "app_idle": {
+            const cond = {
+                type: "app_idle",
+                bundleId: requiredObjectString(input, "bundleId"),
+            };
+            if (typeof input.timeoutMs === "number") {
+                cond.timeoutMs = input.timeoutMs;
+            }
+            return cond;
+        }
+        default:
+            throw new Error(`Unsupported condition type: ${input.type}`);
+    }
+}
+function parseOptionalWaitCondition(input) {
+    if (typeof input === "undefined") {
+        return undefined;
+    }
+    return parseWaitCondition(input);
+}
+function parseExtractFormat(input) {
+    if (input === "text" || input === "table" || input === "json") {
+        return input;
+    }
+    throw new Error("format must be one of: text, table, json");
+}
+function parseOptionalRegion(input) {
+    if (typeof input === "undefined") {
+        return undefined;
+    }
+    if (!isRecord(input)) {
+        throw new Error("region must be an object");
+    }
+    return {
+        x: requiredObjectNumber(input, "x"),
+        y: requiredObjectNumber(input, "y"),
+        width: requiredObjectNumber(input, "width"),
+        height: requiredObjectNumber(input, "height"),
+    };
+}
+function requiredString(input, key) {
+    const value = input[key];
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function requiredStringArray(input, key) {
+    const value = input[key];
+    if (!Array.isArray(value) || !value.every((v) => typeof v === "string")) {
+        throw new Error(`${key} must be an array of strings`);
+    }
+    return value;
+}
+function optionalString(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function optionalNumber(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "number") {
+        throw new Error(`${key} must be a number`);
+    }
+    return value;
+}
+function optionalBoolean(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "boolean") {
+        throw new Error(`${key} must be a boolean`);
+    }
+    return value;
+}
+function requiredObjectString(input, key) {
+    const value = input[key];
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function requiredObjectNumber(input, key) {
+    const value = input[key];
+    if (typeof value !== "number") {
+        throw new Error(`${key} must be a number`);
+    }
+    return value;
+}
+function isRecord(input) {
+    return typeof input === "object" && input !== null;
+}

package/dist/src/mcp-entry.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { PlaceholderAppAdapter } from "./runtime/app-adapter.js";
+import { CdpChromeAdapter } from "./runtime/cdp-chrome-adapter.js";
+import { TimelineLogger } from "./logging/timeline-logger.js";
+import { AutomationRuntimeService } from "./runtime/service.js";
+import { startMcpStdioServer } from "./mcp/mcp-stdio-server.js";
+async function createAdapter() {
+    const adapterType = process.env.SCREENHAND_ADAPTER ?? process.env.AUTOMATOR_ADAPTER ?? "accessibility";
+    switch (adapterType) {
+        case "placeholder":
+            return new PlaceholderAppAdapter();
+        case "cdp":
+            return new CdpChromeAdapter({
+                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
+            });
+        case "composite": {
+            const { BridgeClient } = await import("./native/bridge-client.js");
+            const { CompositeAdapter } = await import("./runtime/composite-adapter.js");
+            const bridge = new BridgeClient();
+            return new CompositeAdapter(bridge, {
+                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
+            });
+        }
+        case "accessibility":
+        default: {
+            const { BridgeClient } = await import("./native/bridge-client.js");
+            const { AccessibilityAdapter } = await import("./runtime/accessibility-adapter.js");
+            const bridge = new BridgeClient();
+            return new AccessibilityAdapter(bridge);
+        }
+    }
+}
+try {
+    const adapter = await createAdapter();
+    const logger = new TimelineLogger();
+    const runtime = new AutomationRuntimeService(adapter, logger);
+    process.stderr.write("ScreenHand MCP server starting...\n");
+    await startMcpStdioServer(runtime);
+    process.stderr.write("ScreenHand MCP server connected.\n");
+}
+catch (e) {
+    process.stderr.write(`ScreenHand startup error: ${e instanceof Error ? e.message : String(e)}\n`);
+    process.exit(1);
+}

package/dist/src/memory/recall.js

@@ -0,0 +1,170 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { MemoryStore } from "./store.js";
+export class RecallEngine {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    /**
+     * Find strategies matching a task description (~0ms, in-memory).
+     * Strategies with high fail rates are penalized.
+     */
+    recallStrategies(query, limit = 5) {
+        const strategies = this.store.readStrategies();
+        if (strategies.length === 0)
+            return [];
+        const queryTokens = tokenize(query);
+        if (queryTokens.length === 0)
+            return [];
+        const scored = strategies.map((s) => {
+            const targetTokens = new Set([
+                ...tokenize(s.task),
+                ...s.tags,
+                ...s.steps.map((step) => step.tool),
+                ...s.steps.flatMap((step) => Object.values(step.params)
+                    .filter((v) => typeof v === "string")
+                    .flatMap(tokenize)),
+            ]);
+            let matches = 0;
+            for (const qt of queryTokens) {
+                for (const tt of targetTokens) {
+                    if (tt.includes(qt) || qt.includes(tt)) {
+                        matches++;
+                        break;
+                    }
+                }
+            }
+            const relevance = matches / queryTokens.length;
+            const ageMs = Date.now() - new Date(s.lastUsed).getTime();
+            const ageDays = ageMs / (1000 * 60 * 60 * 24);
+            const recency = Math.max(0.5, 1.0 - ageDays / 365);
+            const successBoost = 1 + Math.log2(Math.max(1, s.successCount)) * 0.1;
+            // Penalty for strategies that have failed — reduces score proportionally
+            const failCount = s.failCount ?? 0;
+            const totalAttempts = s.successCount + failCount;
+            const reliabilityPenalty = totalAttempts > 0
+                ? s.successCount / totalAttempts
+                : 1;
+            const score = relevance * recency * successBoost * reliabilityPenalty;
+            return { ...s, score };
+        });
+        return scored
+            .filter((s) => s.score > 0)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    /**
+     * O(1) exact match by tool sequence fingerprint.
+     * Returns the strategy if found and it has a positive reliability score.
+     */
+    recallByFingerprint(tools) {
+        const fp = MemoryStore.makeFingerprint(tools);
+        const strategy = this.store.lookupByFingerprint(fp);
+        if (!strategy)
+            return null;
+        // Skip strategies that fail more than they succeed
+        const failCount = strategy.failCount ?? 0;
+        if (failCount > strategy.successCount)
+            return null;
+        return strategy;
+    }
+    /**
+     * Quick error lookup for a tool — used by interceptor on every call (~0ms).
+     * Returns the most relevant error pattern or null.
+     */
+    quickErrorCheck(tool) {
+        const errors = this.store.readErrors();
+        let best = null;
+        for (const e of errors) {
+            if (e.tool === tool && e.resolution) {
+                if (!best || e.occurrences > best.occurrences)
+                    best = e;
+            }
+        }
+        return best;
+    }
+    /**
+     * Quick strategy hint for a tool sequence — used by interceptor.
+     * Tries fingerprint prefix match first (O(1)), then falls back to scan.
+     * Skips unreliable strategies (failCount > successCount).
+     */
+    quickStrategyHint(recentTools) {
+        if (recentTools.length === 0)
+            return null;
+        const strategies = this.store.readStrategies();
+        for (const s of strategies) {
+            if (s.steps.length <= recentTools.length)
+                continue;
+            // Skip unreliable strategies
+            const failCount = s.failCount ?? 0;
+            if (failCount > s.successCount)
+                continue;
+            const strategyToolPrefix = s.steps.slice(0, recentTools.length).map((st) => st.tool);
+            const matches = recentTools.every((t, i) => t === strategyToolPrefix[i]);
+            if (matches) {
+                return {
+                    strategy: s,
+                    nextStep: s.steps[recentTools.length],
+                    fingerprint: s.fingerprint ?? MemoryStore.makeFingerprint(s.steps.map((st) => st.tool)),
+                };
+            }
+        }
+        return null;
+    }
+    /** Find error patterns for a specific tool or all tools */
+    recallErrors(tool, params) {
+        const errors = this.store.readErrors();
+        if (!tool)
+            return errors;
+        let filtered = errors.filter((e) => e.tool === tool);
+        if (params && filtered.length > 1) {
+            const paramStr = JSON.stringify(params).toLowerCase();
+            filtered.sort((a, b) => {
+                const aScore = stringSimilarity(paramStr, JSON.stringify(a.params).toLowerCase());
+                const bScore = stringSimilarity(paramStr, JSON.stringify(b.params).toLowerCase());
+                return bScore - aScore;
+            });
+        }
+        return filtered;
+    }
+}
+/** Tokenize a string into lowercase keywords (3+ chars) */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/[\W_]+/)
+        .filter((w) => w.length >= 3);
+}
+/** Simple string similarity: shared character bigrams / total bigrams */
+function stringSimilarity(a, b) {
+    if (a === b)
+        return 1;
+    if (a.length < 2 || b.length < 2)
+        return 0;
+    const bigramsA = new Set();
+    for (let i = 0; i < a.length - 1; i++)
+        bigramsA.add(a.slice(i, i + 2));
+    let matches = 0;
+    let total = 0;
+    for (let i = 0; i < b.length - 1; i++) {
+        total++;
+        if (bigramsA.has(b.slice(i, i + 2)))
+            matches++;
+    }
+    return total > 0 ? matches / total : 0;
+}

package/dist/src/memory/research.js

@@ -0,0 +1,104 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export function backgroundResearch(store, tool, params, errorMessage) {
+    // Fire-and-forget — never blocks, never throws
+    doResearch(store, tool, params, errorMessage).catch(() => { });
+}
+async function doResearch(store, tool, params, errorMessage) {
+    const query = `macOS automation: "${tool}" failed with "${errorMessage.slice(0, 200)}"`;
+    let resolution = null;
+    const apiKey = process.env["ANTHROPIC_API_KEY"];
+    if (apiKey) {
+        resolution = await tryClaudeAPI(apiKey, query);
+    }
+    if (!resolution) {
+        resolution = await tryDuckDuckGo(query);
+    }
+    if (!resolution)
+        return;
+    // Save resolution to error cache
+    store.appendError({
+        id: "err_research_" + Date.now().toString(36),
+        tool,
+        params,
+        error: errorMessage,
+        resolution,
+        occurrences: 1,
+        lastSeen: new Date().toISOString(),
+    });
+    // Save as a reusable strategy
+    store.appendStrategy({
+        id: "str_research_" + Date.now().toString(36),
+        task: `Fix: ${tool} — ${errorMessage.slice(0, 100)}`,
+        steps: [{ tool, params }],
+        totalDurationMs: 0,
+        successCount: 1,
+        failCount: 0,
+        lastUsed: new Date().toISOString(),
+        tags: [tool, "research", "fix"],
+        fingerprint: "",
+    });
+}
+async function tryClaudeAPI(apiKey, query) {
+    try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 10000);
+        const response = await fetch("https://api.anthropic.com/v1/messages", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-api-key": apiKey,
+                "anthropic-version": "2023-06-01",
+            },
+            body: JSON.stringify({
+                model: "claude-haiku-4-5-20251001",
+                max_tokens: 200,
+                messages: [{
+                        role: "user",
+                        content: `You are a macOS automation expert. Give a brief fix (1-2 sentences) for this error:\n\n${query}`,
+                    }],
+            }),
+            signal: controller.signal,
+        });
+        clearTimeout(timeout);
+        if (!response.ok)
+            return null;
+        const data = await response.json();
+        const text = data.content?.[0]?.text;
+        return text && text.length > 10 ? text.trim() : null;
+    }
+    catch {
+        return null;
+    }
+}
+async function tryDuckDuckGo(query) {
+    try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 5000);
+        const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1`;
+        const response = await fetch(url, { signal: controller.signal });
+        clearTimeout(timeout);
+        if (!response.ok)
+            return null;
+        const data = await response.json();
+        const text = data.AbstractText || data.Abstract || data.Answer;
+        return text && text.length > 10 ? text.trim() : null;
+    }
+    catch {
+        return null;
+    }
+}