screenhand 0.1.1 → 0.3.0

package/dist/src/mcp/server.js

@@ -0,0 +1,363 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class MvpMcpServer {
+    runtime;
+    constructor(runtime) {
+        this.runtime = runtime;
+    }
+    async invoke(request) {
+        switch (request.tool) {
+            case "session_start":
+                return this.runtime.sessionStart(optionalString(request.args, "profile"));
+            case "navigate": {
+                const timeoutMs = optionalNumber(request.args, "timeoutMs");
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    url: requiredString(request.args, "url"),
+                };
+                if (typeof timeoutMs === "number") {
+                    input.timeoutMs = timeoutMs;
+                }
+                return this.runtime.navigate(input);
+            }
+            case "press": {
+                const verify = parseOptionalWaitCondition(request.args.verify);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                };
+                if (verify) {
+                    input.verify = verify;
+                }
+                return this.runtime.press(input);
+            }
+            case "type_into": {
+                const clear = optionalBoolean(request.args, "clear");
+                const verify = parseOptionalWaitCondition(request.args.verify);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                    text: requiredString(request.args, "text"),
+                };
+                if (typeof clear === "boolean") {
+                    input.clear = clear;
+                }
+                if (verify) {
+                    input.verify = verify;
+                }
+                return this.runtime.typeInto(input);
+            }
+            case "wait_for": {
+                const timeoutMs = optionalNumber(request.args, "timeoutMs");
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    condition: parseWaitCondition(request.args.condition),
+                };
+                if (typeof timeoutMs === "number") {
+                    input.timeoutMs = timeoutMs;
+                }
+                return this.runtime.waitFor(input);
+            }
+            case "extract":
+                return this.runtime.extract({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                    format: parseExtractFormat(request.args.format),
+                });
+            case "screenshot": {
+                const region = parseOptionalRegion(request.args.region);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (region) {
+                    input.region = region;
+                }
+                return this.runtime.screenshot(input);
+            }
+            // ── Desktop automation tools ──
+            case "app_launch":
+                return this.runtime.appLaunch({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    bundleId: requiredString(request.args, "bundleId"),
+                });
+            case "app_focus":
+                return this.runtime.appFocus({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    bundleId: requiredString(request.args, "bundleId"),
+                });
+            case "app_list":
+                return this.runtime.appList(requiredString(request.args, "sessionId"));
+            case "window_list":
+                return this.runtime.windowList(requiredString(request.args, "sessionId"));
+            case "menu_click":
+                return this.runtime.menuClick({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    menuPath: requiredStringArray(request.args, "menuPath"),
+                });
+            case "key_combo":
+                return this.runtime.keyCombo({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    keys: requiredStringArray(request.args, "keys"),
+                });
+            case "element_tree": {
+                const maxDepth = optionalNumber(request.args, "maxDepth");
+                const root = request.args.root ? parseTarget(request.args.root) : undefined;
+                const etInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (typeof maxDepth === "number")
+                    etInput.maxDepth = maxDepth;
+                if (root)
+                    etInput.root = root;
+                return this.runtime.elementTree(etInput);
+            }
+            case "observe_start": {
+                const events = request.args.events;
+                const osInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (Array.isArray(events)) {
+                    osInput.events = events;
+                }
+                return this.runtime.observeStart(osInput);
+            }
+            case "observe_stop":
+                return this.runtime.observeStop({
+                    sessionId: requiredString(request.args, "sessionId"),
+                });
+            case "drag":
+                return this.runtime.drag({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    from: parseTarget(request.args.from),
+                    to: parseTarget(request.args.to),
+                });
+            case "scroll": {
+                const scrollTarget = request.args.target ? parseTarget(request.args.target) : undefined;
+                const scrollAmount = optionalNumber(request.args, "amount");
+                const scrollInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    direction: requiredString(request.args, "direction"),
+                };
+                if (scrollTarget)
+                    scrollInput.target = scrollTarget;
+                if (typeof scrollAmount === "number")
+                    scrollInput.amount = scrollAmount;
+                return this.runtime.scroll(scrollInput);
+            }
+            default:
+                throw new Error(`Unsupported tool: ${String(request.tool)}`);
+        }
+    }
+}
+function parseTarget(input) {
+    if (typeof input === "string") {
+        if (input.startsWith("css=")) {
+            return { type: "selector", value: input.slice(4) };
+        }
+        if (input.startsWith("text=")) {
+            return { type: "text", value: input.slice(5), exact: true };
+        }
+        if (input.startsWith("ax_id=")) {
+            return { type: "ax_attribute", attribute: "identifier", value: input.slice(6) };
+        }
+        return { type: "text", value: input };
+    }
+    if (!isRecord(input)) {
+        throw new Error("target must be a string or object");
+    }
+    if (typeof input.selector === "string") {
+        return { type: "selector", value: input.selector };
+    }
+    if (typeof input.text === "string") {
+        return {
+            type: "text",
+            value: input.text,
+            exact: input.exact === true,
+        };
+    }
+    if (typeof input.role === "string" && typeof input.name === "string") {
+        return {
+            type: "role",
+            role: input.role,
+            name: input.name,
+            exact: input.exact === true,
+        };
+    }
+    if (Array.isArray(input.path)) {
+        return { type: "ax_path", path: input.path };
+    }
+    if (typeof input.attribute === "string" && typeof input.value === "string") {
+        return { type: "ax_attribute", attribute: input.attribute, value: input.value };
+    }
+    if (typeof input.x === "number" && typeof input.y === "number") {
+        return { type: "coordinates", x: input.x, y: input.y };
+    }
+    if (typeof input.base64 === "string") {
+        const target = { type: "image", base64: input.base64 };
+        if (typeof input.confidence === "number") {
+            target.confidence = input.confidence;
+        }
+        return target;
+    }
+    throw new Error("target object must contain selector, text, role+name, path, attribute+value, x+y, or base64");
+}
+function parseWaitCondition(input) {
+    if (!isRecord(input) || typeof input.type !== "string") {
+        throw new Error("condition must be an object with a type");
+    }
+    switch (input.type) {
+        case "selector_visible":
+            return {
+                type: "selector_visible",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "selector_hidden":
+            return {
+                type: "selector_hidden",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "url_matches":
+            return {
+                type: "url_matches",
+                regex: requiredObjectString(input, "regex"),
+            };
+        case "text_appears":
+            return {
+                type: "text_appears",
+                text: requiredObjectString(input, "text"),
+            };
+        case "spinner_disappears":
+            return {
+                type: "spinner_disappears",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "element_exists":
+            return {
+                type: "element_exists",
+                target: parseTarget(input.target),
+            };
+        case "element_gone":
+            return {
+                type: "element_gone",
+                target: parseTarget(input.target),
+            };
+        case "window_title_matches":
+            return {
+                type: "window_title_matches",
+                regex: requiredObjectString(input, "regex"),
+            };
+        case "app_idle": {
+            const cond = {
+                type: "app_idle",
+                bundleId: requiredObjectString(input, "bundleId"),
+            };
+            if (typeof input.timeoutMs === "number") {
+                cond.timeoutMs = input.timeoutMs;
+            }
+            return cond;
+        }
+        default:
+            throw new Error(`Unsupported condition type: ${input.type}`);
+    }
+}
+function parseOptionalWaitCondition(input) {
+    if (typeof input === "undefined") {
+        return undefined;
+    }
+    return parseWaitCondition(input);
+}
+function parseExtractFormat(input) {
+    if (input === "text" || input === "table" || input === "json") {
+        return input;
+    }
+    throw new Error("format must be one of: text, table, json");
+}
+function parseOptionalRegion(input) {
+    if (typeof input === "undefined") {
+        return undefined;
+    }
+    if (!isRecord(input)) {
+        throw new Error("region must be an object");
+    }
+    return {
+        x: requiredObjectNumber(input, "x"),
+        y: requiredObjectNumber(input, "y"),
+        width: requiredObjectNumber(input, "width"),
+        height: requiredObjectNumber(input, "height"),
+    };
+}
+function requiredString(input, key) {
+    const value = input[key];
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function requiredStringArray(input, key) {
+    const value = input[key];
+    if (!Array.isArray(value) || !value.every((v) => typeof v === "string")) {
+        throw new Error(`${key} must be an array of strings`);
+    }
+    return value;
+}
+function optionalString(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function optionalNumber(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "number") {
+        throw new Error(`${key} must be a number`);
+    }
+    return value;
+}
+function optionalBoolean(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "boolean") {
+        throw new Error(`${key} must be a boolean`);
+    }
+    return value;
+}
+function requiredObjectString(input, key) {
+    const value = input[key];
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function requiredObjectNumber(input, key) {
+    const value = input[key];
+    if (typeof value !== "number") {
+        throw new Error(`${key} must be a number`);
+    }
+    return value;
+}
+function isRecord(input) {
+    return typeof input === "object" && input !== null;
+}

package/dist/src/mcp-entry.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { PlaceholderAppAdapter } from "./runtime/app-adapter.js";
+import { CdpChromeAdapter } from "./runtime/cdp-chrome-adapter.js";
+import { TimelineLogger } from "./logging/timeline-logger.js";
+import { AutomationRuntimeService } from "./runtime/service.js";
+import { startMcpStdioServer } from "./mcp/mcp-stdio-server.js";
+async function createAdapter() {
+    const adapterType = process.env.SCREENHAND_ADAPTER ?? process.env.AUTOMATOR_ADAPTER ?? "accessibility";
+    switch (adapterType) {
+        case "placeholder":
+            return new PlaceholderAppAdapter();
+        case "cdp":
+            return new CdpChromeAdapter({
+                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
+            });
+        case "composite": {
+            const { BridgeClient } = await import("./native/bridge-client.js");
+            const { CompositeAdapter } = await import("./runtime/composite-adapter.js");
+            const bridge = new BridgeClient();
+            return new CompositeAdapter(bridge, {
+                headless: process.env.SCREENHAND_HEADLESS === "1" || process.env.AUTOMATOR_HEADLESS === "1",
+            });
+        }
+        case "accessibility":
+        default: {
+            const { BridgeClient } = await import("./native/bridge-client.js");
+            const { AccessibilityAdapter } = await import("./runtime/accessibility-adapter.js");
+            const bridge = new BridgeClient();
+            return new AccessibilityAdapter(bridge);
+        }
+    }
+}
+try {
+    const adapter = await createAdapter();
+    const logger = new TimelineLogger();
+    const runtime = new AutomationRuntimeService(adapter, logger);
+    process.stderr.write("ScreenHand MCP server starting...\n");
+    await startMcpStdioServer(runtime);
+    process.stderr.write("ScreenHand MCP server connected.\n");
+}
+catch (e) {
+    process.stderr.write(`ScreenHand startup error: ${e instanceof Error ? e.message : String(e)}\n`);
+    process.exit(1);
+}

package/dist/src/memory/playbook-seeds.js

@@ -0,0 +1,200 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+/**
+ * Playbook Seeds — converts playbook reference knowledge into memory-compatible formats.
+ *
+ * Reads all playbooks from disk and extracts:
+ *   - errors[]       → ErrorPattern[]  (for quickErrorCheck auto-warnings)
+ *   - flows{}        → Learning[]      (for pattern recall)
+ *   - selectors{}    → Learning[]      (for verified selector patterns)
+ *   - policyNotes{}  → Learning[]      (for rate limits and safety)
+ *
+ * Called once during MemoryStore.init() to seed the memory system with
+ * months of team-curated platform knowledge.
+ */
+import fs from "node:fs";
+import path from "node:path";
+// ── Tool name inference from error/solution text ──
+const TOOL_KEYWORDS = {
+    browser_click: ["click", "el.click", ".click()", "button"],
+    browser_human_click: ["human_click", "dispatchMouseEvent", "CDP Input"],
+    browser_fill_form: ["fill_form", "browser_fill_form", "form"],
+    browser_type: ["browser_type", "type into", "typing"],
+    browser_js: ["browser_js", "evaluate", "script", "execCommand"],
+    browser_navigate: ["navigate", "navigation", "url"],
+    browser_dom: ["querySelector", "selector", "DOM"],
+    browser_wait: ["wait", "timeout", "load"],
+    click: ["native click", "coordinates", "screen click"],
+    type_text: ["type_text", "native typing"],
+    scroll: ["scroll"],
+};
+function inferTool(text) {
+    const lower = text.toLowerCase();
+    for (const [tool, keywords] of Object.entries(TOOL_KEYWORDS)) {
+        if (keywords.some(kw => lower.includes(kw.toLowerCase())))
+            return tool;
+    }
+    return "browser_click"; // default — most errors are click-related
+}
+// ── Main seed functions ──
+/**
+ * Read all playbooks from a directory and extract error patterns.
+ * These get loaded into memory's errorsCache so quickErrorCheck() catches them.
+ */
+export function seedErrorsFromPlaybooks(playbooksDir) {
+    const playbooks = loadPlaybooks(playbooksDir);
+    const errors = [];
+    const seen = new Set(); // deduplicate by error text
+    for (const pb of playbooks) {
+        const platform = pb.platform ?? pb.id ?? "unknown";
+        // Extract from errors[]
+        if (pb.errors) {
+            for (const err of pb.errors) {
+                const key = `${platform}::${err.error}`;
+                if (seen.has(key))
+                    continue;
+                seen.add(key);
+                errors.push({
+                    id: `pb_err_${platform}_${errors.length}`,
+                    tool: inferTool(`${err.error} ${err.context} ${err.solution}`),
+                    params: { _source: "playbook", _platform: platform },
+                    error: `[${platform}] ${err.error}`,
+                    resolution: err.solution,
+                    occurrences: err.severity === "high" ? 10 : err.severity === "medium" ? 5 : 2,
+                    lastSeen: new Date().toISOString(),
+                });
+            }
+        }
+        // Extract from flows — tips and why fields often contain error knowledge
+        if (pb.flows) {
+            for (const [flowName, flow] of Object.entries(pb.flows)) {
+                if (flow.why && flow.why.includes("doesn't") || flow.why?.includes("don't") || flow.why?.includes("NOT")) {
+                    const key = `${platform}::${flowName}::why`;
+                    if (seen.has(key))
+                        continue;
+                    seen.add(key);
+                    errors.push({
+                        id: `pb_err_${platform}_flow_${errors.length}`,
+                        tool: inferTool(flow.why ?? ""),
+                        params: { _source: "playbook", _platform: platform, _flow: flowName },
+                        error: `[${platform}/${flowName}] ${flow.why.slice(0, 200)}`,
+                        resolution: flow.steps?.join(" → ") ?? null,
+                        occurrences: 5,
+                        lastSeen: new Date().toISOString(),
+                    });
+                }
+            }
+        }
+    }
+    return errors;
+}
+/**
+ * Read all playbooks and extract learnings (selectors, patterns, policy notes).
+ * These get loaded into memory's learningsCache so queryPatterns() finds them.
+ */
+export function seedLearningsFromPlaybooks(playbooksDir) {
+    const playbooks = loadPlaybooks(playbooksDir);
+    const learnings = [];
+    for (const pb of playbooks) {
+        const platform = pb.platform ?? pb.id ?? "unknown";
+        const reliability = (pb.successCount ?? 0) + (pb.failCount ?? 0) > 0
+            ? (pb.successCount ?? 0) / ((pb.successCount ?? 0) + (pb.failCount ?? 0))
+            : 0.7;
+        // Selectors → learnings (verified working CSS selectors)
+        if (pb.selectors) {
+            for (const [group, sels] of Object.entries(pb.selectors)) {
+                for (const [name, selector] of Object.entries(sels)) {
+                    // Skip notes/annotations (keys starting with _)
+                    if (name.startsWith("_"))
+                        continue;
+                    learnings.push({
+                        scope: `chrome/${platform}`,
+                        pattern: `${group}.${name}: ${selector}`,
+                        method: "cdp",
+                        confidence: reliability,
+                        successCount: Math.max(1, Math.round(reliability * 10)),
+                        failCount: Math.round((1 - reliability) * 10),
+                        lastSeen: new Date().toISOString(),
+                        fix: null,
+                    });
+                }
+            }
+        }
+        // Flow selectors → learnings
+        if (pb.flows) {
+            for (const [flowName, flow] of Object.entries(pb.flows)) {
+                if (flow.selectors) {
+                    for (const [name, selector] of Object.entries(flow.selectors)) {
+                        learnings.push({
+                            scope: `chrome/${platform}/${flowName}`,
+                            pattern: `${name}: ${selector}`,
+                            method: "cdp",
+                            confidence: reliability,
+                            successCount: Math.max(1, Math.round(reliability * 10)),
+                            failCount: Math.round((1 - reliability) * 10),
+                            lastSeen: new Date().toISOString(),
+                            fix: null,
+                        });
+                    }
+                }
+            }
+        }
+        // Policy notes → learnings (rate limits, safety rules)
+        if (pb.policyNotes) {
+            for (const [category, notes] of Object.entries(pb.policyNotes)) {
+                for (const note of notes) {
+                    learnings.push({
+                        scope: `policy/${platform}`,
+                        pattern: `[${category}] ${note}`,
+                        method: "cdp",
+                        confidence: 1.0,
+                        successCount: 10,
+                        failCount: 0,
+                        lastSeen: new Date().toISOString(),
+                        fix: null,
+                    });
+                }
+            }
+        }
+        // Detection expressions → learnings
+        if (pb.detection) {
+            for (const [name, expr] of Object.entries(pb.detection)) {
+                learnings.push({
+                    scope: `chrome/${platform}/detection`,
+                    pattern: `${name}: ${expr}`,
+                    method: "cdp",
+                    confidence: reliability,
+                    successCount: Math.max(1, Math.round(reliability * 10)),
+                    failCount: 0,
+                    lastSeen: new Date().toISOString(),
+                    fix: null,
+                });
+            }
+        }
+    }
+    return learnings;
+}
+// ── Helpers ──
+function loadPlaybooks(dir) {
+    if (!fs.existsSync(dir))
+        return [];
+    const files = fs.readdirSync(dir).filter(f => f.endsWith(".json"));
+    const playbooks = [];
+    for (const file of files) {
+        try {
+            const raw = JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8"));
+            // Ensure it has an id
+            if (!raw.id)
+                raw.id = file.replace(".json", "");
+            if (!raw.platform)
+                raw.platform = raw.id;
+            playbooks.push(raw);
+        }
+        catch {
+            // Skip unparseable files
+        }
+    }
+    return playbooks;
+}