screenhand 0.1.0

package/dist/mcp/mcp-stdio-server.js

@@ -0,0 +1,284 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+// ── Schema building blocks ──
+const TargetSchema = z.union([
+    z.string().describe("Shorthand: text to find, or 'css=...' / 'text=...' / 'ax_id=...' prefix"),
+    z.object({
+        text: z.string(),
+        exact: z.boolean().optional(),
+    }).describe("Find by visible text"),
+    z.object({
+        role: z.string(),
+        name: z.string(),
+        exact: z.boolean().optional(),
+    }).describe("Find by ARIA/AX role and accessible name"),
+    z.object({
+        selector: z.string(),
+    }).describe("Find by CSS selector (browser) or AX identifier (desktop)"),
+    z.object({
+        x: z.number(),
+        y: z.number(),
+    }).describe("Click at screen coordinates"),
+    z.object({
+        attribute: z.string(),
+        value: z.string(),
+    }).describe("Find by accessibility attribute"),
+]);
+const WaitConditionSchema = z.object({
+    type: z.enum([
+        "selector_visible",
+        "selector_hidden",
+        "url_matches",
+        "text_appears",
+        "spinner_disappears",
+        "element_exists",
+        "element_gone",
+        "window_title_matches",
+        "app_idle",
+    ]),
+    selector: z.string().optional(),
+    regex: z.string().optional(),
+    text: z.string().optional(),
+    target: TargetSchema.optional(),
+    bundleId: z.string().optional(),
+    timeoutMs: z.number().optional(),
+}).describe("Condition to wait for");
+const RegionSchema = z.object({
+    x: z.number(),
+    y: z.number(),
+    width: z.number(),
+    height: z.number(),
+});
+// ── Target parser ──
+function parseTarget(input) {
+    if (typeof input === "string") {
+        if (input.startsWith("css="))
+            return { type: "selector", value: input.slice(4) };
+        if (input.startsWith("text="))
+            return { type: "text", value: input.slice(5), exact: true };
+        if (input.startsWith("ax_id="))
+            return { type: "ax_attribute", attribute: "identifier", value: input.slice(6) };
+        return { type: "text", value: input };
+    }
+    const obj = input;
+    if (typeof obj.selector === "string")
+        return { type: "selector", value: obj.selector };
+    if (typeof obj.text === "string")
+        return { type: "text", value: obj.text, exact: obj.exact === true };
+    if (typeof obj.role === "string" && typeof obj.name === "string")
+        return { type: "role", role: obj.role, name: obj.name, exact: obj.exact === true };
+    if (typeof obj.x === "number" && typeof obj.y === "number")
+        return { type: "coordinates", x: obj.x, y: obj.y };
+    if (typeof obj.attribute === "string" && typeof obj.value === "string")
+        return { type: "ax_attribute", attribute: obj.attribute, value: obj.value };
+    throw new Error("Invalid target");
+}
+function parseWaitCondition(input) {
+    const obj = input;
+    const type = obj.type;
+    switch (type) {
+        case "selector_visible": return { type: "selector_visible", selector: obj.selector };
+        case "selector_hidden": return { type: "selector_hidden", selector: obj.selector };
+        case "url_matches": return { type: "url_matches", regex: obj.regex };
+        case "text_appears": return { type: "text_appears", text: obj.text };
+        case "spinner_disappears": return { type: "spinner_disappears", selector: obj.selector };
+        case "element_exists": return { type: "element_exists", target: parseTarget(obj.target) };
+        case "element_gone": return { type: "element_gone", target: parseTarget(obj.target) };
+        case "window_title_matches": return { type: "window_title_matches", regex: obj.regex };
+        case "app_idle": {
+            const cond = { type: "app_idle", bundleId: obj.bundleId };
+            if (typeof obj.timeoutMs === "number")
+                cond.timeoutMs = obj.timeoutMs;
+            return cond;
+        }
+        default: throw new Error(`Unknown wait condition type: ${type}`);
+    }
+}
+// ── Helpers ──
+function ok(data) {
+    return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
+}
+function err(message) {
+    return { content: [{ type: "text", text: message }], isError: true };
+}
+// ── Server builder ──
+export function createMcpStdioServer(runtime) {
+    const mcp = new McpServer({ name: "screenhand", version: "0.1.0" }, {
+        capabilities: { tools: {} },
+        instructions: "ScreenHand gives AI agents eyes and hands on the desktop. Use session_start to begin, then call tools to control apps.",
+    });
+    // ── session_start ──
+    mcp.tool("session_start", "Start a new automation session. Returns a sessionId needed by all other tools. Automatically attaches to the frontmost app.", { profile: z.string().optional().describe("Session profile name (default: 'automation')") }, async ({ profile }) => {
+        try {
+            const session = await runtime.sessionStart(profile);
+            return ok(session);
+        }
+        catch (e) {
+            return err(`Failed to start session: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    });
+    // ── press ──
+    mcp.tool("press", "Click/press a UI element. Finds the element by text, role, selector, or coordinates, then clicks it.", {
+        sessionId: z.string().describe("Session ID from session_start"),
+        target: TargetSchema.describe("What to click — text string, {role, name}, {selector}, or {x, y}"),
+        verify: WaitConditionSchema.optional().describe("Optional condition to verify after clicking"),
+    }, async ({ sessionId, target, verify }) => {
+        const input = { sessionId, target: parseTarget(target) };
+        if (verify)
+            input.verify = parseWaitCondition(verify);
+        const result = await runtime.press(input);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── type_into ──
+    mcp.tool("type_into", "Type text into a UI element (text field, search box, etc). Locates the field, optionally clears it, then types.", {
+        sessionId: z.string(),
+        target: TargetSchema.describe("The input field to type into"),
+        text: z.string().describe("Text to type"),
+        clear: z.boolean().optional().describe("Clear the field first (default: true)"),
+        verify: WaitConditionSchema.optional(),
+    }, async ({ sessionId, target, text, clear, verify }) => {
+        const input = { sessionId, target: parseTarget(target), text };
+        if (typeof clear === "boolean")
+            input.clear = clear;
+        if (verify)
+            input.verify = parseWaitCondition(verify);
+        const result = await runtime.typeInto(input);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── navigate ──
+    mcp.tool("navigate", "Navigate a browser to a URL, or open an app via 'app://com.bundle.id'.", {
+        sessionId: z.string(),
+        url: z.string().describe("URL to navigate to, or 'app://bundleId' to launch an app"),
+        timeoutMs: z.number().optional().describe("Navigation timeout in ms (default: 10000)"),
+    }, async ({ sessionId, url, timeoutMs }) => {
+        const input = { sessionId, url };
+        if (typeof timeoutMs === "number")
+            input.timeoutMs = timeoutMs;
+        const result = await runtime.navigate(input);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── wait_for ──
+    mcp.tool("wait_for", "Wait for a condition: element appears/disappears, text appears, URL changes, window title matches, etc.", {
+        sessionId: z.string(),
+        condition: WaitConditionSchema,
+        timeoutMs: z.number().optional().describe("Timeout in ms (default: 2000)"),
+    }, async ({ sessionId, condition, timeoutMs }) => {
+        const input = { sessionId, condition: parseWaitCondition(condition) };
+        if (typeof timeoutMs === "number")
+            input.timeoutMs = timeoutMs;
+        const result = await runtime.waitFor(input);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── extract ──
+    mcp.tool("extract", "Extract data from a UI element. Returns text content, table data, or structured JSON from the element.", {
+        sessionId: z.string(),
+        target: TargetSchema,
+        format: z.enum(["text", "table", "json"]).describe("Output format"),
+    }, async ({ sessionId, target, format }) => {
+        const result = await runtime.extract({
+            sessionId,
+            target: parseTarget(target),
+            format: format,
+        });
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── screenshot ──
+    mcp.tool("screenshot", "Capture a screenshot of the current app window or a specific screen region. Returns the file path.", {
+        sessionId: z.string(),
+        region: RegionSchema.optional().describe("Optional screen region to capture"),
+    }, async ({ sessionId, region }) => {
+        const input = { sessionId };
+        if (region)
+            input.region = region;
+        const result = await runtime.screenshot(input);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── app_launch ──
+    mcp.tool("app_launch", "Launch a macOS/Windows application by bundle ID (e.g., 'com.apple.Safari', 'com.google.Chrome').", {
+        sessionId: z.string(),
+        bundleId: z.string().describe("macOS bundle ID or Windows process name"),
+    }, async ({ sessionId, bundleId }) => {
+        const result = await runtime.appLaunch({ sessionId, bundleId });
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── app_focus ──
+    mcp.tool("app_focus", "Bring a running application to the foreground.", {
+        sessionId: z.string(),
+        bundleId: z.string(),
+    }, async ({ sessionId, bundleId }) => {
+        const result = await runtime.appFocus({ sessionId, bundleId });
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── app_list ──
+    mcp.tool("app_list", "List all running applications with their bundle IDs, names, and PIDs.", { sessionId: z.string() }, async ({ sessionId }) => {
+        const result = await runtime.appList(sessionId);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── window_list ──
+    mcp.tool("window_list", "List all visible windows with their titles, positions, and sizes.", { sessionId: z.string() }, async ({ sessionId }) => {
+        const result = await runtime.windowList(sessionId);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── menu_click ──
+    mcp.tool("menu_click", "Click a menu item by path. For example ['File', 'Save As...'] clicks File → Save As.", {
+        sessionId: z.string(),
+        menuPath: z.array(z.string()).describe("Menu path, e.g. ['File', 'New Window']"),
+    }, async ({ sessionId, menuPath }) => {
+        const result = await runtime.menuClick({ sessionId, menuPath });
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── key_combo ──
+    mcp.tool("key_combo", "Send a keyboard shortcut. Keys: 'cmd', 'ctrl', 'alt', 'shift', plus any character. E.g. ['cmd', 'c'] for copy.", {
+        sessionId: z.string(),
+        keys: z.array(z.string()).describe("Key combination, e.g. ['cmd', 's']"),
+    }, async ({ sessionId, keys }) => {
+        const result = await runtime.keyCombo({ sessionId, keys });
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── element_tree ──
+    mcp.tool("element_tree", "Get the accessibility element tree of the current app. Useful for understanding the UI structure and finding elements to interact with.", {
+        sessionId: z.string(),
+        maxDepth: z.number().optional().describe("Max tree depth (default: 5)"),
+    }, async ({ sessionId, maxDepth }) => {
+        const input = { sessionId };
+        if (typeof maxDepth === "number")
+            input.maxDepth = maxDepth;
+        const result = await runtime.elementTree(input);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── drag ──
+    mcp.tool("drag", "Drag from one UI element to another.", {
+        sessionId: z.string(),
+        from: TargetSchema.describe("Element to drag from"),
+        to: TargetSchema.describe("Element to drag to"),
+    }, async ({ sessionId, from, to }) => {
+        const result = await runtime.drag({
+            sessionId,
+            from: parseTarget(from),
+            to: parseTarget(to),
+        });
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    // ── scroll ──
+    mcp.tool("scroll", "Scroll in a direction, optionally targeting a specific element.", {
+        sessionId: z.string(),
+        direction: z.enum(["up", "down", "left", "right"]),
+        amount: z.number().optional().describe("Scroll amount (default: 3)"),
+        target: TargetSchema.optional().describe("Element to scroll within"),
+    }, async ({ sessionId, direction, amount, target }) => {
+        const input = { sessionId, direction };
+        if (typeof amount === "number")
+            input.amount = amount;
+        if (target)
+            input.target = parseTarget(target);
+        const result = await runtime.scroll(input);
+        return result.ok ? ok(result) : err(result.error.message);
+    });
+    return mcp;
+}
+export async function startMcpStdioServer(runtime) {
+    const mcp = createMcpStdioServer(runtime);
+    const transport = new StdioServerTransport();
+    await mcp.connect(transport);
+}

package/dist/mcp/server.js

@@ -0,0 +1,347 @@
+export class MvpMcpServer {
+    runtime;
+    constructor(runtime) {
+        this.runtime = runtime;
+    }
+    async invoke(request) {
+        switch (request.tool) {
+            case "session_start":
+                return this.runtime.sessionStart(optionalString(request.args, "profile"));
+            case "navigate": {
+                const timeoutMs = optionalNumber(request.args, "timeoutMs");
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    url: requiredString(request.args, "url"),
+                };
+                if (typeof timeoutMs === "number") {
+                    input.timeoutMs = timeoutMs;
+                }
+                return this.runtime.navigate(input);
+            }
+            case "press": {
+                const verify = parseOptionalWaitCondition(request.args.verify);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                };
+                if (verify) {
+                    input.verify = verify;
+                }
+                return this.runtime.press(input);
+            }
+            case "type_into": {
+                const clear = optionalBoolean(request.args, "clear");
+                const verify = parseOptionalWaitCondition(request.args.verify);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                    text: requiredString(request.args, "text"),
+                };
+                if (typeof clear === "boolean") {
+                    input.clear = clear;
+                }
+                if (verify) {
+                    input.verify = verify;
+                }
+                return this.runtime.typeInto(input);
+            }
+            case "wait_for": {
+                const timeoutMs = optionalNumber(request.args, "timeoutMs");
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    condition: parseWaitCondition(request.args.condition),
+                };
+                if (typeof timeoutMs === "number") {
+                    input.timeoutMs = timeoutMs;
+                }
+                return this.runtime.waitFor(input);
+            }
+            case "extract":
+                return this.runtime.extract({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    target: parseTarget(request.args.target),
+                    format: parseExtractFormat(request.args.format),
+                });
+            case "screenshot": {
+                const region = parseOptionalRegion(request.args.region);
+                const input = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (region) {
+                    input.region = region;
+                }
+                return this.runtime.screenshot(input);
+            }
+            // ── Desktop automation tools ──
+            case "app_launch":
+                return this.runtime.appLaunch({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    bundleId: requiredString(request.args, "bundleId"),
+                });
+            case "app_focus":
+                return this.runtime.appFocus({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    bundleId: requiredString(request.args, "bundleId"),
+                });
+            case "app_list":
+                return this.runtime.appList(requiredString(request.args, "sessionId"));
+            case "window_list":
+                return this.runtime.windowList(requiredString(request.args, "sessionId"));
+            case "menu_click":
+                return this.runtime.menuClick({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    menuPath: requiredStringArray(request.args, "menuPath"),
+                });
+            case "key_combo":
+                return this.runtime.keyCombo({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    keys: requiredStringArray(request.args, "keys"),
+                });
+            case "element_tree": {
+                const maxDepth = optionalNumber(request.args, "maxDepth");
+                const root = request.args.root ? parseTarget(request.args.root) : undefined;
+                const etInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (typeof maxDepth === "number")
+                    etInput.maxDepth = maxDepth;
+                if (root)
+                    etInput.root = root;
+                return this.runtime.elementTree(etInput);
+            }
+            case "observe_start": {
+                const events = request.args.events;
+                const osInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                };
+                if (Array.isArray(events)) {
+                    osInput.events = events;
+                }
+                return this.runtime.observeStart(osInput);
+            }
+            case "observe_stop":
+                return this.runtime.observeStop({
+                    sessionId: requiredString(request.args, "sessionId"),
+                });
+            case "drag":
+                return this.runtime.drag({
+                    sessionId: requiredString(request.args, "sessionId"),
+                    from: parseTarget(request.args.from),
+                    to: parseTarget(request.args.to),
+                });
+            case "scroll": {
+                const scrollTarget = request.args.target ? parseTarget(request.args.target) : undefined;
+                const scrollAmount = optionalNumber(request.args, "amount");
+                const scrollInput = {
+                    sessionId: requiredString(request.args, "sessionId"),
+                    direction: requiredString(request.args, "direction"),
+                };
+                if (scrollTarget)
+                    scrollInput.target = scrollTarget;
+                if (typeof scrollAmount === "number")
+                    scrollInput.amount = scrollAmount;
+                return this.runtime.scroll(scrollInput);
+            }
+            default:
+                throw new Error(`Unsupported tool: ${String(request.tool)}`);
+        }
+    }
+}
+function parseTarget(input) {
+    if (typeof input === "string") {
+        if (input.startsWith("css=")) {
+            return { type: "selector", value: input.slice(4) };
+        }
+        if (input.startsWith("text=")) {
+            return { type: "text", value: input.slice(5), exact: true };
+        }
+        if (input.startsWith("ax_id=")) {
+            return { type: "ax_attribute", attribute: "identifier", value: input.slice(6) };
+        }
+        return { type: "text", value: input };
+    }
+    if (!isRecord(input)) {
+        throw new Error("target must be a string or object");
+    }
+    if (typeof input.selector === "string") {
+        return { type: "selector", value: input.selector };
+    }
+    if (typeof input.text === "string") {
+        return {
+            type: "text",
+            value: input.text,
+            exact: input.exact === true,
+        };
+    }
+    if (typeof input.role === "string" && typeof input.name === "string") {
+        return {
+            type: "role",
+            role: input.role,
+            name: input.name,
+            exact: input.exact === true,
+        };
+    }
+    if (Array.isArray(input.path)) {
+        return { type: "ax_path", path: input.path };
+    }
+    if (typeof input.attribute === "string" && typeof input.value === "string") {
+        return { type: "ax_attribute", attribute: input.attribute, value: input.value };
+    }
+    if (typeof input.x === "number" && typeof input.y === "number") {
+        return { type: "coordinates", x: input.x, y: input.y };
+    }
+    if (typeof input.base64 === "string") {
+        const target = { type: "image", base64: input.base64 };
+        if (typeof input.confidence === "number") {
+            target.confidence = input.confidence;
+        }
+        return target;
+    }
+    throw new Error("target object must contain selector, text, role+name, path, attribute+value, x+y, or base64");
+}
+function parseWaitCondition(input) {
+    if (!isRecord(input) || typeof input.type !== "string") {
+        throw new Error("condition must be an object with a type");
+    }
+    switch (input.type) {
+        case "selector_visible":
+            return {
+                type: "selector_visible",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "selector_hidden":
+            return {
+                type: "selector_hidden",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "url_matches":
+            return {
+                type: "url_matches",
+                regex: requiredObjectString(input, "regex"),
+            };
+        case "text_appears":
+            return {
+                type: "text_appears",
+                text: requiredObjectString(input, "text"),
+            };
+        case "spinner_disappears":
+            return {
+                type: "spinner_disappears",
+                selector: requiredObjectString(input, "selector"),
+            };
+        case "element_exists":
+            return {
+                type: "element_exists",
+                target: parseTarget(input.target),
+            };
+        case "element_gone":
+            return {
+                type: "element_gone",
+                target: parseTarget(input.target),
+            };
+        case "window_title_matches":
+            return {
+                type: "window_title_matches",
+                regex: requiredObjectString(input, "regex"),
+            };
+        case "app_idle": {
+            const cond = {
+                type: "app_idle",
+                bundleId: requiredObjectString(input, "bundleId"),
+            };
+            if (typeof input.timeoutMs === "number") {
+                cond.timeoutMs = input.timeoutMs;
+            }
+            return cond;
+        }
+        default:
+            throw new Error(`Unsupported condition type: ${input.type}`);
+    }
+}
+function parseOptionalWaitCondition(input) {
+    if (typeof input === "undefined") {
+        return undefined;
+    }
+    return parseWaitCondition(input);
+}
+function parseExtractFormat(input) {
+    if (input === "text" || input === "table" || input === "json") {
+        return input;
+    }
+    throw new Error("format must be one of: text, table, json");
+}
+function parseOptionalRegion(input) {
+    if (typeof input === "undefined") {
+        return undefined;
+    }
+    if (!isRecord(input)) {
+        throw new Error("region must be an object");
+    }
+    return {
+        x: requiredObjectNumber(input, "x"),
+        y: requiredObjectNumber(input, "y"),
+        width: requiredObjectNumber(input, "width"),
+        height: requiredObjectNumber(input, "height"),
+    };
+}
+function requiredString(input, key) {
+    const value = input[key];
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function requiredStringArray(input, key) {
+    const value = input[key];
+    if (!Array.isArray(value) || !value.every((v) => typeof v === "string")) {
+        throw new Error(`${key} must be an array of strings`);
+    }
+    return value;
+}
+function optionalString(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function optionalNumber(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "number") {
+        throw new Error(`${key} must be a number`);
+    }
+    return value;
+}
+function optionalBoolean(input, key) {
+    const value = input[key];
+    if (typeof value === "undefined") {
+        return undefined;
+    }
+    if (typeof value !== "boolean") {
+        throw new Error(`${key} must be a boolean`);
+    }
+    return value;
+}
+function requiredObjectString(input, key) {
+    const value = input[key];
+    if (typeof value !== "string") {
+        throw new Error(`${key} must be a string`);
+    }
+    return value;
+}
+function requiredObjectNumber(input, key) {
+    const value = input[key];
+    if (typeof value !== "number") {
+        throw new Error(`${key} must be a number`);
+    }
+    return value;
+}
+function isRecord(input) {
+    return typeof input === "object" && input !== null;
+}