screenhand 0.1.0

package/dist/runtime/vision-adapter.js

@@ -0,0 +1,208 @@
+const POLL_INTERVAL_MS = 200;
+/**
+ * Vision-based adapter for apps with poor/no accessibility support.
+ * Uses screenshots + OCR to locate elements and CG events to interact.
+ */
+export class VisionAdapter {
+    bridge;
+    sessions = new Map();
+    sessionsByProfile = new Map();
+    constructor(bridge) {
+        this.bridge = bridge;
+    }
+    async attach(profile) {
+        const existing = this.sessionsByProfile.get(profile);
+        if (existing)
+            return existing.info;
+        await this.bridge.start();
+        const frontmost = await this.bridge.call("app.frontmost");
+        const info = {
+            sessionId: `vision_session_${profile}_${Date.now()}`,
+            profile,
+            createdAt: new Date().toISOString(),
+            adapterType: "vision",
+        };
+        const state = {
+            info,
+            pid: frontmost.pid,
+            bundleId: frontmost.bundleId,
+            appName: frontmost.name,
+        };
+        this.sessions.set(info.sessionId, state);
+        this.sessionsByProfile.set(profile, state);
+        return info;
+    }
+    async getAppContext(sessionId) {
+        const state = this.requireSession(sessionId);
+        return {
+            bundleId: state.bundleId,
+            appName: state.appName,
+            pid: state.pid,
+            windowTitle: state.appName,
+        };
+    }
+    async getPageMeta(sessionId) {
+        const ctx = await this.getAppContext(sessionId);
+        return {
+            url: `app://${ctx.bundleId}`,
+            title: ctx.appName,
+        };
+    }
+    async navigate(sessionId, url, _timeoutMs) {
+        if (url.startsWith("app://")) {
+            const bundleId = url.slice(6);
+            const result = await this.bridge.call("app.launch", { bundleId });
+            const state = this.requireSession(sessionId);
+            state.pid = result.pid;
+            state.bundleId = result.bundleId;
+            state.appName = result.appName;
+        }
+        return this.getPageMeta(sessionId);
+    }
+    async locate(sessionId, target, timeoutMs) {
+        const state = this.requireSession(sessionId);
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            // Take a screenshot
+            const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+            state.lastScreenshotPath = screenshotResult.path;
+            const searchText = this.getSearchText(target);
+            if (!searchText) {
+                // For coordinate targets, just return coordinates directly
+                if (target.type === "coordinates") {
+                    return {
+                        handleId: `vision_coords_${target.x}_${target.y}`,
+                        locatorUsed: "vision:coordinates",
+                        coordinates: { x: target.x, y: target.y, width: 1, height: 1 },
+                    };
+                }
+                return null;
+            }
+            // OCR the screenshot
+            const matches = await this.bridge.call("vision.findText", {
+                imagePath: screenshotResult.path,
+                searchText,
+            });
+            if (matches.length > 0) {
+                const best = matches.reduce((a, b) => (a.confidence > b.confidence ? a : b));
+                return {
+                    handleId: `vision_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+                    locatorUsed: `vision:text:${searchText}`,
+                    label: best.text,
+                    coordinates: best.bounds,
+                };
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return null;
+    }
+    async click(_sessionId, element) {
+        if (!element.coordinates) {
+            throw new Error("Vision adapter requires coordinates to click");
+        }
+        const cx = element.coordinates.x + element.coordinates.width / 2;
+        const cy = element.coordinates.y + element.coordinates.height / 2;
+        await this.bridge.call("cg.mouseClick", { x: cx, y: cy });
+    }
+    async setValue(_sessionId, element, text, clear) {
+        // Click to focus
+        await this.click(_sessionId, element);
+        await sleep(100);
+        if (clear) {
+            await this.bridge.call("cg.keyCombo", { keys: ["cmd", "a"] });
+            await sleep(50);
+        }
+        await this.bridge.call("cg.typeText", { text });
+    }
+    async getValue(_sessionId, element) {
+        // Vision can't reliably read values; return label if available
+        return element.label ?? "";
+    }
+    async waitFor(sessionId, condition, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            if (condition.type === "text_appears") {
+                const found = await this.locate(sessionId, { type: "text", value: condition.text }, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_exists") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_gone") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (!found)
+                    return true;
+            }
+            else {
+                // Unsupported condition types
+                return false;
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return false;
+    }
+    async extract(sessionId, _target, format) {
+        const state = this.requireSession(sessionId);
+        // Take a fresh screenshot and OCR it
+        const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+        state.lastScreenshotPath = screenshotResult.path;
+        const ocrResult = await this.bridge.call("vision.ocr", { imagePath: screenshotResult.path });
+        if (format === "text") {
+            return ocrResult.text;
+        }
+        if (format === "json") {
+            return ocrResult;
+        }
+        // table format
+        return {
+            headers: ["text", "confidence", "x", "y", "width", "height"],
+            rows: ocrResult.regions.map((r) => [
+                r.text,
+                r.confidence,
+                r.bounds.x,
+                r.bounds.y,
+                r.bounds.width,
+                r.bounds.height,
+            ]),
+        };
+    }
+    async screenshot(_sessionId, region) {
+        const result = await this.bridge.call("cg.captureScreen", region ? { region } : {});
+        return result.path;
+    }
+    async keyCombo(_sessionId, keys) {
+        await this.bridge.call("cg.keyCombo", { keys });
+    }
+    async elementTree(_sessionId, _maxDepth, _root) {
+        throw new Error("Vision adapter does not support elementTree — use accessibility adapter");
+    }
+    // ── Private ──
+    requireSession(sessionId) {
+        const state = this.sessions.get(sessionId);
+        if (!state)
+            throw new Error(`Session not found: ${sessionId}`);
+        return state;
+    }
+    getSearchText(target) {
+        switch (target.type) {
+            case "text":
+                return target.value;
+            case "role":
+                return target.name;
+            case "selector":
+                return target.value;
+            case "ax_attribute":
+                return target.value;
+            case "image":
+            case "coordinates":
+            case "ax_path":
+                return null;
+        }
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/test-mcp-protocol.js

@@ -0,0 +1,138 @@
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const projectRoot = path.resolve(__dirname, "..");
+const tsxBin = path.join(projectRoot, "node_modules", ".bin", "tsx");
+const TIMEOUT_MS = 10_000;
+const proc = spawn(tsxBin, [path.join(projectRoot, "src/mcp-entry.ts")], {
+    stdio: ["pipe", "pipe", "pipe"],
+    env: { ...process.env, SCREENHAND_ADAPTER: "placeholder" },
+    cwd: projectRoot,
+});
+let stderrBuf = "";
+proc.stderr.on("data", (d) => { stderrBuf += d.toString(); });
+// MCP SDK v1.27 uses newline-delimited JSON (NDJSON), not Content-Length framing
+function send(msg) {
+    proc.stdin.write(JSON.stringify(msg) + "\n");
+}
+const rl = createInterface({ input: proc.stdout });
+const lineQueue = [];
+let lineWaiter = null;
+rl.on("line", (line) => {
+    if (lineWaiter) {
+        const w = lineWaiter;
+        lineWaiter = null;
+        w(line);
+    }
+    else {
+        lineQueue.push(line);
+    }
+});
+function readResponse() {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            lineWaiter = null;
+            reject(new Error(`Timeout. stderr: ${stderrBuf.slice(-300)}`));
+        }, TIMEOUT_MS);
+        const handle = (line) => {
+            clearTimeout(timer);
+            resolve(JSON.parse(line));
+        };
+        const queued = lineQueue.shift();
+        if (queued) {
+            clearTimeout(timer);
+            resolve(JSON.parse(queued));
+        }
+        else {
+            lineWaiter = handle;
+        }
+    });
+}
+function fail(msg) {
+    console.error("FAIL:", msg);
+    proc.kill();
+    process.exit(1);
+}
+try {
+    // Wait for server to start
+    await new Promise((r) => setTimeout(r, 2000));
+    // 1. Initialize
+    console.log("Sending initialize...");
+    send({
+        jsonrpc: "2.0",
+        id: 1,
+        method: "initialize",
+        params: {
+            protocolVersion: "2024-11-05",
+            capabilities: {},
+            clientInfo: { name: "test-client", version: "1.0" },
+        },
+    });
+    const initResp = await readResponse();
+    const initResult = initResp.result;
+    if (!initResult)
+        fail(`No init result: ${JSON.stringify(initResp)}`);
+    console.log("=== Initialize ===");
+    console.log(`  Protocol: ${initResult.protocolVersion}`);
+    console.log(`  Server: ${JSON.stringify(initResult.serverInfo)}`);
+    // 2. Send initialized notification
+    send({ jsonrpc: "2.0", method: "notifications/initialized" });
+    await new Promise((r) => setTimeout(r, 300));
+    // 3. List tools
+    console.log("\nListing tools...");
+    send({ jsonrpc: "2.0", id: 2, method: "tools/list", params: {} });
+    const toolsResp = await readResponse();
+    const toolsResult = toolsResp.result;
+    if (!toolsResult)
+        fail(`No tools result: ${JSON.stringify(toolsResp)}`);
+    const tools = toolsResult.tools ?? [];
+    console.log("=== Tools ===");
+    for (const tool of tools) {
+        console.log(`  ${tool.name}: ${(tool.description ?? "").slice(0, 70)}`);
+    }
+    console.log(`\n  Total: ${tools.length} tools`);
+    if (tools.length < 10)
+        fail(`Expected 16 tools, got ${tools.length}`);
+    // 4. Test session_start
+    console.log("\nCalling session_start...");
+    send({
+        jsonrpc: "2.0",
+        id: 3,
+        method: "tools/call",
+        params: { name: "session_start", arguments: {} },
+    });
+    const sessionResp = await readResponse();
+    const sessionResult = sessionResp.result;
+    if (!sessionResult)
+        fail(`No session result: ${JSON.stringify(sessionResp)}`);
+    const sessionContent = sessionResult.content;
+    const sessionData = JSON.parse(sessionContent?.[0]?.text ?? "{}");
+    console.log("=== session_start ===");
+    console.log(`  Session ID: ${sessionData.sessionId}`);
+    console.log(`  Profile: ${sessionData.profile}`);
+    if (!sessionData.sessionId)
+        fail("No sessionId returned");
+    // 5. Test app_list (should work with placeholder)
+    console.log("\nCalling app_list...");
+    send({
+        jsonrpc: "2.0",
+        id: 4,
+        method: "tools/call",
+        params: { name: "app_list", arguments: { sessionId: sessionData.sessionId } },
+    });
+    const appResp = await readResponse();
+    const appResult = appResp.result;
+    console.log("=== app_list ===");
+    const appContent = appResult?.content;
+    const isError = appResult?.isError;
+    console.log(`  isError: ${isError ?? false}`);
+    console.log(`  Response: ${(appContent?.[0]?.text ?? "").slice(0, 100)}`);
+    proc.kill();
+    console.log("\nAll tests passed!");
+    process.exit(0);
+}
+catch (e) {
+    fail(e instanceof Error ? e.message : String(e));
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "screenhand",
+  "version": "0.1.0",
+  "mcpName": "io.github.manushi4/screenhand",
+  "description": "Give AI eyes and hands on your desktop. ScreenHand is an open-source MCP server that lets Claude and other AI agents see your screen, click buttons, type text, and control any app on macOS and Windows.",
+  "homepage": "https://screenhand.com",
+  "type": "module",
+  "bin": {
+    "screenhand": "dist/mcp-entry.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "dev": "tsx src/index.ts",
+    "dev:mcp": "tsx src/mcp-entry.ts",
+    "build": "tsc -p tsconfig.json",
+    "check": "tsc --noEmit -p tsconfig.check.json",
+    "start": "node dist/index.js",
+    "start:mcp": "node dist/mcp-entry.js",
+    "build:native": "cd native/macos-bridge && swift build -c release",
+    "build:native:windows": "cd native/windows-bridge && dotnet build -c Release",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/manushi4/screenhand"
+  },
+  "bugs": {
+    "url": "https://github.com/manushi4/screenhand/issues"
+  },
+  "author": "Khushi Singhal",
+  "license": "MIT",
+  "keywords": [
+    "screenhand",
+    "mcp",
+    "mcp-server",
+    "model-context-protocol",
+    "desktop-automation",
+    "ui-automation",
+    "accessibility",
+    "macos",
+    "windows",
+    "claude",
+    "ai-agent",
+    "ai-tools",
+    "screen-control",
+    "browser-automation",
+    "cdp",
+    "chrome-devtools-protocol",
+    "rpa",
+    "ui-testing",
+    "ocr",
+    "computer-use"
+  ],
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.78.0",
+    "@modelcontextprotocol/sdk": "^1.27.1",
+    "chrome-launcher": "^1.2.1",
+    "chrome-remote-interface": "^0.33.3"
+  },
+  "devDependencies": {
+    "@types/chrome-remote-interface": "^0.31.14",
+    "@types/node": "^22.13.9",
+    "tsx": "^4.19.2",
+    "typescript": "^5.8.2",
+    "vitest": "^3.2.4"
+  }
+}