polarisagi-computer-mcp 0.1.0

package/dist/index.js

@@ -0,0 +1,310 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const readline = __importStar(require("readline"));
+const nut_js_1 = require("@nut-tree-fork/nut-js");
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const fs = __importStar(require("fs/promises"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
+async function captureScreen() {
+    const tmpPath = path.join(os.tmpdir(), `screenshot-${Date.now()}.png`);
+    try {
+        if (process.platform === 'darwin') {
+            await execFileAsync('screencapture', ['-x', tmpPath]);
+        }
+        else if (process.platform === 'win32') {
+            const psScript = `
+            Add-Type -AssemblyName System.Windows.Forms;
+            Add-Type -AssemblyName System.Drawing;
+            $Screen = [System.Windows.Forms.SystemInformation]::VirtualScreen;
+            $Width = $Screen.Width; $Height = $Screen.Height;
+            $Left = $Screen.Left; $Top = $Screen.Top;
+            $bitmap = New-Object System.Drawing.Bitmap $Width, $Height;
+            $graphic = [System.Drawing.Graphics]::FromImage($bitmap);
+            $graphic.CopyFromScreen($Left, $Top, 0, 0, $bitmap.Size);
+            $bitmap.Save('${tmpPath}');
+            `;
+            await execFileAsync('powershell', ['-Command', psScript]);
+        }
+        else {
+            try {
+                // Try scrot first
+                await execFileAsync('scrot', [tmpPath]);
+            }
+            catch (err) {
+                // Fallback to ImageMagick (import)
+                await execFileAsync('import', ['-window', 'root', tmpPath]);
+            }
+        }
+        const buffer = await fs.readFile(tmpPath);
+        return buffer.toString('base64');
+    }
+    finally {
+        try {
+            await fs.unlink(tmpPath);
+        }
+        catch (e) {
+            // ignore
+        }
+    }
+}
+const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+    terminal: false
+});
+function sendResult(id, result) {
+    if (id === undefined || id === null)
+        return;
+    const resp = {
+        jsonrpc: "2.0",
+        id,
+        result
+    };
+    console.log(JSON.stringify(resp));
+}
+function sendError(id, code, message) {
+    const resp = {
+        jsonrpc: "2.0",
+        id,
+        error: { code, message }
+    };
+    console.log(JSON.stringify(resp));
+}
+rl.on('line', async (line) => {
+    if (!line.trim())
+        return;
+    let req;
+    try {
+        req = JSON.parse(line);
+    }
+    catch (e) {
+        sendError(null, -32700, "Parse error: " + e);
+        return;
+    }
+    const id = req.id;
+    try {
+        if (req.method === "initialize") {
+            sendResult(id, {
+                protocolVersion: "2024-11-05",
+                capabilities: { tools: {}, prompts: {} },
+                serverInfo: {
+                    name: "polaris-computer-mcp",
+                    version: "0.1.0"
+                }
+            });
+        }
+        else if (req.method === "prompts/list") {
+            sendResult(id, {
+                prompts: [{
+                        name: "computer_use_guidelines",
+                        description: "Standard system guidelines for AI agents using the computer plugin.",
+                        arguments: []
+                    }]
+            });
+        }
+        else if (req.method === "prompts/get") {
+            const promptName = req.params?.name;
+            if (promptName === "computer_use_guidelines") {
+                sendResult(id, {
+                    description: "System Prompt for Computer Use",
+                    messages: [
+                        {
+                            role: "user",
+                            content: {
+                                type: "text",
+                                text: "You have permission to operate a real computer. Please strictly follow these visual recognition and operation guidelines:\n\n1. Visual Feedback Loop: Before executing any click (`left_click`) or input (`type`/`paste`), you MUST call `screenshot` to get the current screen state. Analyze the screenshot carefully to find the absolute coordinates `[x, y]` of the target UI element before clicking.\n\n2. Handling UI States: UI elements may have loading animations or network delays. After interacting with an element (e.g. clicking 'search'), do not assume the action completed instantly. You MUST call `screenshot` again to verify the new UI state (like checking if a dropdown menu appeared) before proceeding.\n\n3. Text Input Rules: When inputting Chinese characters or long text, you MUST prioritize the `paste` action. This bypasses input method editor (IME) interference and prevents text truncation. Only use `type` or `key` for pure English characters or shortcuts.\n\n4. Error Recovery: If the screenshot does not match your expectations (e.g. click failed or text was typed in the wrong place), analyze the screen, use `mouse_move` to move the focus away, or send `key: escape` to cancel the current state, and retry."
+                            }
+                        }
+                    ]
+                });
+            }
+            else {
+                sendError(id, -32602, "Prompt not found");
+            }
+        }
+        else if (req.method === "tools/list") {
+            sendResult(id, {
+                tools: [{
+                        name: "computer",
+                        description: "Execute computer actions like clicking, typing, and taking screenshots.",
+                        inputSchema: {
+                            type: "object",
+                            properties: {
+                                action: {
+                                    type: "string",
+                                    enum: ["screenshot", "left_click", "right_click", "middle_click", "double_click", "mouse_move", "left_click_drag", "cursor_position", "type", "key", "paste"],
+                                    description: "The action to perform. left_click_drag: press at current position, move to coordinate, release. paste: copy text to clipboard and simulate Cmd+V/Ctrl+V."
+                                },
+                                coordinate: {
+                                    type: "array",
+                                    items: { type: "number" },
+                                    description: "[x, y] coordinates for mouse actions. For left_click_drag, this is the drag destination."
+                                },
+                                text: {
+                                    type: "string",
+                                    description: "Text to type (action=type) or key name to press (action=key)."
+                                }
+                            },
+                            required: ["action"]
+                        }
+                    }]
+            });
+        }
+        else if (req.method === "tools/call") {
+            const params = req.params || {};
+            if (params.name === "computer") {
+                const args = params.arguments || {};
+                const content = await handleComputerUse(args);
+                sendResult(id, { content });
+            }
+            else {
+                sendError(id, -32601, "Tool not found");
+            }
+        }
+        else {
+            if (id !== undefined) {
+                sendError(id, -32601, "Method not found: " + req.method);
+            }
+        }
+    }
+    catch (e) {
+        if (id !== undefined) {
+            sendError(id, -32603, "Execution error: " + e.message);
+        }
+    }
+});
+async function handleComputerUse(args) {
+    const action = args.action;
+    if (!action)
+        throw new Error("Missing action parameter");
+    let x = 0, y = 0;
+    if (Array.isArray(args.coordinate) && args.coordinate.length >= 2) {
+        x = Math.round(Number(args.coordinate[0]) || 0);
+        y = Math.round(Number(args.coordinate[1]) || 0);
+    }
+    switch (action) {
+        case "screenshot":
+            const b64 = await captureScreen();
+            return [{
+                    type: "image",
+                    data: b64,
+                    mimeType: "image/png"
+                }];
+        case "mouse_move":
+            await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+            return [{ type: "text", text: "success" }];
+        case "left_click":
+            await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+            await nut_js_1.mouse.leftClick();
+            return [{ type: "text", text: "success" }];
+        case "right_click":
+            await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+            await nut_js_1.mouse.rightClick();
+            return [{ type: "text", text: "success" }];
+        case "middle_click":
+            await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+            await nut_js_1.mouse.pressButton(nut_js_1.Button.MIDDLE);
+            await nut_js_1.mouse.releaseButton(nut_js_1.Button.MIDDLE);
+            return [{ type: "text", text: "success" }];
+        case "double_click":
+            await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+            await nut_js_1.mouse.leftClick();
+            await nut_js_1.mouse.leftClick();
+            return [{ type: "text", text: "success" }];
+        case "left_click_drag":
+            await nut_js_1.mouse.pressButton(nut_js_1.Button.LEFT);
+            await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+            await nut_js_1.mouse.releaseButton(nut_js_1.Button.LEFT);
+            return [{ type: "text", text: "success" }];
+        case "cursor_position":
+            const pos = await nut_js_1.mouse.getPosition();
+            return [{ type: "text", text: `X: ${pos.x}, Y: ${pos.y}` }];
+        case "type":
+            if (!args.text)
+                throw new Error("Missing text parameter");
+            await nut_js_1.keyboard.type(args.text);
+            return [{ type: "text", text: "success" }];
+        case "paste":
+            if (!args.text)
+                throw new Error("Missing text parameter");
+            await nut_js_1.clipboard.setContent(args.text);
+            // wait a tiny bit to ensure clipboard is synced
+            await new Promise(r => setTimeout(r, 200));
+            if (process.platform === 'darwin') {
+                await nut_js_1.keyboard.pressKey(nut_js_1.Key.LeftSuper, nut_js_1.Key.V);
+                await nut_js_1.keyboard.releaseKey(nut_js_1.Key.LeftSuper, nut_js_1.Key.V);
+            }
+            else {
+                await nut_js_1.keyboard.pressKey(nut_js_1.Key.LeftControl, nut_js_1.Key.V);
+                await nut_js_1.keyboard.releaseKey(nut_js_1.Key.LeftControl, nut_js_1.Key.V);
+            }
+            return [{ type: "text", text: "success" }];
+        case "key":
+            if (!args.text)
+                throw new Error("Missing text parameter");
+            const keyText = args.text.toLowerCase();
+            const keyMap = {
+                "return": nut_js_1.Key.Enter, "enter": nut_js_1.Key.Enter,
+                "escape": nut_js_1.Key.Escape, "esc": nut_js_1.Key.Escape,
+                "tab": nut_js_1.Key.Tab, "space": nut_js_1.Key.Space,
+                "backspace": nut_js_1.Key.Backspace, "delete": nut_js_1.Key.Delete, "del": nut_js_1.Key.Delete,
+                "up": nut_js_1.Key.Up, "down": nut_js_1.Key.Down, "left": nut_js_1.Key.Left, "right": nut_js_1.Key.Right,
+                "home": nut_js_1.Key.Home, "end": nut_js_1.Key.End,
+                "pageup": nut_js_1.Key.PageUp, "page_up": nut_js_1.Key.PageUp,
+                "pagedown": nut_js_1.Key.PageDown, "page_down": nut_js_1.Key.PageDown,
+                "ctrl": nut_js_1.Key.LeftControl, "control": nut_js_1.Key.LeftControl,
+                "alt": nut_js_1.Key.LeftAlt, "shift": nut_js_1.Key.LeftShift,
+                "meta": nut_js_1.Key.LeftSuper, "command": nut_js_1.Key.LeftSuper, "win": nut_js_1.Key.LeftSuper
+            };
+            let nutKey = keyMap[keyText];
+            if (!nutKey && keyText.length === 1) {
+                await nut_js_1.keyboard.type(keyText);
+                return [{ type: "text", text: "success" }];
+            }
+            else if (!nutKey) {
+                throw new Error(`Unsupported key: ${args.text}`);
+            }
+            await nut_js_1.keyboard.pressKey(nutKey);
+            await nut_js_1.keyboard.releaseKey(nutKey);
+            return [{ type: "text", text: "success" }];
+        default:
+            throw new Error(`Action '${action}' is not fully implemented yet`);
+    }
+}

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "polarisagi-computer-mcp",
+  "version": "0.1.0",
+  "description": "TypeScript MCP server for computer automation",
+  "files": [
+    "dist"
+  ],
+  "main": "dist/index.js",
+  "bin": {
+    "polarisagi-computer-mcp": "dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/index.js"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "@nut-tree-fork/nut-js": "^4.2.6"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0"
+  }
+}