screenpipe-mcp 0.4.1 → 0.6.0

package/dist/index.js

@@ -54,11 +54,6 @@ function getCurrentDateInfo() {
         }),
     };
 }
-// Detect OS
-const CURRENT_OS = process.platform;
-const IS_MACOS = CURRENT_OS === "darwin";
-const IS_WINDOWS = CURRENT_OS === "win32";
-const IS_LINUX = CURRENT_OS === "linux";
 // Parse command line arguments
 const args = process.argv.slice(2);
 let port = 3030;
@@ -71,7 +66,7 @@ const SCREENPIPE_API = `http://localhost:${port}`;
 // Initialize server
 const server = new index_js_1.Server({
     name: "screenpipe",
-    version: "0.4.0",
+    version: "0.5.0",
 }, {
     capabilities: {
         tools: {},
@@ -148,54 +143,6 @@ const BASE_TOOLS = [
             },
         },
     },
-    {
-        name: "pixel-control",
-        description: "Control mouse and keyboard at the pixel level. This is a cross-platform tool that works on all operating systems. " +
-            "Use this to type text, press keys, move the mouse, and click buttons.",
-        annotations: {
-            title: "Pixel Control",
-            destructiveHint: true,
-        },
-        inputSchema: {
-            type: "object",
-            properties: {
-                action_type: {
-                    type: "string",
-                    enum: ["WriteText", "KeyPress", "MouseMove", "MouseClick"],
-                    description: "Type of input action to perform",
-                },
-                data: {
-                    oneOf: [
-                        {
-                            type: "string",
-                            description: "Text to type or key to press (for WriteText and KeyPress)",
-                        },
-                        {
-                            type: "object",
-                            properties: {
-                                x: {
-                                    type: "integer",
-                                    description: "X coordinate for mouse movement",
-                                },
-                                y: {
-                                    type: "integer",
-                                    description: "Y coordinate for mouse movement",
-                                },
-                            },
-                            description: "Coordinates for MouseMove",
-                        },
-                        {
-                            type: "string",
-                            enum: ["left", "right", "middle"],
-                            description: "Button to click for MouseClick",
-                        },
-                    ],
-                    description: "Action-specific data",
-                },
-            },
-            required: ["action_type", "data"],
-        },
-    },
     {
         name: "export-video",
         description: "Export a video of screen recordings for a specific time range. " +
@@ -231,229 +178,9 @@ const BASE_TOOLS = [
         },
     },
 ];
-const MACOS_TOOLS = [
-    {
-        name: "find-elements",
-        description: "Find UI elements with a specific role in an application. " +
-            "This tool is especially useful for identifying interactive elements. " +
-            "\n\nMacOS Accessibility Roles Guide:\n" +
-            "- Basic roles: 'button', 'textfield', 'checkbox', 'menu', 'list'\n" +
-            "- MacOS specific roles: 'AXButton', 'AXTextField', 'AXCheckBox', 'AXMenu', etc.\n" +
-            "- Text inputs can be: 'AXTextField', 'AXTextArea', 'AXComboBox', 'AXSearchField'\n" +
-            "- Clickable items: 'AXButton', 'AXMenuItem', 'AXMenuBarItem', 'AXImage', 'AXStaticText'\n" +
-            "- Web content may use: 'AXWebArea', 'AXLink', 'AXHeading', 'AXRadioButton'\n\n" +
-            "Use MacOS Accessibility Inspector app to identify the exact roles in your target application.",
-        annotations: {
-            title: "Find Elements",
-            readOnlyHint: true,
-        },
-        inputSchema: {
-            type: "object",
-            properties: {
-                app: {
-                    type: "string",
-                    description: "The name of the application (e.g., 'Chrome', 'Finder', 'Terminal')",
-                },
-                window: {
-                    type: "string",
-                    description: "The window name or title (optional)",
-                },
-                role: {
-                    type: "string",
-                    description: "The role to search for (e.g., 'button', 'textfield', 'AXButton', 'AXTextField'). For best results, use MacOS AX prefixed roles.",
-                },
-                max_results: {
-                    type: "integer",
-                    description: "Maximum number of elements to return",
-                    default: 10,
-                },
-                max_depth: {
-                    type: "integer",
-                    description: "Maximum depth of element tree to search",
-                },
-                use_background_apps: {
-                    type: "boolean",
-                    description: "Whether to look in background apps",
-                    default: true,
-                },
-                activate_app: {
-                    type: "boolean",
-                    description: "Whether to activate the app before searching",
-                    default: true,
-                },
-            },
-            required: ["app", "role"],
-        },
-    },
-    {
-        name: "click-element",
-        description: "Click an element in an application using its id (MacOS only)",
-        annotations: {
-            title: "Click Element",
-            destructiveHint: true,
-        },
-        inputSchema: {
-            type: "object",
-            properties: {
-                app: {
-                    type: "string",
-                    description: "The name of the application",
-                },
-                window: {
-                    type: "string",
-                    description: "The window name (optional)",
-                },
-                id: {
-                    type: "string",
-                    description: "The id of the element to click",
-                },
-                use_background_apps: {
-                    type: "boolean",
-                    description: "Whether to look in background apps",
-                    default: true,
-                },
-                activate_app: {
-                    type: "boolean",
-                    description: "Whether to activate the app before clicking",
-                    default: true,
-                },
-            },
-            required: ["app", "id"],
-        },
-    },
-    {
-        name: "fill-element",
-        description: "Type text into an element in an application (MacOS only)",
-        annotations: {
-            title: "Fill Element",
-            destructiveHint: true,
-        },
-        inputSchema: {
-            type: "object",
-            properties: {
-                app: {
-                    type: "string",
-                    description: "The name of the application",
-                },
-                window: {
-                    type: "string",
-                    description: "The window name (optional)",
-                },
-                id: {
-                    type: "string",
-                    description: "The id of the element to fill",
-                },
-                text: {
-                    type: "string",
-                    description: "The text to type into the element",
-                },
-                use_background_apps: {
-                    type: "boolean",
-                    description: "Whether to look in background apps",
-                    default: true,
-                },
-                activate_app: {
-                    type: "boolean",
-                    description: "Whether to activate the app before typing",
-                    default: true,
-                },
-            },
-            required: ["app", "id", "text"],
-        },
-    },
-    {
-        name: "scroll-element",
-        description: "Scroll an element in a specific direction (MacOS only)",
-        annotations: {
-            title: "Scroll Element",
-            destructiveHint: true,
-        },
-        inputSchema: {
-            type: "object",
-            properties: {
-                app: {
-                    type: "string",
-                    description: "The name of the application",
-                },
-                window: {
-                    type: "string",
-                    description: "The window name (optional)",
-                },
-                id: {
-                    type: "string",
-                    description: "The id of the element to scroll",
-                },
-                direction: {
-                    type: "string",
-                    enum: ["up", "down", "left", "right"],
-                    description: "The direction to scroll",
-                },
-                amount: {
-                    type: "integer",
-                    description: "The amount to scroll in pixels",
-                },
-                use_background_apps: {
-                    type: "boolean",
-                    description: "Whether to look in background apps",
-                    default: true,
-                },
-                activate_app: {
-                    type: "boolean",
-                    description: "Whether to activate the app before scrolling",
-                    default: true,
-                },
-            },
-            required: ["app", "id", "direction", "amount"],
-        },
-    },
-    {
-        name: "open-application",
-        description: "Open an application by name",
-        annotations: {
-            title: "Open Application",
-            destructiveHint: true,
-        },
-        inputSchema: {
-            type: "object",
-            properties: {
-                app_name: {
-                    type: "string",
-                    description: "The name of the application to open",
-                },
-            },
-            required: ["app_name"],
-        },
-    },
-    {
-        name: "open-url",
-        description: "Open a URL in a browser",
-        annotations: {
-            title: "Open URL",
-            destructiveHint: true,
-        },
-        inputSchema: {
-            type: "object",
-            properties: {
-                url: {
-                    type: "string",
-                    description: "The URL to open",
-                },
-                browser: {
-                    type: "string",
-                    description: "The browser to use (optional)",
-                },
-            },
-            required: ["url"],
-        },
-    },
-];
 // List tools handler
 server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => {
-    const tools = [...BASE_TOOLS];
-    if (IS_MACOS) {
-        tools.push(...MACOS_TOOLS);
-    }
-    return { tools };
+    return { tools: BASE_TOOLS };
 });
 // MCP Resources - provide dynamic context data
 const RESOURCES = [
@@ -469,6 +196,12 @@ const RESOURCES = [
         description: "How to use screenpipe search effectively",
         mimeType: "text/markdown",
     },
+    {
+        uri: "ui://search",
+        name: "Search Dashboard",
+        description: "Interactive search UI for exploring screen recordings and audio transcriptions",
+        mimeType: "text/html",
+    },
 ];
 // List resources handler
 server.setRequestHandler(types_js_1.ListResourcesRequestSchema, async () => {
@@ -541,6 +274,56 @@ server.setRequestHandler(types_js_1.ReadResourceRequestSchema, async (request) =
                     },
                 ],
             };
+        case "ui://search": {
+            // MCP App UI - Interactive search dashboard
+            const uiHtmlPath = path.join(__dirname, "..", "ui", "search.html");
+            let htmlContent;
+            try {
+                htmlContent = fs.readFileSync(uiHtmlPath, "utf-8");
+            }
+            catch {
+                // Fallback: serve embedded minimal UI if file not found
+                htmlContent = `<!DOCTYPE html>
+<html>
+<head>
+  <style>
+    body { font-family: system-ui; background: #0a0a0a; color: #fff; padding: 20px; }
+    input { width: 100%; padding: 10px; margin-bottom: 10px; background: #1a1a1a; border: 1px solid #333; color: #fff; border-radius: 6px; }
+    button { padding: 10px 20px; background: #fff; color: #000; border: none; border-radius: 6px; cursor: pointer; }
+    #results { margin-top: 20px; }
+    .result { background: #1a1a1a; padding: 12px; margin: 8px 0; border-radius: 8px; border: 1px solid #333; }
+  </style>
+</head>
+<body>
+  <h2>screenpipe search</h2>
+  <input id="q" placeholder="search..." onkeydown="if(event.key==='Enter')search()"/>
+  <button onclick="search()">search</button>
+  <div id="results"></div>
+  <script>
+    function search() {
+      window.parent.postMessage({jsonrpc:'2.0',method:'tools/call',params:{name:'search-content',arguments:{q:document.getElementById('q').value,limit:20}}},'*');
+    }
+    window.addEventListener('message',e=>{
+      if(e.data?.result||e.data?.method==='tool/result'){
+        const r=e.data.result||e.data.params?.result;
+        const d=r?.data||r||[];
+        document.getElementById('results').innerHTML=d.map(x=>'<div class="result"><b>'+((x.type||'')+'</b> '+(x.content?.app_name||'')+': '+(x.content?.text||x.content?.transcription||'').substring(0,200))+'</div>').join('');
+      }
+    });
+  </script>
+</body>
+</html>`;
+            }
+            return {
+                contents: [
+                    {
+                        uri,
+                        mimeType: "text/html",
+                        text: htmlContent,
+                    },
+                ],
+            };
+        }
         default:
             throw new Error(`Unknown resource: ${uri}`);
     }
@@ -675,25 +458,6 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
     if (!args) {
         throw new Error("Missing arguments");
     }
-    // Check if the tool is MacOS-only and we're not on MacOS
-    const macosOnlyTools = [
-        "click-element",
-        "fill-element",
-        "find-elements",
-        "scroll-element",
-        "open-application",
-        "open-url",
-    ];
-    if (macosOnlyTools.includes(name) && !IS_MACOS) {
-        return {
-            content: [
-                {
-                    type: "text",
-                    text: `The '${name}' tool is only available on MacOS. Current platform: ${CURRENT_OS}`,
-                },
-            ],
-        };
-    }
     try {
         switch (name) {
             case "search-content": {
@@ -765,47 +529,6 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                 }
                 return { content: contentItems };
             }
-            case "pixel-control": {
-                const action = {
-                    type: args.action_type,
-                    data: args.data,
-                };
-                const response = await fetchAPI("/experimental/operator/pixel", {
-                    method: "POST",
-                    body: JSON.stringify({ action }),
-                });
-                if (!response.ok) {
-                    throw new Error(`HTTP error: ${response.status}`);
-                }
-                const data = await response.json();
-                if (!data.success) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `Failed to perform input control: ${data.error || "unknown error"}`,
-                            },
-                        ],
-                    };
-                }
-                let resultText = "Successfully performed input control action";
-                if (args.action_type === "WriteText") {
-                    resultText = `Successfully typed text: '${args.data}'`;
-                }
-                else if (args.action_type === "KeyPress") {
-                    resultText = `Successfully pressed key: '${args.data}'`;
-                }
-                else if (args.action_type === "MouseMove") {
-                    const coords = args.data;
-                    resultText = `Successfully moved mouse to coordinates: x=${coords.x}, y=${coords.y}`;
-                }
-                else if (args.action_type === "MouseClick") {
-                    resultText = `Successfully clicked ${args.data} mouse button`;
-                }
-                return {
-                    content: [{ type: "text", text: resultText }],
-                };
-            }
             case "export-video": {
                 const startTime = args.start_time;
                 const endTime = args.end_time;
@@ -951,230 +674,6 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                     };
                 }
             }
-            case "click-element": {
-                const selector = {
-                    app_name: args.app,
-                    window_name: args.window,
-                    locator: `#${args.id}`,
-                    use_background_apps: args.use_background_apps ?? true,
-                    activate_app: args.activate_app ?? true,
-                };
-                const response = await fetchAPI("/experimental/operator/click", {
-                    method: "POST",
-                    body: JSON.stringify({ selector }),
-                });
-                if (!response.ok) {
-                    throw new Error(`HTTP error: ${response.status}`);
-                }
-                const data = await response.json();
-                if (!data.success) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `Failed to click element: ${data.error || "unknown error"}`,
-                            },
-                        ],
-                    };
-                }
-                const result = data.result || {};
-                const method = result.method || "unknown";
-                const details = result.details || "click operation completed";
-                return {
-                    content: [
-                        {
-                            type: "text",
-                            text: `Successfully clicked element using ${method}. ${details}`,
-                        },
-                    ],
-                };
-            }
-            case "fill-element": {
-                const selector = {
-                    app_name: args.app,
-                    window_name: args.window,
-                    locator: `#${args.id}`,
-                    use_background_apps: args.use_background_apps ?? true,
-                    activate_app: args.activate_app ?? true,
-                };
-                const response = await fetchAPI("/experimental/operator/type", {
-                    method: "POST",
-                    body: JSON.stringify({ selector, text: args.text || "" }),
-                });
-                if (!response.ok) {
-                    throw new Error(`HTTP error: ${response.status}`);
-                }
-                const data = await response.json();
-                if (!data.success) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `Failed to fill element: ${data.error || "unknown error"}`,
-                            },
-                        ],
-                    };
-                }
-                return {
-                    content: [
-                        { type: "text", text: "Successfully filled element with text" },
-                    ],
-                };
-            }
-            case "find-elements": {
-                const selector = {
-                    app_name: args.app,
-                    window_name: args.window,
-                    locator: args.role || "",
-                    use_background_apps: args.use_background_apps ?? true,
-                    activate_app: args.activate_app ?? true,
-                };
-                const response = await fetchAPI("/experimental/operator", {
-                    method: "POST",
-                    body: JSON.stringify({
-                        selector,
-                        max_results: args.max_results || 10,
-                        max_depth: args.max_depth,
-                    }),
-                });
-                if (!response.ok) {
-                    throw new Error(`HTTP error: ${response.status}`);
-                }
-                const data = await response.json();
-                if (!data.success) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `Failed to find elements: ${data.error || "unknown error"}`,
-                            },
-                        ],
-                    };
-                }
-                const elements = data.data || [];
-                if (elements.length === 0) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `No elements found matching role '${args.role}' in app '${args.app}'`,
-                            },
-                        ],
-                    };
-                }
-                let resultText = `Found ${elements.length} elements matching role '${args.role}' in app '${args.app}':\n\n`;
-                elements.forEach((element, i) => {
-                    resultText +=
-                        `Element ${i + 1}:\n` +
-                            `ID: ${element.id || "N/A"}\n` +
-                            `Role: ${element.role || "N/A"}\n` +
-                            `Text: ${element.text || "N/A"}\n` +
-                            `Description: ${element.description || "N/A"}\n` +
-                            "---\n";
-                });
-                return {
-                    content: [{ type: "text", text: resultText }],
-                };
-            }
-            case "scroll-element": {
-                const selector = {
-                    app_name: args.app,
-                    window_name: args.window,
-                    locator: `#${args.id}`,
-                    use_background_apps: args.use_background_apps ?? true,
-                    activate_app: args.activate_app ?? true,
-                };
-                const response = await fetchAPI("/experimental/operator/scroll", {
-                    method: "POST",
-                    body: JSON.stringify({
-                        selector,
-                        direction: args.direction,
-                        amount: args.amount,
-                    }),
-                });
-                if (!response.ok) {
-                    throw new Error(`HTTP error: ${response.status}`);
-                }
-                const data = await response.json();
-                if (!data.success) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `Failed to scroll element: ${data.error || "unknown error"}`,
-                            },
-                        ],
-                    };
-                }
-                return {
-                    content: [
-                        {
-                            type: "text",
-                            text: `Successfully scrolled element ${args.direction} by ${args.amount} pixels`,
-                        },
-                    ],
-                };
-            }
-            case "open-application": {
-                const response = await fetchAPI("/experimental/operator/open-application", {
-                    method: "POST",
-                    body: JSON.stringify({ app_name: args.app_name || "" }),
-                });
-                if (!response.ok) {
-                    throw new Error(`HTTP error: ${response.status}`);
-                }
-                const data = await response.json();
-                if (!data.success) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `Failed to open application: ${data.error || "unknown error"}`,
-                            },
-                        ],
-                    };
-                }
-                return {
-                    content: [
-                        {
-                            type: "text",
-                            text: `Successfully opened application '${args.app_name}'`,
-                        },
-                    ],
-                };
-            }
-            case "open-url": {
-                const response = await fetchAPI("/experimental/operator/open-url", {
-                    method: "POST",
-                    body: JSON.stringify({
-                        url: args.url || "",
-                        browser: args.browser,
-                    }),
-                });
-                if (!response.ok) {
-                    throw new Error(`HTTP error: ${response.status}`);
-                }
-                const data = await response.json();
-                if (!data.success) {
-                    return {
-                        content: [
-                            {
-                                type: "text",
-                                text: `Failed to open URL: ${data.error || "unknown error"}`,
-                            },
-                        ],
-                    };
-                }
-                const browserInfo = args.browser ? ` using ${args.browser}` : "";
-                return {
-                    content: [
-                        {
-                            type: "text",
-                            text: `Successfully opened URL '${args.url}'${browserInfo}`,
-                        },
-                    ],
-                };
-            }
             default:
                 throw new Error(`Unknown tool: ${name}`);
         }

package/manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": "0.3",
   "name": "screenpipe",
   "display_name": "Screenpipe",
-  "version": "0.3.1",
+  "version": "0.5.0",
   "description": "Search your screen recordings, audio transcriptions, and control your computer with AI",
   "long_description": "Screenpipe is a 24/7 screen and audio recorder that lets you search everything you've seen or heard. This extension connects Claude to your local screenpipe instance, enabling AI-powered search through your digital memory and computer control capabilities.",
   "author": {
@@ -30,6 +30,10 @@
       "name": "search-content",
       "description": "Search through recorded screen content, audio transcriptions, and UI elements"
     },
+    {
+      "name": "export-video",
+      "description": "Export screen recordings as MP4 video for a specific time range"
+    },
     {
       "name": "pixel-control",
       "description": "Control mouse and keyboard (type text, press keys, move mouse, click)"