screenpipe-mcp 0.4.1 → 0.6.0

package/src/index.ts

@@ -30,12 +30,6 @@ function getCurrentDateInfo(): { isoDate: string; localDate: string } {
   };
 }
 
-// Detect OS
-const CURRENT_OS = process.platform;
-const IS_MACOS = CURRENT_OS === "darwin";
-const IS_WINDOWS = CURRENT_OS === "win32";
-const IS_LINUX = CURRENT_OS === "linux";
-
 // Parse command line arguments
 const args = process.argv.slice(2);
 let port = 3030;
@@ -51,7 +45,7 @@ const SCREENPIPE_API = `http://localhost:${port}`;
 const server = new Server(
   {
     name: "screenpipe",
-    version: "0.4.0",
+    version: "0.5.0",
   },
   {
     capabilities: {
@@ -132,56 +126,6 @@ const BASE_TOOLS: Tool[] = [
       },
     },
   },
-  {
-    name: "pixel-control",
-    description:
-      "Control mouse and keyboard at the pixel level. This is a cross-platform tool that works on all operating systems. " +
-      "Use this to type text, press keys, move the mouse, and click buttons.",
-    annotations: {
-      title: "Pixel Control",
-      destructiveHint: true,
-    },
-    inputSchema: {
-      type: "object",
-      properties: {
-        action_type: {
-          type: "string",
-          enum: ["WriteText", "KeyPress", "MouseMove", "MouseClick"],
-          description: "Type of input action to perform",
-        },
-        data: {
-          oneOf: [
-            {
-              type: "string",
-              description:
-                "Text to type or key to press (for WriteText and KeyPress)",
-            },
-            {
-              type: "object",
-              properties: {
-                x: {
-                  type: "integer",
-                  description: "X coordinate for mouse movement",
-                },
-                y: {
-                  type: "integer",
-                  description: "Y coordinate for mouse movement",
-                },
-              },
-              description: "Coordinates for MouseMove",
-            },
-            {
-              type: "string",
-              enum: ["left", "right", "middle"],
-              description: "Button to click for MouseClick",
-            },
-          ],
-          description: "Action-specific data",
-        },
-      },
-      required: ["action_type", "data"],
-    },
-  },
   {
     name: "export-video",
     description:
@@ -222,234 +166,9 @@ const BASE_TOOLS: Tool[] = [
   },
 ];
 
-const MACOS_TOOLS: Tool[] = [
-  {
-    name: "find-elements",
-    description:
-      "Find UI elements with a specific role in an application. " +
-      "This tool is especially useful for identifying interactive elements. " +
-      "\n\nMacOS Accessibility Roles Guide:\n" +
-      "- Basic roles: 'button', 'textfield', 'checkbox', 'menu', 'list'\n" +
-      "- MacOS specific roles: 'AXButton', 'AXTextField', 'AXCheckBox', 'AXMenu', etc.\n" +
-      "- Text inputs can be: 'AXTextField', 'AXTextArea', 'AXComboBox', 'AXSearchField'\n" +
-      "- Clickable items: 'AXButton', 'AXMenuItem', 'AXMenuBarItem', 'AXImage', 'AXStaticText'\n" +
-      "- Web content may use: 'AXWebArea', 'AXLink', 'AXHeading', 'AXRadioButton'\n\n" +
-      "Use MacOS Accessibility Inspector app to identify the exact roles in your target application.",
-    annotations: {
-      title: "Find Elements",
-      readOnlyHint: true,
-    },
-    inputSchema: {
-      type: "object",
-      properties: {
-        app: {
-          type: "string",
-          description:
-            "The name of the application (e.g., 'Chrome', 'Finder', 'Terminal')",
-        },
-        window: {
-          type: "string",
-          description: "The window name or title (optional)",
-        },
-        role: {
-          type: "string",
-          description:
-            "The role to search for (e.g., 'button', 'textfield', 'AXButton', 'AXTextField'). For best results, use MacOS AX prefixed roles.",
-        },
-        max_results: {
-          type: "integer",
-          description: "Maximum number of elements to return",
-          default: 10,
-        },
-        max_depth: {
-          type: "integer",
-          description: "Maximum depth of element tree to search",
-        },
-        use_background_apps: {
-          type: "boolean",
-          description: "Whether to look in background apps",
-          default: true,
-        },
-        activate_app: {
-          type: "boolean",
-          description: "Whether to activate the app before searching",
-          default: true,
-        },
-      },
-      required: ["app", "role"],
-    },
-  },
-  {
-    name: "click-element",
-    description:
-      "Click an element in an application using its id (MacOS only)",
-    annotations: {
-      title: "Click Element",
-      destructiveHint: true,
-    },
-    inputSchema: {
-      type: "object",
-      properties: {
-        app: {
-          type: "string",
-          description: "The name of the application",
-        },
-        window: {
-          type: "string",
-          description: "The window name (optional)",
-        },
-        id: {
-          type: "string",
-          description: "The id of the element to click",
-        },
-        use_background_apps: {
-          type: "boolean",
-          description: "Whether to look in background apps",
-          default: true,
-        },
-        activate_app: {
-          type: "boolean",
-          description: "Whether to activate the app before clicking",
-          default: true,
-        },
-      },
-      required: ["app", "id"],
-    },
-  },
-  {
-    name: "fill-element",
-    description: "Type text into an element in an application (MacOS only)",
-    annotations: {
-      title: "Fill Element",
-      destructiveHint: true,
-    },
-    inputSchema: {
-      type: "object",
-      properties: {
-        app: {
-          type: "string",
-          description: "The name of the application",
-        },
-        window: {
-          type: "string",
-          description: "The window name (optional)",
-        },
-        id: {
-          type: "string",
-          description: "The id of the element to fill",
-        },
-        text: {
-          type: "string",
-          description: "The text to type into the element",
-        },
-        use_background_apps: {
-          type: "boolean",
-          description: "Whether to look in background apps",
-          default: true,
-        },
-        activate_app: {
-          type: "boolean",
-          description: "Whether to activate the app before typing",
-          default: true,
-        },
-      },
-      required: ["app", "id", "text"],
-    },
-  },
-  {
-    name: "scroll-element",
-    description: "Scroll an element in a specific direction (MacOS only)",
-    annotations: {
-      title: "Scroll Element",
-      destructiveHint: true,
-    },
-    inputSchema: {
-      type: "object",
-      properties: {
-        app: {
-          type: "string",
-          description: "The name of the application",
-        },
-        window: {
-          type: "string",
-          description: "The window name (optional)",
-        },
-        id: {
-          type: "string",
-          description: "The id of the element to scroll",
-        },
-        direction: {
-          type: "string",
-          enum: ["up", "down", "left", "right"],
-          description: "The direction to scroll",
-        },
-        amount: {
-          type: "integer",
-          description: "The amount to scroll in pixels",
-        },
-        use_background_apps: {
-          type: "boolean",
-          description: "Whether to look in background apps",
-          default: true,
-        },
-        activate_app: {
-          type: "boolean",
-          description: "Whether to activate the app before scrolling",
-          default: true,
-        },
-      },
-      required: ["app", "id", "direction", "amount"],
-    },
-  },
-  {
-    name: "open-application",
-    description: "Open an application by name",
-    annotations: {
-      title: "Open Application",
-      destructiveHint: true,
-    },
-    inputSchema: {
-      type: "object",
-      properties: {
-        app_name: {
-          type: "string",
-          description: "The name of the application to open",
-        },
-      },
-      required: ["app_name"],
-    },
-  },
-  {
-    name: "open-url",
-    description: "Open a URL in a browser",
-    annotations: {
-      title: "Open URL",
-      destructiveHint: true,
-    },
-    inputSchema: {
-      type: "object",
-      properties: {
-        url: {
-          type: "string",
-          description: "The URL to open",
-        },
-        browser: {
-          type: "string",
-          description: "The browser to use (optional)",
-        },
-      },
-      required: ["url"],
-    },
-  },
-];
-
 // List tools handler
 server.setRequestHandler(ListToolsRequestSchema, async () => {
-  const tools = [...BASE_TOOLS];
-  if (IS_MACOS) {
-    tools.push(...MACOS_TOOLS);
-  }
-  return { tools };
+  return { tools: BASE_TOOLS };
 });
 
 // MCP Resources - provide dynamic context data
@@ -466,6 +185,12 @@ const RESOURCES = [
     description: "How to use screenpipe search effectively",
     mimeType: "text/markdown",
   },
+  {
+    uri: "ui://search",
+    name: "Search Dashboard",
+    description: "Interactive search UI for exploring screen recordings and audio transcriptions",
+    mimeType: "text/html",
+  },
 ];
 
 // List resources handler
@@ -543,6 +268,56 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
         ],
       };
 
+    case "ui://search": {
+      // MCP App UI - Interactive search dashboard
+      const uiHtmlPath = path.join(__dirname, "..", "ui", "search.html");
+      let htmlContent: string;
+      try {
+        htmlContent = fs.readFileSync(uiHtmlPath, "utf-8");
+      } catch {
+        // Fallback: serve embedded minimal UI if file not found
+        htmlContent = `<!DOCTYPE html>
+<html>
+<head>
+  <style>
+    body { font-family: system-ui; background: #0a0a0a; color: #fff; padding: 20px; }
+    input { width: 100%; padding: 10px; margin-bottom: 10px; background: #1a1a1a; border: 1px solid #333; color: #fff; border-radius: 6px; }
+    button { padding: 10px 20px; background: #fff; color: #000; border: none; border-radius: 6px; cursor: pointer; }
+    #results { margin-top: 20px; }
+    .result { background: #1a1a1a; padding: 12px; margin: 8px 0; border-radius: 8px; border: 1px solid #333; }
+  </style>
+</head>
+<body>
+  <h2>screenpipe search</h2>
+  <input id="q" placeholder="search..." onkeydown="if(event.key==='Enter')search()"/>
+  <button onclick="search()">search</button>
+  <div id="results"></div>
+  <script>
+    function search() {
+      window.parent.postMessage({jsonrpc:'2.0',method:'tools/call',params:{name:'search-content',arguments:{q:document.getElementById('q').value,limit:20}}},'*');
+    }
+    window.addEventListener('message',e=>{
+      if(e.data?.result||e.data?.method==='tool/result'){
+        const r=e.data.result||e.data.params?.result;
+        const d=r?.data||r||[];
+        document.getElementById('results').innerHTML=d.map(x=>'<div class="result"><b>'+((x.type||'')+'</b> '+(x.content?.app_name||'')+': '+(x.content?.text||x.content?.transcription||'').substring(0,200))+'</div>').join('');
+      }
+    });
+  </script>
+</body>
+</html>`;
+      }
+      return {
+        contents: [
+          {
+            uri,
+            mimeType: "text/html",
+            text: htmlContent,
+          },
+        ],
+      };
+    }
+
     default:
       throw new Error(`Unknown resource: ${uri}`);
   }
@@ -694,27 +469,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
     throw new Error("Missing arguments");
   }
 
-  // Check if the tool is MacOS-only and we're not on MacOS
-  const macosOnlyTools = [
-    "click-element",
-    "fill-element",
-    "find-elements",
-    "scroll-element",
-    "open-application",
-    "open-url",
-  ];
-
-  if (macosOnlyTools.includes(name) && !IS_MACOS) {
-    return {
-      content: [
-        {
-          type: "text",
-          text: `The '${name}' tool is only available on MacOS. Current platform: ${CURRENT_OS}`,
-        },
-      ],
-    };
-  }
-
   try {
     switch (name) {
       case "search-content": {
@@ -804,50 +558,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         return { content: contentItems };
       }
 
-      case "pixel-control": {
-        const action = {
-          type: args.action_type,
-          data: args.data,
-        };
-
-        const response = await fetchAPI("/experimental/operator/pixel", {
-          method: "POST",
-          body: JSON.stringify({ action }),
-        });
-
-        if (!response.ok) {
-          throw new Error(`HTTP error: ${response.status}`);
-        }
-
-        const data = await response.json();
-        if (!data.success) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Failed to perform input control: ${data.error || "unknown error"}`,
-              },
-            ],
-          };
-        }
-
-        let resultText = "Successfully performed input control action";
-        if (args.action_type === "WriteText") {
-          resultText = `Successfully typed text: '${args.data}'`;
-        } else if (args.action_type === "KeyPress") {
-          resultText = `Successfully pressed key: '${args.data}'`;
-        } else if (args.action_type === "MouseMove") {
-          const coords = args.data as { x: number; y: number };
-          resultText = `Successfully moved mouse to coordinates: x=${coords.x}, y=${coords.y}`;
-        } else if (args.action_type === "MouseClick") {
-          resultText = `Successfully clicked ${args.data} mouse button`;
-        }
-
-        return {
-          content: [{ type: "text", text: resultText }],
-        };
-      }
-
       case "export-video": {
         const startTime = args.start_time as string;
         const endTime = args.end_time as string;
@@ -1014,264 +724,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         }
       }
 
-      case "click-element": {
-        const selector = {
-          app_name: args.app,
-          window_name: args.window,
-          locator: `#${args.id}`,
-          use_background_apps: args.use_background_apps ?? true,
-          activate_app: args.activate_app ?? true,
-        };
-
-        const response = await fetchAPI("/experimental/operator/click", {
-          method: "POST",
-          body: JSON.stringify({ selector }),
-        });
-
-        if (!response.ok) {
-          throw new Error(`HTTP error: ${response.status}`);
-        }
-
-        const data = await response.json();
-        if (!data.success) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Failed to click element: ${data.error || "unknown error"}`,
-              },
-            ],
-          };
-        }
-
-        const result = data.result || {};
-        const method = result.method || "unknown";
-        const details = result.details || "click operation completed";
-
-        return {
-          content: [
-            {
-              type: "text",
-              text: `Successfully clicked element using ${method}. ${details}`,
-            },
-          ],
-        };
-      }
-
-      case "fill-element": {
-        const selector = {
-          app_name: args.app,
-          window_name: args.window,
-          locator: `#${args.id}`,
-          use_background_apps: args.use_background_apps ?? true,
-          activate_app: args.activate_app ?? true,
-        };
-
-        const response = await fetchAPI("/experimental/operator/type", {
-          method: "POST",
-          body: JSON.stringify({ selector, text: args.text || "" }),
-        });
-
-        if (!response.ok) {
-          throw new Error(`HTTP error: ${response.status}`);
-        }
-
-        const data = await response.json();
-        if (!data.success) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Failed to fill element: ${data.error || "unknown error"}`,
-              },
-            ],
-          };
-        }
-
-        return {
-          content: [
-            { type: "text", text: "Successfully filled element with text" },
-          ],
-        };
-      }
-
-      case "find-elements": {
-        const selector = {
-          app_name: args.app,
-          window_name: args.window,
-          locator: args.role || "",
-          use_background_apps: args.use_background_apps ?? true,
-          activate_app: args.activate_app ?? true,
-        };
-
-        const response = await fetchAPI("/experimental/operator", {
-          method: "POST",
-          body: JSON.stringify({
-            selector,
-            max_results: args.max_results || 10,
-            max_depth: args.max_depth,
-          }),
-        });
-
-        if (!response.ok) {
-          throw new Error(`HTTP error: ${response.status}`);
-        }
-
-        const data = await response.json();
-        if (!data.success) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Failed to find elements: ${data.error || "unknown error"}`,
-              },
-            ],
-          };
-        }
-
-        const elements = data.data || [];
-        if (elements.length === 0) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `No elements found matching role '${args.role}' in app '${args.app}'`,
-              },
-            ],
-          };
-        }
-
-        let resultText = `Found ${elements.length} elements matching role '${args.role}' in app '${args.app}':\n\n`;
-        elements.forEach((element: any, i: number) => {
-          resultText +=
-            `Element ${i + 1}:\n` +
-            `ID: ${element.id || "N/A"}\n` +
-            `Role: ${element.role || "N/A"}\n` +
-            `Text: ${element.text || "N/A"}\n` +
-            `Description: ${element.description || "N/A"}\n` +
-            "---\n";
-        });
-
-        return {
-          content: [{ type: "text", text: resultText }],
-        };
-      }
-
-      case "scroll-element": {
-        const selector = {
-          app_name: args.app,
-          window_name: args.window,
-          locator: `#${args.id}`,
-          use_background_apps: args.use_background_apps ?? true,
-          activate_app: args.activate_app ?? true,
-        };
-
-        const response = await fetchAPI("/experimental/operator/scroll", {
-          method: "POST",
-          body: JSON.stringify({
-            selector,
-            direction: args.direction,
-            amount: args.amount,
-          }),
-        });
-
-        if (!response.ok) {
-          throw new Error(`HTTP error: ${response.status}`);
-        }
-
-        const data = await response.json();
-        if (!data.success) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Failed to scroll element: ${data.error || "unknown error"}`,
-              },
-            ],
-          };
-        }
-
-        return {
-          content: [
-            {
-              type: "text",
-              text: `Successfully scrolled element ${args.direction} by ${args.amount} pixels`,
-            },
-          ],
-        };
-      }
-
-      case "open-application": {
-        const response = await fetchAPI(
-          "/experimental/operator/open-application",
-          {
-            method: "POST",
-            body: JSON.stringify({ app_name: args.app_name || "" }),
-          }
-        );
-
-        if (!response.ok) {
-          throw new Error(`HTTP error: ${response.status}`);
-        }
-
-        const data = await response.json();
-        if (!data.success) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Failed to open application: ${data.error || "unknown error"}`,
-              },
-            ],
-          };
-        }
-
-        return {
-          content: [
-            {
-              type: "text",
-              text: `Successfully opened application '${args.app_name}'`,
-            },
-          ],
-        };
-      }
-
-      case "open-url": {
-        const response = await fetchAPI("/experimental/operator/open-url", {
-          method: "POST",
-          body: JSON.stringify({
-            url: args.url || "",
-            browser: args.browser,
-          }),
-        });
-
-        if (!response.ok) {
-          throw new Error(`HTTP error: ${response.status}`);
-        }
-
-        const data = await response.json();
-        if (!data.success) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Failed to open URL: ${data.error || "unknown error"}`,
-              },
-            ],
-          };
-        }
-
-        const browserInfo = args.browser ? ` using ${args.browser}` : "";
-        return {
-          content: [
-            {
-              type: "text",
-              text: `Successfully opened URL '${args.url}'${browserInfo}`,
-            },
-          ],
-        };
-      }
-
       default:
         throw new Error(`Unknown tool: ${name}`);
     }