screenpipe-mcp 0.3.1 → 0.4.1

package/src/index.ts

@@ -5,8 +5,30 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
 import {
   CallToolRequestSchema,
   ListToolsRequestSchema,
+  ListPromptsRequestSchema,
+  GetPromptRequestSchema,
+  ListResourcesRequestSchema,
+  ReadResourceRequestSchema,
   Tool,
 } from "@modelcontextprotocol/sdk/types.js";
+import { WebSocket } from "ws";
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+
+// Helper to get current date in ISO format
+function getCurrentDateInfo(): { isoDate: string; localDate: string } {
+  const now = new Date();
+  return {
+    isoDate: now.toISOString(),
+    localDate: now.toLocaleDateString("en-US", {
+      weekday: "long",
+      year: "numeric",
+      month: "long",
+      day: "numeric",
+    }),
+  };
+}
 
 // Detect OS
 const CURRENT_OS = process.platform;
@@ -29,11 +51,13 @@ const SCREENPIPE_API = `http://localhost:${port}`;
 const server = new Server(
   {
     name: "screenpipe",
-    version: "0.3.1",
+    version: "0.4.0",
   },
   {
     capabilities: {
       tools: {},
+      prompts: {},
+      resources: {},
     },
   }
 );
@@ -43,54 +67,54 @@ const BASE_TOOLS: Tool[] = [
   {
     name: "search-content",
     description:
-      "Search through screenpipe recorded content (OCR text, audio transcriptions, UI elements). " +
-      "Use this to find specific content that has appeared on your screen or been spoken. " +
-      "Results include timestamps, app context, and the content itself. " +
-      "Set include_frames=true to get screenshot images for visual analysis (OCR results only).",
+      "Search screenpipe's recorded content: screen text (OCR), audio transcriptions, and UI elements. " +
+      "Returns timestamped results with app context. " +
+      "Call with no parameters to get recent activity. " +
+      "Use the 'screenpipe://context' resource for current time when building time-based queries.",
+    annotations: {
+      title: "Search Content",
+      readOnlyHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
         q: {
           type: "string",
-          description: "Search query to find in recorded content",
+          description: "Search query. Optional - omit to return all recent content.",
         },
         content_type: {
           type: "string",
           enum: ["all", "ocr", "audio", "ui"],
-          description:
-            "Type of content to search: 'ocr' for screen text, 'audio' for spoken words, 'ui' for UI elements, or 'all' for everything",
+          description: "Content type filter. Default: 'all'",
           default: "all",
         },
         limit: {
           type: "integer",
-          description: "Maximum number of results to return",
+          description: "Max results. Default: 10",
           default: 10,
         },
         offset: {
           type: "integer",
-          description: "Number of results to skip (for pagination)",
+          description: "Skip N results for pagination. Default: 0",
           default: 0,
         },
         start_time: {
           type: "string",
           format: "date-time",
-          description:
-            "Start time in ISO format UTC (e.g. 2024-01-01T00:00:00Z). Filter results from this time onward.",
+          description: "ISO 8601 UTC start time (e.g., 2024-01-15T10:00:00Z)",
         },
         end_time: {
           type: "string",
           format: "date-time",
-          description:
-            "End time in ISO format UTC (e.g. 2024-01-01T00:00:00Z). Filter results up to this time.",
+          description: "ISO 8601 UTC end time (e.g., 2024-01-15T18:00:00Z)",
         },
         app_name: {
           type: "string",
-          description:
-            "Filter by application name (e.g. 'Chrome', 'Safari', 'Terminal')",
+          description: "Filter by app (e.g., 'Google Chrome', 'Slack', 'zoom.us')",
         },
         window_name: {
           type: "string",
-          description: "Filter by window name or title",
+          description: "Filter by window title",
         },
         min_length: {
           type: "integer",
@@ -102,10 +126,7 @@ const BASE_TOOLS: Tool[] = [
         },
         include_frames: {
           type: "boolean",
-          description:
-            "Include screenshot images in results for visual analysis. Only applies to OCR results. " +
-            "When true, returns base64-encoded images that can be analyzed with vision capabilities. " +
-            "Note: Images are limited to ~1MB each. Default: false",
+          description: "Include base64 screenshots (OCR only). Default: false",
           default: false,
         },
       },
@@ -116,6 +137,10 @@ const BASE_TOOLS: Tool[] = [
     description:
       "Control mouse and keyboard at the pixel level. This is a cross-platform tool that works on all operating systems. " +
       "Use this to type text, press keys, move the mouse, and click buttons.",
+    annotations: {
+      title: "Pixel Control",
+      destructiveHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
@@ -157,6 +182,44 @@ const BASE_TOOLS: Tool[] = [
       required: ["action_type", "data"],
     },
   },
+  {
+    name: "export-video",
+    description:
+      "Export a video of screen recordings for a specific time range. " +
+      "Creates an MP4 video from the recorded frames between the start and end times.\n\n" +
+      "IMPORTANT: Use ISO 8601 UTC timestamps (e.g., 2024-01-15T10:00:00Z)\n\n" +
+      "EXAMPLES:\n" +
+      "- Last 30 minutes: Calculate timestamps from current time\n" +
+      "- Specific meeting: Use the meeting's start and end times in UTC",
+    annotations: {
+      title: "Export Video",
+      destructiveHint: true,
+    },
+    inputSchema: {
+      type: "object",
+      properties: {
+        start_time: {
+          type: "string",
+          format: "date-time",
+          description:
+            "Start time in ISO 8601 format UTC. MUST include timezone (Z for UTC). Example: '2024-01-15T10:00:00Z'",
+        },
+        end_time: {
+          type: "string",
+          format: "date-time",
+          description:
+            "End time in ISO 8601 format UTC. MUST include timezone (Z for UTC). Example: '2024-01-15T10:30:00Z'",
+        },
+        fps: {
+          type: "number",
+          description:
+            "Frames per second for the output video. Lower values (0.5-1.0) create smaller files, higher values (5-10) create smoother playback. Default: 1.0",
+          default: 1.0,
+        },
+      },
+      required: ["start_time", "end_time"],
+    },
+  },
 ];
 
 const MACOS_TOOLS: Tool[] = [
@@ -172,6 +235,10 @@ const MACOS_TOOLS: Tool[] = [
       "- Clickable items: 'AXButton', 'AXMenuItem', 'AXMenuBarItem', 'AXImage', 'AXStaticText'\n" +
       "- Web content may use: 'AXWebArea', 'AXLink', 'AXHeading', 'AXRadioButton'\n\n" +
       "Use MacOS Accessibility Inspector app to identify the exact roles in your target application.",
+    annotations: {
+      title: "Find Elements",
+      readOnlyHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
@@ -216,6 +283,10 @@ const MACOS_TOOLS: Tool[] = [
     name: "click-element",
     description:
       "Click an element in an application using its id (MacOS only)",
+    annotations: {
+      title: "Click Element",
+      destructiveHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
@@ -248,6 +319,10 @@ const MACOS_TOOLS: Tool[] = [
   {
     name: "fill-element",
     description: "Type text into an element in an application (MacOS only)",
+    annotations: {
+      title: "Fill Element",
+      destructiveHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
@@ -284,6 +359,10 @@ const MACOS_TOOLS: Tool[] = [
   {
     name: "scroll-element",
     description: "Scroll an element in a specific direction (MacOS only)",
+    annotations: {
+      title: "Scroll Element",
+      destructiveHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
@@ -325,6 +404,10 @@ const MACOS_TOOLS: Tool[] = [
   {
     name: "open-application",
     description: "Open an application by name",
+    annotations: {
+      title: "Open Application",
+      destructiveHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
@@ -339,6 +422,10 @@ const MACOS_TOOLS: Tool[] = [
   {
     name: "open-url",
     description: "Open a URL in a browser",
+    annotations: {
+      title: "Open URL",
+      destructiveHint: true,
+    },
     inputSchema: {
       type: "object",
       properties: {
@@ -365,6 +452,225 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
   return { tools };
 });
 
+// MCP Resources - provide dynamic context data
+const RESOURCES = [
+  {
+    uri: "screenpipe://context",
+    name: "Current Context",
+    description: "Current date/time and pre-computed timestamps for common time ranges",
+    mimeType: "application/json",
+  },
+  {
+    uri: "screenpipe://guide",
+    name: "Usage Guide",
+    description: "How to use screenpipe search effectively",
+    mimeType: "text/markdown",
+  },
+];
+
+// List resources handler
+server.setRequestHandler(ListResourcesRequestSchema, async () => {
+  return { resources: RESOURCES };
+});
+
+// Read resource handler
+server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
+  const { uri } = request.params;
+  const dateInfo = getCurrentDateInfo();
+  const now = Date.now();
+
+  switch (uri) {
+    case "screenpipe://context":
+      return {
+        contents: [
+          {
+            uri,
+            mimeType: "application/json",
+            text: JSON.stringify({
+              current_time: dateInfo.isoDate,
+              current_date_local: dateInfo.localDate,
+              timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+              timestamps: {
+                now: dateInfo.isoDate,
+                one_hour_ago: new Date(now - 60 * 60 * 1000).toISOString(),
+                three_hours_ago: new Date(now - 3 * 60 * 60 * 1000).toISOString(),
+                today_start: `${new Date().toISOString().split("T")[0]}T00:00:00Z`,
+                yesterday_start: `${new Date(now - 24 * 60 * 60 * 1000).toISOString().split("T")[0]}T00:00:00Z`,
+                one_week_ago: new Date(now - 7 * 24 * 60 * 60 * 1000).toISOString(),
+              },
+              common_apps: ["Google Chrome", "Safari", "Slack", "zoom.us", "Microsoft Teams", "Code", "Terminal"],
+            }, null, 2),
+          },
+        ],
+      };
+
+    case "screenpipe://guide":
+      return {
+        contents: [
+          {
+            uri,
+            mimeType: "text/markdown",
+            text: `# Screenpipe Search Guide
+
+## Quick Start
+- **Get recent activity**: Call search-content with no parameters
+- **Search text**: \`{"q": "search term", "content_type": "ocr"}\`
+- **Time filter**: Use start_time/end_time with ISO 8601 UTC timestamps
+
+## Content Types
+- \`ocr\`: Screen text (what you see)
+- \`audio\`: Transcribed speech
+- \`ui\`: UI element interactions
+- \`all\`: Everything (default)
+
+## Key Parameters
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| q | Search query | (none - returns all) |
+| content_type | ocr/audio/ui/all | all |
+| limit | Max results | 10 |
+| start_time | ISO 8601 UTC | (no filter) |
+| end_time | ISO 8601 UTC | (no filter) |
+| app_name | Filter by app | (no filter) |
+| include_frames | Include screenshots | false |
+
+## Tips
+1. Read screenpipe://context first to get current timestamps
+2. Omit \`q\` to get all content (useful for "what was I doing?")
+3. Use \`limit: 50-100\` for comprehensive searches
+4. Combine app_name + time filters for focused results`,
+          },
+        ],
+      };
+
+    default:
+      throw new Error(`Unknown resource: ${uri}`);
+  }
+});
+
+// MCP Prompts - static interaction templates
+const PROMPTS = [
+  {
+    name: "search-recent",
+    description: "Search recent screen activity",
+    arguments: [
+      { name: "query", description: "Optional search term", required: false },
+      { name: "hours", description: "Hours to look back (default: 1)", required: false },
+    ],
+  },
+  {
+    name: "find-in-app",
+    description: "Find content from a specific application",
+    arguments: [
+      { name: "app", description: "App name (e.g., Chrome, Slack)", required: true },
+      { name: "query", description: "Optional search term", required: false },
+    ],
+  },
+  {
+    name: "meeting-notes",
+    description: "Get audio transcriptions from meetings",
+    arguments: [
+      { name: "hours", description: "Hours to look back (default: 3)", required: false },
+    ],
+  },
+];
+
+// List prompts handler
+server.setRequestHandler(ListPromptsRequestSchema, async () => {
+  return { prompts: PROMPTS };
+});
+
+// Get prompt handler
+server.setRequestHandler(GetPromptRequestSchema, async (request) => {
+  const { name, arguments: promptArgs } = request.params;
+  const dateInfo = getCurrentDateInfo();
+  const now = Date.now();
+
+  switch (name) {
+    case "search-recent": {
+      const query = promptArgs?.query || "";
+      const hours = parseInt(promptArgs?.hours || "1", 10);
+      const startTime = new Date(now - hours * 60 * 60 * 1000).toISOString();
+
+      return {
+        description: `Search recent activity (last ${hours} hour${hours > 1 ? "s" : ""})`,
+        messages: [
+          {
+            role: "user" as const,
+            content: {
+              type: "text" as const,
+              text: `Search screenpipe for recent activity.
+
+Current time: ${dateInfo.isoDate}
+
+Use search-content with:
+${query ? `- q: "${query}"` : "- No query filter (get all content)"}
+- start_time: "${startTime}"
+- limit: 50`,
+            },
+          },
+        ],
+      };
+    }
+
+    case "find-in-app": {
+      const app = promptArgs?.app || "Google Chrome";
+      const query = promptArgs?.query || "";
+
+      return {
+        description: `Find content from ${app}`,
+        messages: [
+          {
+            role: "user" as const,
+            content: {
+              type: "text" as const,
+              text: `Search screenpipe for content from ${app}.
+
+Current time: ${dateInfo.isoDate}
+
+Use search-content with:
+- app_name: "${app}"
+${query ? `- q: "${query}"` : "- No query filter"}
+- content_type: "ocr"
+- limit: 50`,
+            },
+          },
+        ],
+      };
+    }
+
+    case "meeting-notes": {
+      const hours = parseInt(promptArgs?.hours || "3", 10);
+      const startTime = new Date(now - hours * 60 * 60 * 1000).toISOString();
+
+      return {
+        description: `Get meeting transcriptions (last ${hours} hours)`,
+        messages: [
+          {
+            role: "user" as const,
+            content: {
+              type: "text" as const,
+              text: `Get audio transcriptions from recent meetings.
+
+Current time: ${dateInfo.isoDate}
+
+Use search-content with:
+- content_type: "audio"
+- start_time: "${startTime}"
+- limit: 100
+
+Common meeting apps: zoom.us, Microsoft Teams, Google Meet, Slack`,
+            },
+          },
+        ],
+      };
+    }
+
+    default:
+      throw new Error(`Unknown prompt: ${name}`);
+  }
+});
+
 // Helper function to make HTTP requests
 async function fetchAPI(
   endpoint: string,
@@ -427,10 +733,16 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 
         const data = await response.json();
         const results = data.data || [];
+        const pagination = data.pagination || {};
 
         if (results.length === 0) {
           return {
-            content: [{ type: "text", text: "No results found" }],
+            content: [
+              {
+                type: "text",
+                text: "No results found. Try: broader search terms, different content_type, or wider time range.",
+              },
+            ],
           };
         }
 
@@ -448,64 +760,45 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           if (!content) continue;
 
           if (result.type === "OCR") {
-            const textResult =
-              `OCR Text: ${content.text || "N/A"}\n` +
-              `App: ${content.app_name || "N/A"}\n` +
-              `Window: ${content.window_name || "N/A"}\n` +
-              `Time: ${content.timestamp || "N/A"}\n` +
-              `Frame ID: ${content.frame_id || "N/A"}\n` +
-              "---";
-            formattedResults.push(textResult);
-
-            // Collect frame if available and requested
+            formattedResults.push(
+              `[OCR] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
+              `${content.timestamp || ""}\n` +
+              `${content.text || ""}`
+            );
             if (includeFrames && content.frame) {
               images.push({
                 data: content.frame,
-                context: `Screenshot from ${content.app_name || "unknown"} - ${content.window_name || "unknown"} at ${content.timestamp || "unknown"}`,
+                context: `${content.app_name} at ${content.timestamp}`,
               });
             }
           } else if (result.type === "Audio") {
             formattedResults.push(
-              `Audio Transcription: ${content.transcription || "N/A"}\n` +
-                `Device: ${content.device_name || "N/A"}\n` +
-                `Time: ${content.timestamp || "N/A"}\n` +
-                "---"
+              `[Audio] ${content.device_name || "?"}\n` +
+              `${content.timestamp || ""}\n` +
+              `${content.transcription || ""}`
             );
           } else if (result.type === "UI") {
             formattedResults.push(
-              `UI Text: ${content.text || "N/A"}\n` +
-                `App: ${content.app_name || "N/A"}\n` +
-                `Window: ${content.window_name || "N/A"}\n` +
-                `Time: ${content.timestamp || "N/A"}\n` +
-                "---"
+              `[UI] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
+              `${content.timestamp || ""}\n` +
+              `${content.text || ""}`
             );
           }
         }
 
-        // Add text results
+        // Header with pagination info
+        const header = `Results: ${results.length}/${pagination.total || "?"}` +
+          (pagination.total > results.length ? ` (use offset=${(pagination.offset || 0) + results.length} for more)` : "");
+
         contentItems.push({
           type: "text",
-          text:
-            "Search Results:\n\n" +
-            formattedResults.join("\n") +
-            (images.length > 0
-              ? `\n\n${images.length} screenshot(s) included below for visual analysis:`
-              : ""),
+          text: header + "\n\n" + formattedResults.join("\n---\n"),
         });
 
-        // Add images if requested and available
+        // Add images if requested
         for (const img of images) {
-          // Add context for the image
-          contentItems.push({
-            type: "text",
-            text: `\n📷 ${img.context}`,
-          });
-          // Add the image itself
-          contentItems.push({
-            type: "image",
-            data: img.data,
-            mimeType: "image/png",
-          });
+          contentItems.push({ type: "text", text: `\n📷 ${img.context}` });
+          contentItems.push({ type: "image", data: img.data, mimeType: "image/png" });
         }
 
         return { content: contentItems };
@@ -555,6 +848,172 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         };
       }
 
+      case "export-video": {
+        const startTime = args.start_time as string;
+        const endTime = args.end_time as string;
+        const fps = (args.fps as number) || 1.0;
+
+        // Validate time inputs
+        if (!startTime || !endTime) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "Error: Both start_time and end_time are required in ISO 8601 format (e.g., '2024-01-15T10:00:00Z')",
+              },
+            ],
+          };
+        }
+
+        // Step 1: Query the search API to get frame IDs for the time range
+        const searchParams = new URLSearchParams({
+          content_type: "ocr",
+          start_time: startTime,
+          end_time: endTime,
+          limit: "10000", // Get all frames in range
+        });
+
+        const searchResponse = await fetchAPI(`/search?${searchParams.toString()}`);
+        if (!searchResponse.ok) {
+          throw new Error(`Failed to search for frames: HTTP ${searchResponse.status}`);
+        }
+
+        const searchData = await searchResponse.json();
+        const results = searchData.data || [];
+
+        if (results.length === 0) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `No screen recordings found between ${startTime} and ${endTime}. Make sure screenpipe was recording during this time period.`,
+              },
+            ],
+          };
+        }
+
+        // Extract unique frame IDs from OCR results
+        const frameIds: number[] = [];
+        const seenIds = new Set<number>();
+        for (const result of results) {
+          if (result.type === "OCR" && result.content?.frame_id) {
+            const frameId = result.content.frame_id;
+            if (!seenIds.has(frameId)) {
+              seenIds.add(frameId);
+              frameIds.push(frameId);
+            }
+          }
+        }
+
+        if (frameIds.length === 0) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Found ${results.length} results but no valid frame IDs. The recordings may be audio-only.`,
+              },
+            ],
+          };
+        }
+
+        // Sort frame IDs
+        frameIds.sort((a, b) => a - b);
+
+        // Step 2: Connect to WebSocket and export video
+        const wsUrl = `ws://localhost:${port}/frames/export?frame_ids=${frameIds.join(",")}&fps=${fps}`;
+
+        const exportResult = await new Promise<{
+          success: boolean;
+          filePath?: string;
+          error?: string;
+          frameCount?: number;
+        }>((resolve) => {
+          const ws = new WebSocket(wsUrl);
+          let resolved = false;
+
+          const timeout = setTimeout(() => {
+            if (!resolved) {
+              resolved = true;
+              ws.close();
+              resolve({ success: false, error: "Export timed out after 5 minutes" });
+            }
+          }, 5 * 60 * 1000); // 5 minute timeout
+
+          ws.on("error", (error) => {
+            if (!resolved) {
+              resolved = true;
+              clearTimeout(timeout);
+              resolve({ success: false, error: `WebSocket error: ${error.message}` });
+            }
+          });
+
+          ws.on("close", () => {
+            if (!resolved) {
+              resolved = true;
+              clearTimeout(timeout);
+              resolve({ success: false, error: "Connection closed unexpectedly" });
+            }
+          });
+
+          ws.on("message", (data) => {
+            try {
+              const message = JSON.parse(data.toString());
+
+              if (message.status === "completed" && message.video_data) {
+                // Save video to temp file
+                const tempDir = os.tmpdir();
+                const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+                const filename = `screenpipe_export_${timestamp}.mp4`;
+                const filePath = path.join(tempDir, filename);
+
+                fs.writeFileSync(filePath, Buffer.from(message.video_data));
+
+                resolved = true;
+                clearTimeout(timeout);
+                ws.close();
+                resolve({
+                  success: true,
+                  filePath,
+                  frameCount: frameIds.length,
+                });
+              } else if (message.status === "error") {
+                resolved = true;
+                clearTimeout(timeout);
+                ws.close();
+                resolve({ success: false, error: message.error || "Export failed" });
+              }
+              // Ignore "extracting" and "encoding" status updates
+            } catch (parseError) {
+              // Ignore parse errors for progress messages
+            }
+          });
+        });
+
+        if (exportResult.success && exportResult.filePath) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Successfully exported video!\n\n` +
+                  `File: ${exportResult.filePath}\n` +
+                  `Frames: ${exportResult.frameCount}\n` +
+                  `Time range: ${startTime} to ${endTime}\n` +
+                  `FPS: ${fps}`,
+              },
+            ],
+          };
+        } else {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Failed to export video: ${exportResult.error}`,
+              },
+            ],
+          };
+        }
+      }
+
       case "click-element": {
         const selector = {
           app_name: args.app,