screenpipe-mcp 0.3.1 → 0.4.0

package/dist/index.js

@@ -1,9 +1,59 @@
 #!/usr/bin/env node
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 const index_js_1 = require("@modelcontextprotocol/sdk/server/index.js");
 const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
 const types_js_1 = require("@modelcontextprotocol/sdk/types.js");
+const ws_1 = require("ws");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+// Helper to get current date in ISO format
+function getCurrentDateInfo() {
+    const now = new Date();
+    return {
+        isoDate: now.toISOString(),
+        localDate: now.toLocaleDateString("en-US", {
+            weekday: "long",
+            year: "numeric",
+            month: "long",
+            day: "numeric",
+        }),
+    };
+}
 // Detect OS
 const CURRENT_OS = process.platform;
 const IS_MACOS = CURRENT_OS === "darwin";
@@ -21,60 +71,62 @@ const SCREENPIPE_API = `http://localhost:${port}`;
 // Initialize server
 const server = new index_js_1.Server({
     name: "screenpipe",
-    version: "0.3.1",
+    version: "0.4.0",
 }, {
     capabilities: {
         tools: {},
+        prompts: {},
+        resources: {},
     },
 });
 // Tool definitions
 const BASE_TOOLS = [
     {
         name: "search-content",
-        description: "Search through screenpipe recorded content (OCR text, audio transcriptions, UI elements). " +
-            "Use this to find specific content that has appeared on your screen or been spoken. " +
-            "Results include timestamps, app context, and the content itself. " +
-            "Set include_frames=true to get screenshot images for visual analysis (OCR results only).",
+        description: "Search screenpipe's recorded content: screen text (OCR), audio transcriptions, and UI elements. " +
+            "Returns timestamped results with app context. " +
+            "Call with no parameters to get recent activity. " +
+            "Use the 'screenpipe://context' resource for current time when building time-based queries.",
         inputSchema: {
             type: "object",
             properties: {
                 q: {
                     type: "string",
-                    description: "Search query to find in recorded content",
+                    description: "Search query. Optional - omit to return all recent content.",
                 },
                 content_type: {
                     type: "string",
                     enum: ["all", "ocr", "audio", "ui"],
-                    description: "Type of content to search: 'ocr' for screen text, 'audio' for spoken words, 'ui' for UI elements, or 'all' for everything",
+                    description: "Content type filter. Default: 'all'",
                     default: "all",
                 },
                 limit: {
                     type: "integer",
-                    description: "Maximum number of results to return",
+                    description: "Max results. Default: 10",
                     default: 10,
                 },
                 offset: {
                     type: "integer",
-                    description: "Number of results to skip (for pagination)",
+                    description: "Skip N results for pagination. Default: 0",
                     default: 0,
                 },
                 start_time: {
                     type: "string",
                     format: "date-time",
-                    description: "Start time in ISO format UTC (e.g. 2024-01-01T00:00:00Z). Filter results from this time onward.",
+                    description: "ISO 8601 UTC start time (e.g., 2024-01-15T10:00:00Z)",
                 },
                 end_time: {
                     type: "string",
                     format: "date-time",
-                    description: "End time in ISO format UTC (e.g. 2024-01-01T00:00:00Z). Filter results up to this time.",
+                    description: "ISO 8601 UTC end time (e.g., 2024-01-15T18:00:00Z)",
                 },
                 app_name: {
                     type: "string",
-                    description: "Filter by application name (e.g. 'Chrome', 'Safari', 'Terminal')",
+                    description: "Filter by app (e.g., 'Google Chrome', 'Slack', 'zoom.us')",
                 },
                 window_name: {
                     type: "string",
-                    description: "Filter by window name or title",
+                    description: "Filter by window title",
                 },
                 min_length: {
                     type: "integer",
@@ -86,9 +138,7 @@ const BASE_TOOLS = [
                 },
                 include_frames: {
                     type: "boolean",
-                    description: "Include screenshot images in results for visual analysis. Only applies to OCR results. " +
-                        "When true, returns base64-encoded images that can be analyzed with vision capabilities. " +
-                        "Note: Images are limited to ~1MB each. Default: false",
+                    description: "Include base64 screenshots (OCR only). Default: false",
                     default: false,
                 },
             },
@@ -138,6 +188,36 @@ const BASE_TOOLS = [
             required: ["action_type", "data"],
         },
     },
+    {
+        name: "export-video",
+        description: "Export a video of screen recordings for a specific time range. " +
+            "Creates an MP4 video from the recorded frames between the start and end times.\n\n" +
+            "IMPORTANT: Use ISO 8601 UTC timestamps (e.g., 2024-01-15T10:00:00Z)\n\n" +
+            "EXAMPLES:\n" +
+            "- Last 30 minutes: Calculate timestamps from current time\n" +
+            "- Specific meeting: Use the meeting's start and end times in UTC",
+        inputSchema: {
+            type: "object",
+            properties: {
+                start_time: {
+                    type: "string",
+                    format: "date-time",
+                    description: "Start time in ISO 8601 format UTC. MUST include timezone (Z for UTC). Example: '2024-01-15T10:00:00Z'",
+                },
+                end_time: {
+                    type: "string",
+                    format: "date-time",
+                    description: "End time in ISO 8601 format UTC. MUST include timezone (Z for UTC). Example: '2024-01-15T10:30:00Z'",
+                },
+                fps: {
+                    type: "number",
+                    description: "Frames per second for the output video. Lower values (0.5-1.0) create smaller files, higher values (5-10) create smoother playback. Default: 1.0",
+                    default: 1.0,
+                },
+            },
+            required: ["start_time", "end_time"],
+        },
+    },
 ];
 const MACOS_TOOLS = [
     {
@@ -339,6 +419,209 @@ server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => {
     }
     return { tools };
 });
+// MCP Resources - provide dynamic context data
+const RESOURCES = [
+    {
+        uri: "screenpipe://context",
+        name: "Current Context",
+        description: "Current date/time and pre-computed timestamps for common time ranges",
+        mimeType: "application/json",
+    },
+    {
+        uri: "screenpipe://guide",
+        name: "Usage Guide",
+        description: "How to use screenpipe search effectively",
+        mimeType: "text/markdown",
+    },
+];
+// List resources handler
+server.setRequestHandler(types_js_1.ListResourcesRequestSchema, async () => {
+    return { resources: RESOURCES };
+});
+// Read resource handler
+server.setRequestHandler(types_js_1.ReadResourceRequestSchema, async (request) => {
+    const { uri } = request.params;
+    const dateInfo = getCurrentDateInfo();
+    const now = Date.now();
+    switch (uri) {
+        case "screenpipe://context":
+            return {
+                contents: [
+                    {
+                        uri,
+                        mimeType: "application/json",
+                        text: JSON.stringify({
+                            current_time: dateInfo.isoDate,
+                            current_date_local: dateInfo.localDate,
+                            timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+                            timestamps: {
+                                now: dateInfo.isoDate,
+                                one_hour_ago: new Date(now - 60 * 60 * 1000).toISOString(),
+                                three_hours_ago: new Date(now - 3 * 60 * 60 * 1000).toISOString(),
+                                today_start: `${new Date().toISOString().split("T")[0]}T00:00:00Z`,
+                                yesterday_start: `${new Date(now - 24 * 60 * 60 * 1000).toISOString().split("T")[0]}T00:00:00Z`,
+                                one_week_ago: new Date(now - 7 * 24 * 60 * 60 * 1000).toISOString(),
+                            },
+                            common_apps: ["Google Chrome", "Safari", "Slack", "zoom.us", "Microsoft Teams", "Code", "Terminal"],
+                        }, null, 2),
+                    },
+                ],
+            };
+        case "screenpipe://guide":
+            return {
+                contents: [
+                    {
+                        uri,
+                        mimeType: "text/markdown",
+                        text: `# Screenpipe Search Guide
+
+## Quick Start
+- **Get recent activity**: Call search-content with no parameters
+- **Search text**: \`{"q": "search term", "content_type": "ocr"}\`
+- **Time filter**: Use start_time/end_time with ISO 8601 UTC timestamps
+
+## Content Types
+- \`ocr\`: Screen text (what you see)
+- \`audio\`: Transcribed speech
+- \`ui\`: UI element interactions
+- \`all\`: Everything (default)
+
+## Key Parameters
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| q | Search query | (none - returns all) |
+| content_type | ocr/audio/ui/all | all |
+| limit | Max results | 10 |
+| start_time | ISO 8601 UTC | (no filter) |
+| end_time | ISO 8601 UTC | (no filter) |
+| app_name | Filter by app | (no filter) |
+| include_frames | Include screenshots | false |
+
+## Tips
+1. Read screenpipe://context first to get current timestamps
+2. Omit \`q\` to get all content (useful for "what was I doing?")
+3. Use \`limit: 50-100\` for comprehensive searches
+4. Combine app_name + time filters for focused results`,
+                    },
+                ],
+            };
+        default:
+            throw new Error(`Unknown resource: ${uri}`);
+    }
+});
+// MCP Prompts - static interaction templates
+const PROMPTS = [
+    {
+        name: "search-recent",
+        description: "Search recent screen activity",
+        arguments: [
+            { name: "query", description: "Optional search term", required: false },
+            { name: "hours", description: "Hours to look back (default: 1)", required: false },
+        ],
+    },
+    {
+        name: "find-in-app",
+        description: "Find content from a specific application",
+        arguments: [
+            { name: "app", description: "App name (e.g., Chrome, Slack)", required: true },
+            { name: "query", description: "Optional search term", required: false },
+        ],
+    },
+    {
+        name: "meeting-notes",
+        description: "Get audio transcriptions from meetings",
+        arguments: [
+            { name: "hours", description: "Hours to look back (default: 3)", required: false },
+        ],
+    },
+];
+// List prompts handler
+server.setRequestHandler(types_js_1.ListPromptsRequestSchema, async () => {
+    return { prompts: PROMPTS };
+});
+// Get prompt handler
+server.setRequestHandler(types_js_1.GetPromptRequestSchema, async (request) => {
+    const { name, arguments: promptArgs } = request.params;
+    const dateInfo = getCurrentDateInfo();
+    const now = Date.now();
+    switch (name) {
+        case "search-recent": {
+            const query = promptArgs?.query || "";
+            const hours = parseInt(promptArgs?.hours || "1", 10);
+            const startTime = new Date(now - hours * 60 * 60 * 1000).toISOString();
+            return {
+                description: `Search recent activity (last ${hours} hour${hours > 1 ? "s" : ""})`,
+                messages: [
+                    {
+                        role: "user",
+                        content: {
+                            type: "text",
+                            text: `Search screenpipe for recent activity.
+
+Current time: ${dateInfo.isoDate}
+
+Use search-content with:
+${query ? `- q: "${query}"` : "- No query filter (get all content)"}
+- start_time: "${startTime}"
+- limit: 50`,
+                        },
+                    },
+                ],
+            };
+        }
+        case "find-in-app": {
+            const app = promptArgs?.app || "Google Chrome";
+            const query = promptArgs?.query || "";
+            return {
+                description: `Find content from ${app}`,
+                messages: [
+                    {
+                        role: "user",
+                        content: {
+                            type: "text",
+                            text: `Search screenpipe for content from ${app}.
+
+Current time: ${dateInfo.isoDate}
+
+Use search-content with:
+- app_name: "${app}"
+${query ? `- q: "${query}"` : "- No query filter"}
+- content_type: "ocr"
+- limit: 50`,
+                        },
+                    },
+                ],
+            };
+        }
+        case "meeting-notes": {
+            const hours = parseInt(promptArgs?.hours || "3", 10);
+            const startTime = new Date(now - hours * 60 * 60 * 1000).toISOString();
+            return {
+                description: `Get meeting transcriptions (last ${hours} hours)`,
+                messages: [
+                    {
+                        role: "user",
+                        content: {
+                            type: "text",
+                            text: `Get audio transcriptions from recent meetings.
+
+Current time: ${dateInfo.isoDate}
+
+Use search-content with:
+- content_type: "audio"
+- start_time: "${startTime}"
+- limit: 100
+
+Common meeting apps: zoom.us, Microsoft Teams, Google Meet, Slack`,
+                        },
+                    },
+                ],
+            };
+        }
+        default:
+            throw new Error(`Unknown prompt: ${name}`);
+    }
+});
 // Helper function to make HTTP requests
 async function fetchAPI(endpoint, options = {}) {
     const url = `${SCREENPIPE_API}${endpoint}`;
@@ -391,9 +674,15 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                 }
                 const data = await response.json();
                 const results = data.data || [];
+                const pagination = data.pagination || {};
                 if (results.length === 0) {
                     return {
-                        content: [{ type: "text", text: "No results found" }],
+                        content: [
+                            {
+                                type: "text",
+                                text: "No results found. Try: broader search terms, different content_type, or wider time range.",
+                            },
+                        ],
                     };
                 }
                 // Build content array with text and optional images
@@ -405,57 +694,38 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                     if (!content)
                         continue;
                     if (result.type === "OCR") {
-                        const textResult = `OCR Text: ${content.text || "N/A"}\n` +
-                            `App: ${content.app_name || "N/A"}\n` +
-                            `Window: ${content.window_name || "N/A"}\n` +
-                            `Time: ${content.timestamp || "N/A"}\n` +
-                            `Frame ID: ${content.frame_id || "N/A"}\n` +
-                            "---";
-                        formattedResults.push(textResult);
-                        // Collect frame if available and requested
+                        formattedResults.push(`[OCR] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
+                            `${content.timestamp || ""}\n` +
+                            `${content.text || ""}`);
                         if (includeFrames && content.frame) {
                             images.push({
                                 data: content.frame,
-                                context: `Screenshot from ${content.app_name || "unknown"} - ${content.window_name || "unknown"} at ${content.timestamp || "unknown"}`,
+                                context: `${content.app_name} at ${content.timestamp}`,
                             });
                         }
                     }
                     else if (result.type === "Audio") {
-                        formattedResults.push(`Audio Transcription: ${content.transcription || "N/A"}\n` +
-                            `Device: ${content.device_name || "N/A"}\n` +
-                            `Time: ${content.timestamp || "N/A"}\n` +
-                            "---");
+                        formattedResults.push(`[Audio] ${content.device_name || "?"}\n` +
+                            `${content.timestamp || ""}\n` +
+                            `${content.transcription || ""}`);
                     }
                     else if (result.type === "UI") {
-                        formattedResults.push(`UI Text: ${content.text || "N/A"}\n` +
-                            `App: ${content.app_name || "N/A"}\n` +
-                            `Window: ${content.window_name || "N/A"}\n` +
-                            `Time: ${content.timestamp || "N/A"}\n` +
-                            "---");
+                        formattedResults.push(`[UI] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
+                            `${content.timestamp || ""}\n` +
+                            `${content.text || ""}`);
                     }
                 }
-                // Add text results
+                // Header with pagination info
+                const header = `Results: ${results.length}/${pagination.total || "?"}` +
+                    (pagination.total > results.length ? ` (use offset=${(pagination.offset || 0) + results.length} for more)` : "");
                 contentItems.push({
                     type: "text",
-                    text: "Search Results:\n\n" +
-                        formattedResults.join("\n") +
-                        (images.length > 0
-                            ? `\n\n${images.length} screenshot(s) included below for visual analysis:`
-                            : ""),
+                    text: header + "\n\n" + formattedResults.join("\n---\n"),
                 });
-                // Add images if requested and available
+                // Add images if requested
                 for (const img of images) {
-                    // Add context for the image
-                    contentItems.push({
-                        type: "text",
-                        text: `\n📷 ${img.context}`,
-                    });
-                    // Add the image itself
-                    contentItems.push({
-                        type: "image",
-                        data: img.data,
-                        mimeType: "image/png",
-                    });
+                    contentItems.push({ type: "text", text: `\n📷 ${img.context}` });
+                    contentItems.push({ type: "image", data: img.data, mimeType: "image/png" });
                 }
                 return { content: contentItems };
             }
@@ -500,6 +770,151 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                     content: [{ type: "text", text: resultText }],
                 };
             }
+            case "export-video": {
+                const startTime = args.start_time;
+                const endTime = args.end_time;
+                const fps = args.fps || 1.0;
+                // Validate time inputs
+                if (!startTime || !endTime) {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: "Error: Both start_time and end_time are required in ISO 8601 format (e.g., '2024-01-15T10:00:00Z')",
+                            },
+                        ],
+                    };
+                }
+                // Step 1: Query the search API to get frame IDs for the time range
+                const searchParams = new URLSearchParams({
+                    content_type: "ocr",
+                    start_time: startTime,
+                    end_time: endTime,
+                    limit: "10000", // Get all frames in range
+                });
+                const searchResponse = await fetchAPI(`/search?${searchParams.toString()}`);
+                if (!searchResponse.ok) {
+                    throw new Error(`Failed to search for frames: HTTP ${searchResponse.status}`);
+                }
+                const searchData = await searchResponse.json();
+                const results = searchData.data || [];
+                if (results.length === 0) {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: `No screen recordings found between ${startTime} and ${endTime}. Make sure screenpipe was recording during this time period.`,
+                            },
+                        ],
+                    };
+                }
+                // Extract unique frame IDs from OCR results
+                const frameIds = [];
+                const seenIds = new Set();
+                for (const result of results) {
+                    if (result.type === "OCR" && result.content?.frame_id) {
+                        const frameId = result.content.frame_id;
+                        if (!seenIds.has(frameId)) {
+                            seenIds.add(frameId);
+                            frameIds.push(frameId);
+                        }
+                    }
+                }
+                if (frameIds.length === 0) {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: `Found ${results.length} results but no valid frame IDs. The recordings may be audio-only.`,
+                            },
+                        ],
+                    };
+                }
+                // Sort frame IDs
+                frameIds.sort((a, b) => a - b);
+                // Step 2: Connect to WebSocket and export video
+                const wsUrl = `ws://localhost:${port}/frames/export?frame_ids=${frameIds.join(",")}&fps=${fps}`;
+                const exportResult = await new Promise((resolve) => {
+                    const ws = new ws_1.WebSocket(wsUrl);
+                    let resolved = false;
+                    const timeout = setTimeout(() => {
+                        if (!resolved) {
+                            resolved = true;
+                            ws.close();
+                            resolve({ success: false, error: "Export timed out after 5 minutes" });
+                        }
+                    }, 5 * 60 * 1000); // 5 minute timeout
+                    ws.on("error", (error) => {
+                        if (!resolved) {
+                            resolved = true;
+                            clearTimeout(timeout);
+                            resolve({ success: false, error: `WebSocket error: ${error.message}` });
+                        }
+                    });
+                    ws.on("close", () => {
+                        if (!resolved) {
+                            resolved = true;
+                            clearTimeout(timeout);
+                            resolve({ success: false, error: "Connection closed unexpectedly" });
+                        }
+                    });
+                    ws.on("message", (data) => {
+                        try {
+                            const message = JSON.parse(data.toString());
+                            if (message.status === "completed" && message.video_data) {
+                                // Save video to temp file
+                                const tempDir = os.tmpdir();
+                                const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+                                const filename = `screenpipe_export_${timestamp}.mp4`;
+                                const filePath = path.join(tempDir, filename);
+                                fs.writeFileSync(filePath, Buffer.from(message.video_data));
+                                resolved = true;
+                                clearTimeout(timeout);
+                                ws.close();
+                                resolve({
+                                    success: true,
+                                    filePath,
+                                    frameCount: frameIds.length,
+                                });
+                            }
+                            else if (message.status === "error") {
+                                resolved = true;
+                                clearTimeout(timeout);
+                                ws.close();
+                                resolve({ success: false, error: message.error || "Export failed" });
+                            }
+                            // Ignore "extracting" and "encoding" status updates
+                        }
+                        catch (parseError) {
+                            // Ignore parse errors for progress messages
+                        }
+                    });
+                });
+                if (exportResult.success && exportResult.filePath) {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: `Successfully exported video!\n\n` +
+                                    `File: ${exportResult.filePath}\n` +
+                                    `Frames: ${exportResult.frameCount}\n` +
+                                    `Time range: ${startTime} to ${endTime}\n` +
+                                    `FPS: ${fps}`,
+                            },
+                        ],
+                    };
+                }
+                else {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: `Failed to export video: ${exportResult.error}`,
+                            },
+                        ],
+                    };
+                }
+            }
             case "click-element": {
                 const selector = {
                     app_name: args.app,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screenpipe-mcp",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "MCP server for screenpipe - search your screen recordings, audio transcriptions, and control your computer",
   "main": "dist/index.js",
   "bin": {
@@ -10,6 +10,8 @@
     "build": "tsc",
     "start": "node dist/index.js",
     "dev": "ts-node src/index.ts",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "prepublishOnly": "npm run build"
   },
   "keywords": [
@@ -25,12 +27,15 @@
   "author": "Mediar AI",
   "license": "MIT",
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.0.0"
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "ws": "^8.18.0"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",
+    "@types/ws": "^8.5.13",
     "typescript": "^5.0.0",
-    "ts-node": "^10.9.0"
+    "ts-node": "^10.9.0",
+    "vitest": "^2.1.0"
   },
   "engines": {
     "node": ">=18.0.0"