screenpipe-mcp 0.18.9 → 0.18.11

package/src/index.ts

@@ -12,7 +12,6 @@ import {
   ReadResourceRequestSchema,
   Tool,
 } from "@modelcontextprotocol/sdk/types.js";
-import { WebSocket } from "ws";
 import * as fs from "fs";
 import * as path from "path";
 import * as os from "os";
@@ -303,6 +302,11 @@ const TOOLS: Tool[] = [
         },
         speaker_ids: { type: "string", description: "Comma-separated speaker IDs to filter audio" },
         speaker_name: { type: "string", description: "Filter audio by speaker name (case-insensitive partial match)" },
+        tags: {
+          type: "string",
+          description:
+            "Comma-separated tags; returns only items carrying ALL of them (e.g. 'person:ada,project:atlas'). Works for screen + audio (content_type 'ocr'/'audio'/'all', tags written by add-tags) AND memories (content_type 'memory', tags written by update-memory). Same tag string links across all three, so two items sharing a tag are connected. Use namespaced tags (person:, project:, topic:) to link people/projects/topics. content_type 'input' and 'accessibility' have no tags and return nothing when this is set.",
+        },
         max_content_length: {
           type: "integer",
           description: "Truncate each result's text via middle-truncation. Use 200-500 to keep responses compact.",
@@ -389,15 +393,22 @@ const TOOLS: Tool[] = [
   {
     name: "export-video",
     description:
-      "Export an MP4 video of screen recordings for a time range. " +
-      "Returns the file path. Can take a few minutes for long ranges.",
+      "Export an MP4 of screen recordings for a time range, with synced microphone audio. " +
+      "Frames are placed at their real timestamps, so the clip's duration matches the " +
+      "wall-clock span you requested (not a sped-up timelapse). Returns the file path. " +
+      "Can take a few minutes for long ranges.",
     annotations: { title: "Export Video", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
     inputSchema: {
       type: "object",
       properties: {
-        start_time: { type: "string", description: "ISO 8601 UTC or relative" },
-        end_time: { type: "string", description: "ISO 8601 UTC or relative" },
-        fps: { type: "number", description: "Output FPS (default 1.0). Higher = smoother but larger file.", default: 1.0 },
+        start_time: { type: "string", description: 'ISO 8601 UTC or relative (e.g. "5m ago", "now")' },
+        end_time: { type: "string", description: 'ISO 8601 UTC or relative (e.g. "5m ago", "now")' },
+        output_path: {
+          type: "string",
+          description:
+            "Optional absolute path for the MP4 (e.g. ~/Downloads/clip.mp4). " +
+            "Defaults to the screenpipe data dir's exports/ folder.",
+        },
       },
       required: ["start_time", "end_time"],
     },
@@ -414,7 +425,7 @@ const TOOLS: Tool[] = [
       properties: {
         id: { type: "integer", description: "Memory ID — omit to create new, provide to update/delete" },
         content: { type: "string", description: "Memory text (required for creation)" },
-        tags: { type: "array", items: { type: "string" }, description: "Categorization tags (e.g. ['work', 'project-x'])" },
+        tags: { type: "array", items: { type: "string" }, description: "Tags. Prefer namespaced (person:ada, project:atlas, topic:pricing) so this memory links to the same people/projects you tag on frames/audio. Retrieve with search-content content_type='memory' tags='person:ada'." },
         importance: { type: "number", description: "0.0 (trivial) to 1.0 (critical). Default 0.5." },
         source_context: { type: "object", description: "Optional metadata linking to source (app, timestamp, etc.)" },
         delete: { type: "boolean", description: "Set true to delete the memory identified by id" },
@@ -477,14 +488,18 @@ const TOOLS: Tool[] = [
   {
     name: "add-tags",
     description:
-      "Add tags to a content item (vision frame or audio chunk) for organization and retrieval.",
+      "Tag a screen frame (vision) or audio chunk (audio) so it can be retrieved later. " +
+      "Tags are a shared linking layer: use namespaced tags (person:ada, project:atlas, topic:pricing) to connect a capture to a person, project, or topic. " +
+      "The SAME tag string also works on memories (via update-memory), so tagging a frame and a memory with person:ada links them. " +
+      "Retrieve later with search-content tags='person:ada' (add content_type+start_time/end_time to scope to a timeframe). " +
+      "Note: frames are pruned by retention, so for durable links prefer tagging a memory; tag frames/audio for shorter-term recall.",
     annotations: { title: "Add Tags", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
     inputSchema: {
       type: "object",
       properties: {
-        content_type: { type: "string", enum: ["vision", "audio"], description: "Type of content to tag" },
-        id: { type: "integer", description: "Content item ID" },
-        tags: { type: "array", items: { type: "string" }, description: "Tags to add" },
+        content_type: { type: "string", enum: ["vision", "audio"], description: "vision = screen frame, audio = audio chunk. Get the id from search-content results (frame_id / chunk_id)." },
+        id: { type: "integer", description: "Content item ID (OCR result frame_id, or audio result chunk_id)" },
+        tags: { type: "array", items: { type: "string" }, description: "Tags to add. Prefer namespaced: person:<name>, project:<name>, topic:<name>." },
       },
       required: ["content_type", "id", "tags"],
     },
@@ -1333,7 +1348,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       case "export-video": {
         const startTime = normalizeTime(args.start_time as string);
         const endTime = normalizeTime(args.end_time as string);
-        const fps = (args.fps as number) || 1.0;
 
         if (!startTime || !endTime) {
           return {
@@ -1341,128 +1355,51 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
           };
         }
 
-        // Get frame IDs for the time range
-        const searchParams = new URLSearchParams({
-          content_type: "ocr",
-          start_time: startTime,
-          end_time: endTime,
-          limit: "10000",
-        });
-
-        const searchResponse = await callAPI(`/search?${searchParams.toString()}`);
-        const searchData = await searchResponse.json();
-        const results = searchData.data || [];
-
-        if (results.length === 0) {
+        // A real-time MP4 with synced microphone audio, rendered server-side by the
+        // engine export core (the `screenpipe export` CLI's HTTP twin). MCP runs on the
+        // same host as the backend, so the returned path is a local file. Frames sit at
+        // their real timestamps, so the clip duration matches the wall-clock span.
+        try {
+          const body: Record<string, unknown> = { start: startTime, end: endTime };
+          if (typeof args.output_path === "string" && args.output_path.trim()) {
+            body.output_path = args.output_path;
+          }
+          const response = await callAPI("/export", {
+            method: "POST",
+            body: JSON.stringify(body),
+          });
+          const data = (await response.json()) as {
+            output_path: string;
+            frame_count: number;
+            audio_chunk_count: number;
+            duration_secs: number;
+            file_size_bytes: number;
+          };
+          const sizeMb = data.file_size_bytes
+            ? (data.file_size_bytes / (1024 * 1024)).toFixed(1)
+            : null;
           return {
             content: [
               {
                 type: "text",
-                text: `No screen recordings found between ${startTime} and ${endTime}.`,
+                text:
+                  `Video exported (with audio): ${data.output_path}\n` +
+                  `${data.frame_count ?? 0} frames | ${data.audio_chunk_count ?? 0} audio chunks` +
+                  (sizeMb ? ` | ${sizeMb} MB` : "") +
+                  (data.duration_secs ? ` | ${data.duration_secs}s` : "") +
+                  ` | ${startTime} → ${endTime}`,
               },
             ],
           };
-        }
-
-        const frameIds: number[] = [];
-        const seenIds = new Set<number>();
-        for (const result of results) {
-          if (result.type === "OCR" && result.content?.frame_id) {
-            const frameId = result.content.frame_id;
-            if (!seenIds.has(frameId)) {
-              seenIds.add(frameId);
-              frameIds.push(frameId);
-            }
-          }
-        }
-
-        if (frameIds.length === 0) {
-          return {
-            content: [{ type: "text", text: "No valid frame IDs found (audio-only?)." }],
-          };
-        }
-
-        frameIds.sort((a, b) => a - b);
-
-        const wsUrl = `ws://localhost:${port}/frames/export?fps=${fps}`;
-
-        const exportResult = await new Promise<{
-          success: boolean;
-          filePath?: string;
-          error?: string;
-          frameCount?: number;
-        }>((resolve) => {
-          const ws = new WebSocket(wsUrl);
-          let resolved = false;
-
-          const timeout = setTimeout(() => {
-            if (!resolved) {
-              resolved = true;
-              ws.close();
-              resolve({ success: false, error: "Export timed out after 5 minutes" });
-            }
-          }, 5 * 60 * 1000);
-
-          ws.on("open", () => {
-            ws.send(JSON.stringify({ frame_ids: frameIds }));
-          });
-
-          ws.on("error", (error) => {
-            if (!resolved) {
-              resolved = true;
-              clearTimeout(timeout);
-              resolve({ success: false, error: `WebSocket error: ${error.message}` });
-            }
-          });
-
-          ws.on("close", () => {
-            if (!resolved) {
-              resolved = true;
-              clearTimeout(timeout);
-              resolve({ success: false, error: "Connection closed unexpectedly" });
-            }
-          });
-
-          ws.on("message", (data) => {
-            try {
-              const message = JSON.parse(data.toString());
-              if (message.status === "completed" && message.video_data) {
-                const tempDir = os.tmpdir();
-                const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
-                const filename = `screenpipe_export_${timestamp}.mp4`;
-                const filePath = path.join(tempDir, filename);
-                fs.writeFileSync(filePath, Buffer.from(message.video_data));
-                resolved = true;
-                clearTimeout(timeout);
-                ws.close();
-                resolve({ success: true, filePath, frameCount: frameIds.length });
-              } else if (message.status === "error") {
-                resolved = true;
-                clearTimeout(timeout);
-                ws.close();
-                resolve({ success: false, error: message.error || "Export failed" });
-              }
-            } catch {
-              // Ignore parse errors for progress messages
-            }
-          });
-        });
-
-        if (exportResult.success && exportResult.filePath) {
+        } catch (err) {
           return {
             content: [
               {
                 type: "text",
-                text:
-                  `Video exported: ${exportResult.filePath}\n` +
-                  `Frames: ${exportResult.frameCount} | ${startTime} → ${endTime} | ${fps} fps`,
+                text: `Export failed: ${err instanceof Error ? err.message : String(err)}`,
               },
             ],
           };
-        } else {
-          return {
-            content: [{ type: "text", text: `Export failed: ${exportResult.error}` }],
-          };
         }
       }