screenpipe-mcp 0.6.0 → 0.8.0

package/README.md

@@ -1,6 +1,6 @@
 # Screenpipe MCP Server
 
-<a href="https://www.pulsemcp.com/servers/mediar-ai-screenpipe"><img src="https://www.pulsemcp.com/badge/top-pick/mediar-ai-screenpipe" width="400" alt="PulseMCP Badge"></a>
+<a href="https://www.pulsemcp.com/servers/screenpipe-screenpipe"><img src="https://www.pulsemcp.com/badge/top-pick/screenpipe-screenpipe" width="400" alt="PulseMCP Badge"></a>
 
 <br/>
 
@@ -33,8 +33,8 @@ The easiest way to use screenpipe-mcp is with npx. Edit your Claude Desktop conf
 Clone and build from source:
 
 ```bash
-git clone https://github.com/mediar-ai/screenpipe
-cd screenpipe/screenpipe-integrations/screenpipe-mcp
+git clone https://github.com/screenpipe/screenpipe
+cd screenpipe/crates/screenpipe-integrations/screenpipe-mcp
 npm install
 npm run build
 ```
@@ -64,32 +64,42 @@ npx @modelcontextprotocol/inspector npx screenpipe-mcp
 
 ## Available Tools
 
-### Cross-Platform
-
-- **search-content** - Search through recorded screen content, audio transcriptions, and UI elements
-  - Full text search with content type filtering (OCR/Audio/UI)
-  - Time range and app/window filtering
-  - Pagination support
-
-- **pixel-control** - Control mouse and keyboard
-  - Type text, press keys, move mouse, click
-
-### macOS Only
-
-- **find-elements** - Find UI elements in applications by role
-- **click-element** - Click UI elements by accessibility ID
-- **fill-element** - Type text into UI elements
-- **scroll-element** - Scroll UI elements
-- **open-application** - Open applications by name
-- **open-url** - Open URLs in default browser
+### search-content
+Search through recorded screen content (OCR) and audio transcriptions:
+- Full text search with content type filtering (OCR/Audio/UI)
+- Time range and app/window filtering
+- Speaker filtering (by ID or name)
+- Pagination support
+
+### search-ui-events (macOS)
+Search UI input events captured via accessibility APIs. This is the third data modality alongside vision and audio:
+- **Event types**: `click`, `text`, `scroll`, `key`, `app_switch`, `window_focus`, `clipboard`
+- Filter by app, window, time range
+- `text` events show aggregated keyboard input (what was typed)
+- `click` events include accessibility element labels
+- `clipboard` events show copy/paste content
+
+### get-ui-event-stats (macOS)
+Get aggregated statistics of UI events:
+- Event counts grouped by app and event type
+- Useful for productivity analysis and app usage tracking
+
+### export-video
+Export screen recordings as video files:
+- Specify time range with start/end times
+- Configurable FPS for output video
 
 ## Example Queries in Claude
 
 - "Search for any mentions of 'rust' in my screen recordings"
 - "Find audio transcriptions from the last hour"
 - "Show me what was on my screen in VSCode yesterday"
-- "Open Safari and go to github.com"
-- "Find the search button in Chrome and click it"
+- "Export a video of my screen from 2-3pm today"
+- "Find what John said in our meeting about the database"
+- "What did I type in Slack today?" (uses search-ui-events)
+- "Show me my app usage statistics for the past 3 hours"
+- "What did I copy to clipboard recently?"
+- "Which apps did I switch between most today?"
 
 ## Requirements
 

package/dist/index.js

@@ -66,7 +66,7 @@ const SCREENPIPE_API = `http://localhost:${port}`;
 // Initialize server
 const server = new index_js_1.Server({
     name: "screenpipe",
-    version: "0.5.0",
+    version: "0.7.0",
 }, {
     capabilities: {
         tools: {},
@@ -81,7 +81,11 @@ const BASE_TOOLS = [
         description: "Search screenpipe's recorded content: screen text (OCR), audio transcriptions, and UI elements. " +
             "Returns timestamped results with app context. " +
             "Call with no parameters to get recent activity. " +
-            "Use the 'screenpipe://context' resource for current time when building time-based queries.",
+            "Use the 'screenpipe://context' resource for current time when building time-based queries.\n\n" +
+            "DEEP LINKS: When referencing specific moments in results, create clickable timeline links:\n" +
+            "Format: [readable time](screenpipe://timeline?timestamp=ISO8601_TIMESTAMP)\n" +
+            "Example: [10:30 AM](screenpipe://timeline?timestamp=2024-01-15T18:30:00Z)\n" +
+            "Users can click these links to jump directly to that moment in their timeline.",
         annotations: {
             title: "Search Content",
             readOnlyHint: true,
@@ -96,7 +100,7 @@ const BASE_TOOLS = [
                 content_type: {
                     type: "string",
                     enum: ["all", "ocr", "audio", "ui"],
-                    description: "Content type filter. Default: 'all'",
+                    description: "Content type filter: 'ocr' (screen text), 'audio' (transcriptions), 'ui' (legacy UI monitoring), 'all'. Default: 'all'. For keyboard/mouse/accessibility events, use search-ui-events tool instead.",
                     default: "all",
                 },
                 limit: {
@@ -140,6 +144,14 @@ const BASE_TOOLS = [
                     description: "Include base64 screenshots (OCR only). Default: false",
                     default: false,
                 },
+                speaker_ids: {
+                    type: "string",
+                    description: "Comma-separated speaker IDs to filter audio results (e.g., '1,2,3')",
+                },
+                speaker_name: {
+                    type: "string",
+                    description: "Filter audio by speaker name (case-insensitive partial match)",
+                },
             },
         },
     },
@@ -177,6 +189,84 @@ const BASE_TOOLS = [
             required: ["start_time", "end_time"],
         },
     },
+    {
+        name: "search-ui-events",
+        description: "Search UI input events captured via accessibility APIs (macOS). " +
+            "This is the third modality alongside vision (OCR) and audio. " +
+            "Captures: mouse clicks, keyboard text input, scroll events, app/window switches, clipboard operations. " +
+            "Events include app context, element info (accessibility labels), and precise timestamps. " +
+            "Great for understanding user workflow, what was typed, clicked, or copied.",
+        annotations: {
+            title: "Search UI Events (Accessibility)",
+            readOnlyHint: true,
+        },
+        inputSchema: {
+            type: "object",
+            properties: {
+                q: {
+                    type: "string",
+                    description: "Search query for text content, app name, window title. Optional - omit to return recent events.",
+                },
+                event_type: {
+                    type: "string",
+                    enum: ["click", "text", "scroll", "key", "app_switch", "window_focus", "clipboard"],
+                    description: "Filter by event type. 'text' = aggregated keyboard input, 'click' = mouse clicks with element context, 'app_switch'/'window_focus' = app usage tracking, 'clipboard' = copy/paste events.",
+                },
+                app_name: {
+                    type: "string",
+                    description: "Filter by application name (e.g., 'Google Chrome', 'Slack', 'Code')",
+                },
+                window_name: {
+                    type: "string",
+                    description: "Filter by window title",
+                },
+                start_time: {
+                    type: "string",
+                    format: "date-time",
+                    description: "ISO 8601 UTC start time (e.g., 2024-01-15T10:00:00Z)",
+                },
+                end_time: {
+                    type: "string",
+                    format: "date-time",
+                    description: "ISO 8601 UTC end time (e.g., 2024-01-15T18:00:00Z)",
+                },
+                limit: {
+                    type: "integer",
+                    description: "Max results. Default: 50",
+                    default: 50,
+                },
+                offset: {
+                    type: "integer",
+                    description: "Skip N results for pagination. Default: 0",
+                    default: 0,
+                },
+            },
+        },
+    },
+    {
+        name: "get-ui-event-stats",
+        description: "Get aggregated statistics of UI events by app and event type. " +
+            "Useful for understanding app usage patterns, productivity analysis, or finding which apps were used most.",
+        annotations: {
+            title: "UI Event Statistics",
+            readOnlyHint: true,
+        },
+        inputSchema: {
+            type: "object",
+            properties: {
+                start_time: {
+                    type: "string",
+                    format: "date-time",
+                    description: "ISO 8601 UTC start time for stats period",
+                },
+                end_time: {
+                    type: "string",
+                    format: "date-time",
+                    description: "ISO 8601 UTC end time for stats period",
+                },
+            },
+        },
+    },
 ];
 // List tools handler
 server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => {
@@ -244,18 +334,20 @@ server.setRequestHandler(types_js_1.ReadResourceRequestSchema, async (request) =
                         mimeType: "text/markdown",
                         text: `# Screenpipe Search Guide
 
+## Three Data Modalities
+
+Screenpipe captures three types of data:
+1. **Vision (OCR)** - Screen text from screenshots
+2. **Audio** - Transcribed speech from microphone/system audio
+3. **UI Events (Accessibility)** - Keyboard input, mouse clicks, app switches, clipboard (macOS)
+
 ## Quick Start
 - **Get recent activity**: Call search-content with no parameters
 - **Search text**: \`{"q": "search term", "content_type": "ocr"}\`
-- **Time filter**: Use start_time/end_time with ISO 8601 UTC timestamps
-
-## Content Types
-- \`ocr\`: Screen text (what you see)
-- \`audio\`: Transcribed speech
-- \`ui\`: UI element interactions
-- \`all\`: Everything (default)
+- **Get keyboard input**: Use search-ui-events with \`event_type: "text"\`
+- **Track app usage**: Use get-ui-event-stats for aggregated data
 
-## Key Parameters
+## search-content (Vision + Audio)
 | Parameter | Description | Default |
 |-----------|-------------|---------|
 | q | Search query | (none - returns all) |
@@ -266,11 +358,39 @@ server.setRequestHandler(types_js_1.ReadResourceRequestSchema, async (request) =
 | app_name | Filter by app | (no filter) |
 | include_frames | Include screenshots | false |
 
+## search-ui-events (Accessibility Data)
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| q | Search text content, app, window | (none) |
+| event_type | click/text/scroll/key/app_switch/window_focus/clipboard | (all types) |
+| app_name | Filter by application | (no filter) |
+| limit | Max results | 50 |
+
+### Event Types
+- \`text\`: Aggregated keyboard input (what was typed)
+- \`click\`: Mouse clicks with element context (accessibility labels)
+- \`app_switch\`: When user switched applications
+- \`window_focus\`: When window focus changed
+- \`clipboard\`: Copy/paste operations
+- \`scroll\`: Scroll events with delta values
+
 ## Tips
 1. Read screenpipe://context first to get current timestamps
-2. Omit \`q\` to get all content (useful for "what was I doing?")
-3. Use \`limit: 50-100\` for comprehensive searches
-4. Combine app_name + time filters for focused results`,
+2. Use search-ui-events for "what did I type?" queries
+3. Use get-ui-event-stats to understand app usage patterns
+4. Combine search-content (what was on screen) with search-ui-events (what was done)
+
+## Timeline Deep Links
+When showing search results to users, create clickable links to specific moments:
+
+**Format:** \`[readable time](screenpipe://timeline?timestamp=ISO8601_TIMESTAMP)\`
+
+**Examples:**
+- \`[10:30 AM](screenpipe://timeline?timestamp=2024-01-15T18:30:00Z)\`
+- \`[yesterday at 3pm](screenpipe://timeline?timestamp=2024-01-14T15:00:00Z)\`
+
+Users can click these links to jump directly to that moment in their screenpipe timeline.
+Always use the exact timestamp from search results when creating these links.`,
                     },
                 ],
             };
@@ -592,7 +712,8 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                 // Sort frame IDs
                 frameIds.sort((a, b) => a - b);
                 // Step 2: Connect to WebSocket and export video
-                const wsUrl = `ws://localhost:${port}/frames/export?frame_ids=${frameIds.join(",")}&fps=${fps}`;
+                // Send frame_ids in message body to avoid URL length limits
+                const wsUrl = `ws://localhost:${port}/frames/export?fps=${fps}`;
                 const exportResult = await new Promise((resolve) => {
                     const ws = new ws_1.WebSocket(wsUrl);
                     let resolved = false;
@@ -603,6 +724,10 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                             resolve({ success: false, error: "Export timed out after 5 minutes" });
                         }
                     }, 5 * 60 * 1000); // 5 minute timeout
+                    ws.on("open", () => {
+                        // Send frame_ids in message body to avoid URL length limits
+                        ws.send(JSON.stringify({ frame_ids: frameIds }));
+                    });
                     ws.on("error", (error) => {
                         if (!resolved) {
                             resolved = true;
@@ -674,6 +799,123 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
                     };
                 }
             }
+            case "search-ui-events": {
+                const params = new URLSearchParams();
+                for (const [key, value] of Object.entries(args)) {
+                    if (value !== null && value !== undefined) {
+                        // Map event_type to the API parameter
+                        params.append(key, String(value));
+                    }
+                }
+                const response = await fetchAPI(`/ui-events?${params.toString()}`);
+                if (!response.ok) {
+                    throw new Error(`HTTP error: ${response.status}`);
+                }
+                const data = await response.json();
+                const events = data.data || [];
+                const pagination = data.pagination || {};
+                if (events.length === 0) {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: "No UI events found. This feature requires:\n" +
+                                    "1. macOS with Accessibility permissions granted\n" +
+                                    "2. UI Events enabled in screenpipe settings\n" +
+                                    "Try: broader time range or different event_type filter.",
+                            },
+                        ],
+                    };
+                }
+                const formattedEvents = [];
+                for (const event of events) {
+                    const parts = [
+                        `[${event.event_type?.toUpperCase() || "?"}]`,
+                        event.app_name || "?",
+                        event.window_title ? `| ${event.window_title}` : "",
+                    ];
+                    let details = "";
+                    if (event.event_type === "text" && event.text_content) {
+                        details = `Text: "${event.text_content}"`;
+                    }
+                    else if (event.event_type === "click") {
+                        details = `Click at (${event.x || 0}, ${event.y || 0})`;
+                        if (event.element?.label) {
+                            details += ` on "${event.element.label}"`;
+                        }
+                    }
+                    else if (event.event_type === "clipboard" && event.text_content) {
+                        details = `Clipboard: "${event.text_content.substring(0, 100)}${event.text_content.length > 100 ? "..." : ""}"`;
+                    }
+                    else if (event.event_type === "app_switch" || event.event_type === "window_focus") {
+                        details = `Switched to: ${event.app_name}${event.window_title ? ` - ${event.window_title}` : ""}`;
+                    }
+                    else if (event.event_type === "scroll") {
+                        details = `Scroll: dx=${event.delta_x || 0}, dy=${event.delta_y || 0}`;
+                    }
+                    formattedEvents.push(`${parts.join(" ")}\n` +
+                        `${event.timestamp || ""}\n` +
+                        `${details}`);
+                }
+                const header = `UI Events: ${events.length}/${pagination.total || "?"}` +
+                    (pagination.total > events.length ? ` (use offset=${(pagination.offset || 0) + events.length} for more)` : "");
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: header + "\n\n" + formattedEvents.join("\n---\n"),
+                        },
+                    ],
+                };
+            }
+            case "get-ui-event-stats": {
+                const params = new URLSearchParams();
+                if (args.start_time)
+                    params.append("start_time", String(args.start_time));
+                if (args.end_time)
+                    params.append("end_time", String(args.end_time));
+                const response = await fetchAPI(`/ui-events/stats?${params.toString()}`);
+                if (!response.ok) {
+                    throw new Error(`HTTP error: ${response.status}`);
+                }
+                const stats = await response.json();
+                if (!stats || stats.length === 0) {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: "No UI event statistics available. UI Events may not be enabled or no events have been captured yet.",
+                            },
+                        ],
+                    };
+                }
+                // Group by app
+                const byApp = {};
+                for (const stat of stats) {
+                    const app = stat.app_name || "Unknown";
+                    if (!byApp[app]) {
+                        byApp[app] = { app, events: {}, total: 0 };
+                    }
+                    byApp[app].events[stat.event_type] = stat.count;
+                    byApp[app].total += stat.count;
+                }
+                // Sort by total events
+                const sorted = Object.values(byApp).sort((a, b) => b.total - a.total);
+                const lines = sorted.map(({ app, events, total }) => {
+                    const eventDetails = Object.entries(events)
+                        .map(([type, count]) => `${type}: ${count}`)
+                        .join(", ");
+                    return `${app}: ${total} events (${eventDetails})`;
+                });
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `UI Event Statistics:\n\n${lines.join("\n")}`,
+                        },
+                    ],
+                };
+            }
             default:
                 throw new Error(`Unknown tool: ${name}`);
         }

package/manifest.json

@@ -2,20 +2,20 @@
   "manifest_version": "0.3",
   "name": "screenpipe",
   "display_name": "Screenpipe",
-  "version": "0.5.0",
-  "description": "Search your screen recordings, audio transcriptions, and control your computer with AI",
-  "long_description": "Screenpipe is a 24/7 screen and audio recorder that lets you search everything you've seen or heard. This extension connects Claude to your local screenpipe instance, enabling AI-powered search through your digital memory and computer control capabilities.",
+  "version": "0.8.0",
+  "description": "Search your screen recordings and audio transcriptions with AI",
+  "long_description": "Screenpipe is a 24/7 screen and audio recorder that lets you search everything you've seen or heard. This extension connects Claude to your local screenpipe instance, enabling AI-powered search through your digital memory.",
   "author": {
     "name": "screenpipe",
     "url": "https://screenpi.pe"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/mediar-ai/screenpipe"
+    "url": "https://github.com/screenpipe/screenpipe"
   },
   "homepage": "https://screenpi.pe",
-  "documentation": "https://github.com/mediar-ai/screenpipe/tree/main/screenpipe-integrations/screenpipe-mcp",
-  "support": "https://github.com/mediar-ai/screenpipe/issues",
+  "documentation": "https://github.com/screenpipe/screenpipe/tree/main/crates/screenpipe-integrations/screenpipe-mcp",
+  "support": "https://github.com/screenpipe/screenpipe/issues",
   "license": "MIT",
   "server": {
     "type": "node",
@@ -28,39 +28,11 @@
   "tools": [
     {
       "name": "search-content",
-      "description": "Search through recorded screen content, audio transcriptions, and UI elements"
+      "description": "Search through recorded screen content, audio transcriptions, and UI elements with speaker filtering"
     },
     {
       "name": "export-video",
       "description": "Export screen recordings as MP4 video for a specific time range"
-    },
-    {
-      "name": "pixel-control",
-      "description": "Control mouse and keyboard (type text, press keys, move mouse, click)"
-    },
-    {
-      "name": "find-elements",
-      "description": "Find UI elements in applications by role (macOS only)"
-    },
-    {
-      "name": "click-element",
-      "description": "Click UI elements by ID (macOS only)"
-    },
-    {
-      "name": "fill-element",
-      "description": "Type text into UI elements (macOS only)"
-    },
-    {
-      "name": "scroll-element",
-      "description": "Scroll UI elements (macOS only)"
-    },
-    {
-      "name": "open-application",
-      "description": "Open applications by name (macOS only)"
-    },
-    {
-      "name": "open-url",
-      "description": "Open URLs in browser (macOS only)"
     }
   ],
   "compatibility": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screenpipe-mcp",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "MCP server for screenpipe - search your screen recordings and audio transcriptions",
   "main": "dist/index.js",
   "bin": {

package/src/index.ts

@@ -45,7 +45,7 @@ const SCREENPIPE_API = `http://localhost:${port}`;
 const server = new Server(
   {
     name: "screenpipe",
-    version: "0.5.0",
+    version: "0.7.0",
   },
   {
     capabilities: {
@@ -64,7 +64,11 @@ const BASE_TOOLS: Tool[] = [
       "Search screenpipe's recorded content: screen text (OCR), audio transcriptions, and UI elements. " +
       "Returns timestamped results with app context. " +
       "Call with no parameters to get recent activity. " +
-      "Use the 'screenpipe://context' resource for current time when building time-based queries.",
+      "Use the 'screenpipe://context' resource for current time when building time-based queries.\n\n" +
+      "DEEP LINKS: When referencing specific moments in results, create clickable timeline links:\n" +
+      "Format: [readable time](screenpipe://timeline?timestamp=ISO8601_TIMESTAMP)\n" +
+      "Example: [10:30 AM](screenpipe://timeline?timestamp=2024-01-15T18:30:00Z)\n" +
+      "Users can click these links to jump directly to that moment in their timeline.",
     annotations: {
       title: "Search Content",
       readOnlyHint: true,
@@ -79,7 +83,7 @@ const BASE_TOOLS: Tool[] = [
         content_type: {
           type: "string",
           enum: ["all", "ocr", "audio", "ui"],
-          description: "Content type filter. Default: 'all'",
+          description: "Content type filter: 'ocr' (screen text), 'audio' (transcriptions), 'ui' (legacy UI monitoring), 'all'. Default: 'all'. For keyboard/mouse/accessibility events, use search-ui-events tool instead.",
           default: "all",
         },
         limit: {
@@ -123,6 +127,14 @@ const BASE_TOOLS: Tool[] = [
           description: "Include base64 screenshots (OCR only). Default: false",
           default: false,
         },
+        speaker_ids: {
+          type: "string",
+          description: "Comma-separated speaker IDs to filter audio results (e.g., '1,2,3')",
+        },
+        speaker_name: {
+          type: "string",
+          description: "Filter audio by speaker name (case-insensitive partial match)",
+        },
       },
     },
   },
@@ -164,6 +176,86 @@ const BASE_TOOLS: Tool[] = [
       required: ["start_time", "end_time"],
     },
   },
+  {
+    name: "search-ui-events",
+    description:
+      "Search UI input events captured via accessibility APIs (macOS). " +
+      "This is the third modality alongside vision (OCR) and audio. " +
+      "Captures: mouse clicks, keyboard text input, scroll events, app/window switches, clipboard operations. " +
+      "Events include app context, element info (accessibility labels), and precise timestamps. " +
+      "Great for understanding user workflow, what was typed, clicked, or copied.",
+    annotations: {
+      title: "Search UI Events (Accessibility)",
+      readOnlyHint: true,
+    },
+    inputSchema: {
+      type: "object",
+      properties: {
+        q: {
+          type: "string",
+          description: "Search query for text content, app name, window title. Optional - omit to return recent events.",
+        },
+        event_type: {
+          type: "string",
+          enum: ["click", "text", "scroll", "key", "app_switch", "window_focus", "clipboard"],
+          description: "Filter by event type. 'text' = aggregated keyboard input, 'click' = mouse clicks with element context, 'app_switch'/'window_focus' = app usage tracking, 'clipboard' = copy/paste events.",
+        },
+        app_name: {
+          type: "string",
+          description: "Filter by application name (e.g., 'Google Chrome', 'Slack', 'Code')",
+        },
+        window_name: {
+          type: "string",
+          description: "Filter by window title",
+        },
+        start_time: {
+          type: "string",
+          format: "date-time",
+          description: "ISO 8601 UTC start time (e.g., 2024-01-15T10:00:00Z)",
+        },
+        end_time: {
+          type: "string",
+          format: "date-time",
+          description: "ISO 8601 UTC end time (e.g., 2024-01-15T18:00:00Z)",
+        },
+        limit: {
+          type: "integer",
+          description: "Max results. Default: 50",
+          default: 50,
+        },
+        offset: {
+          type: "integer",
+          description: "Skip N results for pagination. Default: 0",
+          default: 0,
+        },
+      },
+    },
+  },
+  {
+    name: "get-ui-event-stats",
+    description:
+      "Get aggregated statistics of UI events by app and event type. " +
+      "Useful for understanding app usage patterns, productivity analysis, or finding which apps were used most.",
+    annotations: {
+      title: "UI Event Statistics",
+      readOnlyHint: true,
+    },
+    inputSchema: {
+      type: "object",
+      properties: {
+        start_time: {
+          type: "string",
+          format: "date-time",
+          description: "ISO 8601 UTC start time for stats period",
+        },
+        end_time: {
+          type: "string",
+          format: "date-time",
+          description: "ISO 8601 UTC end time for stats period",
+        },
+      },
+    },
+  },
 ];
 
 // List tools handler
@@ -237,18 +329,20 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
             mimeType: "text/markdown",
             text: `# Screenpipe Search Guide
 
+## Three Data Modalities
+
+Screenpipe captures three types of data:
+1. **Vision (OCR)** - Screen text from screenshots
+2. **Audio** - Transcribed speech from microphone/system audio
+3. **UI Events (Accessibility)** - Keyboard input, mouse clicks, app switches, clipboard (macOS)
+
 ## Quick Start
 - **Get recent activity**: Call search-content with no parameters
 - **Search text**: \`{"q": "search term", "content_type": "ocr"}\`
-- **Time filter**: Use start_time/end_time with ISO 8601 UTC timestamps
-
-## Content Types
-- \`ocr\`: Screen text (what you see)
-- \`audio\`: Transcribed speech
-- \`ui\`: UI element interactions
-- \`all\`: Everything (default)
+- **Get keyboard input**: Use search-ui-events with \`event_type: "text"\`
+- **Track app usage**: Use get-ui-event-stats for aggregated data
 
-## Key Parameters
+## search-content (Vision + Audio)
 | Parameter | Description | Default |
 |-----------|-------------|---------|
 | q | Search query | (none - returns all) |
@@ -259,11 +353,39 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
 | app_name | Filter by app | (no filter) |
 | include_frames | Include screenshots | false |
 
+## search-ui-events (Accessibility Data)
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| q | Search text content, app, window | (none) |
+| event_type | click/text/scroll/key/app_switch/window_focus/clipboard | (all types) |
+| app_name | Filter by application | (no filter) |
+| limit | Max results | 50 |
+
+### Event Types
+- \`text\`: Aggregated keyboard input (what was typed)
+- \`click\`: Mouse clicks with element context (accessibility labels)
+- \`app_switch\`: When user switched applications
+- \`window_focus\`: When window focus changed
+- \`clipboard\`: Copy/paste operations
+- \`scroll\`: Scroll events with delta values
+
 ## Tips
 1. Read screenpipe://context first to get current timestamps
-2. Omit \`q\` to get all content (useful for "what was I doing?")
-3. Use \`limit: 50-100\` for comprehensive searches
-4. Combine app_name + time filters for focused results`,
+2. Use search-ui-events for "what did I type?" queries
+3. Use get-ui-event-stats to understand app usage patterns
+4. Combine search-content (what was on screen) with search-ui-events (what was done)
+
+## Timeline Deep Links
+When showing search results to users, create clickable links to specific moments:
+
+**Format:** \`[readable time](screenpipe://timeline?timestamp=ISO8601_TIMESTAMP)\`
+
+**Examples:**
+- \`[10:30 AM](screenpipe://timeline?timestamp=2024-01-15T18:30:00Z)\`
+- \`[yesterday at 3pm](screenpipe://timeline?timestamp=2024-01-14T15:00:00Z)\`
+
+Users can click these links to jump directly to that moment in their screenpipe timeline.
+Always use the exact timestamp from search results when creating these links.`,
           },
         ],
       };
@@ -630,7 +752,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         frameIds.sort((a, b) => a - b);
 
         // Step 2: Connect to WebSocket and export video
-        const wsUrl = `ws://localhost:${port}/frames/export?frame_ids=${frameIds.join(",")}&fps=${fps}`;
+        // Send frame_ids in message body to avoid URL length limits
+        const wsUrl = `ws://localhost:${port}/frames/export?fps=${fps}`;
 
         const exportResult = await new Promise<{
           success: boolean;
@@ -649,6 +772,11 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             }
           }, 5 * 60 * 1000); // 5 minute timeout
 
+          ws.on("open", () => {
+            // Send frame_ids in message body to avoid URL length limits
+            ws.send(JSON.stringify({ frame_ids: frameIds }));
+          });
+
           ws.on("error", (error) => {
             if (!resolved) {
               resolved = true;
@@ -724,6 +852,136 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         }
       }
 
+      case "search-ui-events": {
+        const params = new URLSearchParams();
+        for (const [key, value] of Object.entries(args)) {
+          if (value !== null && value !== undefined) {
+            // Map event_type to the API parameter
+            params.append(key, String(value));
+          }
+        }
+
+        const response = await fetchAPI(`/ui-events?${params.toString()}`);
+        if (!response.ok) {
+          throw new Error(`HTTP error: ${response.status}`);
+        }
+
+        const data = await response.json();
+        const events = data.data || [];
+        const pagination = data.pagination || {};
+
+        if (events.length === 0) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "No UI events found. This feature requires:\n" +
+                  "1. macOS with Accessibility permissions granted\n" +
+                  "2. UI Events enabled in screenpipe settings\n" +
+                  "Try: broader time range or different event_type filter.",
+              },
+            ],
+          };
+        }
+
+        const formattedEvents: string[] = [];
+        for (const event of events) {
+          const parts = [
+            `[${event.event_type?.toUpperCase() || "?"}]`,
+            event.app_name || "?",
+            event.window_title ? `| ${event.window_title}` : "",
+          ];
+          
+          let details = "";
+          if (event.event_type === "text" && event.text_content) {
+            details = `Text: "${event.text_content}"`;
+          } else if (event.event_type === "click") {
+            details = `Click at (${event.x || 0}, ${event.y || 0})`;
+            if (event.element?.label) {
+              details += ` on "${event.element.label}"`;
+            }
+          } else if (event.event_type === "clipboard" && event.text_content) {
+            details = `Clipboard: "${event.text_content.substring(0, 100)}${event.text_content.length > 100 ? "..." : ""}"`;
+          } else if (event.event_type === "app_switch" || event.event_type === "window_focus") {
+            details = `Switched to: ${event.app_name}${event.window_title ? ` - ${event.window_title}` : ""}`;
+          } else if (event.event_type === "scroll") {
+            details = `Scroll: dx=${event.delta_x || 0}, dy=${event.delta_y || 0}`;
+          }
+
+          formattedEvents.push(
+            `${parts.join(" ")}\n` +
+            `${event.timestamp || ""}\n` +
+            `${details}`
+          );
+        }
+
+        const header = `UI Events: ${events.length}/${pagination.total || "?"}` +
+          (pagination.total > events.length ? ` (use offset=${(pagination.offset || 0) + events.length} for more)` : "");
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: header + "\n\n" + formattedEvents.join("\n---\n"),
+            },
+          ],
+        };
+      }
+
+      case "get-ui-event-stats": {
+        const params = new URLSearchParams();
+        if (args.start_time) params.append("start_time", String(args.start_time));
+        if (args.end_time) params.append("end_time", String(args.end_time));
+
+        const response = await fetchAPI(`/ui-events/stats?${params.toString()}`);
+        if (!response.ok) {
+          throw new Error(`HTTP error: ${response.status}`);
+        }
+
+        const stats = await response.json();
+
+        if (!stats || stats.length === 0) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "No UI event statistics available. UI Events may not be enabled or no events have been captured yet.",
+              },
+            ],
+          };
+        }
+
+        // Group by app
+        const byApp: Record<string, { app: string; events: Record<string, number>; total: number }> = {};
+        for (const stat of stats) {
+          const app = stat.app_name || "Unknown";
+          if (!byApp[app]) {
+            byApp[app] = { app, events: {}, total: 0 };
+          }
+          byApp[app].events[stat.event_type] = stat.count;
+          byApp[app].total += stat.count;
+        }
+
+        // Sort by total events
+        const sorted = Object.values(byApp).sort((a, b) => b.total - a.total);
+
+        const lines = sorted.map(({ app, events, total }) => {
+          const eventDetails = Object.entries(events)
+            .map(([type, count]) => `${type}: ${count}`)
+            .join(", ");
+          return `${app}: ${total} events (${eventDetails})`;
+        });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: `UI Event Statistics:\n\n${lines.join("\n")}`,
+            },
+          ],
+        };
+      }
+
       default:
         throw new Error(`Unknown tool: ${name}`);
     }