screenpipe-mcp 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.js +499 -59
  2. package/package.json +1 -1
  3. package/src/index.ts +513 -58
package/src/index.ts CHANGED
@@ -43,64 +43,69 @@ const server = new Server(
43
43
  );
44
44
 
45
45
  // ---------------------------------------------------------------------------
46
- // Tools — minimal descriptions, no behavioral guidance (that belongs in resources)
46
+ // Tools
47
47
  // ---------------------------------------------------------------------------
48
48
  const TOOLS: Tool[] = [
49
49
  {
50
50
  name: "search-content",
51
51
  description:
52
52
  "Search screen text, audio transcriptions, input events, and memories. " +
53
- "Returns timestamped results with app context. Call with no params for recent activity.",
54
- annotations: { title: "Search Content", readOnlyHint: true },
53
+ "Returns timestamped results with app context. " +
54
+ "IMPORTANT: prefer activity-summary for broad questions ('what was I doing?'). " +
55
+ "Use search-content only when you need specific text/content. " +
56
+ "Start with limit=5, increase only if needed. Results can be large — use max_content_length=500 to truncate.",
57
+ annotations: { title: "Search Content", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
55
58
  inputSchema: {
56
59
  type: "object",
57
60
  properties: {
58
61
  q: {
59
62
  type: "string",
60
- description: "Full-text search query. Omit to return all content in time range.",
63
+ description: "Full-text search query. Omit to return all content in time range. Avoid for audio — transcriptions are noisy, q filters too aggressively.",
61
64
  },
62
65
  content_type: {
63
66
  type: "string",
64
67
  enum: ["all", "ocr", "audio", "input", "accessibility", "memory"],
65
- description: "Filter by content type. Default: 'all'.",
68
+ description: "Filter by content type. 'accessibility' is preferred for screen text (OS-native). 'ocr' is fallback for apps without accessibility support. Default: 'all'.",
66
69
  default: "all",
67
70
  },
68
- limit: { type: "integer", description: "Max results (default 10)", default: 10 },
69
- offset: { type: "integer", description: "Pagination offset", default: 0 },
71
+ limit: { type: "integer", description: "Max results (default 10, max 20). Start with 5 for exploration.", default: 10 },
72
+ offset: { type: "integer", description: "Pagination offset. Use when results say 'use offset=N for more'.", default: 0 },
70
73
  start_time: {
71
74
  type: "string",
72
- description: "ISO 8601 UTC or relative (e.g. '2h ago')",
75
+ description: "ISO 8601 UTC or relative (e.g. '2h ago', '1d ago'). Always provide to avoid scanning entire history.",
73
76
  },
74
77
  end_time: {
75
78
  type: "string",
76
- description: "ISO 8601 UTC or relative (e.g. 'now')",
79
+ description: "ISO 8601 UTC or relative (e.g. 'now'). Defaults to now.",
77
80
  },
78
- app_name: { type: "string", description: "Filter by app name" },
79
- window_name: { type: "string", description: "Filter by window title" },
80
- min_length: { type: "integer", description: "Min content length" },
81
- max_length: { type: "integer", description: "Max content length" },
81
+ app_name: { type: "string", description: "Filter by app name (e.g. 'Google Chrome', 'Slack', 'zoom.us'). Case-sensitive." },
82
+ window_name: { type: "string", description: "Filter by window title substring" },
83
+ min_length: { type: "integer", description: "Min content length in characters" },
84
+ max_length: { type: "integer", description: "Max content length in characters" },
82
85
  include_frames: {
83
86
  type: "boolean",
84
- description: "Include base64 screenshots (OCR only)",
87
+ description: "Include base64 screenshots (OCR only). Warning: large response.",
85
88
  default: false,
86
89
  },
87
- speaker_ids: { type: "string", description: "Comma-separated speaker IDs" },
88
- speaker_name: { type: "string", description: "Filter audio by speaker name" },
90
+ speaker_ids: { type: "string", description: "Comma-separated speaker IDs to filter audio" },
91
+ speaker_name: { type: "string", description: "Filter audio by speaker name (case-insensitive partial match)" },
89
92
  max_content_length: {
90
93
  type: "integer",
91
- description: "Truncate each result via middle-truncation",
94
+ description: "Truncate each result's text via middle-truncation. Use 200-500 to keep responses compact.",
92
95
  },
93
96
  },
94
97
  },
95
98
  },
96
99
  {
97
100
  name: "list-meetings",
98
- description: "List detected meetings (Zoom, Teams, Meet, etc.) with duration, app, and attendees.",
99
- annotations: { title: "List Meetings", readOnlyHint: true },
101
+ description:
102
+ "List detected meetings (Zoom, Teams, Meet, etc.) with duration, app, and attendees. " +
103
+ "Only available when screenpipe runs in smart transcription mode.",
104
+ annotations: { title: "List Meetings", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
100
105
  inputSchema: {
101
106
  type: "object",
102
107
  properties: {
103
- start_time: { type: "string", description: "ISO 8601 UTC or relative" },
108
+ start_time: { type: "string", description: "ISO 8601 UTC or relative (e.g. '1d ago')" },
104
109
  end_time: { type: "string", description: "ISO 8601 UTC or relative" },
105
110
  limit: { type: "integer", description: "Max results (default 20)", default: 20 },
106
111
  offset: { type: "integer", description: "Pagination offset", default: 0 },
@@ -111,14 +116,15 @@ const TOOLS: Tool[] = [
111
116
  name: "activity-summary",
112
117
  description:
113
118
  "Lightweight activity overview (~200-500 tokens): app usage with active minutes, audio speakers, recent texts. " +
114
- "Use for 'how long on X?', 'which apps?', 'what was I doing?' questions.",
115
- annotations: { title: "Activity Summary", readOnlyHint: true },
119
+ "USE THIS FIRST for broad questions: 'what was I doing?', 'how long on X?', 'which apps?'. " +
120
+ "Only escalate to search-content if you need specific text content.",
121
+ annotations: { title: "Activity Summary", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
116
122
  inputSchema: {
117
123
  type: "object",
118
124
  properties: {
119
- start_time: { type: "string", description: "ISO 8601 UTC or relative" },
120
- end_time: { type: "string", description: "ISO 8601 UTC or relative" },
121
- app_name: { type: "string", description: "Optional app name filter" },
125
+ start_time: { type: "string", description: "ISO 8601 UTC or relative (e.g. '3h ago')" },
126
+ end_time: { type: "string", description: "ISO 8601 UTC or relative (e.g. 'now')" },
127
+ app_name: { type: "string", description: "Optional app name filter to focus on one app" },
122
128
  },
123
129
  required: ["start_time", "end_time"],
124
130
  },
@@ -127,23 +133,24 @@ const TOOLS: Tool[] = [
127
133
  name: "search-elements",
128
134
  description:
129
135
  "Search UI elements (buttons, links, text fields) from the accessibility tree. " +
130
- "Lighter than search-content for targeted UI lookups.",
131
- annotations: { title: "Search Elements", readOnlyHint: true },
136
+ "Lighter than search-content for targeted UI lookups. " +
137
+ "Use when you need to find specific UI controls or page structure, not general content.",
138
+ annotations: { title: "Search Elements", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
132
139
  inputSchema: {
133
140
  type: "object",
134
141
  properties: {
135
142
  q: { type: "string", description: "Full-text search on element text" },
136
- frame_id: { type: "integer", description: "Filter to specific frame" },
143
+ frame_id: { type: "integer", description: "Filter to specific frame ID from search results" },
137
144
  source: {
138
145
  type: "string",
139
146
  enum: ["accessibility", "ocr"],
140
- description: "Element source filter",
147
+ description: "Element source. 'accessibility' is preferred (OS-native tree). 'ocr' for apps without a11y.",
141
148
  },
142
- role: { type: "string", description: "Element role (e.g. AXButton, AXLink)" },
149
+ role: { type: "string", description: "Element role filter (e.g. 'AXButton', 'AXLink', 'AXTextField')" },
143
150
  start_time: { type: "string", description: "ISO 8601 UTC or relative" },
144
151
  end_time: { type: "string", description: "ISO 8601 UTC or relative" },
145
152
  app_name: { type: "string", description: "Filter by app name" },
146
- limit: { type: "integer", description: "Max results (default 50)", default: 50 },
153
+ limit: { type: "integer", description: "Max results (default 50). Start with 10-20.", default: 50 },
147
154
  offset: { type: "integer", description: "Pagination offset", default: 0 },
148
155
  },
149
156
  },
@@ -151,26 +158,29 @@ const TOOLS: Tool[] = [
151
158
  {
152
159
  name: "frame-context",
153
160
  description:
154
- "Get accessibility text, parsed tree nodes, and URLs for a specific frame ID.",
155
- annotations: { title: "Frame Context", readOnlyHint: true },
161
+ "Get full accessibility text, parsed tree nodes, and URLs for a specific frame ID. " +
162
+ "Use after search-content to get detailed context for a specific moment.",
163
+ annotations: { title: "Frame Context", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
156
164
  inputSchema: {
157
165
  type: "object",
158
166
  properties: {
159
- frame_id: { type: "integer", description: "Frame ID from search results" },
167
+ frame_id: { type: "integer", description: "Frame ID from search-content results (content.frame_id field)" },
160
168
  },
161
169
  required: ["frame_id"],
162
170
  },
163
171
  },
164
172
  {
165
173
  name: "export-video",
166
- description: "Export an MP4 video of screen recordings for a time range.",
167
- annotations: { title: "Export Video", destructiveHint: true },
174
+ description:
175
+ "Export an MP4 video of screen recordings for a time range. " +
176
+ "Returns the file path. Can take a few minutes for long ranges.",
177
+ annotations: { title: "Export Video", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
168
178
  inputSchema: {
169
179
  type: "object",
170
180
  properties: {
171
181
  start_time: { type: "string", description: "ISO 8601 UTC or relative" },
172
182
  end_time: { type: "string", description: "ISO 8601 UTC or relative" },
173
- fps: { type: "number", description: "Output FPS (default 1.0)", default: 1.0 },
183
+ fps: { type: "number", description: "Output FPS (default 1.0). Higher = smoother but larger file.", default: 1.0 },
174
184
  },
175
185
  required: ["start_time", "end_time"],
176
186
  },
@@ -178,37 +188,38 @@ const TOOLS: Tool[] = [
178
188
  {
179
189
  name: "update-memory",
180
190
  description:
181
- "Create, update, or delete a persistent memory (facts, preferences, decisions). " +
182
- "Retrieve memories via search-content with content_type='memory'.",
183
- annotations: { title: "Update Memory", destructiveHint: false },
191
+ "Create, update, or delete a persistent memory (facts, preferences, decisions the user wants to remember). " +
192
+ "To retrieve memories, use search-content with content_type='memory'. " +
193
+ "To create: provide content + tags. To update: provide id + fields to change. To delete: provide id + delete=true.",
194
+ annotations: { title: "Update Memory", readOnlyHint: false, destructiveHint: false, openWorldHint: false, idempotentHint: true },
184
195
  inputSchema: {
185
196
  type: "object",
186
197
  properties: {
187
- id: { type: "integer", description: "Memory ID (omit to create new)" },
188
- content: { type: "string", description: "Memory text" },
189
- tags: { type: "array", items: { type: "string" }, description: "Categorization tags" },
190
- importance: { type: "number", description: "0.0-1.0 (default 0.5)" },
191
- source_context: { type: "object", description: "Optional source data links" },
192
- delete: { type: "boolean", description: "Delete the memory identified by id" },
198
+ id: { type: "integer", description: "Memory ID omit to create new, provide to update/delete" },
199
+ content: { type: "string", description: "Memory text (required for creation)" },
200
+ tags: { type: "array", items: { type: "string" }, description: "Categorization tags (e.g. ['work', 'project-x'])" },
201
+ importance: { type: "number", description: "0.0 (trivial) to 1.0 (critical). Default 0.5." },
202
+ source_context: { type: "object", description: "Optional metadata linking to source (app, timestamp, etc.)" },
203
+ delete: { type: "boolean", description: "Set true to delete the memory identified by id" },
193
204
  },
194
205
  },
195
206
  },
196
207
  {
197
208
  name: "send-notification",
198
209
  description:
199
- "Send a notification to the screenpipe desktop UI with optional action buttons. " +
200
- "Actions can re-run pipes with context, call API endpoints, or open deep links.",
201
- annotations: { title: "Send Notification", destructiveHint: false },
210
+ "Send a notification to the screenpipe desktop UI. " +
211
+ "Use to alert the user about findings, completed tasks, or actions needing attention.",
212
+ annotations: { title: "Send Notification", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
202
213
  inputSchema: {
203
214
  type: "object",
204
215
  properties: {
205
- title: { type: "string", description: "Notification title" },
216
+ title: { type: "string", description: "Notification title (short, descriptive)" },
206
217
  body: { type: "string", description: "Notification body (markdown supported)" },
207
- pipe_name: { type: "string", description: "Name of the pipe sending this notification" },
208
- timeout_secs: { type: "integer", description: "Auto-dismiss seconds (default 20)", default: 20 },
218
+ pipe_name: { type: "string", description: "Name of the pipe/tool sending this notification" },
219
+ timeout_secs: { type: "integer", description: "Auto-dismiss after N seconds (default 20). Use 0 for persistent.", default: 20 },
209
220
  actions: {
210
221
  type: "array",
211
- description: "Up to 5 action buttons",
222
+ description: "Up to 5 action buttons. Each needs id, label, type ('pipe'|'api'|'deeplink'|'dismiss').",
212
223
  items: {
213
224
  type: "object",
214
225
  properties: {
@@ -217,6 +228,7 @@ const TOOLS: Tool[] = [
217
228
  type: { type: "string", enum: ["pipe", "api", "deeplink", "dismiss"], description: "Action type" },
218
229
  pipe: { type: "string", description: "Pipe name to run (type=pipe)" },
219
230
  context: { type: "object", description: "Context passed to pipe (type=pipe)" },
231
+ open_in_chat: { type: "boolean", description: "Open pipe run in chat UI instead of background (type=pipe)" },
220
232
  url: { type: "string", description: "URL for api/deeplink actions" },
221
233
  },
222
234
  required: ["id", "label", "type"],
@@ -226,6 +238,167 @@ const TOOLS: Tool[] = [
226
238
  required: ["title", "pipe_name"],
227
239
  },
228
240
  },
241
+ {
242
+ name: "health-check",
243
+ description:
244
+ "Check if screenpipe is running and healthy. Returns recording status, frame/audio stats, timestamps.",
245
+ annotations: { title: "Health Check", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
246
+ inputSchema: { type: "object", properties: {} },
247
+ },
248
+ {
249
+ name: "list-audio-devices",
250
+ description: "List available audio input/output devices for recording.",
251
+ annotations: { title: "List Audio Devices", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
252
+ inputSchema: { type: "object", properties: {} },
253
+ },
254
+ {
255
+ name: "list-monitors",
256
+ description: "List available monitors/screens for capture.",
257
+ annotations: { title: "List Monitors", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
258
+ inputSchema: { type: "object", properties: {} },
259
+ },
260
+ {
261
+ name: "add-tags",
262
+ description:
263
+ "Add tags to a content item (vision frame or audio chunk) for organization and retrieval.",
264
+ annotations: { title: "Add Tags", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
265
+ inputSchema: {
266
+ type: "object",
267
+ properties: {
268
+ content_type: { type: "string", enum: ["vision", "audio"], description: "Type of content to tag" },
269
+ id: { type: "integer", description: "Content item ID" },
270
+ tags: { type: "array", items: { type: "string" }, description: "Tags to add" },
271
+ },
272
+ required: ["content_type", "id", "tags"],
273
+ },
274
+ },
275
+ {
276
+ name: "search-speakers",
277
+ description: "Search for speakers by name prefix. Returns speaker ID, name, and metadata.",
278
+ annotations: { title: "Search Speakers", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
279
+ inputSchema: {
280
+ type: "object",
281
+ properties: {
282
+ name: { type: "string", description: "Speaker name prefix to search for (case-insensitive)" },
283
+ },
284
+ },
285
+ },
286
+ {
287
+ name: "list-unnamed-speakers",
288
+ description: "List speakers that haven't been named yet. Useful for speaker identification workflow.",
289
+ annotations: { title: "List Unnamed Speakers", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
290
+ inputSchema: {
291
+ type: "object",
292
+ properties: {
293
+ limit: { type: "integer", description: "Max results (default 10)", default: 10 },
294
+ offset: { type: "integer", description: "Pagination offset", default: 0 },
295
+ },
296
+ },
297
+ },
298
+ {
299
+ name: "update-speaker",
300
+ description: "Rename a speaker or update their metadata.",
301
+ annotations: { title: "Update Speaker", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
302
+ inputSchema: {
303
+ type: "object",
304
+ properties: {
305
+ id: { type: "integer", description: "Speaker ID" },
306
+ name: { type: "string", description: "New speaker name" },
307
+ metadata: { type: "string", description: "JSON metadata string" },
308
+ },
309
+ required: ["id"],
310
+ },
311
+ },
312
+ {
313
+ name: "merge-speakers",
314
+ description: "Merge two speakers into one (e.g. when the same person was detected as different speakers).",
315
+ annotations: { title: "Merge Speakers", readOnlyHint: false, destructiveHint: true, openWorldHint: false },
316
+ inputSchema: {
317
+ type: "object",
318
+ properties: {
319
+ speaker_to_keep: { type: "integer", description: "Speaker ID to keep" },
320
+ speaker_to_merge: { type: "integer", description: "Speaker ID to merge into the kept one" },
321
+ },
322
+ required: ["speaker_to_keep", "speaker_to_merge"],
323
+ },
324
+ },
325
+ {
326
+ name: "start-meeting",
327
+ description: "Manually start a meeting recording session.",
328
+ annotations: { title: "Start Meeting", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
329
+ inputSchema: {
330
+ type: "object",
331
+ properties: {
332
+ app: { type: "string", description: "App name (default 'manual')", default: "manual" },
333
+ title: { type: "string", description: "Meeting title" },
334
+ attendees: { type: "string", description: "Comma-separated attendee names" },
335
+ },
336
+ },
337
+ },
338
+ {
339
+ name: "stop-meeting",
340
+ description: "Stop the current manual meeting recording session.",
341
+ annotations: { title: "Stop Meeting", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
342
+ inputSchema: { type: "object", properties: {} },
343
+ },
344
+ {
345
+ name: "get-meeting",
346
+ description: "Get details of a specific meeting by ID, including transcription and attendees.",
347
+ annotations: { title: "Get Meeting", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
348
+ inputSchema: {
349
+ type: "object",
350
+ properties: {
351
+ id: { type: "integer", description: "Meeting ID" },
352
+ },
353
+ required: ["id"],
354
+ },
355
+ },
356
+ {
357
+ name: "keyword-search",
358
+ description:
359
+ "Fast keyword search using FTS index. Faster than search-content for exact keyword matching. " +
360
+ "Returns frame IDs and matched text.",
361
+ annotations: { title: "Keyword Search", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
362
+ inputSchema: {
363
+ type: "object",
364
+ properties: {
365
+ q: { type: "string", description: "Keyword search query" },
366
+ content_type: { type: "string", enum: ["ocr", "audio", "all"], description: "Content type filter", default: "all" },
367
+ start_time: { type: "string", description: "ISO 8601 UTC or relative" },
368
+ end_time: { type: "string", description: "ISO 8601 UTC or relative" },
369
+ app_name: { type: "string", description: "Filter by app name" },
370
+ limit: { type: "integer", description: "Max results (default 20)", default: 20 },
371
+ offset: { type: "integer", description: "Pagination offset", default: 0 },
372
+ },
373
+ required: ["q"],
374
+ },
375
+ },
376
+ {
377
+ name: "get-frame-elements",
378
+ description:
379
+ "Get all UI elements for a specific frame. More targeted than search-elements when you already have a frame_id.",
380
+ annotations: { title: "Get Frame Elements", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
381
+ inputSchema: {
382
+ type: "object",
383
+ properties: {
384
+ frame_id: { type: "integer", description: "Frame ID" },
385
+ },
386
+ required: ["frame_id"],
387
+ },
388
+ },
389
+ {
390
+ name: "control-recording",
391
+ description:
392
+ "Start or stop audio/screen recording. Use to pause/resume capture.",
393
+ annotations: { title: "Control Recording", readOnlyHint: false, destructiveHint: false, openWorldHint: false },
394
+ inputSchema: {
395
+ type: "object",
396
+ properties: {
397
+ action: { type: "string", enum: ["start-audio", "stop-audio"], description: "Recording action" },
398
+ },
399
+ required: ["action"],
400
+ },
401
+ },
229
402
  ];
230
403
 
231
404
  server.setRequestHandler(ListToolsRequestSchema, async () => {
@@ -242,6 +415,12 @@ const RESOURCES = [
242
415
  description: "Current date/time, timezone, and pre-computed timestamps for common time ranges",
243
416
  mimeType: "application/json",
244
417
  },
418
+ {
419
+ uri: "screenpipe://guide",
420
+ name: "Usage Guide",
421
+ description: "How to use screenpipe tools effectively — search strategy, progressive disclosure, and common patterns",
422
+ mimeType: "text/markdown",
423
+ },
245
424
  ];
246
425
 
247
426
  server.setRequestHandler(ListResourcesRequestSchema, async () => {
@@ -286,6 +465,52 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
286
465
  };
287
466
  }
288
467
 
468
+ if (uri === "screenpipe://guide") {
469
+ return {
470
+ contents: [
471
+ {
472
+ uri,
473
+ mimeType: "text/markdown",
474
+ text: `# Screenpipe Usage Guide
475
+
476
+ ## Progressive Disclosure — start light, escalate only when needed
477
+
478
+ | Step | Tool | When to use |
479
+ |------|------|-------------|
480
+ | 1 | activity-summary | Broad questions: "what was I doing?", "which apps?", "how long on X?" |
481
+ | 2 | search-content | Need specific text, transcriptions, or content |
482
+ | 3 | search-elements | Need UI structure — buttons, links, form fields |
483
+ | 4 | frame-context | Need full detail for a specific moment (use frame_id from step 2) |
484
+
485
+ ## Search Strategy
486
+
487
+ - **Always provide start_time** — without it, search scans the entire history
488
+ - **Start with limit=5** — increase only if you need more results
489
+ - **Use max_content_length=500** to keep responses compact
490
+ - **Don't use q for audio** — transcriptions are noisy, q filters too aggressively. Search audio by time range and speaker instead
491
+ - **app_name is case-sensitive** — use exact names: "Google Chrome" not "chrome"
492
+ - **content_type=accessibility is preferred** for screen text (OS-native). ocr is fallback for apps without accessibility support
493
+
494
+ ## Common Patterns
495
+
496
+ - "What was I doing for the last 2 hours?" → activity-summary with start_time='2h ago'
497
+ - "What did I discuss in my meeting?" → list-meetings to find it, then search-content with audio + that time range
498
+ - "Find when I was on Twitter" → search-content with app_name='Arc' (or the browser name), q='twitter'
499
+ - "Remember that I prefer X" → update-memory with content describing the preference
500
+ - "What do you remember about X?" → search-content with content_type='memory', q='X'
501
+
502
+ ## Deep Links
503
+
504
+ When referencing specific moments in results, create clickable links:
505
+ - Frame: [10:30 AM — Chrome](screenpipe://frame/{frame_id}) — use frame_id from search results
506
+ - Timeline: [meeting at 3pm](screenpipe://timeline?timestamp=2024-01-15T15:00:00Z) — use exact timestamp from results
507
+ Never fabricate IDs or timestamps — only use values from actual results.
508
+ `,
509
+ },
510
+ ],
511
+ };
512
+ }
513
+
289
514
  throw new Error(`Unknown resource: ${uri}`);
290
515
  });
291
516
 
@@ -787,19 +1012,249 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
787
1012
  case "send-notification": {
788
1013
  const notifBody: Record<string, unknown> = {
789
1014
  title: args.title,
790
- pipe_name: args.pipe_name,
1015
+ body: args.body || "",
1016
+ type: "pipe",
791
1017
  };
792
- if (args.body) notifBody.body = args.body;
793
- if (args.timeout_secs) notifBody.timeout_secs = args.timeout_secs;
1018
+ if (args.timeout_secs) notifBody.timeout = Number(args.timeout_secs) * 1000;
794
1019
  if (args.actions) notifBody.actions = args.actions;
795
- const notifResponse = await fetchAPI("/notify", {
1020
+ const notifResponse = await fetch("http://localhost:11435/notify", {
796
1021
  method: "POST",
1022
+ headers: { "Content-Type": "application/json" },
797
1023
  body: JSON.stringify(notifBody),
798
1024
  });
799
1025
  if (!notifResponse.ok) throw new Error(`HTTP error: ${notifResponse.status}`);
800
1026
  const notifResult = await notifResponse.json();
801
1027
  return {
802
- content: [{ type: "text", text: `Notification sent (id: ${notifResult.id})` }],
1028
+ content: [{ type: "text", text: `Notification sent: ${notifResult.message}` }],
1029
+ };
1030
+ }
1031
+
1032
+ case "health-check": {
1033
+ const response = await fetchAPI("/health");
1034
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1035
+ const data = await response.json();
1036
+ return {
1037
+ content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
1038
+ };
1039
+ }
1040
+
1041
+ case "list-audio-devices": {
1042
+ const response = await fetchAPI("/audio/list");
1043
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1044
+ const devices = await response.json();
1045
+ if (!Array.isArray(devices) || devices.length === 0) {
1046
+ return { content: [{ type: "text", text: "No audio devices found." }] };
1047
+ }
1048
+ const formatted = devices.map(
1049
+ (d: { name: string; is_default: boolean; device_type?: string }) =>
1050
+ `${d.is_default ? "* " : " "}${d.name}${d.device_type ? ` (${d.device_type})` : ""}`
1051
+ );
1052
+ return {
1053
+ content: [{ type: "text", text: `Audio devices:\n${formatted.join("\n")}` }],
1054
+ };
1055
+ }
1056
+
1057
+ case "list-monitors": {
1058
+ const response = await fetchAPI("/vision/list");
1059
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1060
+ const monitors = await response.json();
1061
+ if (!Array.isArray(monitors) || monitors.length === 0) {
1062
+ return { content: [{ type: "text", text: "No monitors found." }] };
1063
+ }
1064
+ const formatted = monitors.map(
1065
+ (m: { id: number; name?: string; width?: number; height?: number; is_default?: boolean }) =>
1066
+ `${m.is_default ? "* " : " "}Monitor ${m.id}${m.name ? `: ${m.name}` : ""}${m.width ? ` (${m.width}x${m.height})` : ""}`
1067
+ );
1068
+ return {
1069
+ content: [{ type: "text", text: `Monitors:\n${formatted.join("\n")}` }],
1070
+ };
1071
+ }
1072
+
1073
+ case "add-tags": {
1074
+ const contentType = args.content_type as string;
1075
+ const id = args.id as number;
1076
+ const tags = args.tags as string[];
1077
+ if (!contentType || !id || !tags) {
1078
+ return { content: [{ type: "text", text: "Error: content_type, id, and tags are required" }] };
1079
+ }
1080
+ const response = await fetchAPI(`/tags/${contentType}/${id}`, {
1081
+ method: "POST",
1082
+ body: JSON.stringify({ tags }),
1083
+ });
1084
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1085
+ return {
1086
+ content: [{ type: "text", text: `Tags added to ${contentType}/${id}: ${tags.join(", ")}` }],
1087
+ };
1088
+ }
1089
+
1090
+ case "search-speakers": {
1091
+ const nameQuery = args.name as string;
1092
+ if (!nameQuery) {
1093
+ return { content: [{ type: "text", text: "Error: name is required" }] };
1094
+ }
1095
+ const response = await fetchAPI(`/speakers/search?name=${encodeURIComponent(nameQuery)}`);
1096
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1097
+ const speakers = await response.json();
1098
+ if (!Array.isArray(speakers) || speakers.length === 0) {
1099
+ return { content: [{ type: "text", text: "No speakers found." }] };
1100
+ }
1101
+ const formatted = speakers.map(
1102
+ (s: { id: number; name: string; metadata?: string }) =>
1103
+ `#${s.id} ${s.name}${s.metadata ? ` — ${s.metadata}` : ""}`
1104
+ );
1105
+ return {
1106
+ content: [{ type: "text", text: `Speakers:\n${formatted.join("\n")}` }],
1107
+ };
1108
+ }
1109
+
1110
+ case "list-unnamed-speakers": {
1111
+ const limit = (args.limit as number) || 10;
1112
+ const offset = (args.offset as number) || 0;
1113
+ const response = await fetchAPI(`/speakers/unnamed?limit=${limit}&offset=${offset}`);
1114
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1115
+ const speakers = await response.json();
1116
+ if (!Array.isArray(speakers) || speakers.length === 0) {
1117
+ return { content: [{ type: "text", text: "No unnamed speakers found." }] };
1118
+ }
1119
+ const formatted = speakers.map(
1120
+ (s: { id: number; name: string }) => `#${s.id} ${s.name}`
1121
+ );
1122
+ return {
1123
+ content: [{ type: "text", text: `Unnamed speakers:\n${formatted.join("\n")}` }],
1124
+ };
1125
+ }
1126
+
1127
+ case "update-speaker": {
1128
+ const speakerId = args.id as number;
1129
+ if (!speakerId) {
1130
+ return { content: [{ type: "text", text: "Error: id is required" }] };
1131
+ }
1132
+ const body: Record<string, unknown> = { id: speakerId };
1133
+ if (args.name !== undefined) body.name = args.name;
1134
+ if (args.metadata !== undefined) body.metadata = args.metadata;
1135
+ const response = await fetchAPI("/speakers/update", {
1136
+ method: "POST",
1137
+ body: JSON.stringify(body),
1138
+ });
1139
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1140
+ return {
1141
+ content: [{ type: "text", text: `Speaker ${speakerId} updated.` }],
1142
+ };
1143
+ }
1144
+
1145
+ case "merge-speakers": {
1146
+ const keepId = args.speaker_to_keep as number;
1147
+ const mergeId = args.speaker_to_merge as number;
1148
+ if (!keepId || !mergeId) {
1149
+ return { content: [{ type: "text", text: "Error: speaker_to_keep and speaker_to_merge are required" }] };
1150
+ }
1151
+ const response = await fetchAPI("/speakers/merge", {
1152
+ method: "POST",
1153
+ body: JSON.stringify({ speaker_to_keep: keepId, speaker_to_merge: mergeId }),
1154
+ });
1155
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1156
+ return {
1157
+ content: [{ type: "text", text: `Merged speaker ${mergeId} into ${keepId}.` }],
1158
+ };
1159
+ }
1160
+
1161
+ case "start-meeting": {
1162
+ const body: Record<string, unknown> = {};
1163
+ if (args.app) body.app = args.app;
1164
+ if (args.title) body.title = args.title;
1165
+ if (args.attendees) body.attendees = args.attendees;
1166
+ const response = await fetchAPI("/meetings/start", {
1167
+ method: "POST",
1168
+ body: JSON.stringify(body),
1169
+ });
1170
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1171
+ const meeting = await response.json();
1172
+ return {
1173
+ content: [{ type: "text", text: `Meeting started (id: ${meeting.id || "ok"}).` }],
1174
+ };
1175
+ }
1176
+
1177
+ case "stop-meeting": {
1178
+ const response = await fetchAPI("/meetings/stop", { method: "POST" });
1179
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1180
+ return {
1181
+ content: [{ type: "text", text: "Meeting stopped." }],
1182
+ };
1183
+ }
1184
+
1185
+ case "get-meeting": {
1186
+ const meetingId = args.id as number;
1187
+ if (!meetingId) {
1188
+ return { content: [{ type: "text", text: "Error: id is required" }] };
1189
+ }
1190
+ const response = await fetchAPI(`/meetings/${meetingId}`);
1191
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1192
+ const meeting = await response.json();
1193
+ return {
1194
+ content: [{ type: "text", text: JSON.stringify(meeting, null, 2) }],
1195
+ };
1196
+ }
1197
+
1198
+ case "keyword-search": {
1199
+ const params = new URLSearchParams();
1200
+ for (const [key, value] of Object.entries(args)) {
1201
+ if (value !== null && value !== undefined) {
1202
+ params.append(key, String(value));
1203
+ }
1204
+ }
1205
+ const response = await fetchAPI(`/search/keyword?${params.toString()}`);
1206
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1207
+ const data = await response.json();
1208
+ const results = data.data || [];
1209
+ if (results.length === 0) {
1210
+ return { content: [{ type: "text", text: "No keyword search results found." }] };
1211
+ }
1212
+ const formatted = results.map((r: Record<string, unknown>) => {
1213
+ const content = r.content as Record<string, unknown> | undefined;
1214
+ return `[${r.type}] ${content?.app_name || "?"} | ${content?.timestamp || ""}\n${content?.text || content?.transcription || ""}`;
1215
+ });
1216
+ return {
1217
+ content: [{ type: "text", text: `Results: ${results.length}\n\n${formatted.join("\n---\n")}` }],
1218
+ };
1219
+ }
1220
+
1221
+ case "get-frame-elements": {
1222
+ const frameId = args.frame_id as number;
1223
+ if (!frameId) {
1224
+ return { content: [{ type: "text", text: "Error: frame_id is required" }] };
1225
+ }
1226
+ const response = await fetchAPI(`/frames/${frameId}/elements`);
1227
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1228
+ const elements = await response.json();
1229
+ if (!Array.isArray(elements) || elements.length === 0) {
1230
+ return { content: [{ type: "text", text: `No elements found for frame ${frameId}.` }] };
1231
+ }
1232
+ const formatted = elements.map(
1233
+ (e: { role: string; text: string | null; depth: number; source: string }) => {
1234
+ const indent = " ".repeat(Math.min(e.depth, 5));
1235
+ return `${indent}[${e.source}:${e.role}] ${e.text || "(no text)"}`;
1236
+ }
1237
+ );
1238
+ return {
1239
+ content: [{ type: "text", text: `Frame ${frameId} elements (${elements.length}):\n${formatted.join("\n")}` }],
1240
+ };
1241
+ }
1242
+
1243
+ case "control-recording": {
1244
+ const action = args.action as string;
1245
+ if (!action) {
1246
+ return { content: [{ type: "text", text: "Error: action is required" }] };
1247
+ }
1248
+ let endpoint: string;
1249
+ if (action === "start-audio") endpoint = "/audio/start";
1250
+ else if (action === "stop-audio") endpoint = "/audio/stop";
1251
+ else {
1252
+ return { content: [{ type: "text", text: `Error: unknown action '${action}'` }] };
1253
+ }
1254
+ const response = await fetchAPI(endpoint, { method: "POST" });
1255
+ if (!response.ok) throw new Error(`HTTP error: ${response.status}`);
1256
+ return {
1257
+ content: [{ type: "text", text: `Recording action '${action}' executed.` }],
803
1258
  };
804
1259
  }
805
1260