screenpipe-mcp 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +365 -1303
- package/package.json +1 -1
- package/src/index.ts +466 -1430
package/dist/index.js
CHANGED
|
@@ -44,19 +44,6 @@ const ws_1 = require("ws");
|
|
|
44
44
|
const fs = __importStar(require("fs"));
|
|
45
45
|
const path = __importStar(require("path"));
|
|
46
46
|
const os = __importStar(require("os"));
|
|
47
|
-
// Helper to get current date in ISO format
|
|
48
|
-
function getCurrentDateInfo() {
|
|
49
|
-
const now = new Date();
|
|
50
|
-
return {
|
|
51
|
-
isoDate: now.toISOString(),
|
|
52
|
-
localDate: now.toLocaleDateString("en-US", {
|
|
53
|
-
weekday: "long",
|
|
54
|
-
year: "numeric",
|
|
55
|
-
month: "long",
|
|
56
|
-
day: "numeric",
|
|
57
|
-
}),
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
47
|
// Parse command line arguments
|
|
61
48
|
const args = process.argv.slice(2);
|
|
62
49
|
let port = 3030;
|
|
@@ -69,1199 +56,246 @@ const SCREENPIPE_API = `http://localhost:${port}`;
|
|
|
69
56
|
// Initialize server
|
|
70
57
|
const server = new index_js_1.Server({
|
|
71
58
|
name: "screenpipe",
|
|
72
|
-
version: "0.
|
|
59
|
+
version: "0.9.0",
|
|
73
60
|
}, {
|
|
74
61
|
capabilities: {
|
|
75
62
|
tools: {},
|
|
76
|
-
prompts: {},
|
|
77
63
|
resources: {},
|
|
78
64
|
},
|
|
79
65
|
});
|
|
80
|
-
//
|
|
81
|
-
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
// Tools — minimal descriptions, no behavioral guidance (that belongs in resources)
|
|
68
|
+
// ---------------------------------------------------------------------------
|
|
69
|
+
const TOOLS = [
|
|
82
70
|
{
|
|
83
71
|
name: "search-content",
|
|
84
|
-
description: "Search
|
|
85
|
-
"Returns timestamped results with app context. "
|
|
86
|
-
|
|
87
|
-
"Use the 'screenpipe://context' resource for current time when building time-based queries.\n\n" +
|
|
88
|
-
"WHEN TO USE WHICH content_type:\n" +
|
|
89
|
-
"- For meetings/calls/conversations: content_type='audio', do NOT use q param (transcriptions are noisy, q filters too aggressively)\n" +
|
|
90
|
-
"- For screen text/reading: content_type='all' or 'accessibility'\n" +
|
|
91
|
-
"- For time spent/app usage questions: use activity-summary tool instead (this tool returns content, not time stats)\n\n" +
|
|
92
|
-
"SEARCH STRATEGY: First search with ONLY time params (start_time/end_time) — no q, no app_name, no content_type. " +
|
|
93
|
-
"This gives ground truth of what's recorded. Scan results to find correct app_name values, then narrow with filters using exact observed values. " +
|
|
94
|
-
"App names are case-sensitive (e.g. 'Discord' vs 'Discord.exe'). " +
|
|
95
|
-
"The q param searches captured text, NOT app names. NEVER report 'no data' after one filtered search — verify with unfiltered time-only search first.\n\n" +
|
|
96
|
-
"DEEP LINKS: When referencing specific moments, create clickable links using IDs from search results:\n" +
|
|
97
|
-
"- OCR results (PREFERRED): [10:30 AM — Chrome](screenpipe://frame/12345) — use content.frame_id from the result\n" +
|
|
98
|
-
"- Audio results: [meeting at 3pm](screenpipe://timeline?timestamp=2024-01-15T15:00:00Z) — use exact timestamp from result\n" +
|
|
99
|
-
"NEVER fabricate frame IDs or timestamps — only use values from actual search results.",
|
|
100
|
-
annotations: {
|
|
101
|
-
title: "Search Content",
|
|
102
|
-
readOnlyHint: true,
|
|
103
|
-
},
|
|
72
|
+
description: "Search screen text, audio transcriptions, input events, and memories. " +
|
|
73
|
+
"Returns timestamped results with app context. Call with no params for recent activity.",
|
|
74
|
+
annotations: { title: "Search Content", readOnlyHint: true },
|
|
104
75
|
inputSchema: {
|
|
105
76
|
type: "object",
|
|
106
77
|
properties: {
|
|
107
78
|
q: {
|
|
108
79
|
type: "string",
|
|
109
|
-
description: "
|
|
80
|
+
description: "Full-text search query. Omit to return all content in time range.",
|
|
110
81
|
},
|
|
111
82
|
content_type: {
|
|
112
83
|
type: "string",
|
|
113
|
-
enum: ["all", "ocr", "audio", "input", "accessibility"],
|
|
114
|
-
description: "
|
|
84
|
+
enum: ["all", "ocr", "audio", "input", "accessibility", "memory"],
|
|
85
|
+
description: "Filter by content type. Default: 'all'.",
|
|
115
86
|
default: "all",
|
|
116
87
|
},
|
|
117
|
-
limit: {
|
|
118
|
-
|
|
119
|
-
description: "Max results. Default: 10",
|
|
120
|
-
default: 10,
|
|
121
|
-
},
|
|
122
|
-
offset: {
|
|
123
|
-
type: "integer",
|
|
124
|
-
description: "Skip N results for pagination. Default: 0",
|
|
125
|
-
default: 0,
|
|
126
|
-
},
|
|
88
|
+
limit: { type: "integer", description: "Max results (default 10)", default: 10 },
|
|
89
|
+
offset: { type: "integer", description: "Pagination offset", default: 0 },
|
|
127
90
|
start_time: {
|
|
128
91
|
type: "string",
|
|
129
|
-
|
|
130
|
-
description: "Start time: ISO 8601 UTC (e.g., 2024-01-15T10:00:00Z) or relative (e.g., '16h ago', '2d ago', 'now')",
|
|
92
|
+
description: "ISO 8601 UTC or relative (e.g. '2h ago')",
|
|
131
93
|
},
|
|
132
94
|
end_time: {
|
|
133
95
|
type: "string",
|
|
134
|
-
|
|
135
|
-
description: "End time: ISO 8601 UTC (e.g., 2024-01-15T18:00:00Z) or relative (e.g., 'now', '1h ago')",
|
|
136
|
-
},
|
|
137
|
-
app_name: {
|
|
138
|
-
type: "string",
|
|
139
|
-
description: "Filter by app (e.g., 'Google Chrome', 'Slack', 'zoom.us')",
|
|
140
|
-
},
|
|
141
|
-
window_name: {
|
|
142
|
-
type: "string",
|
|
143
|
-
description: "Filter by window title",
|
|
144
|
-
},
|
|
145
|
-
min_length: {
|
|
146
|
-
type: "integer",
|
|
147
|
-
description: "Minimum content length in characters",
|
|
148
|
-
},
|
|
149
|
-
max_length: {
|
|
150
|
-
type: "integer",
|
|
151
|
-
description: "Maximum content length in characters",
|
|
96
|
+
description: "ISO 8601 UTC or relative (e.g. 'now')",
|
|
152
97
|
},
|
|
98
|
+
app_name: { type: "string", description: "Filter by app name" },
|
|
99
|
+
window_name: { type: "string", description: "Filter by window title" },
|
|
100
|
+
min_length: { type: "integer", description: "Min content length" },
|
|
101
|
+
max_length: { type: "integer", description: "Max content length" },
|
|
153
102
|
include_frames: {
|
|
154
103
|
type: "boolean",
|
|
155
|
-
description: "Include base64 screenshots (OCR only)
|
|
104
|
+
description: "Include base64 screenshots (OCR only)",
|
|
156
105
|
default: false,
|
|
157
106
|
},
|
|
158
|
-
speaker_ids: {
|
|
159
|
-
|
|
160
|
-
description: "Comma-separated speaker IDs to filter audio results (e.g., '1,2,3')",
|
|
161
|
-
},
|
|
162
|
-
speaker_name: {
|
|
163
|
-
type: "string",
|
|
164
|
-
description: "Filter audio by speaker name (case-insensitive partial match)",
|
|
165
|
-
},
|
|
107
|
+
speaker_ids: { type: "string", description: "Comma-separated speaker IDs" },
|
|
108
|
+
speaker_name: { type: "string", description: "Filter audio by speaker name" },
|
|
166
109
|
max_content_length: {
|
|
167
110
|
type: "integer",
|
|
168
|
-
description: "Truncate each result
|
|
111
|
+
description: "Truncate each result via middle-truncation",
|
|
169
112
|
},
|
|
170
113
|
},
|
|
171
114
|
},
|
|
172
115
|
},
|
|
173
116
|
{
|
|
174
|
-
name: "
|
|
175
|
-
description: "
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
"
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
117
|
+
name: "list-meetings",
|
|
118
|
+
description: "List detected meetings (Zoom, Teams, Meet, etc.) with duration, app, and attendees.",
|
|
119
|
+
annotations: { title: "List Meetings", readOnlyHint: true },
|
|
120
|
+
inputSchema: {
|
|
121
|
+
type: "object",
|
|
122
|
+
properties: {
|
|
123
|
+
start_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
124
|
+
end_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
125
|
+
limit: { type: "integer", description: "Max results (default 20)", default: 20 },
|
|
126
|
+
offset: { type: "integer", description: "Pagination offset", default: 0 },
|
|
127
|
+
},
|
|
184
128
|
},
|
|
129
|
+
},
|
|
130
|
+
{
|
|
131
|
+
name: "activity-summary",
|
|
132
|
+
description: "Lightweight activity overview (~200-500 tokens): app usage with active minutes, audio speakers, recent texts. " +
|
|
133
|
+
"Use for 'how long on X?', 'which apps?', 'what was I doing?' questions.",
|
|
134
|
+
annotations: { title: "Activity Summary", readOnlyHint: true },
|
|
185
135
|
inputSchema: {
|
|
186
136
|
type: "object",
|
|
187
137
|
properties: {
|
|
188
|
-
start_time: {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
description: "Start time: ISO 8601 UTC (e.g., '2024-01-15T10:00:00Z') or relative (e.g., '16h ago', 'now')",
|
|
192
|
-
},
|
|
193
|
-
end_time: {
|
|
194
|
-
type: "string",
|
|
195
|
-
format: "date-time",
|
|
196
|
-
description: "End time: ISO 8601 UTC (e.g., '2024-01-15T10:30:00Z') or relative (e.g., 'now', '1h ago')",
|
|
197
|
-
},
|
|
198
|
-
fps: {
|
|
199
|
-
type: "number",
|
|
200
|
-
description: "Frames per second for the output video. Lower values (0.5-1.0) create smaller files, higher values (5-10) create smoother playback. Default: 1.0",
|
|
201
|
-
default: 1.0,
|
|
202
|
-
},
|
|
138
|
+
start_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
139
|
+
end_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
140
|
+
app_name: { type: "string", description: "Optional app name filter" },
|
|
203
141
|
},
|
|
204
142
|
required: ["start_time", "end_time"],
|
|
205
143
|
},
|
|
206
144
|
},
|
|
207
145
|
{
|
|
208
|
-
name: "
|
|
209
|
-
description: "
|
|
210
|
-
"
|
|
211
|
-
|
|
212
|
-
annotations: {
|
|
213
|
-
title: "List Meetings",
|
|
214
|
-
readOnlyHint: true,
|
|
215
|
-
},
|
|
146
|
+
name: "search-elements",
|
|
147
|
+
description: "Search UI elements (buttons, links, text fields) from the accessibility tree. " +
|
|
148
|
+
"Lighter than search-content for targeted UI lookups.",
|
|
149
|
+
annotations: { title: "Search Elements", readOnlyHint: true },
|
|
216
150
|
inputSchema: {
|
|
217
151
|
type: "object",
|
|
218
152
|
properties: {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
description: "Start filter: ISO 8601 UTC (e.g., 2024-01-15T10:00:00Z) or relative (e.g., '16h ago', 'now')",
|
|
223
|
-
},
|
|
224
|
-
end_time: {
|
|
153
|
+
q: { type: "string", description: "Full-text search on element text" },
|
|
154
|
+
frame_id: { type: "integer", description: "Filter to specific frame" },
|
|
155
|
+
source: {
|
|
225
156
|
type: "string",
|
|
226
|
-
|
|
227
|
-
description: "
|
|
228
|
-
},
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
},
|
|
234
|
-
offset: {
|
|
235
|
-
type: "integer",
|
|
236
|
-
description: "Skip N results for pagination. Default: 0",
|
|
237
|
-
default: 0,
|
|
238
|
-
},
|
|
157
|
+
enum: ["accessibility", "ocr"],
|
|
158
|
+
description: "Element source filter",
|
|
159
|
+
},
|
|
160
|
+
role: { type: "string", description: "Element role (e.g. AXButton, AXLink)" },
|
|
161
|
+
start_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
162
|
+
end_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
163
|
+
app_name: { type: "string", description: "Filter by app name" },
|
|
164
|
+
limit: { type: "integer", description: "Max results (default 50)", default: 50 },
|
|
165
|
+
offset: { type: "integer", description: "Pagination offset", default: 0 },
|
|
239
166
|
},
|
|
240
167
|
},
|
|
241
168
|
},
|
|
242
169
|
{
|
|
243
|
-
name: "
|
|
244
|
-
description: "Get
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
"- Any time-spent or app-usage question\n\n" +
|
|
253
|
-
"WARNING: Do NOT estimate time from raw frame counts or SQL queries — those are inaccurate. " +
|
|
254
|
-
"This endpoint calculates actual active session time correctly.",
|
|
255
|
-
annotations: {
|
|
256
|
-
title: "Activity Summary",
|
|
257
|
-
readOnlyHint: true,
|
|
170
|
+
name: "frame-context",
|
|
171
|
+
description: "Get accessibility text, parsed tree nodes, and URLs for a specific frame ID.",
|
|
172
|
+
annotations: { title: "Frame Context", readOnlyHint: true },
|
|
173
|
+
inputSchema: {
|
|
174
|
+
type: "object",
|
|
175
|
+
properties: {
|
|
176
|
+
frame_id: { type: "integer", description: "Frame ID from search results" },
|
|
177
|
+
},
|
|
178
|
+
required: ["frame_id"],
|
|
258
179
|
},
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
name: "export-video",
|
|
183
|
+
description: "Export an MP4 video of screen recordings for a time range.",
|
|
184
|
+
annotations: { title: "Export Video", destructiveHint: true },
|
|
259
185
|
inputSchema: {
|
|
260
186
|
type: "object",
|
|
261
187
|
properties: {
|
|
262
|
-
start_time: {
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
description: "Start of time range: ISO 8601 UTC (e.g., 2024-01-15T10:00:00Z) or relative (e.g., '16h ago', 'now')",
|
|
266
|
-
},
|
|
267
|
-
end_time: {
|
|
268
|
-
type: "string",
|
|
269
|
-
format: "date-time",
|
|
270
|
-
description: "End of time range: ISO 8601 UTC (e.g., 2024-01-15T18:00:00Z) or relative (e.g., 'now', '1h ago')",
|
|
271
|
-
},
|
|
272
|
-
app_name: {
|
|
273
|
-
type: "string",
|
|
274
|
-
description: "Optional app name filter (e.g., 'Google Chrome', 'VS Code')",
|
|
275
|
-
},
|
|
188
|
+
start_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
189
|
+
end_time: { type: "string", description: "ISO 8601 UTC or relative" },
|
|
190
|
+
fps: { type: "number", description: "Output FPS (default 1.0)", default: 1.0 },
|
|
276
191
|
},
|
|
277
192
|
required: ["start_time", "end_time"],
|
|
278
193
|
},
|
|
279
194
|
},
|
|
280
195
|
{
|
|
281
|
-
name: "
|
|
282
|
-
description: "
|
|
283
|
-
"
|
|
284
|
-
|
|
285
|
-
"Use for: finding specific buttons, links, text fields, or UI components. " +
|
|
286
|
-
"Prefer this over search-content when you need structural UI detail rather than full screen text.",
|
|
287
|
-
annotations: {
|
|
288
|
-
title: "Search Elements",
|
|
289
|
-
readOnlyHint: true,
|
|
290
|
-
},
|
|
196
|
+
name: "update-memory",
|
|
197
|
+
description: "Create, update, or delete a persistent memory (facts, preferences, decisions). " +
|
|
198
|
+
"Retrieve memories via search-content with content_type='memory'.",
|
|
199
|
+
annotations: { title: "Update Memory", destructiveHint: false },
|
|
291
200
|
inputSchema: {
|
|
292
201
|
type: "object",
|
|
293
202
|
properties: {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
},
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
description: "Filter to elements from a specific frame",
|
|
301
|
-
},
|
|
302
|
-
source: {
|
|
303
|
-
type: "string",
|
|
304
|
-
enum: ["accessibility", "ocr"],
|
|
305
|
-
description: "Filter by element source: 'accessibility' (structured tree) or 'ocr' (text blocks)",
|
|
306
|
-
},
|
|
307
|
-
role: {
|
|
308
|
-
type: "string",
|
|
309
|
-
description: "Filter by element role (e.g., 'AXButton', 'AXStaticText', 'AXLink', 'AXTextField', 'line')",
|
|
310
|
-
},
|
|
311
|
-
start_time: {
|
|
312
|
-
type: "string",
|
|
313
|
-
format: "date-time",
|
|
314
|
-
description: "Start time: ISO 8601 UTC or relative (e.g., '16h ago', 'now')",
|
|
315
|
-
},
|
|
316
|
-
end_time: {
|
|
317
|
-
type: "string",
|
|
318
|
-
format: "date-time",
|
|
319
|
-
description: "End time: ISO 8601 UTC or relative (e.g., 'now', '1h ago')",
|
|
320
|
-
},
|
|
321
|
-
app_name: {
|
|
322
|
-
type: "string",
|
|
323
|
-
description: "Filter by app name",
|
|
324
|
-
},
|
|
325
|
-
limit: {
|
|
326
|
-
type: "integer",
|
|
327
|
-
description: "Max results. Default: 50",
|
|
328
|
-
default: 50,
|
|
329
|
-
},
|
|
330
|
-
offset: {
|
|
331
|
-
type: "integer",
|
|
332
|
-
description: "Skip N results for pagination. Default: 0",
|
|
333
|
-
default: 0,
|
|
334
|
-
},
|
|
203
|
+
id: { type: "integer", description: "Memory ID (omit to create new)" },
|
|
204
|
+
content: { type: "string", description: "Memory text" },
|
|
205
|
+
tags: { type: "array", items: { type: "string" }, description: "Categorization tags" },
|
|
206
|
+
importance: { type: "number", description: "0.0-1.0 (default 0.5)" },
|
|
207
|
+
source_context: { type: "object", description: "Optional source data links" },
|
|
208
|
+
delete: { type: "boolean", description: "Delete the memory identified by id" },
|
|
335
209
|
},
|
|
336
210
|
},
|
|
337
211
|
},
|
|
338
212
|
{
|
|
339
|
-
name: "
|
|
340
|
-
description: "
|
|
341
|
-
"
|
|
342
|
-
|
|
343
|
-
annotations: {
|
|
344
|
-
title: "Frame Context",
|
|
345
|
-
readOnlyHint: true,
|
|
346
|
-
},
|
|
213
|
+
name: "send-notification",
|
|
214
|
+
description: "Send a notification to the screenpipe desktop UI with optional action buttons. " +
|
|
215
|
+
"Actions can re-run pipes with context, call API endpoints, or open deep links.",
|
|
216
|
+
annotations: { title: "Send Notification", destructiveHint: false },
|
|
347
217
|
inputSchema: {
|
|
348
218
|
type: "object",
|
|
349
219
|
properties: {
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
220
|
+
title: { type: "string", description: "Notification title" },
|
|
221
|
+
body: { type: "string", description: "Notification body (markdown supported)" },
|
|
222
|
+
pipe_name: { type: "string", description: "Name of the pipe sending this notification" },
|
|
223
|
+
timeout_secs: { type: "integer", description: "Auto-dismiss seconds (default 20)", default: 20 },
|
|
224
|
+
actions: {
|
|
225
|
+
type: "array",
|
|
226
|
+
description: "Up to 5 action buttons",
|
|
227
|
+
items: {
|
|
228
|
+
type: "object",
|
|
229
|
+
properties: {
|
|
230
|
+
id: { type: "string", description: "Unique action ID" },
|
|
231
|
+
label: { type: "string", description: "Button label" },
|
|
232
|
+
type: { type: "string", enum: ["pipe", "api", "deeplink", "dismiss"], description: "Action type" },
|
|
233
|
+
pipe: { type: "string", description: "Pipe name to run (type=pipe)" },
|
|
234
|
+
context: { type: "object", description: "Context passed to pipe (type=pipe)" },
|
|
235
|
+
url: { type: "string", description: "URL for api/deeplink actions" },
|
|
236
|
+
},
|
|
237
|
+
required: ["id", "label", "type"],
|
|
238
|
+
},
|
|
353
239
|
},
|
|
354
240
|
},
|
|
355
|
-
required: ["
|
|
241
|
+
required: ["title", "pipe_name"],
|
|
356
242
|
},
|
|
357
243
|
},
|
|
358
244
|
];
|
|
359
|
-
// List tools handler
|
|
360
245
|
server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => {
|
|
361
|
-
return { tools:
|
|
246
|
+
return { tools: TOOLS };
|
|
362
247
|
});
|
|
363
|
-
//
|
|
248
|
+
// ---------------------------------------------------------------------------
|
|
249
|
+
// Resources — dynamic context only (no duplicated reference docs)
|
|
250
|
+
// ---------------------------------------------------------------------------
|
|
364
251
|
const RESOURCES = [
|
|
365
252
|
{
|
|
366
253
|
uri: "screenpipe://context",
|
|
367
254
|
name: "Current Context",
|
|
368
|
-
description: "Current date/time and pre-computed timestamps for common time ranges",
|
|
255
|
+
description: "Current date/time, timezone, and pre-computed timestamps for common time ranges",
|
|
369
256
|
mimeType: "application/json",
|
|
370
257
|
},
|
|
371
|
-
{
|
|
372
|
-
uri: "screenpipe://guide",
|
|
373
|
-
name: "Usage Guide",
|
|
374
|
-
description: "How to use screenpipe search effectively",
|
|
375
|
-
mimeType: "text/markdown",
|
|
376
|
-
},
|
|
377
|
-
{
|
|
378
|
-
uri: "ui://search",
|
|
379
|
-
name: "Search Dashboard",
|
|
380
|
-
description: "Interactive search UI for exploring screen recordings and audio transcriptions",
|
|
381
|
-
mimeType: "text/html",
|
|
382
|
-
},
|
|
383
|
-
{
|
|
384
|
-
uri: "screenpipe://pipe-creation-guide",
|
|
385
|
-
name: "Pipe Creation Guide",
|
|
386
|
-
description: "How to create screenpipe pipes (scheduled AI automations): format, YAML frontmatter, schedule syntax, API parameters, and example templates",
|
|
387
|
-
mimeType: "text/markdown",
|
|
388
|
-
},
|
|
389
|
-
{
|
|
390
|
-
uri: "screenpipe://api-reference",
|
|
391
|
-
name: "REST API Reference",
|
|
392
|
-
description: "Full screenpipe REST API reference: search, activity-summary, elements, frames, export, retranscribe, raw SQL, connections, speakers (60+ endpoints)",
|
|
393
|
-
mimeType: "text/markdown",
|
|
394
|
-
},
|
|
395
|
-
{
|
|
396
|
-
uri: "screenpipe://cli-reference",
|
|
397
|
-
name: "CLI Reference",
|
|
398
|
-
description: "Screenpipe CLI commands: pipe management (list, enable, run, install, delete) and connection management (Telegram, Slack, Discord, etc.)",
|
|
399
|
-
mimeType: "text/markdown",
|
|
400
|
-
},
|
|
401
258
|
];
|
|
402
|
-
// List resources handler
|
|
403
259
|
server.setRequestHandler(types_js_1.ListResourcesRequestSchema, async () => {
|
|
404
260
|
return { resources: RESOURCES };
|
|
405
261
|
});
|
|
406
|
-
// Read resource handler
|
|
407
262
|
server.setRequestHandler(types_js_1.ReadResourceRequestSchema, async (request) => {
|
|
408
263
|
const { uri } = request.params;
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
],
|
|
434
|
-
};
|
|
435
|
-
case "screenpipe://guide":
|
|
436
|
-
return {
|
|
437
|
-
contents: [
|
|
438
|
-
{
|
|
439
|
-
uri,
|
|
440
|
-
mimeType: "text/markdown",
|
|
441
|
-
text: `# Screenpipe Search Guide
|
|
442
|
-
|
|
443
|
-
## Data Modalities
|
|
444
|
-
|
|
445
|
-
Screenpipe captures four types of data:
|
|
446
|
-
1. **Accessibility** - Screen text via accessibility APIs (primary, preferred for screen content)
|
|
447
|
-
2. **OCR** - Screen text from screenshots (legacy fallback for apps without accessibility support)
|
|
448
|
-
3. **Audio** - Transcribed speech from microphone/system audio
|
|
449
|
-
4. **Input** - Keyboard input, mouse clicks, app switches, clipboard (macOS)
|
|
450
|
-
|
|
451
|
-
## Quick Start
|
|
452
|
-
- **Get recent activity**: Call search-content with no parameters
|
|
453
|
-
- **Search screen text**: \`{"q": "search term", "content_type": "all"}\`
|
|
454
|
-
- **Get keyboard input**: \`{"content_type": "input"}\`
|
|
455
|
-
- **Get audio only**: \`{"content_type": "audio"}\`
|
|
456
|
-
|
|
457
|
-
## Common User Requests → Correct Tool Choice
|
|
458
|
-
| User says | Use this tool | Key params |
|
|
459
|
-
|-----------|--------------|------------|
|
|
460
|
-
| "summarize my meeting/call" | search-content | content_type:"audio", NO q param, start_time |
|
|
461
|
-
| "what did they/I say about X" | search-content | content_type:"audio", NO q param (scan results manually) |
|
|
462
|
-
| "how long on X" / "which apps" / "time spent" | activity-summary | start_time, end_time |
|
|
463
|
-
| "what was I doing" | activity-summary | start_time, end_time (then drill into search-content) |
|
|
464
|
-
| "what was I reading/looking at" | search-content | content_type:"all", start_time |
|
|
465
|
-
|
|
466
|
-
## Behavior Rules
|
|
467
|
-
- Act immediately on clear requests. NEVER ask "what time range?" or "which content type?" when the intent is obvious.
|
|
468
|
-
- If search returns empty, silently retry with wider time range or fewer filters. Do NOT ask the user what to change.
|
|
469
|
-
- For meetings: ALWAYS use content_type:"audio" and do NOT use the q param. Transcriptions are noisy — q filters too aggressively and misses relevant content.
|
|
470
|
-
|
|
471
|
-
## search-content
|
|
472
|
-
| Parameter | Description | Default |
|
|
473
|
-
|-----------|-------------|---------|
|
|
474
|
-
| q | Search query | (none - returns all) |
|
|
475
|
-
| content_type | all/ocr/audio/input/accessibility | all |
|
|
476
|
-
| limit | Max results | 10 |
|
|
477
|
-
| start_time | ISO 8601 UTC or relative (e.g. '16h ago') | (no filter) |
|
|
478
|
-
| end_time | ISO 8601 UTC or relative (e.g. 'now') | (no filter) |
|
|
479
|
-
| app_name | Filter by app | (no filter) |
|
|
480
|
-
| include_frames | Include screenshots | false |
|
|
481
|
-
|
|
482
|
-
## Search Strategy (MANDATORY)
|
|
483
|
-
1. First search: ONLY use time params (start_time/end_time). No q, no app_name, no content_type. This gives ground truth of what's recorded.
|
|
484
|
-
2. Scan results to find correct app_name values and content patterns.
|
|
485
|
-
3. Only THEN narrow with filters using exact observed values. App names are case-sensitive and may differ from user input (e.g. "Discord" vs "Discord.exe").
|
|
486
|
-
4. The q param searches captured text (accessibility/OCR), NOT app names — an app can be visible without its name in the captured text.
|
|
487
|
-
5. NEVER report "no data found" after one filtered search. Verify with unfiltered time-only search first.
|
|
488
|
-
|
|
489
|
-
## Progressive Disclosure (Token-Efficient Strategy)
|
|
490
|
-
1. **Start with activity-summary** (~200 tokens) for broad questions ("what was I doing?")
|
|
491
|
-
2. **Narrow with search-content** (~500-1000 tokens) using filters from step 1
|
|
492
|
-
3. **Drill into search-elements** (~200 tokens each) for structural UI detail (buttons, links)
|
|
493
|
-
4. **Fetch frame-context** for URLs and accessibility tree of specific frames
|
|
494
|
-
5. **Screenshots** (include_frames=true) only when text isn't enough
|
|
495
|
-
|
|
496
|
-
## Chat History
|
|
497
|
-
Previous screenpipe chat conversations are stored as individual JSON files in ~/.screenpipe/chats/{conversation-id}.json
|
|
498
|
-
Each file contains: id, title, messages[], createdAt, updatedAt. You can read these files to reference or search previous conversations.
|
|
499
|
-
|
|
500
|
-
## Speaker Management
|
|
501
|
-
screenpipe auto-identifies speakers in audio. API endpoints for managing them:
|
|
502
|
-
- \`GET /speakers/unnamed?limit=10\` — list unnamed speakers
|
|
503
|
-
- \`GET /speakers/search?name=John\` — search by name
|
|
504
|
-
- \`POST /speakers/update\` with \`{"id": 5, "name": "John"}\` — rename a speaker
|
|
505
|
-
- \`POST /speakers/merge\` with \`{"speaker_to_keep_id": 1, "speaker_to_merge_id": 2}\` — merge duplicates
|
|
506
|
-
- \`GET /speakers/similar?speaker_id=5\` — find similar speakers for merging
|
|
507
|
-
- \`POST /speakers/reassign\` — reassign audio chunk to different speaker
|
|
508
|
-
|
|
509
|
-
## Tips
|
|
510
|
-
1. Read screenpipe://context first to get current timestamps
|
|
511
|
-
2. Use activity-summary before search-content for broad overview questions
|
|
512
|
-
3. Use search-elements instead of search-content for targeted UI lookups (10x lighter)
|
|
513
|
-
4. Use content_type=input for "what did I type?" queries
|
|
514
|
-
5. Use content_type=accessibility for accessibility tree text
|
|
515
|
-
6. For large aggregations (e.g. "what apps did I use today?"), paginate with offset or suggest the user run raw SQL via \`curl -X POST http://localhost:3030/raw_sql\` for efficient GROUP BY queries
|
|
516
|
-
|
|
517
|
-
## Deep Links (Clickable References)
|
|
518
|
-
When showing search results to users, create clickable links so they can jump to that exact moment.
|
|
519
|
-
|
|
520
|
-
**ALWAYS prefer frame-based links for OCR results** (frame IDs are exact DB keys):
|
|
521
|
-
- \`[10:30 AM — Chrome](screenpipe://frame/12345)\` — use \`content.frame_id\` from OCR results
|
|
522
|
-
|
|
523
|
-
**Use timestamp links only for audio results** (which have no frame_id):
|
|
524
|
-
- \`[meeting at 3pm](screenpipe://timeline?timestamp=2024-01-15T15:00:00Z)\` — use exact \`timestamp\` from audio results
|
|
525
|
-
|
|
526
|
-
**NEVER fabricate frame IDs or timestamps.** Only use values copied from actual search results.`,
|
|
527
|
-
},
|
|
528
|
-
],
|
|
529
|
-
};
|
|
530
|
-
case "ui://search": {
|
|
531
|
-
// MCP App UI - Interactive search dashboard
|
|
532
|
-
const uiHtmlPath = path.join(__dirname, "..", "ui", "search.html");
|
|
533
|
-
let htmlContent;
|
|
534
|
-
try {
|
|
535
|
-
htmlContent = fs.readFileSync(uiHtmlPath, "utf-8");
|
|
536
|
-
}
|
|
537
|
-
catch {
|
|
538
|
-
// Fallback: serve embedded minimal UI if file not found
|
|
539
|
-
htmlContent = `<!DOCTYPE html>
|
|
540
|
-
<html>
|
|
541
|
-
<head>
|
|
542
|
-
<style>
|
|
543
|
-
body { font-family: system-ui; background: #0a0a0a; color: #fff; padding: 20px; }
|
|
544
|
-
input { width: 100%; padding: 10px; margin-bottom: 10px; background: #1a1a1a; border: 1px solid #333; color: #fff; border-radius: 6px; }
|
|
545
|
-
button { padding: 10px 20px; background: #fff; color: #000; border: none; border-radius: 6px; cursor: pointer; }
|
|
546
|
-
#results { margin-top: 20px; }
|
|
547
|
-
.result { background: #1a1a1a; padding: 12px; margin: 8px 0; border-radius: 8px; border: 1px solid #333; }
|
|
548
|
-
</style>
|
|
549
|
-
</head>
|
|
550
|
-
<body>
|
|
551
|
-
<h2>screenpipe search</h2>
|
|
552
|
-
<input id="q" placeholder="search..." onkeydown="if(event.key==='Enter')search()"/>
|
|
553
|
-
<button onclick="search()">search</button>
|
|
554
|
-
<div id="results"></div>
|
|
555
|
-
<script>
|
|
556
|
-
function search() {
|
|
557
|
-
window.parent.postMessage({jsonrpc:'2.0',method:'tools/call',params:{name:'search-content',arguments:{q:document.getElementById('q').value,limit:20}}},'*');
|
|
558
|
-
}
|
|
559
|
-
window.addEventListener('message',e=>{
|
|
560
|
-
if(e.data?.result||e.data?.method==='tool/result'){
|
|
561
|
-
const r=e.data.result||e.data.params?.result;
|
|
562
|
-
const d=r?.data||r||[];
|
|
563
|
-
document.getElementById('results').innerHTML=d.map(x=>'<div class="result"><b>'+((x.type||'')+'</b> '+(x.content?.app_name||'')+': '+(x.content?.text||x.content?.transcription||'').substring(0,200))+'</div>').join('');
|
|
564
|
-
}
|
|
565
|
-
});
|
|
566
|
-
</script>
|
|
567
|
-
</body>
|
|
568
|
-
</html>`;
|
|
569
|
-
}
|
|
570
|
-
return {
|
|
571
|
-
contents: [
|
|
572
|
-
{
|
|
573
|
-
uri,
|
|
574
|
-
mimeType: "text/html",
|
|
575
|
-
text: htmlContent,
|
|
576
|
-
},
|
|
577
|
-
],
|
|
578
|
-
};
|
|
579
|
-
}
|
|
580
|
-
case "screenpipe://pipe-creation-guide":
|
|
581
|
-
return {
|
|
582
|
-
contents: [
|
|
583
|
-
{
|
|
584
|
-
uri,
|
|
585
|
-
mimeType: "text/markdown",
|
|
586
|
-
text: `# Screenpipe Pipe Creation Guide
|
|
587
|
-
|
|
588
|
-
## What is a pipe?
|
|
589
|
-
|
|
590
|
-
A pipe is a scheduled AI agent defined as a single markdown file: \`~/.screenpipe/pipes/{name}/pipe.md\`
|
|
591
|
-
Every N minutes, screenpipe runs a coding agent (like pi or claude-code) with the pipe's prompt.
|
|
592
|
-
The agent can query your screen data, write files, call external APIs, send notifications, etc.
|
|
593
|
-
|
|
594
|
-
## pipe.md format
|
|
595
|
-
|
|
596
|
-
The file starts with YAML frontmatter on the very first line (no blank lines before it), then the prompt body:
|
|
597
|
-
|
|
598
|
-
\`\`\`markdown
|
|
599
|
-
---
|
|
600
|
-
schedule: every 30m
|
|
601
|
-
enabled: true
|
|
602
|
-
---
|
|
603
|
-
|
|
604
|
-
Your prompt instructions here...
|
|
605
|
-
\`\`\`
|
|
606
|
-
|
|
607
|
-
### Config fields
|
|
608
|
-
|
|
609
|
-
| Field | Values | Description |
|
|
610
|
-
|-------|--------|-------------|
|
|
611
|
-
| \`schedule\` | \`every 30m\`, \`every 1h\`, \`every day at 9am\`, \`every monday at 9am\`, \`manual\`, or cron: \`*/30 * * * *\` | When to run |
|
|
612
|
-
| \`enabled\` | \`true\` / \`false\` | Whether the pipe is active |
|
|
613
|
-
| \`preset\` | AI preset name (e.g. \`Oai\`) | Which AI model to use |
|
|
614
|
-
| \`history\` | \`true\` / \`false\` | Include previous output as context |
|
|
615
|
-
| \`connections\` | list of connection IDs | Required integrations (e.g. \`obsidian\`, \`telegram\`) |
|
|
616
|
-
|
|
617
|
-
## Context header
|
|
618
|
-
|
|
619
|
-
Before execution, screenpipe prepends a context header to the prompt with:
|
|
620
|
-
- Time range (start/end timestamps based on the schedule interval)
|
|
621
|
-
- Current date and user's timezone
|
|
622
|
-
- Screenpipe API base URL (http://localhost:3030)
|
|
623
|
-
- Output directory
|
|
624
|
-
|
|
625
|
-
The AI agent uses this context to query the right time range. No template variables needed in the prompt.
|
|
626
|
-
|
|
627
|
-
## Screenpipe search API
|
|
628
|
-
|
|
629
|
-
The agent queries screen data via the local REST API:
|
|
630
|
-
|
|
631
|
-
\`\`\`
|
|
632
|
-
curl "http://localhost:3030/search?limit=20&content_type=all&start_time=<ISO8601>&end_time=<ISO8601>"
|
|
633
|
-
\`\`\`
|
|
634
|
-
|
|
635
|
-
### Query parameters
|
|
636
|
-
|
|
637
|
-
| Parameter | Description |
|
|
638
|
-
|-----------|-------------|
|
|
639
|
-
| \`q\` | Text search query (optional — skip for audio, transcriptions are noisy) |
|
|
640
|
-
| \`content_type\` | \`all\`, \`ocr\`, \`audio\`, \`input\`, \`accessibility\` (prefer \`all\` or \`accessibility\`) |
|
|
641
|
-
| \`limit\` | Max results (default 20) |
|
|
642
|
-
| \`offset\` | Pagination offset |
|
|
643
|
-
| \`start_time\` / \`end_time\` | ISO 8601 timestamps or relative (\`1h ago\`, \`now\`) |
|
|
644
|
-
| \`app_name\` | Filter by app (e.g. \`Google Chrome\`, \`Slack\`) |
|
|
645
|
-
| \`window_name\` | Filter by window title |
|
|
646
|
-
| \`browser_url\` | Filter by URL |
|
|
647
|
-
| \`min_length\` / \`max_length\` | Filter by text length |
|
|
648
|
-
| \`speaker_name\` | Filter audio by speaker |
|
|
649
|
-
|
|
650
|
-
Other useful endpoints:
|
|
651
|
-
- \`GET /activity-summary?start_time=...&end_time=...\` — lightweight overview (~200 tokens)
|
|
652
|
-
- \`GET /elements?q=...&role=AXButton&start_time=...\` — UI elements
|
|
653
|
-
- \`GET /connections/{id}\` — get integration credentials (telegram, slack, obsidian, etc.)
|
|
654
|
-
- \`POST /raw_sql\` — run SQL queries (always include LIMIT)
|
|
655
|
-
|
|
656
|
-
Full API reference: read the \`screenpipe://api-reference\` resource.
|
|
657
|
-
|
|
658
|
-
## Installing and running
|
|
659
|
-
|
|
660
|
-
After creating the pipe.md file:
|
|
661
|
-
|
|
662
|
-
\`\`\`bash
|
|
663
|
-
bunx screenpipe@latest pipe install ~/.screenpipe/pipes/my-pipe
|
|
664
|
-
bunx screenpipe@latest pipe enable my-pipe
|
|
665
|
-
bunx screenpipe@latest pipe run my-pipe # test immediately
|
|
666
|
-
\`\`\`
|
|
667
|
-
|
|
668
|
-
## Example pipes
|
|
669
|
-
|
|
670
|
-
### Daily recap (manual trigger)
|
|
671
|
-
\`\`\`markdown
|
|
672
|
-
---
|
|
673
|
-
schedule: manual
|
|
674
|
-
enabled: true
|
|
675
|
-
---
|
|
676
|
-
|
|
677
|
-
Analyze my screen and audio recordings from today (last 16 hours). Use limit=10 per search, max 5 searches total.
|
|
678
|
-
|
|
679
|
-
## Summary
|
|
680
|
-
One sentence: what I mainly did today.
|
|
681
|
-
|
|
682
|
-
## Accomplishments
|
|
683
|
-
- Top 3 things I finished, with timestamps
|
|
684
|
-
|
|
685
|
-
## Key Moments
|
|
686
|
-
- Important things I saw, said, or heard
|
|
687
|
-
|
|
688
|
-
## Unfinished Work
|
|
689
|
-
- What I should continue tomorrow
|
|
690
|
-
\`\`\`
|
|
691
|
-
|
|
692
|
-
### Obsidian sync (every hour)
|
|
693
|
-
\`\`\`markdown
|
|
694
|
-
---
|
|
695
|
-
schedule: every 1h
|
|
696
|
-
enabled: true
|
|
697
|
-
connections:
|
|
698
|
-
- obsidian
|
|
699
|
-
---
|
|
700
|
-
|
|
701
|
-
Sync screenpipe activity to Obsidian vault as a daily note.
|
|
702
|
-
|
|
703
|
-
1. Get vault path from GET http://localhost:3030/connections/obsidian
|
|
704
|
-
2. Read existing daily note (merge into it)
|
|
705
|
-
3. Query search API in 30-minute chunks with min_length=50
|
|
706
|
-
4. Synthesize activities, extract action items, write note
|
|
707
|
-
\`\`\`
|
|
708
|
-
|
|
709
|
-
### Slack standup (every weekday at 9am)
|
|
710
|
-
\`\`\`markdown
|
|
711
|
-
---
|
|
712
|
-
schedule: every weekday at 9am
|
|
713
|
-
enabled: true
|
|
714
|
-
connections:
|
|
715
|
-
- slack
|
|
716
|
-
---
|
|
717
|
-
|
|
718
|
-
Generate standup update from yesterday's activity and post to Slack.
|
|
719
|
-
|
|
720
|
-
1. Query activity-summary for yesterday
|
|
721
|
-
2. Search for key accomplishments and blockers
|
|
722
|
-
3. Format as: Done / Doing / Blocked
|
|
723
|
-
4. POST to Slack webhook from GET http://localhost:3030/connections/slack
|
|
724
|
-
\`\`\`
|
|
725
|
-
|
|
726
|
-
## Optimization tips
|
|
727
|
-
|
|
728
|
-
- Be specific about expected output format
|
|
729
|
-
- Give step-by-step instructions
|
|
730
|
-
- Add error handling: "if API returns empty, try content_type=accessibility instead of ocr"
|
|
731
|
-
- Add validation: "before writing, verify you have at least 3 entries"
|
|
732
|
-
- Specify exact file paths, API parameters, output structure
|
|
733
|
-
- Keep search limit low (10-20) and use time ranges from the context header
|
|
734
|
-
- Use \`min_length=50\` to skip noisy OCR fragments`,
|
|
735
|
-
},
|
|
736
|
-
],
|
|
737
|
-
};
|
|
738
|
-
case "screenpipe://api-reference":
|
|
739
|
-
return {
|
|
740
|
-
contents: [
|
|
741
|
-
{
|
|
742
|
-
uri,
|
|
743
|
-
mimeType: "text/markdown",
|
|
744
|
-
text: `# Screenpipe REST API Reference
|
|
745
|
-
|
|
746
|
-
Local REST API at \`http://localhost:3030\`. Full reference (60+ endpoints): https://docs.screenpi.pe/llms-full.txt
|
|
747
|
-
|
|
748
|
-
## Shell
|
|
749
|
-
|
|
750
|
-
- **macOS/Linux** → \`bash\`, \`curl\`
|
|
751
|
-
- **Windows** → \`powershell\`, \`curl.exe\` (not the alias)
|
|
752
|
-
|
|
753
|
-
## Context Window Protection
|
|
754
|
-
|
|
755
|
-
API responses can be large. Always write curl output to a file first (\`curl ... -o /tmp/sp_result.json\`), check size (\`wc -c\`), and if over 5KB read only the first 50-100 lines. Extract what you need with \`jq\`. NEVER dump full large responses into context.
|
|
756
|
-
|
|
757
|
-
---
|
|
758
|
-
|
|
759
|
-
## 1. Search — \`GET /search\`
|
|
760
|
-
|
|
761
|
-
\`\`\`bash
|
|
762
|
-
curl "http://localhost:3030/search?q=QUERY&content_type=all&limit=10&start_time=1h%20ago"
|
|
763
|
-
\`\`\`
|
|
764
|
-
|
|
765
|
-
### Parameters
|
|
766
|
-
|
|
767
|
-
| Parameter | Type | Required | Description |
|
|
768
|
-
|-----------|------|----------|-------------|
|
|
769
|
-
| \`q\` | string | No | Keywords. Do NOT use for audio — transcriptions are noisy. |
|
|
770
|
-
| \`content_type\` | string | No | \`all\` (default), \`ocr\`, \`audio\`, \`input\`, \`accessibility\` |
|
|
771
|
-
| \`limit\` | integer | No | Max 1-20. Default: 10 |
|
|
772
|
-
| \`offset\` | integer | No | Pagination. Default: 0 |
|
|
773
|
-
| \`start_time\` | ISO 8601 or relative | **Yes** | \`2024-01-15T10:00:00Z\` or \`16h ago\`, \`2d ago\`, \`30m ago\` |
|
|
774
|
-
| \`end_time\` | ISO 8601 or relative | No | Defaults to now. \`now\`, \`1h ago\` |
|
|
775
|
-
| \`app_name\` | string | No | e.g. "Google Chrome", "Slack", "zoom.us" |
|
|
776
|
-
| \`window_name\` | string | No | Window title substring |
|
|
777
|
-
| \`speaker_name\` | string | No | Filter audio by speaker (case-insensitive partial) |
|
|
778
|
-
| \`focused\` | boolean | No | Only focused windows |
|
|
779
|
-
|
|
780
|
-
### Critical Rules
|
|
781
|
-
|
|
782
|
-
1. **ALWAYS include \`start_time\`** — queries without time bounds WILL timeout
|
|
783
|
-
2. **Start with 1-2 hour ranges** — expand only if no results
|
|
784
|
-
3. **Use \`app_name\`** when user mentions a specific app
|
|
785
|
-
4. **"recent"** = 30 min. **"today"** = since midnight. **"yesterday"** = yesterday's range
|
|
786
|
-
|
|
787
|
-
### Response Format
|
|
788
|
-
|
|
789
|
-
\`\`\`json
|
|
790
|
-
{
|
|
791
|
-
"data": [
|
|
792
|
-
{"type": "OCR", "content": {"frame_id": 12345, "text": "...", "timestamp": "...", "app_name": "Chrome"}},
|
|
793
|
-
{"type": "Audio", "content": {"chunk_id": 678, "transcription": "...", "timestamp": "...", "speaker": {"name": "John"}}},
|
|
794
|
-
{"type": "UI", "content": {"id": 999, "text": "Clicked Submit", "timestamp": "...", "app_name": "Safari"}}
|
|
795
|
-
],
|
|
796
|
-
"pagination": {"limit": 10, "offset": 0, "total": 42}
|
|
797
|
-
}
|
|
798
|
-
\`\`\`
|
|
799
|
-
|
|
800
|
-
---
|
|
801
|
-
|
|
802
|
-
## 2. Activity Summary — \`GET /activity-summary\`
|
|
803
|
-
|
|
804
|
-
\`\`\`bash
|
|
805
|
-
curl "http://localhost:3030/activity-summary?start_time=1h%20ago&end_time=now"
|
|
806
|
-
\`\`\`
|
|
807
|
-
|
|
808
|
-
Returns app usage with \`active_minutes\`, first/last seen, recent texts, audio summary. ~200-500 tokens. Best starting point.
|
|
809
|
-
|
|
810
|
-
---
|
|
811
|
-
|
|
812
|
-
## 3. Elements — \`GET /elements\`
|
|
813
|
-
|
|
814
|
-
Lightweight FTS search across UI elements (~100-500 bytes each).
|
|
815
|
-
|
|
816
|
-
\`\`\`bash
|
|
817
|
-
curl "http://localhost:3030/elements?q=Submit&role=AXButton&start_time=1h%20ago&limit=10"
|
|
818
|
-
\`\`\`
|
|
819
|
-
|
|
820
|
-
Parameters: \`q\`, \`frame_id\`, \`source\` (\`accessibility\`|\`ocr\`), \`role\`, \`start_time\`, \`end_time\`, \`app_name\`, \`limit\`, \`offset\`.
|
|
821
|
-
|
|
822
|
-
### Frame Context — \`GET /frames/{id}/context\`
|
|
823
|
-
|
|
824
|
-
Returns accessibility text, parsed nodes, and extracted URLs for a frame.
|
|
825
|
-
|
|
826
|
-
Common roles: \`AXButton\`, \`AXStaticText\`, \`AXLink\`, \`AXTextField\`, \`AXTextArea\`, \`AXMenuItem\`, \`AXCheckBox\`
|
|
827
|
-
|
|
828
|
-
---
|
|
829
|
-
|
|
830
|
-
## 4. Frames — \`GET /frames/{frame_id}\`
|
|
831
|
-
|
|
832
|
-
Returns raw PNG screenshot. Never fetch more than 2-3 per query.
|
|
833
|
-
|
|
834
|
-
---
|
|
835
|
-
|
|
836
|
-
## 5. Media Export — \`POST /frames/export\`
|
|
837
|
-
|
|
838
|
-
\`\`\`bash
|
|
839
|
-
curl -X POST http://localhost:3030/frames/export \\
|
|
840
|
-
-H "Content-Type: application/json" \\
|
|
841
|
-
-d '{"start_time": "5m ago", "end_time": "now", "fps": 1.0}'
|
|
842
|
-
\`\`\`
|
|
843
|
-
|
|
844
|
-
FPS guidelines: 5min→1.0, 30min→0.5, 1h→0.2, 2h+→0.1. Max 10,000 frames.
|
|
845
|
-
|
|
846
|
-
---
|
|
847
|
-
|
|
848
|
-
## 6. Retranscribe — \`POST /audio/retranscribe\`
|
|
849
|
-
|
|
850
|
-
\`\`\`bash
|
|
851
|
-
curl -X POST http://localhost:3030/audio/retranscribe \\
|
|
852
|
-
-H "Content-Type: application/json" \\
|
|
853
|
-
-d '{"start": "1h ago", "end": "now"}'
|
|
854
|
-
\`\`\`
|
|
855
|
-
|
|
856
|
-
Optional: \`engine\`, \`vocabulary\` (array of \`{"word": "...", "replacement": "..."}\`), \`prompt\` (topic context).
|
|
857
|
-
|
|
858
|
-
---
|
|
859
|
-
|
|
860
|
-
## 7. Raw SQL — \`POST /raw_sql\`
|
|
861
|
-
|
|
862
|
-
\`\`\`bash
|
|
863
|
-
curl -X POST http://localhost:3030/raw_sql \\
|
|
864
|
-
-H "Content-Type: application/json" \\
|
|
865
|
-
-d '{"query": "SELECT ... LIMIT 100"}'
|
|
866
|
-
\`\`\`
|
|
867
|
-
|
|
868
|
-
Every SELECT needs LIMIT. Always filter by time. Read-only.
|
|
869
|
-
|
|
870
|
-
### Schema
|
|
871
|
-
|
|
872
|
-
| Table | Key Columns | Time Column |
|
|
873
|
-
|-------|-------------|-------------|
|
|
874
|
-
| \`frames\` | \`app_name\`, \`window_name\`, \`browser_url\`, \`focused\` | \`timestamp\` |
|
|
875
|
-
| \`ocr_text\` | \`text\`, \`app_name\`, \`window_name\` | join via \`frame_id\` |
|
|
876
|
-
| \`elements\` | \`source\`, \`role\`, \`text\` | join via \`frame_id\` |
|
|
877
|
-
| \`audio_transcriptions\` | \`transcription\`, \`device\`, \`speaker_id\`, \`is_input_device\` | \`timestamp\` |
|
|
878
|
-
| \`speakers\` | \`name\`, \`metadata\` | — |
|
|
879
|
-
| \`ui_events\` | \`event_type\`, \`app_name\`, \`window_title\`, \`browser_url\` | \`timestamp\` |
|
|
880
|
-
| \`accessibility\` | \`app_name\`, \`window_name\`, \`text_content\` | \`timestamp\` |
|
|
881
|
-
|
|
882
|
-
### Example Queries
|
|
883
|
-
|
|
884
|
-
\`\`\`sql
|
|
885
|
-
-- Most used apps (last 24h)
|
|
886
|
-
SELECT app_name, COUNT(*) as frames FROM frames
|
|
887
|
-
WHERE timestamp > datetime('now', '-24 hours') AND app_name IS NOT NULL
|
|
888
|
-
GROUP BY app_name ORDER BY frames DESC LIMIT 20
|
|
889
|
-
|
|
890
|
-
-- Speaker stats
|
|
891
|
-
SELECT COALESCE(NULLIF(s.name, ''), 'Unknown') as speaker, COUNT(*) as segments
|
|
892
|
-
FROM audio_transcriptions at LEFT JOIN speakers s ON at.speaker_id = s.id
|
|
893
|
-
WHERE at.timestamp > datetime('now', '-24 hours')
|
|
894
|
-
GROUP BY at.speaker_id ORDER BY segments DESC LIMIT 20
|
|
895
|
-
\`\`\`
|
|
896
|
-
|
|
897
|
-
---
|
|
898
|
-
|
|
899
|
-
## 8. Connections — \`GET /connections\`
|
|
900
|
-
|
|
901
|
-
\`\`\`bash
|
|
902
|
-
curl http://localhost:3030/connections # List all
|
|
903
|
-
curl http://localhost:3030/connections/telegram # Get credentials
|
|
904
|
-
\`\`\`
|
|
905
|
-
|
|
906
|
-
Services: Telegram (\`bot_token\` + \`chat_id\`), Slack (\`webhook_url\`), Discord (\`webhook_url\`), Todoist (\`api_token\`), Teams (\`webhook_url\`), Email (SMTP config).
|
|
907
|
-
|
|
908
|
-
---
|
|
909
|
-
|
|
910
|
-
## 9. Speakers
|
|
911
|
-
|
|
912
|
-
\`\`\`bash
|
|
913
|
-
curl "http://localhost:3030/speakers/search?name=John"
|
|
914
|
-
curl "http://localhost:3030/speakers/unnamed?limit=10"
|
|
915
|
-
curl -X POST http://localhost:3030/speakers/update -H "Content-Type: application/json" -d '{"id": 5, "name": "John"}'
|
|
916
|
-
curl -X POST http://localhost:3030/speakers/merge -H "Content-Type: application/json" -d '{"speaker_to_keep_id": 1, "speaker_to_merge_id": 2}'
|
|
917
|
-
\`\`\`
|
|
918
|
-
|
|
919
|
-
---
|
|
920
|
-
|
|
921
|
-
## 10. Other Endpoints
|
|
922
|
-
|
|
923
|
-
\`\`\`bash
|
|
924
|
-
curl http://localhost:3030/health # Health check
|
|
925
|
-
curl http://localhost:3030/audio/list # Audio devices
|
|
926
|
-
curl http://localhost:3030/vision/list # Monitors
|
|
927
|
-
\`\`\`
|
|
928
|
-
|
|
929
|
-
## Pipes API
|
|
930
|
-
|
|
931
|
-
\`\`\`bash
|
|
932
|
-
curl http://localhost:3030/pipes/list # List all pipes
|
|
933
|
-
curl -X POST http://localhost:3030/pipes/enable -d '{"name":"..."}' # Enable
|
|
934
|
-
curl -X POST http://localhost:3030/pipes/disable -d '{"name":"..."}' # Disable
|
|
935
|
-
curl -X POST http://localhost:3030/pipes/run -d '{"name":"..."}' # Run once
|
|
936
|
-
curl "http://localhost:3030/pipes/{name}/executions?limit=5" # Execution history
|
|
937
|
-
\`\`\`
|
|
938
|
-
|
|
939
|
-
## Deep Links
|
|
940
|
-
|
|
941
|
-
\`\`\`markdown
|
|
942
|
-
[10:30 AM — Chrome](screenpipe://frame/12345) # OCR results (use frame_id)
|
|
943
|
-
[meeting at 3pm](screenpipe://timeline?timestamp=ISO8601) # Audio results (use timestamp)
|
|
944
|
-
\`\`\`
|
|
945
|
-
|
|
946
|
-
Only use IDs/timestamps from actual search results. Never fabricate.`,
|
|
947
|
-
},
|
|
948
|
-
],
|
|
949
|
-
};
|
|
950
|
-
case "screenpipe://cli-reference":
|
|
951
|
-
return {
|
|
952
|
-
contents: [
|
|
953
|
-
{
|
|
954
|
-
uri,
|
|
955
|
-
mimeType: "text/markdown",
|
|
956
|
-
text: `# Screenpipe CLI Reference
|
|
957
|
-
|
|
958
|
-
Use \`bunx screenpipe@latest\` to run CLI commands (or \`npx screenpipe@latest\`). No separate install needed.
|
|
959
|
-
|
|
960
|
-
## Shell
|
|
961
|
-
|
|
962
|
-
- **macOS/Linux** → \`bash\`
|
|
963
|
-
- **Windows** → \`powershell\`
|
|
964
|
-
|
|
965
|
-
---
|
|
966
|
-
|
|
967
|
-
## Pipe Management
|
|
968
|
-
|
|
969
|
-
Pipes are markdown-based AI automations. Each pipe lives at \`~/.screenpipe/pipes/<name>/pipe.md\`.
|
|
970
|
-
|
|
971
|
-
### Commands
|
|
972
|
-
|
|
973
|
-
\`\`\`bash
|
|
974
|
-
bunx screenpipe@latest pipe list # List all pipes (compact table)
|
|
975
|
-
bunx screenpipe@latest pipe enable <name> # Enable a pipe
|
|
976
|
-
bunx screenpipe@latest pipe disable <name> # Disable a pipe
|
|
977
|
-
bunx screenpipe@latest pipe run <name> # Run once immediately (for testing)
|
|
978
|
-
bunx screenpipe@latest pipe logs <name> # View execution logs
|
|
979
|
-
bunx screenpipe@latest pipe install <url-or-path> # Install from GitHub or local path
|
|
980
|
-
bunx screenpipe@latest pipe delete <name> # Delete a pipe
|
|
981
|
-
bunx screenpipe@latest pipe models list # View AI model presets
|
|
982
|
-
\`\`\`
|
|
983
|
-
|
|
984
|
-
### Creating a Pipe
|
|
985
|
-
|
|
986
|
-
Create \`~/.screenpipe/pipes/<name>/pipe.md\` with YAML frontmatter + prompt:
|
|
987
|
-
|
|
988
|
-
\`\`\`markdown
|
|
989
|
-
---
|
|
990
|
-
schedule: every 30m
|
|
991
|
-
enabled: true
|
|
992
|
-
preset: Oai
|
|
993
|
-
---
|
|
994
|
-
|
|
995
|
-
Your prompt instructions here. The AI agent executes this on schedule.
|
|
996
|
-
\`\`\`
|
|
997
|
-
|
|
998
|
-
**Schedule syntax**: \`every 30m\`, \`every 1h\`, \`every day at 9am\`, \`every monday at 9am\`, \`manual\`, or cron: \`*/30 * * * *\`
|
|
999
|
-
|
|
1000
|
-
**Config fields**: \`schedule\`, \`enabled\` (bool), \`preset\` (AI preset name), \`history\` (bool — include previous output), \`connections\` (list of required integrations)
|
|
1001
|
-
|
|
1002
|
-
After creating:
|
|
1003
|
-
\`\`\`bash
|
|
1004
|
-
bunx screenpipe@latest pipe install ~/.screenpipe/pipes/my-pipe
|
|
1005
|
-
bunx screenpipe@latest pipe enable my-pipe
|
|
1006
|
-
bunx screenpipe@latest pipe run my-pipe # test immediately
|
|
1007
|
-
\`\`\`
|
|
1008
|
-
|
|
1009
|
-
### Editing Config
|
|
1010
|
-
|
|
1011
|
-
Edit frontmatter in the pipe.md file directly, or via API:
|
|
1012
|
-
|
|
1013
|
-
\`\`\`bash
|
|
1014
|
-
curl -X POST http://localhost:3030/pipes/<name>/config \\
|
|
1015
|
-
-H "Content-Type: application/json" \\
|
|
1016
|
-
-d '{"config": {"schedule": "every 1h", "enabled": true}}'
|
|
1017
|
-
\`\`\`
|
|
1018
|
-
|
|
1019
|
-
### Rules
|
|
1020
|
-
|
|
1021
|
-
1. Use \`pipe list\` (not \`--json\`) — table output is compact
|
|
1022
|
-
2. Never dump full pipe JSON — can be 15MB+
|
|
1023
|
-
3. Check logs first when debugging: \`pipe logs <name>\`
|
|
1024
|
-
4. Use \`pipe run <name>\` to test before waiting for schedule
|
|
1025
|
-
|
|
1026
|
-
---
|
|
1027
|
-
|
|
1028
|
-
## Connection Management
|
|
1029
|
-
|
|
1030
|
-
Manage integrations (Telegram, Slack, Discord, Email, Todoist, Teams) from the CLI.
|
|
1031
|
-
|
|
1032
|
-
### Commands
|
|
1033
|
-
|
|
1034
|
-
\`\`\`bash
|
|
1035
|
-
bunx screenpipe@latest connection list # List all connections + status
|
|
1036
|
-
bunx screenpipe@latest connection list --json # JSON output
|
|
1037
|
-
bunx screenpipe@latest connection get <id> # Show saved credentials
|
|
1038
|
-
bunx screenpipe@latest connection set <id> key=val # Save credentials
|
|
1039
|
-
bunx screenpipe@latest connection test <id> # Test a connection
|
|
1040
|
-
bunx screenpipe@latest connection remove <id> # Remove credentials
|
|
1041
|
-
\`\`\`
|
|
1042
|
-
|
|
1043
|
-
### Examples
|
|
1044
|
-
|
|
1045
|
-
\`\`\`bash
|
|
1046
|
-
# Set up Telegram
|
|
1047
|
-
bunx screenpipe@latest connection set telegram bot_token=123456:ABC-DEF chat_id=5776185278
|
|
1048
|
-
|
|
1049
|
-
# Set up Slack webhook
|
|
1050
|
-
bunx screenpipe@latest connection set slack webhook_url=https://hooks.slack.com/services/...
|
|
1051
|
-
|
|
1052
|
-
# Verify it works
|
|
1053
|
-
bunx screenpipe@latest connection test telegram
|
|
1054
|
-
\`\`\`
|
|
1055
|
-
|
|
1056
|
-
Connection IDs: \`telegram\`, \`slack\`, \`discord\`, \`email\`, \`todoist\`, \`teams\`, \`google-calendar\`, \`apple-intelligence\`, \`openclaw\`, \`obsidian\`
|
|
1057
|
-
|
|
1058
|
-
Credentials are stored locally at \`~/.screenpipe/connections.json\`.`,
|
|
1059
|
-
},
|
|
1060
|
-
],
|
|
1061
|
-
};
|
|
1062
|
-
default:
|
|
1063
|
-
throw new Error(`Unknown resource: ${uri}`);
|
|
1064
|
-
}
|
|
1065
|
-
});
|
|
1066
|
-
// MCP Prompts - static interaction templates
|
|
1067
|
-
const PROMPTS = [
|
|
1068
|
-
{
|
|
1069
|
-
name: "search-recent",
|
|
1070
|
-
description: "Search recent screen activity",
|
|
1071
|
-
arguments: [
|
|
1072
|
-
{ name: "query", description: "Optional search term", required: false },
|
|
1073
|
-
{ name: "hours", description: "Hours to look back (default: 1)", required: false },
|
|
1074
|
-
],
|
|
1075
|
-
},
|
|
1076
|
-
{
|
|
1077
|
-
name: "find-in-app",
|
|
1078
|
-
description: "Find content from a specific application",
|
|
1079
|
-
arguments: [
|
|
1080
|
-
{ name: "app", description: "App name (e.g., Chrome, Slack)", required: true },
|
|
1081
|
-
{ name: "query", description: "Optional search term", required: false },
|
|
1082
|
-
],
|
|
1083
|
-
},
|
|
1084
|
-
{
|
|
1085
|
-
name: "meeting-notes",
|
|
1086
|
-
description: "Get audio transcriptions from meetings",
|
|
1087
|
-
arguments: [
|
|
1088
|
-
{ name: "hours", description: "Hours to look back (default: 3)", required: false },
|
|
1089
|
-
],
|
|
1090
|
-
},
|
|
1091
|
-
{
|
|
1092
|
-
name: "create-pipe",
|
|
1093
|
-
description: "Create a new screenpipe pipe (scheduled AI automation)",
|
|
1094
|
-
arguments: [
|
|
1095
|
-
{ name: "description", description: "What the pipe should do", required: true },
|
|
1096
|
-
{ name: "schedule", description: "Schedule (e.g., 'every 30m', 'every day at 9am', 'manual')", required: false },
|
|
1097
|
-
],
|
|
1098
|
-
},
|
|
1099
|
-
];
|
|
1100
|
-
// List prompts handler
|
|
1101
|
-
server.setRequestHandler(types_js_1.ListPromptsRequestSchema, async () => {
|
|
1102
|
-
return { prompts: PROMPTS };
|
|
1103
|
-
});
|
|
1104
|
-
// Get prompt handler
|
|
1105
|
-
server.setRequestHandler(types_js_1.GetPromptRequestSchema, async (request) => {
|
|
1106
|
-
const { name, arguments: promptArgs } = request.params;
|
|
1107
|
-
const dateInfo = getCurrentDateInfo();
|
|
1108
|
-
const now = Date.now();
|
|
1109
|
-
switch (name) {
|
|
1110
|
-
case "search-recent": {
|
|
1111
|
-
const query = promptArgs?.query || "";
|
|
1112
|
-
const hours = parseInt(promptArgs?.hours || "1", 10);
|
|
1113
|
-
const startTime = new Date(now - hours * 60 * 60 * 1000).toISOString();
|
|
1114
|
-
return {
|
|
1115
|
-
description: `Search recent activity (last ${hours} hour${hours > 1 ? "s" : ""})`,
|
|
1116
|
-
messages: [
|
|
1117
|
-
{
|
|
1118
|
-
role: "user",
|
|
1119
|
-
content: {
|
|
1120
|
-
type: "text",
|
|
1121
|
-
text: `Search screenpipe for recent activity.
|
|
1122
|
-
|
|
1123
|
-
Current time: ${dateInfo.isoDate}
|
|
1124
|
-
|
|
1125
|
-
Use search-content with:
|
|
1126
|
-
${query ? `- q: "${query}"` : "- No query filter (get all content)"}
|
|
1127
|
-
- start_time: "${startTime}"
|
|
1128
|
-
- limit: 50`,
|
|
1129
|
-
},
|
|
1130
|
-
},
|
|
1131
|
-
],
|
|
1132
|
-
};
|
|
1133
|
-
}
|
|
1134
|
-
case "find-in-app": {
|
|
1135
|
-
const app = promptArgs?.app || "Google Chrome";
|
|
1136
|
-
const query = promptArgs?.query || "";
|
|
1137
|
-
return {
|
|
1138
|
-
description: `Find content from ${app}`,
|
|
1139
|
-
messages: [
|
|
1140
|
-
{
|
|
1141
|
-
role: "user",
|
|
1142
|
-
content: {
|
|
1143
|
-
type: "text",
|
|
1144
|
-
text: `Search screenpipe for content from ${app}.
|
|
1145
|
-
|
|
1146
|
-
Current time: ${dateInfo.isoDate}
|
|
1147
|
-
|
|
1148
|
-
Use search-content with:
|
|
1149
|
-
- app_name: "${app}"
|
|
1150
|
-
${query ? `- q: "${query}"` : "- No query filter"}
|
|
1151
|
-
- content_type: "all"
|
|
1152
|
-
- limit: 50`,
|
|
1153
|
-
},
|
|
1154
|
-
},
|
|
1155
|
-
],
|
|
1156
|
-
};
|
|
1157
|
-
}
|
|
1158
|
-
case "meeting-notes": {
|
|
1159
|
-
const hours = parseInt(promptArgs?.hours || "3", 10);
|
|
1160
|
-
const startTime = new Date(now - hours * 60 * 60 * 1000).toISOString();
|
|
1161
|
-
return {
|
|
1162
|
-
description: `Get meeting transcriptions (last ${hours} hours)`,
|
|
1163
|
-
messages: [
|
|
1164
|
-
{
|
|
1165
|
-
role: "user",
|
|
1166
|
-
content: {
|
|
1167
|
-
type: "text",
|
|
1168
|
-
text: `Get audio transcriptions from recent meetings.
|
|
1169
|
-
|
|
1170
|
-
Current time: ${dateInfo.isoDate}
|
|
1171
|
-
|
|
1172
|
-
Use search-content with:
|
|
1173
|
-
- content_type: "audio"
|
|
1174
|
-
- start_time: "${startTime}"
|
|
1175
|
-
- limit: 100
|
|
1176
|
-
|
|
1177
|
-
Common meeting apps: zoom.us, Microsoft Teams, Google Meet, Slack`,
|
|
1178
|
-
},
|
|
1179
|
-
},
|
|
1180
|
-
],
|
|
1181
|
-
};
|
|
1182
|
-
}
|
|
1183
|
-
case "create-pipe": {
|
|
1184
|
-
const description = promptArgs?.description || "a useful automation";
|
|
1185
|
-
const schedule = promptArgs?.schedule || "every 30m";
|
|
1186
|
-
return {
|
|
1187
|
-
description: `Create a new screenpipe pipe: ${description}`,
|
|
1188
|
-
messages: [
|
|
1189
|
-
{
|
|
1190
|
-
role: "user",
|
|
1191
|
-
content: {
|
|
1192
|
-
type: "text",
|
|
1193
|
-
text: `Create a new screenpipe pipe based on this description: "${description}"
|
|
1194
|
-
Schedule: ${schedule}
|
|
1195
|
-
|
|
1196
|
-
## How to create a pipe
|
|
1197
|
-
|
|
1198
|
-
A pipe is a TypeScript file that runs on a schedule or manually. It uses the screenpipe API to access screen/audio data and can send notifications, call AI, etc.
|
|
1199
|
-
|
|
1200
|
-
### Pipe structure
|
|
1201
|
-
\`\`\`typescript
|
|
1202
|
-
const pipe = () => import("https://raw.githubusercontent.com/nichochar/screenpipe/refs/heads/main/pipes/pipe-modules/pipe-core/index.ts");
|
|
1203
|
-
|
|
1204
|
-
async function main() {
|
|
1205
|
-
const sp = await pipe();
|
|
1206
|
-
|
|
1207
|
-
// Query recent screen/audio data
|
|
1208
|
-
const results = await sp.queryScreenpipe({
|
|
1209
|
-
q: "search term",
|
|
1210
|
-
contentType: "all", // "ocr" | "audio" | "all" | "ui"
|
|
1211
|
-
limit: 50,
|
|
1212
|
-
startTime: new Date(Date.now() - 30 * 60 * 1000).toISOString(),
|
|
1213
|
-
endTime: new Date().toISOString(),
|
|
1214
|
-
});
|
|
1215
|
-
|
|
1216
|
-
// Send notification
|
|
1217
|
-
await sp.sendDesktopNotification({ title: "Title", body: "Body" });
|
|
1218
|
-
|
|
1219
|
-
// Call AI (uses user's configured AI provider)
|
|
1220
|
-
const response = await sp.generateText({
|
|
1221
|
-
messages: [{ role: "user", content: "Analyze this data..." }],
|
|
1222
|
-
});
|
|
1223
|
-
}
|
|
1224
|
-
|
|
1225
|
-
main();
|
|
1226
|
-
\`\`\`
|
|
1227
|
-
|
|
1228
|
-
### Key APIs available in pipes
|
|
1229
|
-
- \`queryScreenpipe(params)\` - Search screen text (OCR/UI), audio transcriptions
|
|
1230
|
-
- \`sendDesktopNotification({ title, body })\` - System notifications
|
|
1231
|
-
- \`generateText({ messages, model? })\` - AI text generation
|
|
1232
|
-
- \`generateObject({ messages, schema, model? })\` - AI structured output
|
|
1233
|
-
- \`loadPipeConfig()\` - Load pipe configuration
|
|
1234
|
-
- \`fetch()\` - HTTP requests to external services
|
|
1235
|
-
|
|
1236
|
-
### pipe.json config
|
|
1237
|
-
\`\`\`json
|
|
1238
|
-
{
|
|
1239
|
-
"cron": "${schedule === "manual" ? "" : schedule.replace("every ", "*/").replace("m", " * * * *").replace("h", " * * *")}",
|
|
1240
|
-
"is_nextjs": false,
|
|
1241
|
-
"fields": [
|
|
1242
|
-
{ "name": "setting_name", "type": "string", "default": "value", "description": "Setting description" }
|
|
1243
|
-
]
|
|
1244
|
-
}
|
|
1245
|
-
\`\`\`
|
|
1246
|
-
|
|
1247
|
-
### Important notes
|
|
1248
|
-
- Use \`contentType: "ui"\` for accessibility/structured text, \`"ocr"\` for raw screen text
|
|
1249
|
-
- Always handle empty results gracefully
|
|
1250
|
-
- Use \`startTime\`/\`endTime\` to scope queries
|
|
1251
|
-
- Pipes run in Bun runtime with full TypeScript support
|
|
1252
|
-
- For scheduled pipes, keep execution fast (< 30s)
|
|
1253
|
-
|
|
1254
|
-
Create the pipe with the necessary files (pipe.ts and pipe.json). Follow the patterns above exactly.`,
|
|
264
|
+
if (uri === "screenpipe://context") {
|
|
265
|
+
const now = new Date();
|
|
266
|
+
const ms = now.getTime();
|
|
267
|
+
return {
|
|
268
|
+
contents: [
|
|
269
|
+
{
|
|
270
|
+
uri,
|
|
271
|
+
mimeType: "application/json",
|
|
272
|
+
text: JSON.stringify({
|
|
273
|
+
current_time: now.toISOString(),
|
|
274
|
+
current_date_local: now.toLocaleDateString("en-US", {
|
|
275
|
+
weekday: "long",
|
|
276
|
+
year: "numeric",
|
|
277
|
+
month: "long",
|
|
278
|
+
day: "numeric",
|
|
279
|
+
}),
|
|
280
|
+
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
281
|
+
timestamps: {
|
|
282
|
+
now: now.toISOString(),
|
|
283
|
+
one_hour_ago: new Date(ms - 60 * 60 * 1000).toISOString(),
|
|
284
|
+
three_hours_ago: new Date(ms - 3 * 60 * 60 * 1000).toISOString(),
|
|
285
|
+
today_start: `${now.toISOString().split("T")[0]}T00:00:00Z`,
|
|
286
|
+
yesterday_start: `${new Date(ms - 24 * 60 * 60 * 1000).toISOString().split("T")[0]}T00:00:00Z`,
|
|
287
|
+
one_week_ago: new Date(ms - 7 * 24 * 60 * 60 * 1000).toISOString(),
|
|
1255
288
|
},
|
|
1256
|
-
},
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
}
|
|
1260
|
-
default:
|
|
1261
|
-
throw new Error(`Unknown prompt: ${name}`);
|
|
289
|
+
}, null, 2),
|
|
290
|
+
},
|
|
291
|
+
],
|
|
292
|
+
};
|
|
1262
293
|
}
|
|
294
|
+
throw new Error(`Unknown resource: ${uri}`);
|
|
1263
295
|
});
|
|
1264
|
-
//
|
|
296
|
+
// ---------------------------------------------------------------------------
|
|
297
|
+
// Helper
|
|
298
|
+
// ---------------------------------------------------------------------------
|
|
1265
299
|
async function fetchAPI(endpoint, options = {}) {
|
|
1266
300
|
const url = `${SCREENPIPE_API}${endpoint}`;
|
|
1267
301
|
return fetch(url, {
|
|
@@ -1272,7 +306,9 @@ async function fetchAPI(endpoint, options = {}) {
|
|
|
1272
306
|
},
|
|
1273
307
|
});
|
|
1274
308
|
}
|
|
1275
|
-
//
|
|
309
|
+
// ---------------------------------------------------------------------------
|
|
310
|
+
// Tool handlers
|
|
311
|
+
// ---------------------------------------------------------------------------
|
|
1276
312
|
server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
1277
313
|
const { name, arguments: args } = request.params;
|
|
1278
314
|
if (!args) {
|
|
@@ -1289,9 +325,8 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1289
325
|
}
|
|
1290
326
|
}
|
|
1291
327
|
const response = await fetchAPI(`/search?${params.toString()}`);
|
|
1292
|
-
if (!response.ok)
|
|
328
|
+
if (!response.ok)
|
|
1293
329
|
throw new Error(`HTTP error: ${response.status}`);
|
|
1294
|
-
}
|
|
1295
330
|
const data = await response.json();
|
|
1296
331
|
const results = data.data || [];
|
|
1297
332
|
const pagination = data.pagination || {};
|
|
@@ -1300,12 +335,11 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1300
335
|
content: [
|
|
1301
336
|
{
|
|
1302
337
|
type: "text",
|
|
1303
|
-
text: "No results found. Try: broader
|
|
338
|
+
text: "No results found. Try: broader terms, different content_type, or wider time range.",
|
|
1304
339
|
},
|
|
1305
340
|
],
|
|
1306
341
|
};
|
|
1307
342
|
}
|
|
1308
|
-
// Build content array with text and optional images
|
|
1309
343
|
const contentItems = [];
|
|
1310
344
|
const formattedResults = [];
|
|
1311
345
|
const images = [];
|
|
@@ -1338,42 +372,173 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1338
372
|
`${content.timestamp || ""}\n` +
|
|
1339
373
|
`${content.text || ""}`);
|
|
1340
374
|
}
|
|
375
|
+
else if (result.type === "Memory") {
|
|
376
|
+
const tagsStr = content.tags?.length ? ` [${content.tags.join(", ")}]` : "";
|
|
377
|
+
const importance = content.importance != null ? ` (importance: ${content.importance})` : "";
|
|
378
|
+
formattedResults.push(`[Memory #${content.id}]${tagsStr}${importance}\n` +
|
|
379
|
+
`${content.created_at || ""}\n` +
|
|
380
|
+
`${content.content || ""}`);
|
|
381
|
+
}
|
|
1341
382
|
}
|
|
1342
|
-
// Header with pagination info
|
|
1343
383
|
const header = `Results: ${results.length}/${pagination.total || "?"}` +
|
|
1344
|
-
(pagination.total > results.length
|
|
384
|
+
(pagination.total > results.length
|
|
385
|
+
? ` (use offset=${(pagination.offset || 0) + results.length} for more)`
|
|
386
|
+
: "");
|
|
1345
387
|
contentItems.push({
|
|
1346
388
|
type: "text",
|
|
1347
389
|
text: header + "\n\n" + formattedResults.join("\n---\n"),
|
|
1348
390
|
});
|
|
1349
|
-
// Add images if requested
|
|
1350
391
|
for (const img of images) {
|
|
1351
392
|
contentItems.push({ type: "text", text: `\n📷 ${img.context}` });
|
|
1352
393
|
contentItems.push({ type: "image", data: img.data, mimeType: "image/png" });
|
|
1353
394
|
}
|
|
1354
395
|
return { content: contentItems };
|
|
1355
396
|
}
|
|
1356
|
-
case "
|
|
1357
|
-
const
|
|
1358
|
-
const
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
397
|
+
case "list-meetings": {
|
|
398
|
+
const params = new URLSearchParams();
|
|
399
|
+
for (const [key, value] of Object.entries(args)) {
|
|
400
|
+
if (value !== null && value !== undefined) {
|
|
401
|
+
params.append(key, String(value));
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
const response = await fetchAPI(`/meetings?${params.toString()}`);
|
|
405
|
+
if (!response.ok)
|
|
406
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
407
|
+
const meetings = await response.json();
|
|
408
|
+
if (!Array.isArray(meetings) || meetings.length === 0) {
|
|
409
|
+
return {
|
|
410
|
+
content: [{ type: "text", text: "No meetings found in the given time range." }],
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
const formatted = meetings.map((m) => {
|
|
414
|
+
const start = m.meeting_start;
|
|
415
|
+
const end = m.meeting_end || "ongoing";
|
|
416
|
+
const app = m.meeting_app;
|
|
417
|
+
const title = m.title ? ` — ${m.title}` : "";
|
|
418
|
+
const attendees = m.attendees ? `\nAttendees: ${m.attendees}` : "";
|
|
419
|
+
return `[${m.detection_source}] ${app}${title}\n ${start} → ${end}${attendees}`;
|
|
420
|
+
});
|
|
421
|
+
return {
|
|
422
|
+
content: [
|
|
423
|
+
{ type: "text", text: `Meetings: ${meetings.length}\n\n${formatted.join("\n---\n")}` },
|
|
424
|
+
],
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
case "activity-summary": {
|
|
428
|
+
const params = new URLSearchParams();
|
|
429
|
+
for (const [key, value] of Object.entries(args)) {
|
|
430
|
+
if (value !== null && value !== undefined) {
|
|
431
|
+
params.append(key, String(value));
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
const response = await fetchAPI(`/activity-summary?${params.toString()}`);
|
|
435
|
+
if (!response.ok)
|
|
436
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
437
|
+
const data = await response.json();
|
|
438
|
+
const appsLines = (data.apps || []).map((a) => {
|
|
439
|
+
const timeSpan = a.first_seen && a.last_seen
|
|
440
|
+
? `, ${a.first_seen.slice(11, 16)}–${a.last_seen.slice(11, 16)} UTC`
|
|
441
|
+
: "";
|
|
442
|
+
return ` ${a.name}: ${a.minutes} min (${a.frame_count} frames${timeSpan})`;
|
|
443
|
+
});
|
|
444
|
+
const speakerLines = (data.audio_summary?.speakers || []).map((s) => ` ${s.name}: ${s.segment_count} segments`);
|
|
445
|
+
const textLines = (data.recent_texts || []).map((t) => ` [${t.app_name}] ${t.text}`);
|
|
446
|
+
const summary = [
|
|
447
|
+
`Activity Summary (${data.time_range?.start} → ${data.time_range?.end})`,
|
|
448
|
+
`Total frames: ${data.total_frames}`,
|
|
449
|
+
"",
|
|
450
|
+
"Apps:",
|
|
451
|
+
...(appsLines.length ? appsLines : [" (none)"]),
|
|
452
|
+
"",
|
|
453
|
+
`Audio: ${data.audio_summary?.segment_count || 0} segments`,
|
|
454
|
+
...(speakerLines.length ? speakerLines : []),
|
|
455
|
+
"",
|
|
456
|
+
"Recent texts:",
|
|
457
|
+
...(textLines.length ? textLines.slice(0, 10) : [" (none)"]),
|
|
458
|
+
].join("\n");
|
|
459
|
+
return { content: [{ type: "text", text: summary }] };
|
|
460
|
+
}
|
|
461
|
+
case "search-elements": {
|
|
462
|
+
const params = new URLSearchParams();
|
|
463
|
+
for (const [key, value] of Object.entries(args)) {
|
|
464
|
+
if (value !== null && value !== undefined) {
|
|
465
|
+
params.append(key, String(value));
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
const response = await fetchAPI(`/elements?${params.toString()}`);
|
|
469
|
+
if (!response.ok)
|
|
470
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
471
|
+
const data = await response.json();
|
|
472
|
+
const elements = data.data || [];
|
|
473
|
+
const pagination = data.pagination || {};
|
|
474
|
+
if (elements.length === 0) {
|
|
1362
475
|
return {
|
|
1363
476
|
content: [
|
|
1364
477
|
{
|
|
1365
478
|
type: "text",
|
|
1366
|
-
text: "
|
|
479
|
+
text: "No elements found. Try: broader search, different role/source, or wider time range.",
|
|
1367
480
|
},
|
|
1368
481
|
],
|
|
1369
482
|
};
|
|
1370
483
|
}
|
|
1371
|
-
|
|
484
|
+
const formatted = elements.map((e) => {
|
|
485
|
+
const boundsStr = e.bounds
|
|
486
|
+
? ` [${e.bounds.left.toFixed(2)},${e.bounds.top.toFixed(2)} ${e.bounds.width.toFixed(2)}x${e.bounds.height.toFixed(2)}]`
|
|
487
|
+
: "";
|
|
488
|
+
return `[${e.source}] ${e.role} (frame:${e.frame_id}, depth:${e.depth})${boundsStr}\n ${e.text || "(no text)"}`;
|
|
489
|
+
});
|
|
490
|
+
const header = `Elements: ${elements.length}/${pagination.total || "?"}` +
|
|
491
|
+
(pagination.total > elements.length
|
|
492
|
+
? ` (use offset=${(pagination.offset || 0) + elements.length} for more)`
|
|
493
|
+
: "");
|
|
494
|
+
return {
|
|
495
|
+
content: [{ type: "text", text: header + "\n\n" + formatted.join("\n---\n") }],
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
case "frame-context": {
|
|
499
|
+
const frameId = args.frame_id;
|
|
500
|
+
if (!frameId) {
|
|
501
|
+
return { content: [{ type: "text", text: "Error: frame_id is required" }] };
|
|
502
|
+
}
|
|
503
|
+
const response = await fetchAPI(`/frames/${frameId}/context`);
|
|
504
|
+
if (!response.ok)
|
|
505
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
506
|
+
const data = await response.json();
|
|
507
|
+
const lines = [`Frame ${data.frame_id} (source: ${data.text_source})`];
|
|
508
|
+
if (data.urls?.length) {
|
|
509
|
+
lines.push("", "URLs:", ...data.urls.map((u) => ` ${u}`));
|
|
510
|
+
}
|
|
511
|
+
if (data.nodes?.length) {
|
|
512
|
+
lines.push("", `Nodes: ${data.nodes.length}`);
|
|
513
|
+
for (const node of data.nodes.slice(0, 50)) {
|
|
514
|
+
const indent = " ".repeat(Math.min(node.depth, 5));
|
|
515
|
+
lines.push(`${indent}[${node.role}] ${node.text}`);
|
|
516
|
+
}
|
|
517
|
+
if (data.nodes.length > 50) {
|
|
518
|
+
lines.push(` ... and ${data.nodes.length - 50} more nodes`);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
if (data.text) {
|
|
522
|
+
const truncated = data.text.length > 2000 ? data.text.substring(0, 2000) + "..." : data.text;
|
|
523
|
+
lines.push("", "Full text:", truncated);
|
|
524
|
+
}
|
|
525
|
+
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
526
|
+
}
|
|
527
|
+
case "export-video": {
|
|
528
|
+
const startTime = args.start_time;
|
|
529
|
+
const endTime = args.end_time;
|
|
530
|
+
const fps = args.fps || 1.0;
|
|
531
|
+
if (!startTime || !endTime) {
|
|
532
|
+
return {
|
|
533
|
+
content: [{ type: "text", text: "Error: start_time and end_time are required" }],
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
// Get frame IDs for the time range
|
|
1372
537
|
const searchParams = new URLSearchParams({
|
|
1373
538
|
content_type: "ocr",
|
|
1374
539
|
start_time: startTime,
|
|
1375
540
|
end_time: endTime,
|
|
1376
|
-
limit: "10000",
|
|
541
|
+
limit: "10000",
|
|
1377
542
|
});
|
|
1378
543
|
const searchResponse = await fetchAPI(`/search?${searchParams.toString()}`);
|
|
1379
544
|
if (!searchResponse.ok) {
|
|
@@ -1386,12 +551,11 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1386
551
|
content: [
|
|
1387
552
|
{
|
|
1388
553
|
type: "text",
|
|
1389
|
-
text: `No screen recordings found between ${startTime} and ${endTime}
|
|
554
|
+
text: `No screen recordings found between ${startTime} and ${endTime}.`,
|
|
1390
555
|
},
|
|
1391
556
|
],
|
|
1392
557
|
};
|
|
1393
558
|
}
|
|
1394
|
-
// Extract unique frame IDs from OCR results
|
|
1395
559
|
const frameIds = [];
|
|
1396
560
|
const seenIds = new Set();
|
|
1397
561
|
for (const result of results) {
|
|
@@ -1405,18 +569,10 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1405
569
|
}
|
|
1406
570
|
if (frameIds.length === 0) {
|
|
1407
571
|
return {
|
|
1408
|
-
content: [
|
|
1409
|
-
{
|
|
1410
|
-
type: "text",
|
|
1411
|
-
text: `Found ${results.length} results but no valid frame IDs. The recordings may be audio-only.`,
|
|
1412
|
-
},
|
|
1413
|
-
],
|
|
572
|
+
content: [{ type: "text", text: "No valid frame IDs found (audio-only?)." }],
|
|
1414
573
|
};
|
|
1415
574
|
}
|
|
1416
|
-
// Sort frame IDs
|
|
1417
575
|
frameIds.sort((a, b) => a - b);
|
|
1418
|
-
// Step 2: Connect to WebSocket and export video
|
|
1419
|
-
// Send frame_ids in message body to avoid URL length limits
|
|
1420
576
|
const wsUrl = `ws://localhost:${port}/frames/export?fps=${fps}`;
|
|
1421
577
|
const exportResult = await new Promise((resolve) => {
|
|
1422
578
|
const ws = new ws_1.WebSocket(wsUrl);
|
|
@@ -1427,9 +583,8 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1427
583
|
ws.close();
|
|
1428
584
|
resolve({ success: false, error: "Export timed out after 5 minutes" });
|
|
1429
585
|
}
|
|
1430
|
-
}, 5 * 60 * 1000);
|
|
586
|
+
}, 5 * 60 * 1000);
|
|
1431
587
|
ws.on("open", () => {
|
|
1432
|
-
// Send frame_ids in message body to avoid URL length limits
|
|
1433
588
|
ws.send(JSON.stringify({ frame_ids: frameIds }));
|
|
1434
589
|
});
|
|
1435
590
|
ws.on("error", (error) => {
|
|
@@ -1450,7 +605,6 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1450
605
|
try {
|
|
1451
606
|
const message = JSON.parse(data.toString());
|
|
1452
607
|
if (message.status === "completed" && message.video_data) {
|
|
1453
|
-
// Save video to temp file
|
|
1454
608
|
const tempDir = os.tmpdir();
|
|
1455
609
|
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
1456
610
|
const filename = `screenpipe_export_${timestamp}.mp4`;
|
|
@@ -1459,11 +613,7 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1459
613
|
resolved = true;
|
|
1460
614
|
clearTimeout(timeout);
|
|
1461
615
|
ws.close();
|
|
1462
|
-
resolve({
|
|
1463
|
-
success: true,
|
|
1464
|
-
filePath,
|
|
1465
|
-
frameCount: frameIds.length,
|
|
1466
|
-
});
|
|
616
|
+
resolve({ success: true, filePath, frameCount: frameIds.length });
|
|
1467
617
|
}
|
|
1468
618
|
else if (message.status === "error") {
|
|
1469
619
|
resolved = true;
|
|
@@ -1471,9 +621,8 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1471
621
|
ws.close();
|
|
1472
622
|
resolve({ success: false, error: message.error || "Export failed" });
|
|
1473
623
|
}
|
|
1474
|
-
// Ignore "extracting" and "encoding" status updates
|
|
1475
624
|
}
|
|
1476
|
-
catch
|
|
625
|
+
catch {
|
|
1477
626
|
// Ignore parse errors for progress messages
|
|
1478
627
|
}
|
|
1479
628
|
});
|
|
@@ -1483,176 +632,94 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1483
632
|
content: [
|
|
1484
633
|
{
|
|
1485
634
|
type: "text",
|
|
1486
|
-
text: `
|
|
1487
|
-
`
|
|
1488
|
-
`Frames: ${exportResult.frameCount}\n` +
|
|
1489
|
-
`Time range: ${startTime} to ${endTime}\n` +
|
|
1490
|
-
`FPS: ${fps}`,
|
|
635
|
+
text: `Video exported: ${exportResult.filePath}\n` +
|
|
636
|
+
`Frames: ${exportResult.frameCount} | ${startTime} → ${endTime} | ${fps} fps`,
|
|
1491
637
|
},
|
|
1492
638
|
],
|
|
1493
639
|
};
|
|
1494
640
|
}
|
|
1495
641
|
else {
|
|
1496
642
|
return {
|
|
1497
|
-
content: [
|
|
1498
|
-
{
|
|
1499
|
-
type: "text",
|
|
1500
|
-
text: `Failed to export video: ${exportResult.error}`,
|
|
1501
|
-
},
|
|
1502
|
-
],
|
|
643
|
+
content: [{ type: "text", text: `Export failed: ${exportResult.error}` }],
|
|
1503
644
|
};
|
|
1504
645
|
}
|
|
1505
646
|
}
|
|
1506
|
-
case "
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
if (
|
|
1510
|
-
|
|
1511
|
-
}
|
|
647
|
+
case "update-memory": {
|
|
648
|
+
if (args.delete && args.id) {
|
|
649
|
+
const response = await fetchAPI(`/memories/${args.id}`, { method: "DELETE" });
|
|
650
|
+
if (!response.ok)
|
|
651
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
652
|
+
return { content: [{ type: "text", text: `Memory ${args.id} deleted.` }] };
|
|
1512
653
|
}
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
654
|
+
if (args.id) {
|
|
655
|
+
const body = {};
|
|
656
|
+
if (args.content !== undefined)
|
|
657
|
+
body.content = args.content;
|
|
658
|
+
if (args.tags !== undefined)
|
|
659
|
+
body.tags = args.tags;
|
|
660
|
+
if (args.importance !== undefined)
|
|
661
|
+
body.importance = args.importance;
|
|
662
|
+
if (args.source_context !== undefined)
|
|
663
|
+
body.source_context = args.source_context;
|
|
664
|
+
const response = await fetchAPI(`/memories/${args.id}`, {
|
|
665
|
+
method: "PUT",
|
|
666
|
+
body: JSON.stringify(body),
|
|
667
|
+
});
|
|
668
|
+
if (!response.ok)
|
|
669
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
670
|
+
const memory = await response.json();
|
|
671
|
+
return {
|
|
672
|
+
content: [{ type: "text", text: `Memory ${memory.id} updated: "${memory.content}"` }],
|
|
673
|
+
};
|
|
1516
674
|
}
|
|
1517
|
-
|
|
1518
|
-
if (!Array.isArray(meetings) || meetings.length === 0) {
|
|
675
|
+
if (!args.content) {
|
|
1519
676
|
return {
|
|
1520
|
-
content: [
|
|
1521
|
-
{
|
|
1522
|
-
type: "text",
|
|
1523
|
-
text: "No meetings found. Make sure screenpipe is running in smart transcription mode.",
|
|
1524
|
-
},
|
|
1525
|
-
],
|
|
677
|
+
content: [{ type: "text", text: "Error: 'content' is required to create a memory" }],
|
|
1526
678
|
};
|
|
1527
679
|
}
|
|
1528
|
-
const
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
680
|
+
const memoryBody = {
|
|
681
|
+
content: args.content,
|
|
682
|
+
source: "mcp",
|
|
683
|
+
tags: args.tags || [],
|
|
684
|
+
importance: args.importance ?? 0.5,
|
|
685
|
+
};
|
|
686
|
+
if (args.source_context)
|
|
687
|
+
memoryBody.source_context = args.source_context;
|
|
688
|
+
const memoryResponse = await fetchAPI("/memories", {
|
|
689
|
+
method: "POST",
|
|
690
|
+
body: JSON.stringify(memoryBody),
|
|
1535
691
|
});
|
|
692
|
+
if (!memoryResponse.ok)
|
|
693
|
+
throw new Error(`HTTP error: ${memoryResponse.status}`);
|
|
694
|
+
const newMemory = await memoryResponse.json();
|
|
1536
695
|
return {
|
|
1537
696
|
content: [
|
|
1538
|
-
{
|
|
1539
|
-
type: "text",
|
|
1540
|
-
text: `Meetings: ${meetings.length}\n\n${formatted.join("\n---\n")}`,
|
|
1541
|
-
},
|
|
697
|
+
{ type: "text", text: `Memory created (id: ${newMemory.id}): "${newMemory.content}"` },
|
|
1542
698
|
],
|
|
1543
699
|
};
|
|
1544
700
|
}
|
|
1545
|
-
case "
|
|
1546
|
-
const
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
const
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
const timeSpan = a.first_seen && a.last_seen
|
|
1560
|
-
? `, ${a.first_seen.slice(11, 16)}–${a.last_seen.slice(11, 16)} UTC`
|
|
1561
|
-
: "";
|
|
1562
|
-
return ` ${a.name}: ${a.minutes} min (${a.frame_count} frames${timeSpan})`;
|
|
1563
|
-
});
|
|
1564
|
-
// Format audio
|
|
1565
|
-
const speakerLines = (data.audio_summary?.speakers || []).map((s) => ` ${s.name}: ${s.segment_count} segments`);
|
|
1566
|
-
// Format recent texts
|
|
1567
|
-
const textLines = (data.recent_texts || []).map((t) => ` [${t.app_name}] ${t.text}`);
|
|
1568
|
-
const summary = [
|
|
1569
|
-
`Activity Summary (${data.time_range?.start} → ${data.time_range?.end})`,
|
|
1570
|
-
`Total frames: ${data.total_frames}`,
|
|
1571
|
-
"",
|
|
1572
|
-
"Apps:",
|
|
1573
|
-
...(appsLines.length ? appsLines : [" (none)"]),
|
|
1574
|
-
"",
|
|
1575
|
-
`Audio: ${data.audio_summary?.segment_count || 0} segments`,
|
|
1576
|
-
...(speakerLines.length ? speakerLines : []),
|
|
1577
|
-
"",
|
|
1578
|
-
"Recent texts:",
|
|
1579
|
-
...(textLines.length ? textLines.slice(0, 10) : [" (none)"]),
|
|
1580
|
-
].join("\n");
|
|
1581
|
-
return { content: [{ type: "text", text: summary }] };
|
|
1582
|
-
}
|
|
1583
|
-
case "search-elements": {
|
|
1584
|
-
const params = new URLSearchParams();
|
|
1585
|
-
for (const [key, value] of Object.entries(args)) {
|
|
1586
|
-
if (value !== null && value !== undefined) {
|
|
1587
|
-
params.append(key, String(value));
|
|
1588
|
-
}
|
|
1589
|
-
}
|
|
1590
|
-
const response = await fetchAPI(`/elements?${params.toString()}`);
|
|
1591
|
-
if (!response.ok) {
|
|
1592
|
-
throw new Error(`HTTP error: ${response.status}`);
|
|
1593
|
-
}
|
|
1594
|
-
const data = await response.json();
|
|
1595
|
-
const elements = data.data || [];
|
|
1596
|
-
const pagination = data.pagination || {};
|
|
1597
|
-
if (elements.length === 0) {
|
|
1598
|
-
return {
|
|
1599
|
-
content: [
|
|
1600
|
-
{
|
|
1601
|
-
type: "text",
|
|
1602
|
-
text: "No elements found. Try: broader search, different role/source, or wider time range.",
|
|
1603
|
-
},
|
|
1604
|
-
],
|
|
1605
|
-
};
|
|
1606
|
-
}
|
|
1607
|
-
const formatted = elements.map((e) => {
|
|
1608
|
-
const boundsStr = e.bounds
|
|
1609
|
-
? ` [${e.bounds.left.toFixed(2)},${e.bounds.top.toFixed(2)} ${e.bounds.width.toFixed(2)}x${e.bounds.height.toFixed(2)}]`
|
|
1610
|
-
: "";
|
|
1611
|
-
return `[${e.source}] ${e.role} (frame:${e.frame_id}, depth:${e.depth})${boundsStr}\n ${e.text || "(no text)"}`;
|
|
701
|
+
case "send-notification": {
|
|
702
|
+
const notifBody = {
|
|
703
|
+
title: args.title,
|
|
704
|
+
pipe_name: args.pipe_name,
|
|
705
|
+
};
|
|
706
|
+
if (args.body)
|
|
707
|
+
notifBody.body = args.body;
|
|
708
|
+
if (args.timeout_secs)
|
|
709
|
+
notifBody.timeout_secs = args.timeout_secs;
|
|
710
|
+
if (args.actions)
|
|
711
|
+
notifBody.actions = args.actions;
|
|
712
|
+
const notifResponse = await fetchAPI("/notify", {
|
|
713
|
+
method: "POST",
|
|
714
|
+
body: JSON.stringify(notifBody),
|
|
1612
715
|
});
|
|
1613
|
-
|
|
1614
|
-
(
|
|
1615
|
-
|
|
1616
|
-
: "");
|
|
716
|
+
if (!notifResponse.ok)
|
|
717
|
+
throw new Error(`HTTP error: ${notifResponse.status}`);
|
|
718
|
+
const notifResult = await notifResponse.json();
|
|
1617
719
|
return {
|
|
1618
|
-
content: [{ type: "text", text:
|
|
720
|
+
content: [{ type: "text", text: `Notification sent (id: ${notifResult.id})` }],
|
|
1619
721
|
};
|
|
1620
722
|
}
|
|
1621
|
-
case "frame-context": {
|
|
1622
|
-
const frameId = args.frame_id;
|
|
1623
|
-
if (!frameId) {
|
|
1624
|
-
return {
|
|
1625
|
-
content: [{ type: "text", text: "Error: frame_id is required" }],
|
|
1626
|
-
};
|
|
1627
|
-
}
|
|
1628
|
-
const response = await fetchAPI(`/frames/${frameId}/context`);
|
|
1629
|
-
if (!response.ok) {
|
|
1630
|
-
throw new Error(`HTTP error: ${response.status}`);
|
|
1631
|
-
}
|
|
1632
|
-
const data = await response.json();
|
|
1633
|
-
const lines = [
|
|
1634
|
-
`Frame ${data.frame_id} (source: ${data.text_source})`,
|
|
1635
|
-
];
|
|
1636
|
-
if (data.urls?.length) {
|
|
1637
|
-
lines.push("", "URLs:", ...data.urls.map((u) => ` ${u}`));
|
|
1638
|
-
}
|
|
1639
|
-
if (data.nodes?.length) {
|
|
1640
|
-
lines.push("", `Nodes: ${data.nodes.length}`);
|
|
1641
|
-
for (const node of data.nodes.slice(0, 50)) {
|
|
1642
|
-
const indent = " ".repeat(Math.min(node.depth, 5));
|
|
1643
|
-
lines.push(`${indent}[${node.role}] ${node.text}`);
|
|
1644
|
-
}
|
|
1645
|
-
if (data.nodes.length > 50) {
|
|
1646
|
-
lines.push(` ... and ${data.nodes.length - 50} more nodes`);
|
|
1647
|
-
}
|
|
1648
|
-
}
|
|
1649
|
-
if (data.text) {
|
|
1650
|
-
// Truncate to avoid massive outputs
|
|
1651
|
-
const truncated = data.text.length > 2000 ? data.text.substring(0, 2000) + "..." : data.text;
|
|
1652
|
-
lines.push("", "Full text:", truncated);
|
|
1653
|
-
}
|
|
1654
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
1655
|
-
}
|
|
1656
723
|
default:
|
|
1657
724
|
throw new Error(`Unknown tool: ${name}`);
|
|
1658
725
|
}
|
|
@@ -1660,12 +727,7 @@ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
|
1660
727
|
catch (error) {
|
|
1661
728
|
const errorMessage = error instanceof Error ? error.message : "Unknown error";
|
|
1662
729
|
return {
|
|
1663
|
-
content: [
|
|
1664
|
-
{
|
|
1665
|
-
type: "text",
|
|
1666
|
-
text: `Error executing ${name}: ${errorMessage}`,
|
|
1667
|
-
},
|
|
1668
|
-
],
|
|
730
|
+
content: [{ type: "text", text: `Error executing ${name}: ${errorMessage}` }],
|
|
1669
731
|
};
|
|
1670
732
|
}
|
|
1671
733
|
});
|