screenpipe-mcp 0.19.1 → 0.19.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -12
- package/bun.lock +1 -3
- package/dist/cli.js +52886 -44
- package/dist/http-server.js +50456 -328
- package/dist/index.js +51266 -2078
- package/package.json +4 -6
- package/scripts/assert-pack-contents.js +60 -28
- package/scripts/build.mjs +34 -0
- package/server.json +2 -2
- package/src/activity-summary-format.test.ts +109 -0
- package/src/activity-summary-format.ts +253 -0
- package/src/activity-summary-tool.test.ts +133 -0
- package/src/activity-summary-tool.ts +79 -0
- package/src/api-base.test.ts +39 -0
- package/src/api-base.ts +24 -0
- package/src/http-server.ts +6 -5
- package/src/index.ts +202 -207
- package/src/pack-contents.test.ts +53 -17
- package/src/self-contained-pack.test.ts +115 -0
- package/src/stdio-startup.test.ts +223 -5
- package/src/team-frame.test.ts +86 -0
- package/src/team-frame.ts +78 -0
- package/src/time-normalization.test.ts +94 -0
- package/src/time-normalization.ts +90 -0
- package/vitest.config.ts +7 -0
- package/dist/cli.d.ts +0 -2
- package/dist/element-format.d.ts +0 -8
- package/dist/element-format.js +0 -18
- package/dist/element-format.test.d.ts +0 -1
- package/dist/element-format.test.js +0 -20
- package/dist/export-video.test.d.ts +0 -1
- package/dist/export-video.test.js +0 -156
- package/dist/http-server.d.ts +0 -65
- package/dist/http-server.test.d.ts +0 -1
- package/dist/http-server.test.js +0 -232
- package/dist/index.d.ts +0 -2
- package/dist/notification-request.d.ts +0 -3
- package/dist/notification-request.js +0 -28
- package/dist/notification-request.test.d.ts +0 -1
- package/dist/notification-request.test.js +0 -47
- package/dist/pack-contents.test.d.ts +0 -1
- package/dist/pack-contents.test.js +0 -119
- package/dist/qualified-value.d.ts +0 -15
- package/dist/qualified-value.js +0 -53
- package/dist/qualified-value.test.d.ts +0 -1
- package/dist/qualified-value.test.js +0 -52
- package/dist/stdio-startup.test.d.ts +0 -1
- package/dist/stdio-startup.test.js +0 -273
- package/dist/team-config.d.ts +0 -19
- package/dist/team-config.js +0 -99
- package/dist/team-config.test.d.ts +0 -1
- package/dist/team-config.test.js +0 -90
- package/dist/telemetry.d.ts +0 -24
- package/dist/telemetry.js +0 -228
- package/dist/telemetry.test.d.ts +0 -1
- package/dist/telemetry.test.js +0 -130
- package/dist/version.d.ts +0 -1
- package/dist/version.js +0 -27
- package/dist/version.test.d.ts +0 -1
- package/dist/version.test.js +0 -144
package/src/index.ts
CHANGED
|
@@ -31,8 +31,16 @@ import {
|
|
|
31
31
|
resolveMcpClient,
|
|
32
32
|
} from "./qualified-value";
|
|
33
33
|
import { discoverTeamApiBase, discoverTeamToken } from "./team-config";
|
|
34
|
+
import { teamFrameContent, teamFramePath } from "./team-frame";
|
|
34
35
|
import { PKG_VERSION } from "./version";
|
|
35
36
|
import { formatForElementPurpose } from "./element-format";
|
|
37
|
+
import { buildActivitySummaryResult } from "./activity-summary-tool";
|
|
38
|
+
import {
|
|
39
|
+
localContextDayStarts,
|
|
40
|
+
normalizeTime,
|
|
41
|
+
normalizeTimeFields,
|
|
42
|
+
} from "./time-normalization";
|
|
43
|
+
import { resolveScreenpipeApiBase } from "./api-base";
|
|
36
44
|
|
|
37
45
|
initMcpTelemetry({ transport: "stdio" });
|
|
38
46
|
|
|
@@ -61,14 +69,11 @@ for (let i = 0; i < args.length; i++) {
|
|
|
61
69
|
// screenpipe (e.g. an agent on a VPS reading a synced copy of your data),
|
|
62
70
|
// not just localhost. Priority:
|
|
63
71
|
// 1. --screenpipe-url / --screenpipe-api-url flag
|
|
64
|
-
// 2.
|
|
65
|
-
// 3.
|
|
66
|
-
// 4.
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
process.env.SCREENPIPE_API_URL ||
|
|
70
|
-
`http://${host}:${port}`
|
|
71
|
-
).replace(/\/+$/, "");
|
|
72
|
+
// 2. SCREENPIPE_LOCAL_API_URL / PORT from the launching desktop instance
|
|
73
|
+
// 3. SCREENPIPE_API_URL env (set by `screenpipe agent setup --api-url`)
|
|
74
|
+
// 4. --screenpipe-host (+ --port) → http://host:port
|
|
75
|
+
// 5. default http://localhost:<port>
|
|
76
|
+
const SCREENPIPE_API = resolveScreenpipeApiBase({ baseOverride, host, port });
|
|
72
77
|
|
|
73
78
|
// Discover the local API key, in priority order:
|
|
74
79
|
//
|
|
@@ -81,14 +86,13 @@ const SCREENPIPE_API = (
|
|
|
81
86
|
// 3. CLI via node-adjacent npx — for dev environments that have node but
|
|
82
87
|
// not the desktop app.
|
|
83
88
|
// 4. CLI via PATH-based npx — last CLI fallback.
|
|
84
|
-
// 5. Direct sqlite3 read of ~/.screenpipe/db.sqlite — plaintext entries
|
|
85
|
-
// only (encrypted entries need the keychain, which only the CLI can
|
|
86
|
-
// reach). Kept as a final last-resort for users who have screenpipe
|
|
87
|
-
// *data* but no working CLI install (rare). Demoted below the CLI
|
|
88
|
-
// paths because it reimplements logic that lives in `auth_key.rs` and
|
|
89
|
-
// can silently drift on storage-format changes.
|
|
90
89
|
//
|
|
91
|
-
//
|
|
90
|
+
// The MCP process never opens Screenpipe's SQLite files. The desktop app and
|
|
91
|
+
// CLI own database access, locking, WAL handling, and secret-store decoding;
|
|
92
|
+
// bypassing those boundaries here can race the recorder or drift from the
|
|
93
|
+
// encrypted storage format.
|
|
94
|
+
//
|
|
95
|
+
// If all 4 miss we log a loud stderr warning so it surfaces in the host's
|
|
92
96
|
// MCP log instead of the user just seeing 403s with no explanation.
|
|
93
97
|
async function discoverApiKey(): Promise<string> {
|
|
94
98
|
const envKey = process.env.SCREENPIPE_LOCAL_API_KEY || process.env.SCREENPIPE_API_KEY;
|
|
@@ -219,51 +223,7 @@ async function discoverApiKey(): Promise<string> {
|
|
|
219
223
|
}
|
|
220
224
|
} catch {}
|
|
221
225
|
|
|
222
|
-
//
|
|
223
|
-
// entries only — encrypted ones live behind the keychain, which the
|
|
224
|
-
// CLI paths above already cover. Used when the user has screenpipe
|
|
225
|
-
// data on disk but no working CLI install.
|
|
226
|
-
const sqliteCandidates: string[] =
|
|
227
|
-
process.platform === "win32"
|
|
228
|
-
? ["sqlite3.exe", "C:\\Windows\\System32\\sqlite3.exe"]
|
|
229
|
-
: process.platform === "darwin"
|
|
230
|
-
? ["sqlite3", "/usr/bin/sqlite3", "/opt/homebrew/bin/sqlite3", "/usr/local/bin/sqlite3"]
|
|
231
|
-
: ["sqlite3", "/usr/bin/sqlite3", "/usr/local/bin/sqlite3"];
|
|
232
|
-
try {
|
|
233
|
-
const dbPath = path.join(home, ".screenpipe", "db.sqlite");
|
|
234
|
-
if (fs.existsSync(dbPath)) {
|
|
235
|
-
let row: string | null = null;
|
|
236
|
-
for (const candidate of sqliteCandidates) {
|
|
237
|
-
if (budgetLeft() <= 0) break;
|
|
238
|
-
try {
|
|
239
|
-
const { stdout } = await execFileAsync(
|
|
240
|
-
candidate,
|
|
241
|
-
[dbPath, "SELECT hex(nonce), value FROM secrets WHERE key = 'api_auth_key';"],
|
|
242
|
-
{ timeout: Math.min(5000, budgetLeft()), encoding: "utf-8" },
|
|
243
|
-
);
|
|
244
|
-
row = String(stdout).trim();
|
|
245
|
-
break;
|
|
246
|
-
} catch {
|
|
247
|
-
// try next candidate
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
if (row) {
|
|
251
|
-
const sepIdx = row.indexOf("|");
|
|
252
|
-
const nonceHex = sepIdx >= 0 ? row.substring(0, sepIdx) : "";
|
|
253
|
-
const value = sepIdx >= 0 ? row.substring(sepIdx + 1) : row;
|
|
254
|
-
const isPlaintext = !nonceHex || /^0+$/.test(nonceHex);
|
|
255
|
-
if (isPlaintext && value) {
|
|
256
|
-
const decoded = Buffer.from(value, "base64").toString("utf-8");
|
|
257
|
-
if (decoded && decoded.startsWith("sp-")) return decoded;
|
|
258
|
-
if (value.startsWith("sp-")) return value;
|
|
259
|
-
}
|
|
260
|
-
// Encrypted — only the CLI paths above can decrypt this; we
|
|
261
|
-
// already tried them.
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
} catch {}
|
|
265
|
-
|
|
266
|
-
// All five paths missed. Log loudly to stderr so the host's MCP
|
|
226
|
+
// All four paths missed. Log loudly to stderr so the host's MCP
|
|
267
227
|
// panel surfaces this instead of the user seeing cryptic 403s from
|
|
268
228
|
// the screenpipe server on every tool call.
|
|
269
229
|
process.stderr.write(
|
|
@@ -272,14 +232,14 @@ async function discoverApiKey(): Promise<string> {
|
|
|
272
232
|
" - env vars (SCREENPIPE_LOCAL_API_KEY / SCREENPIPE_API_KEY) not set",
|
|
273
233
|
" - bundled `bun` from screenpipe.app not found at any known install path",
|
|
274
234
|
" - npx fallback unavailable",
|
|
275
|
-
" - direct sqlite3 read of ~/.screenpipe/db.sqlite failed",
|
|
276
235
|
"Fix: set SCREENPIPE_LOCAL_API_KEY in your MCP launcher's env block,",
|
|
277
|
-
"or install the screenpipe desktop app (https://screenpi.pe)
|
|
236
|
+
"or install the screenpipe desktop app (https://screenpi.pe) so its CLI",
|
|
237
|
+
"can resolve the key without bypassing Screenpipe's database boundary.",
|
|
278
238
|
"",
|
|
279
239
|
].join("\n"),
|
|
280
240
|
);
|
|
281
|
-
// This is a user-side misconfiguration (no key set + no desktop app / CLI
|
|
282
|
-
//
|
|
241
|
+
// This is a user-side misconfiguration (no key set + no desktop app / CLI),
|
|
242
|
+
// not a screenpipe defect — the stderr hint above tells the user
|
|
283
243
|
// how to fix it. Log it as `info` for activation signal, and throttle to one
|
|
284
244
|
// event per machine per day so a respawning MCP host can't escalate it.
|
|
285
245
|
captureMcpMessage("api key discovery failed", "info", {
|
|
@@ -290,7 +250,7 @@ async function discoverApiKey(): Promise<string> {
|
|
|
290
250
|
}
|
|
291
251
|
|
|
292
252
|
// API key is resolved LAZILY, never at module load. `discoverApiKey()` can run
|
|
293
|
-
// several subprocess fallbacks (bundled bun
|
|
253
|
+
// several subprocess fallbacks (bundled bun and npx) that, on a cold cache
|
|
294
254
|
// or restricted PATH, take many seconds. Running that synchronously at module
|
|
295
255
|
// scope used to block the entire module body from finishing — which meant
|
|
296
256
|
// `main()` (and therefore `server.connect()`) was never reached until discovery
|
|
@@ -391,7 +351,7 @@ const TOOLS: Tool[] = [
|
|
|
391
351
|
properties: {
|
|
392
352
|
q: {
|
|
393
353
|
type: "string",
|
|
394
|
-
description: "Full-text search query. Omit to return all content in time range. Avoid for audio — transcriptions are noisy, q filters too aggressively.",
|
|
354
|
+
description: "Full-text search query. Omit to return all content in time range. Avoid for audio — transcriptions are noisy, q filters too aggressively. For an attached activity episode, its generated title/summary are labels, not query terms: use the exact time range and artifact anchors with q omitted.",
|
|
395
355
|
},
|
|
396
356
|
content_type: {
|
|
397
357
|
type: "string",
|
|
@@ -404,11 +364,11 @@ const TOOLS: Tool[] = [
|
|
|
404
364
|
offset: { type: "integer", description: "Pagination offset. Use when results say 'use offset=N for more'.", default: 0 },
|
|
405
365
|
start_time: {
|
|
406
366
|
type: "string",
|
|
407
|
-
description: "Accepted: ISO 8601 ('2024-01-15T10:00:00Z'),
|
|
367
|
+
description: "Accepted: ISO 8601 ('2024-01-15T10:00:00Z'), relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD'). Always provide to avoid scanning entire history.",
|
|
408
368
|
},
|
|
409
369
|
end_time: {
|
|
410
370
|
type: "string",
|
|
411
|
-
description: "ISO 8601
|
|
371
|
+
description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD'). Defaults to now.",
|
|
412
372
|
},
|
|
413
373
|
app_name: { type: "string", description: "Filter by app name (e.g. 'Google Chrome', 'Slack', 'zoom.us'). Case-sensitive." },
|
|
414
374
|
window_name: { type: "string", description: "Filter by window title substring" },
|
|
@@ -441,6 +401,36 @@ const TOOLS: Tool[] = [
|
|
|
441
401
|
},
|
|
442
402
|
},
|
|
443
403
|
},
|
|
404
|
+
{
|
|
405
|
+
name: "synced-devices",
|
|
406
|
+
description:
|
|
407
|
+
"List this signed-in user's Screenpipe devices that have uploaded Data Sync records, including each device name and last sync time. " +
|
|
408
|
+
"USE WHEN: the user asks what devices are available, names another device, or asks a cross-device question and you need the exact device_name filter. " +
|
|
409
|
+
"This never accepts an account or bucket identifier; the local app forwards the signed-in user's identity.",
|
|
410
|
+
annotations: { title: "Synced Devices", readOnlyHint: true, openWorldHint: true, idempotentHint: true },
|
|
411
|
+
inputSchema: { type: "object", properties: {} },
|
|
412
|
+
},
|
|
413
|
+
{
|
|
414
|
+
name: "search-synced-content",
|
|
415
|
+
description:
|
|
416
|
+
"Search Data Sync records from this signed-in user's devices. Results include device name, device ID, and timestamp for attribution. " +
|
|
417
|
+
"USE WHEN: the user asks about another/named device, asks across devices, or local search does not cover the requested machine. " +
|
|
418
|
+
"For the current machine only, use search-content. Start with a narrow time range and limit=10.",
|
|
419
|
+
annotations: { title: "Search Synced Content", readOnlyHint: true, openWorldHint: true, idempotentHint: true },
|
|
420
|
+
inputSchema: {
|
|
421
|
+
type: "object",
|
|
422
|
+
properties: {
|
|
423
|
+
q: { type: "string", description: "Case-insensitive substring query. Omit to return all matching records in the window." },
|
|
424
|
+
device_name: { type: "string", description: "Exact device name from synced-devices." },
|
|
425
|
+
device_id: { type: "string", description: "Exact device ID from synced-devices." },
|
|
426
|
+
app_name: { type: "string", description: "Exact app name, case-insensitive." },
|
|
427
|
+
since: { type: "string", description: "ISO 8601 lower bound." },
|
|
428
|
+
until: { type: "string", description: "ISO 8601 upper bound." },
|
|
429
|
+
since_hours_ago: { type: "number", description: "Alternative relative time window in hours." },
|
|
430
|
+
limit: { type: "integer", description: "Max results (default 50, max 200).", default: 50 },
|
|
431
|
+
},
|
|
432
|
+
},
|
|
433
|
+
},
|
|
444
434
|
{
|
|
445
435
|
name: "list-meetings",
|
|
446
436
|
description:
|
|
@@ -454,8 +444,8 @@ const TOOLS: Tool[] = [
|
|
|
454
444
|
inputSchema: {
|
|
455
445
|
type: "object",
|
|
456
446
|
properties: {
|
|
457
|
-
start_time: { type: "string", description: "ISO 8601
|
|
458
|
-
end_time: { type: "string", description: "ISO 8601
|
|
447
|
+
start_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD'). Omit when searching by q — it filters all history." },
|
|
448
|
+
end_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
459
449
|
q: { type: "string", description: "Case-insensitive substring filter on title, attendees (names/emails), and note. Searches all history." },
|
|
460
450
|
limit: { type: "integer", description: "Max results (default 20)", default: 20 },
|
|
461
451
|
offset: { type: "integer", description: "Pagination offset", default: 0 },
|
|
@@ -465,17 +455,31 @@ const TOOLS: Tool[] = [
|
|
|
465
455
|
{
|
|
466
456
|
name: "activity-summary",
|
|
467
457
|
description:
|
|
468
|
-
"Rich activity overview:
|
|
458
|
+
"Rich activity overview: authoritative active minutes, app/window time, edited document paths, key text, and audio transcriptions, with optional parsed task context when available. " +
|
|
469
459
|
"USE WHEN: any broad question about what the user did — 'what was I doing?', 'how long on X?', 'which apps?', 'recap my morning'. " +
|
|
470
460
|
"This is almost always the right first call for time-range questions — usually sufficient without follow-up searches. " +
|
|
461
|
+
"Use parsed/path evidence to identify tasks, but only active-minute fields for duration; frame and row counts are never time. " +
|
|
471
462
|
"DO NOT USE for: finding a specific keyword (use keyword-search) or a specific UI control (use search-elements).",
|
|
472
463
|
annotations: { title: "Activity Summary", readOnlyHint: true, openWorldHint: false, idempotentHint: true },
|
|
473
464
|
inputSchema: {
|
|
474
465
|
type: "object",
|
|
475
466
|
properties: {
|
|
476
|
-
start_time: { type: "string", description: "ISO 8601
|
|
477
|
-
end_time: { type: "string", description: "ISO 8601
|
|
467
|
+
start_time: { type: "string", description: "ISO 8601, relative (e.g. '3h ago'), or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
468
|
+
end_time: { type: "string", description: "ISO 8601, relative (e.g. 'now'), or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
478
469
|
app_name: { type: "string", description: "Optional app name filter to focus on one app" },
|
|
470
|
+
include_parsed_context: {
|
|
471
|
+
type: "boolean",
|
|
472
|
+
description:
|
|
473
|
+
"Optionally include a bounded parsed-context sample for identifying projects and tasks. Parsed capture is experimental and may be disabled or unsupported. Context only; never use row counts as duration.",
|
|
474
|
+
default: false,
|
|
475
|
+
},
|
|
476
|
+
parsed_context_limit: {
|
|
477
|
+
type: "integer",
|
|
478
|
+
minimum: 1,
|
|
479
|
+
maximum: 20,
|
|
480
|
+
description: "Maximum parsed-context rows (default 10, max 20).",
|
|
481
|
+
default: 10,
|
|
482
|
+
},
|
|
479
483
|
},
|
|
480
484
|
required: ["start_time", "end_time"],
|
|
481
485
|
},
|
|
@@ -498,8 +502,8 @@ const TOOLS: Tool[] = [
|
|
|
498
502
|
description: "Element source. 'accessibility' is preferred (OS-native tree). 'ocr' for apps without a11y.",
|
|
499
503
|
},
|
|
500
504
|
role: { type: "string", description: "Element role filter (e.g. 'AXButton', 'AXLink', 'AXTextField')" },
|
|
501
|
-
start_time: { type: "string", description: "ISO 8601
|
|
502
|
-
end_time: { type: "string", description: "ISO 8601
|
|
505
|
+
start_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
506
|
+
end_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
503
507
|
app_name: { type: "string", description: "Filter by app name" },
|
|
504
508
|
purpose: {
|
|
505
509
|
type: "string",
|
|
@@ -537,8 +541,8 @@ const TOOLS: Tool[] = [
|
|
|
537
541
|
inputSchema: {
|
|
538
542
|
type: "object",
|
|
539
543
|
properties: {
|
|
540
|
-
start_time: { type: "string", description:
|
|
541
|
-
end_time: { type: "string", description:
|
|
544
|
+
start_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
545
|
+
end_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
542
546
|
output_path: {
|
|
543
547
|
type: "string",
|
|
544
548
|
description:
|
|
@@ -626,7 +630,7 @@ const TOOLS: Tool[] = [
|
|
|
626
630
|
priority: {
|
|
627
631
|
type: "string",
|
|
628
632
|
enum: ["high", "normal", "low"],
|
|
629
|
-
description: "High
|
|
633
|
+
description: "Every priority appears in the top-right panel. High also appears in the focused Priority view, normal (default) stays in All, and low is toast-only by default.",
|
|
630
634
|
default: "normal",
|
|
631
635
|
},
|
|
632
636
|
timeout_secs: { type: "integer", description: "Auto-dismiss after N seconds (default 20). Use 0 for persistent.", default: 20 },
|
|
@@ -823,8 +827,8 @@ const TOOLS: Tool[] = [
|
|
|
823
827
|
type: "object",
|
|
824
828
|
properties: {
|
|
825
829
|
q: { type: "string", description: "Keyword query (FTS5 syntax: quoted phrases, AND/OR, prefix*)" },
|
|
826
|
-
start_time: { type: "string", description: "ISO 8601
|
|
827
|
-
end_time: { type: "string", description: "
|
|
830
|
+
start_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
831
|
+
end_time: { type: "string", description: "ISO 8601, relative time, or local calendar ('today', 'yesterday', 'tomorrow', 'YYYY-MM-DD')" },
|
|
828
832
|
app_name: { type: "string", description: "Filter by exact app name (case-sensitive, e.g. 'Google Chrome')" },
|
|
829
833
|
limit: { type: "integer", description: "Max results (default 20)", default: 20 },
|
|
830
834
|
offset: { type: "integer", description: "Pagination offset", default: 0 },
|
|
@@ -1033,6 +1037,36 @@ const TEAM_TOOLS: Tool[] = [
|
|
|
1033
1037
|
},
|
|
1034
1038
|
},
|
|
1035
1039
|
},
|
|
1040
|
+
{
|
|
1041
|
+
name: "team-frame",
|
|
1042
|
+
description:
|
|
1043
|
+
"Read one PII-redacted team screenshot. Use device_id and frame_id from " +
|
|
1044
|
+
"team-search or team-records. Returns actual JPEG image content when the " +
|
|
1045
|
+
"device has uploaded it, or an explicit unavailable result. Never claim " +
|
|
1046
|
+
"to have seen a frame unless this tool returns image content. " +
|
|
1047
|
+
"Auth: enterprise admin token with read:records.",
|
|
1048
|
+
annotations: { title: "Team Frame", readOnlyHint: true, openWorldHint: true, idempotentHint: true },
|
|
1049
|
+
inputSchema: {
|
|
1050
|
+
type: "object",
|
|
1051
|
+
properties: {
|
|
1052
|
+
device_id: {
|
|
1053
|
+
type: "string",
|
|
1054
|
+
minLength: 1,
|
|
1055
|
+
maxLength: 64,
|
|
1056
|
+
pattern: "^[A-Za-z0-9_-]+$",
|
|
1057
|
+
description: "Device ID from team-search or team-devices.",
|
|
1058
|
+
},
|
|
1059
|
+
frame_id: {
|
|
1060
|
+
type: "integer",
|
|
1061
|
+
minimum: 1,
|
|
1062
|
+
maximum: 999999999999999,
|
|
1063
|
+
description: "Frame ID from team-search or team-records.",
|
|
1064
|
+
},
|
|
1065
|
+
},
|
|
1066
|
+
required: ["device_id", "frame_id"],
|
|
1067
|
+
additionalProperties: false,
|
|
1068
|
+
},
|
|
1069
|
+
},
|
|
1036
1070
|
];
|
|
1037
1071
|
|
|
1038
1072
|
// Pipe-output kinds map to /workflows/generated, raw kinds map to /records.
|
|
@@ -1080,6 +1114,7 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
|
|
|
1080
1114
|
if (uri === "screenpipe://context") {
|
|
1081
1115
|
const now = new Date();
|
|
1082
1116
|
const ms = now.getTime();
|
|
1117
|
+
const dayStarts = localContextDayStarts(now);
|
|
1083
1118
|
return {
|
|
1084
1119
|
contents: [
|
|
1085
1120
|
{
|
|
@@ -1099,8 +1134,7 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
|
|
|
1099
1134
|
now: now.toISOString(),
|
|
1100
1135
|
one_hour_ago: new Date(ms - 60 * 60 * 1000).toISOString(),
|
|
1101
1136
|
three_hours_ago: new Date(ms - 3 * 60 * 60 * 1000).toISOString(),
|
|
1102
|
-
|
|
1103
|
-
yesterday_start: `${new Date(ms - 24 * 60 * 60 * 1000).toISOString().split("T")[0]}T00:00:00Z`,
|
|
1137
|
+
...dayStarts,
|
|
1104
1138
|
one_week_ago: new Date(ms - 7 * 24 * 60 * 60 * 1000).toISOString(),
|
|
1105
1139
|
},
|
|
1106
1140
|
},
|
|
@@ -1129,6 +1163,10 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
|
|
|
1129
1163
|
| 3 | search-elements | Need UI structure: buttons, links, form fields |
|
|
1130
1164
|
| 4 | frame-context | Need full detail for a specific moment (use frame_id from step 2) |
|
|
1131
1165
|
|
|
1166
|
+
For another/named device or an across-device question, use synced-devices to
|
|
1167
|
+
resolve the device name, then search-synced-content. Keep search-content for the
|
|
1168
|
+
current machine. Synced results must be attributed with their device and timestamp.
|
|
1169
|
+
|
|
1132
1170
|
## Search Strategy
|
|
1133
1171
|
|
|
1134
1172
|
- **Always provide start_time** — without it, search scans the entire history
|
|
@@ -1144,6 +1182,8 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
|
|
|
1144
1182
|
- "What did I discuss in my meeting?" → list-meetings to find it, then get-meeting with include_transcript=true
|
|
1145
1183
|
- "When did I last talk to <person>?" → list-meetings with q=<name or email>, NO start_time (q searches all history)
|
|
1146
1184
|
- "Find when I was on Twitter" → search-content with app_name='Arc' (or the browser name), q='twitter'
|
|
1185
|
+
- "What was I doing on my MacBook this morning?" → synced-devices, then search-synced-content with device_name='MacBook' and the requested time window
|
|
1186
|
+
- "Find this across my devices" → search-synced-content with the requested time window and no device filter
|
|
1147
1187
|
- "Remember that I prefer X" → update-memory with content describing the preference
|
|
1148
1188
|
- "What do you remember about X?" → search-content with content_type='memory', q='X'
|
|
1149
1189
|
- "Automate X every day / on a schedule" → read the screenpipe://guide/pipes resource, then create-pipe (a scheduled AI automation)
|
|
@@ -1153,6 +1193,7 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
|
|
|
1153
1193
|
When referencing specific moments in results, create clickable links:
|
|
1154
1194
|
- Frame: [10:30 AM — Chrome](screenpipe://frame/{frame_id}) — use frame_id from search results
|
|
1155
1195
|
- Timeline: [meeting at 3pm](screenpipe://timeline?timestamp=2024-01-15T15:00:00Z) — use exact timestamp from results
|
|
1196
|
+
- Chat: [crm](screenpipe://chat/{conversationId}) — use a real conversation id
|
|
1156
1197
|
Never fabricate IDs or timestamps — only use values from actual results.
|
|
1157
1198
|
`,
|
|
1158
1199
|
},
|
|
@@ -1282,6 +1323,7 @@ async function fetchAPI(
|
|
|
1282
1323
|
headers: {
|
|
1283
1324
|
"Content-Type": "application/json",
|
|
1284
1325
|
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
1326
|
+
"x-screenpipe-client": "mcp",
|
|
1285
1327
|
...options.headers,
|
|
1286
1328
|
},
|
|
1287
1329
|
});
|
|
@@ -1318,42 +1360,6 @@ const qualifiedValue = createMcpQualifiedValueReporter((payload) =>
|
|
|
1318
1360
|
),
|
|
1319
1361
|
);
|
|
1320
1362
|
|
|
1321
|
-
// Server's deserialize_flexible_datetime accepts ISO 8601 + "Nh ago" / "Nd ago"
|
|
1322
|
-
// / "Nw ago" / "now". Models also try "yesterday", "today", and bare dates
|
|
1323
|
-
// ("2026-05-17") — normalize those here so the request doesn't 400.
|
|
1324
|
-
function normalizeTime(input: string | undefined): string | undefined {
|
|
1325
|
-
if (!input) return input;
|
|
1326
|
-
const s = input.trim();
|
|
1327
|
-
if (!s) return input;
|
|
1328
|
-
const lower = s.toLowerCase();
|
|
1329
|
-
if (lower === "yesterday") return "1d ago";
|
|
1330
|
-
if (lower === "today") {
|
|
1331
|
-
return `${new Date().toISOString().split("T")[0]}T00:00:00Z`;
|
|
1332
|
-
}
|
|
1333
|
-
if (lower === "tomorrow") {
|
|
1334
|
-
const t = new Date();
|
|
1335
|
-
t.setUTCDate(t.getUTCDate() + 1);
|
|
1336
|
-
return `${t.toISOString().split("T")[0]}T00:00:00Z`;
|
|
1337
|
-
}
|
|
1338
|
-
// Bare YYYY-MM-DD → start of day UTC
|
|
1339
|
-
if (/^\d{4}-\d{2}-\d{2}$/.test(s)) return `${s}T00:00:00Z`;
|
|
1340
|
-
return s;
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
// Apply normalizeTime to start_time/end_time fields in an args object.
|
|
1344
|
-
// Returns a new object — does not mutate the input.
|
|
1345
|
-
function normalizeTimeFields(
|
|
1346
|
-
args: Record<string, unknown>,
|
|
1347
|
-
): Record<string, unknown> {
|
|
1348
|
-
const out = { ...args };
|
|
1349
|
-
for (const k of ["start_time", "end_time"] as const) {
|
|
1350
|
-
if (typeof out[k] === "string") {
|
|
1351
|
-
out[k] = normalizeTime(out[k] as string);
|
|
1352
|
-
}
|
|
1353
|
-
}
|
|
1354
|
-
return out;
|
|
1355
|
-
}
|
|
1356
|
-
|
|
1357
1363
|
// Zone label for a timestamp's HH:MM slice. The server serializes timestamps in
|
|
1358
1364
|
// its LOCAL timezone (e.g. "...T09:03:44+05:30"), so the HH:MM is already local —
|
|
1359
1365
|
// derive the label from the string's own offset instead of hardcoding "UTC"
|
|
@@ -1533,6 +1539,69 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
1533
1539
|
return { content: [{ type: "text", text: trimmed || "(no logs yet)" }] };
|
|
1534
1540
|
}
|
|
1535
1541
|
|
|
1542
|
+
case "synced-devices": {
|
|
1543
|
+
const response = await callAPI("/data-sync/devices");
|
|
1544
|
+
const data = await response.json();
|
|
1545
|
+
const devices = Array.isArray(data.devices) ? data.devices : [];
|
|
1546
|
+
if (devices.length === 0) {
|
|
1547
|
+
return {
|
|
1548
|
+
content: [{
|
|
1549
|
+
type: "text",
|
|
1550
|
+
text: data.enabled === false
|
|
1551
|
+
? "Data Sync is off and no synced devices are available. Enable Data Sync in Screenpipe settings on the devices you want to query."
|
|
1552
|
+
: "No synced devices are available yet. Enable Data Sync and let a device complete its first upload.",
|
|
1553
|
+
}],
|
|
1554
|
+
};
|
|
1555
|
+
}
|
|
1556
|
+
return {
|
|
1557
|
+
content: [{
|
|
1558
|
+
type: "text",
|
|
1559
|
+
text: devices
|
|
1560
|
+
.map((device: any) =>
|
|
1561
|
+
`${device.device_name} (${device.device_id}) — last synced ${device.last_synced_at}` +
|
|
1562
|
+
`${device.platform ? ` — ${device.platform}` : ""}`
|
|
1563
|
+
)
|
|
1564
|
+
.join("\n"),
|
|
1565
|
+
}],
|
|
1566
|
+
};
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
case "search-synced-content": {
|
|
1570
|
+
const params = new URLSearchParams();
|
|
1571
|
+
for (const key of ["q", "device_name", "device_id", "app_name", "since", "until", "since_hours_ago", "limit"]) {
|
|
1572
|
+
const value = args[key];
|
|
1573
|
+
if (value !== null && value !== undefined && value !== "") {
|
|
1574
|
+
params.set(key, String(value));
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
const response = await callAPI(`/data-sync/search?${params.toString()}`);
|
|
1578
|
+
const data = await response.json();
|
|
1579
|
+
const results = Array.isArray(data.results) ? data.results : [];
|
|
1580
|
+
if (results.length === 0) {
|
|
1581
|
+
return {
|
|
1582
|
+
content: [{
|
|
1583
|
+
type: "text",
|
|
1584
|
+
text: data.enabled === false
|
|
1585
|
+
? "No matching synced records. Data Sync is currently off, so no new records are uploading."
|
|
1586
|
+
: "No matching synced records. Try a wider time range, confirm the device with synced-devices, or use a broader query.",
|
|
1587
|
+
}],
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1590
|
+
const prefix = data.truncated
|
|
1591
|
+
? "Results are truncated; narrow the device, time range, or query.\n\n"
|
|
1592
|
+
: "";
|
|
1593
|
+
return {
|
|
1594
|
+
content: [{
|
|
1595
|
+
type: "text",
|
|
1596
|
+
text: prefix + results.map((record: any) => {
|
|
1597
|
+
const content = record.text || record.transcription || record.content || "";
|
|
1598
|
+
return `[${record.device || record.device_id || "unknown device"}] ${record.t || "unknown time"}` +
|
|
1599
|
+
`${record.app ? ` — ${record.app}` : ""}\n${truncateMiddle(String(content), DEFAULT_SEARCH_CONTENT_TRUNCATE)}`;
|
|
1600
|
+
}).join("\n\n"),
|
|
1601
|
+
}],
|
|
1602
|
+
};
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1536
1605
|
case "search-content": {
|
|
1537
1606
|
const includeFrames = args.include_frames === true;
|
|
1538
1607
|
const normalized = normalizeTimeFields(args);
|
|
@@ -1713,94 +1782,11 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
1713
1782
|
}
|
|
1714
1783
|
|
|
1715
1784
|
case "activity-summary": {
|
|
1716
|
-
const
|
|
1717
|
-
|
|
1718
|
-
for (const [key, value] of Object.entries(normalized)) {
|
|
1719
|
-
if (value !== null && value !== undefined) {
|
|
1720
|
-
params.append(key, String(value));
|
|
1721
|
-
}
|
|
1722
|
-
}
|
|
1723
|
-
|
|
1724
|
-
const response = await callAPI(`/activity-summary?${params.toString()}`);
|
|
1725
|
-
|
|
1726
|
-
const data = await response.json();
|
|
1727
|
-
|
|
1728
|
-
if (
|
|
1729
|
-
(data.total_frames ?? 0) > 0 ||
|
|
1730
|
-
(data.audio_summary?.segment_count ?? 0) > 0 ||
|
|
1731
|
-
(data.apps?.length ?? 0) > 0
|
|
1732
|
-
) {
|
|
1785
|
+
const result = await buildActivitySummaryResult(args, callAPI);
|
|
1786
|
+
if (result.hasArtifact) {
|
|
1733
1787
|
qualifiedValue.artifactResult();
|
|
1734
1788
|
}
|
|
1735
|
-
|
|
1736
|
-
const appsLines = (data.apps || []).map(
|
|
1737
|
-
(a: {
|
|
1738
|
-
name: string;
|
|
1739
|
-
frame_count: number;
|
|
1740
|
-
minutes: number;
|
|
1741
|
-
first_seen?: string;
|
|
1742
|
-
last_seen?: string;
|
|
1743
|
-
}) => {
|
|
1744
|
-
const timeSpan =
|
|
1745
|
-
a.first_seen && a.last_seen
|
|
1746
|
-
? `, ${a.first_seen.slice(11, 16)}–${a.last_seen.slice(11, 16)}${zoneSuffix(a.first_seen)}`
|
|
1747
|
-
: "";
|
|
1748
|
-
return ` ${a.name}: ${a.minutes} min (${a.frame_count} frames${timeSpan})`;
|
|
1749
|
-
}
|
|
1750
|
-
);
|
|
1751
|
-
|
|
1752
|
-
// Window/tab activity — what pages/documents were open
|
|
1753
|
-
const windowLines = (data.windows || []).map(
|
|
1754
|
-
(w: {
|
|
1755
|
-
app_name: string;
|
|
1756
|
-
window_name: string;
|
|
1757
|
-
browser_url: string;
|
|
1758
|
-
minutes: number;
|
|
1759
|
-
frame_count: number;
|
|
1760
|
-
}) => {
|
|
1761
|
-
const url = w.browser_url ? ` (${w.browser_url})` : "";
|
|
1762
|
-
return ` [${w.app_name}] ${w.window_name}${url} — ${w.minutes} min`;
|
|
1763
|
-
}
|
|
1764
|
-
);
|
|
1765
|
-
|
|
1766
|
-
const speakerLines = (data.audio_summary?.speakers || []).map(
|
|
1767
|
-
(s: { name: string; segment_count: number }) =>
|
|
1768
|
-
` ${s.name}: ${s.segment_count} segments`
|
|
1769
|
-
);
|
|
1770
|
-
|
|
1771
|
-
// Actual audio transcriptions (not just counts)
|
|
1772
|
-
const transcriptLines = (data.audio_summary?.top_transcriptions || []).map(
|
|
1773
|
-
(t: { transcription: string; speaker: string; device: string; timestamp: string }) =>
|
|
1774
|
-
` [${t.speaker}, ${t.timestamp.slice(11, 19)}] ${t.transcription}`
|
|
1775
|
-
);
|
|
1776
|
-
|
|
1777
|
-
// Key text content sampled across the time range
|
|
1778
|
-
const textLines = (data.key_texts || data.recent_texts || []).map(
|
|
1779
|
-
(t: { text: string; app_name: string; window_name?: string; timestamp: string }) => {
|
|
1780
|
-
const win = t.window_name ? ` | ${t.window_name}` : "";
|
|
1781
|
-
return ` [${t.app_name}${win}, ${t.timestamp.slice(11, 19)}] ${t.text}`;
|
|
1782
|
-
}
|
|
1783
|
-
);
|
|
1784
|
-
|
|
1785
|
-
const summary = [
|
|
1786
|
-
`Activity Summary (${data.time_range?.start} → ${data.time_range?.end})`,
|
|
1787
|
-
`Total frames: ${data.total_frames}`,
|
|
1788
|
-
"",
|
|
1789
|
-
"Apps:",
|
|
1790
|
-
...(appsLines.length ? appsLines : [" (none)"]),
|
|
1791
|
-
"",
|
|
1792
|
-
"Windows & Tabs:",
|
|
1793
|
-
...(windowLines.length ? windowLines.slice(0, 20) : [" (none)"]),
|
|
1794
|
-
"",
|
|
1795
|
-
`Audio: ${data.audio_summary?.segment_count || 0} segments`,
|
|
1796
|
-
...(speakerLines.length ? speakerLines : []),
|
|
1797
|
-
...(transcriptLines.length ? ["", "Audio transcriptions:", ...transcriptLines.slice(0, 15)] : []),
|
|
1798
|
-
"",
|
|
1799
|
-
"Key content (sampled across time range):",
|
|
1800
|
-
...(textLines.length ? textLines.slice(0, 20) : [" (none)"]),
|
|
1801
|
-
].join("\n");
|
|
1802
|
-
|
|
1803
|
-
return { content: [{ type: "text", text: summary }] };
|
|
1789
|
+
return { content: [{ type: "text", text: result.text }] };
|
|
1804
1790
|
}
|
|
1805
1791
|
|
|
1806
1792
|
case "search-elements": {
|
|
@@ -1879,8 +1865,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
1879
1865
|
}
|
|
1880
1866
|
|
|
1881
1867
|
case "export-video": {
|
|
1882
|
-
const
|
|
1883
|
-
const
|
|
1868
|
+
const now = new Date();
|
|
1869
|
+
const startTime = normalizeTime(args.start_time as string, now);
|
|
1870
|
+
const endTime = normalizeTime(args.end_time as string, now);
|
|
1884
1871
|
|
|
1885
1872
|
if (!startTime || !endTime) {
|
|
1886
1873
|
return {
|
|
@@ -2384,7 +2371,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
2384
2371
|
// ---------------------------------------------------------------------
|
|
2385
2372
|
case "team-search":
|
|
2386
2373
|
case "team-devices":
|
|
2387
|
-
case "team-records":
|
|
2374
|
+
case "team-records":
|
|
2375
|
+
case "team-frame": {
|
|
2388
2376
|
if (!TEAM_TOKEN) {
|
|
2389
2377
|
return {
|
|
2390
2378
|
content: [
|
|
@@ -2405,6 +2393,13 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
2405
2393
|
],
|
|
2406
2394
|
};
|
|
2407
2395
|
}
|
|
2396
|
+
if (name === "team-frame") {
|
|
2397
|
+
const deviceId = args.device_id;
|
|
2398
|
+
const frameId = args.frame_id;
|
|
2399
|
+
const path = teamFramePath(deviceId, frameId);
|
|
2400
|
+
const response = await fetchTeam(path);
|
|
2401
|
+
return teamFrameContent(response, deviceId as string, frameId as number);
|
|
2402
|
+
}
|
|
2408
2403
|
// Map MCP tool name → /api/enterprise/v1 path. team-records also
|
|
2409
2404
|
// routes synthesized pipe outputs (kind=sop|skill|...) to the
|
|
2410
2405
|
// workflows endpoint so callers see one tool surface for "give me
|