screenpipe-mcp 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http-server.d.ts +8 -0
- package/dist/http-server.js +203 -0
- package/dist/index.js +36 -0
- package/package.json +3 -1
- package/src/http-server.ts +243 -0
- package/src/index.ts +36 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* HTTP Server for Screenpipe MCP
|
|
5
|
+
*
|
|
6
|
+
* This allows web apps to call MCP tools over HTTP instead of stdio.
|
|
7
|
+
* Run with: npx ts-node src/http-server.ts --port 3031
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
const http_1 = require("http");
|
|
11
|
+
const index_js_1 = require("@modelcontextprotocol/sdk/server/index.js");
|
|
12
|
+
const streamableHttp_js_1 = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
|
|
13
|
+
const types_js_1 = require("@modelcontextprotocol/sdk/types.js");
|
|
14
|
+
// Parse command line arguments
|
|
15
|
+
const args = process.argv.slice(2);
|
|
16
|
+
let mcpPort = 3031;
|
|
17
|
+
let screenpipePort = 3030;
|
|
18
|
+
for (let i = 0; i < args.length; i++) {
|
|
19
|
+
if (args[i] === "--port" && args[i + 1]) {
|
|
20
|
+
mcpPort = parseInt(args[i + 1], 10);
|
|
21
|
+
}
|
|
22
|
+
if (args[i] === "--screenpipe-port" && args[i + 1]) {
|
|
23
|
+
screenpipePort = parseInt(args[i + 1], 10);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const SCREENPIPE_API = `http://localhost:${screenpipePort}`;
|
|
27
|
+
// Tool definitions
|
|
28
|
+
const TOOLS = [
|
|
29
|
+
{
|
|
30
|
+
name: "search_content",
|
|
31
|
+
description: "Search screenpipe's recorded content: screen text (OCR), audio transcriptions, and UI elements. " +
|
|
32
|
+
"Returns timestamped results with app context. " +
|
|
33
|
+
"Call with no parameters to get recent activity.",
|
|
34
|
+
inputSchema: {
|
|
35
|
+
type: "object",
|
|
36
|
+
properties: {
|
|
37
|
+
q: {
|
|
38
|
+
type: "string",
|
|
39
|
+
description: "Search query. Optional - omit to return all recent content.",
|
|
40
|
+
},
|
|
41
|
+
content_type: {
|
|
42
|
+
type: "string",
|
|
43
|
+
enum: ["all", "ocr", "audio", "ui"],
|
|
44
|
+
description: "Content type filter. Default: 'all'",
|
|
45
|
+
},
|
|
46
|
+
limit: {
|
|
47
|
+
type: "integer",
|
|
48
|
+
description: "Max results. Default: 10",
|
|
49
|
+
},
|
|
50
|
+
offset: {
|
|
51
|
+
type: "integer",
|
|
52
|
+
description: "Skip N results for pagination. Default: 0",
|
|
53
|
+
},
|
|
54
|
+
start_time: {
|
|
55
|
+
type: "string",
|
|
56
|
+
description: "ISO 8601 UTC start time (e.g., 2024-01-15T10:00:00Z)",
|
|
57
|
+
},
|
|
58
|
+
end_time: {
|
|
59
|
+
type: "string",
|
|
60
|
+
description: "ISO 8601 UTC end time (e.g., 2024-01-15T18:00:00Z)",
|
|
61
|
+
},
|
|
62
|
+
app_name: {
|
|
63
|
+
type: "string",
|
|
64
|
+
description: "Filter by app (e.g., 'Google Chrome', 'Slack', 'zoom.us')",
|
|
65
|
+
},
|
|
66
|
+
window_name: {
|
|
67
|
+
type: "string",
|
|
68
|
+
description: "Filter by window title",
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
];
|
|
74
|
+
// Helper function to make HTTP requests
|
|
75
|
+
async function fetchAPI(endpoint, options = {}) {
|
|
76
|
+
const url = `${SCREENPIPE_API}${endpoint}`;
|
|
77
|
+
return fetch(url, {
|
|
78
|
+
...options,
|
|
79
|
+
headers: {
|
|
80
|
+
"Content-Type": "application/json",
|
|
81
|
+
...options.headers,
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
// Create MCP server
|
|
86
|
+
const server = new index_js_1.Server({
|
|
87
|
+
name: "screenpipe-http",
|
|
88
|
+
version: "0.1.0",
|
|
89
|
+
}, {
|
|
90
|
+
capabilities: {
|
|
91
|
+
tools: {},
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
// List tools handler
|
|
95
|
+
server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => {
|
|
96
|
+
return { tools: TOOLS };
|
|
97
|
+
});
|
|
98
|
+
// Call tool handler
|
|
99
|
+
server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
100
|
+
const { name, arguments: args } = request.params;
|
|
101
|
+
if (!args) {
|
|
102
|
+
throw new Error("Missing arguments");
|
|
103
|
+
}
|
|
104
|
+
if (name === "search_content") {
|
|
105
|
+
const params = new URLSearchParams();
|
|
106
|
+
for (const [key, value] of Object.entries(args)) {
|
|
107
|
+
if (value !== null && value !== undefined) {
|
|
108
|
+
params.append(key, String(value));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const response = await fetchAPI(`/search?${params.toString()}`);
|
|
112
|
+
if (!response.ok) {
|
|
113
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
114
|
+
}
|
|
115
|
+
const data = await response.json();
|
|
116
|
+
const results = data.data || [];
|
|
117
|
+
const pagination = data.pagination || {};
|
|
118
|
+
if (results.length === 0) {
|
|
119
|
+
return {
|
|
120
|
+
content: [
|
|
121
|
+
{
|
|
122
|
+
type: "text",
|
|
123
|
+
text: "No results found. Try: broader search terms, different content_type, or wider time range.",
|
|
124
|
+
},
|
|
125
|
+
],
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const formattedResults = [];
|
|
129
|
+
for (const result of results) {
|
|
130
|
+
const content = result.content;
|
|
131
|
+
if (!content)
|
|
132
|
+
continue;
|
|
133
|
+
if (result.type === "OCR") {
|
|
134
|
+
formattedResults.push(`[OCR] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
|
|
135
|
+
`${content.timestamp || ""}\n` +
|
|
136
|
+
`${content.text || ""}`);
|
|
137
|
+
}
|
|
138
|
+
else if (result.type === "Audio") {
|
|
139
|
+
formattedResults.push(`[Audio] ${content.device_name || "?"}\n` +
|
|
140
|
+
`${content.timestamp || ""}\n` +
|
|
141
|
+
`${content.transcription || ""}`);
|
|
142
|
+
}
|
|
143
|
+
else if (result.type === "UI") {
|
|
144
|
+
formattedResults.push(`[UI] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
|
|
145
|
+
`${content.timestamp || ""}\n` +
|
|
146
|
+
`${content.text || ""}`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const header = `Results: ${results.length}/${pagination.total || "?"}` +
|
|
150
|
+
(pagination.total > results.length ? ` (use offset=${(pagination.offset || 0) + results.length} for more)` : "");
|
|
151
|
+
return {
|
|
152
|
+
content: [
|
|
153
|
+
{
|
|
154
|
+
type: "text",
|
|
155
|
+
text: header + "\n\n" + formattedResults.join("\n---\n"),
|
|
156
|
+
},
|
|
157
|
+
],
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
throw new Error(`Unknown tool: ${name}`);
|
|
161
|
+
});
|
|
162
|
+
// Create HTTP server with MCP transport
|
|
163
|
+
const transports = new Map();
|
|
164
|
+
const httpServer = (0, http_1.createServer)(async (req, res) => {
|
|
165
|
+
// CORS headers
|
|
166
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
167
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
168
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization, mcp-session-id");
|
|
169
|
+
if (req.method === "OPTIONS") {
|
|
170
|
+
res.writeHead(204);
|
|
171
|
+
res.end();
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
// Health check
|
|
175
|
+
if (req.url === "/health") {
|
|
176
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
177
|
+
res.end(JSON.stringify({ status: "ok" }));
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
// MCP endpoint
|
|
181
|
+
if (req.url === "/mcp" || req.url?.startsWith("/mcp?")) {
|
|
182
|
+
const sessionId = req.headers["mcp-session-id"];
|
|
183
|
+
let transport = sessionId ? transports.get(sessionId) : undefined;
|
|
184
|
+
if (!transport) {
|
|
185
|
+
transport = new streamableHttp_js_1.StreamableHTTPServerTransport({
|
|
186
|
+
sessionIdGenerator: () => crypto.randomUUID(),
|
|
187
|
+
});
|
|
188
|
+
await server.connect(transport);
|
|
189
|
+
if (transport.sessionId) {
|
|
190
|
+
transports.set(transport.sessionId, transport);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
await transport.handleRequest(req, res);
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
197
|
+
res.end(JSON.stringify({ error: "Not found" }));
|
|
198
|
+
});
|
|
199
|
+
httpServer.listen(mcpPort, () => {
|
|
200
|
+
console.log(`Screenpipe MCP HTTP server running on http://localhost:${mcpPort}`);
|
|
201
|
+
console.log(`MCP endpoint: http://localhost:${mcpPort}/mcp`);
|
|
202
|
+
console.log(`Health check: http://localhost:${mcpPort}/health`);
|
|
203
|
+
});
|
package/dist/index.js
CHANGED
|
@@ -87,6 +87,10 @@ const BASE_TOOLS = [
|
|
|
87
87
|
"Returns timestamped results with app context. " +
|
|
88
88
|
"Call with no parameters to get recent activity. " +
|
|
89
89
|
"Use the 'screenpipe://context' resource for current time when building time-based queries.",
|
|
90
|
+
annotations: {
|
|
91
|
+
title: "Search Content",
|
|
92
|
+
readOnlyHint: true,
|
|
93
|
+
},
|
|
90
94
|
inputSchema: {
|
|
91
95
|
type: "object",
|
|
92
96
|
properties: {
|
|
@@ -148,6 +152,10 @@ const BASE_TOOLS = [
|
|
|
148
152
|
name: "pixel-control",
|
|
149
153
|
description: "Control mouse and keyboard at the pixel level. This is a cross-platform tool that works on all operating systems. " +
|
|
150
154
|
"Use this to type text, press keys, move the mouse, and click buttons.",
|
|
155
|
+
annotations: {
|
|
156
|
+
title: "Pixel Control",
|
|
157
|
+
destructiveHint: true,
|
|
158
|
+
},
|
|
151
159
|
inputSchema: {
|
|
152
160
|
type: "object",
|
|
153
161
|
properties: {
|
|
@@ -196,6 +204,10 @@ const BASE_TOOLS = [
|
|
|
196
204
|
"EXAMPLES:\n" +
|
|
197
205
|
"- Last 30 minutes: Calculate timestamps from current time\n" +
|
|
198
206
|
"- Specific meeting: Use the meeting's start and end times in UTC",
|
|
207
|
+
annotations: {
|
|
208
|
+
title: "Export Video",
|
|
209
|
+
destructiveHint: true,
|
|
210
|
+
},
|
|
199
211
|
inputSchema: {
|
|
200
212
|
type: "object",
|
|
201
213
|
properties: {
|
|
@@ -231,6 +243,10 @@ const MACOS_TOOLS = [
|
|
|
231
243
|
"- Clickable items: 'AXButton', 'AXMenuItem', 'AXMenuBarItem', 'AXImage', 'AXStaticText'\n" +
|
|
232
244
|
"- Web content may use: 'AXWebArea', 'AXLink', 'AXHeading', 'AXRadioButton'\n\n" +
|
|
233
245
|
"Use MacOS Accessibility Inspector app to identify the exact roles in your target application.",
|
|
246
|
+
annotations: {
|
|
247
|
+
title: "Find Elements",
|
|
248
|
+
readOnlyHint: true,
|
|
249
|
+
},
|
|
234
250
|
inputSchema: {
|
|
235
251
|
type: "object",
|
|
236
252
|
properties: {
|
|
@@ -272,6 +288,10 @@ const MACOS_TOOLS = [
|
|
|
272
288
|
{
|
|
273
289
|
name: "click-element",
|
|
274
290
|
description: "Click an element in an application using its id (MacOS only)",
|
|
291
|
+
annotations: {
|
|
292
|
+
title: "Click Element",
|
|
293
|
+
destructiveHint: true,
|
|
294
|
+
},
|
|
275
295
|
inputSchema: {
|
|
276
296
|
type: "object",
|
|
277
297
|
properties: {
|
|
@@ -304,6 +324,10 @@ const MACOS_TOOLS = [
|
|
|
304
324
|
{
|
|
305
325
|
name: "fill-element",
|
|
306
326
|
description: "Type text into an element in an application (MacOS only)",
|
|
327
|
+
annotations: {
|
|
328
|
+
title: "Fill Element",
|
|
329
|
+
destructiveHint: true,
|
|
330
|
+
},
|
|
307
331
|
inputSchema: {
|
|
308
332
|
type: "object",
|
|
309
333
|
properties: {
|
|
@@ -340,6 +364,10 @@ const MACOS_TOOLS = [
|
|
|
340
364
|
{
|
|
341
365
|
name: "scroll-element",
|
|
342
366
|
description: "Scroll an element in a specific direction (MacOS only)",
|
|
367
|
+
annotations: {
|
|
368
|
+
title: "Scroll Element",
|
|
369
|
+
destructiveHint: true,
|
|
370
|
+
},
|
|
343
371
|
inputSchema: {
|
|
344
372
|
type: "object",
|
|
345
373
|
properties: {
|
|
@@ -381,6 +409,10 @@ const MACOS_TOOLS = [
|
|
|
381
409
|
{
|
|
382
410
|
name: "open-application",
|
|
383
411
|
description: "Open an application by name",
|
|
412
|
+
annotations: {
|
|
413
|
+
title: "Open Application",
|
|
414
|
+
destructiveHint: true,
|
|
415
|
+
},
|
|
384
416
|
inputSchema: {
|
|
385
417
|
type: "object",
|
|
386
418
|
properties: {
|
|
@@ -395,6 +427,10 @@ const MACOS_TOOLS = [
|
|
|
395
427
|
{
|
|
396
428
|
name: "open-url",
|
|
397
429
|
description: "Open a URL in a browser",
|
|
430
|
+
annotations: {
|
|
431
|
+
title: "Open URL",
|
|
432
|
+
destructiveHint: true,
|
|
433
|
+
},
|
|
398
434
|
inputSchema: {
|
|
399
435
|
type: "object",
|
|
400
436
|
properties: {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "screenpipe-mcp",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "MCP server for screenpipe - search your screen recordings, audio transcriptions, and control your computer",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
"scripts": {
|
|
10
10
|
"build": "tsc",
|
|
11
11
|
"start": "node dist/index.js",
|
|
12
|
+
"start:http": "node dist/http-server.js",
|
|
12
13
|
"dev": "ts-node src/index.ts",
|
|
14
|
+
"dev:http": "ts-node src/http-server.ts",
|
|
13
15
|
"test": "vitest run",
|
|
14
16
|
"test:watch": "vitest",
|
|
15
17
|
"prepublishOnly": "npm run build"
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* HTTP Server for Screenpipe MCP
|
|
5
|
+
*
|
|
6
|
+
* This allows web apps to call MCP tools over HTTP instead of stdio.
|
|
7
|
+
* Run with: npx ts-node src/http-server.ts --port 3031
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { createServer } from "http";
|
|
11
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
12
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
13
|
+
import {
|
|
14
|
+
CallToolRequestSchema,
|
|
15
|
+
ListToolsRequestSchema,
|
|
16
|
+
} from "@modelcontextprotocol/sdk/types.js";
|
|
17
|
+
|
|
18
|
+
// Parse command line arguments
|
|
19
|
+
const args = process.argv.slice(2);
|
|
20
|
+
let mcpPort = 3031;
|
|
21
|
+
let screenpipePort = 3030;
|
|
22
|
+
|
|
23
|
+
for (let i = 0; i < args.length; i++) {
|
|
24
|
+
if (args[i] === "--port" && args[i + 1]) {
|
|
25
|
+
mcpPort = parseInt(args[i + 1], 10);
|
|
26
|
+
}
|
|
27
|
+
if (args[i] === "--screenpipe-port" && args[i + 1]) {
|
|
28
|
+
screenpipePort = parseInt(args[i + 1], 10);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const SCREENPIPE_API = `http://localhost:${screenpipePort}`;
|
|
33
|
+
|
|
34
|
+
// Tool definitions
|
|
35
|
+
const TOOLS = [
|
|
36
|
+
{
|
|
37
|
+
name: "search_content",
|
|
38
|
+
description:
|
|
39
|
+
"Search screenpipe's recorded content: screen text (OCR), audio transcriptions, and UI elements. " +
|
|
40
|
+
"Returns timestamped results with app context. " +
|
|
41
|
+
"Call with no parameters to get recent activity.",
|
|
42
|
+
inputSchema: {
|
|
43
|
+
type: "object" as const,
|
|
44
|
+
properties: {
|
|
45
|
+
q: {
|
|
46
|
+
type: "string",
|
|
47
|
+
description: "Search query. Optional - omit to return all recent content.",
|
|
48
|
+
},
|
|
49
|
+
content_type: {
|
|
50
|
+
type: "string",
|
|
51
|
+
enum: ["all", "ocr", "audio", "ui"],
|
|
52
|
+
description: "Content type filter. Default: 'all'",
|
|
53
|
+
},
|
|
54
|
+
limit: {
|
|
55
|
+
type: "integer",
|
|
56
|
+
description: "Max results. Default: 10",
|
|
57
|
+
},
|
|
58
|
+
offset: {
|
|
59
|
+
type: "integer",
|
|
60
|
+
description: "Skip N results for pagination. Default: 0",
|
|
61
|
+
},
|
|
62
|
+
start_time: {
|
|
63
|
+
type: "string",
|
|
64
|
+
description: "ISO 8601 UTC start time (e.g., 2024-01-15T10:00:00Z)",
|
|
65
|
+
},
|
|
66
|
+
end_time: {
|
|
67
|
+
type: "string",
|
|
68
|
+
description: "ISO 8601 UTC end time (e.g., 2024-01-15T18:00:00Z)",
|
|
69
|
+
},
|
|
70
|
+
app_name: {
|
|
71
|
+
type: "string",
|
|
72
|
+
description: "Filter by app (e.g., 'Google Chrome', 'Slack', 'zoom.us')",
|
|
73
|
+
},
|
|
74
|
+
window_name: {
|
|
75
|
+
type: "string",
|
|
76
|
+
description: "Filter by window title",
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
];
|
|
82
|
+
|
|
83
|
+
// Helper function to make HTTP requests
|
|
84
|
+
async function fetchAPI(endpoint: string, options: RequestInit = {}): Promise<Response> {
|
|
85
|
+
const url = `${SCREENPIPE_API}${endpoint}`;
|
|
86
|
+
return fetch(url, {
|
|
87
|
+
...options,
|
|
88
|
+
headers: {
|
|
89
|
+
"Content-Type": "application/json",
|
|
90
|
+
...options.headers,
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Create MCP server
|
|
96
|
+
const server = new Server(
|
|
97
|
+
{
|
|
98
|
+
name: "screenpipe-http",
|
|
99
|
+
version: "0.1.0",
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
capabilities: {
|
|
103
|
+
tools: {},
|
|
104
|
+
},
|
|
105
|
+
}
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
// List tools handler
|
|
109
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
110
|
+
return { tools: TOOLS };
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
// Call tool handler
|
|
114
|
+
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
115
|
+
const { name, arguments: args } = request.params;
|
|
116
|
+
|
|
117
|
+
if (!args) {
|
|
118
|
+
throw new Error("Missing arguments");
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (name === "search_content") {
|
|
122
|
+
const params = new URLSearchParams();
|
|
123
|
+
for (const [key, value] of Object.entries(args)) {
|
|
124
|
+
if (value !== null && value !== undefined) {
|
|
125
|
+
params.append(key, String(value));
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const response = await fetchAPI(`/search?${params.toString()}`);
|
|
130
|
+
if (!response.ok) {
|
|
131
|
+
throw new Error(`HTTP error: ${response.status}`);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const data = await response.json();
|
|
135
|
+
const results = data.data || [];
|
|
136
|
+
const pagination = data.pagination || {};
|
|
137
|
+
|
|
138
|
+
if (results.length === 0) {
|
|
139
|
+
return {
|
|
140
|
+
content: [
|
|
141
|
+
{
|
|
142
|
+
type: "text",
|
|
143
|
+
text: "No results found. Try: broader search terms, different content_type, or wider time range.",
|
|
144
|
+
},
|
|
145
|
+
],
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const formattedResults: string[] = [];
|
|
150
|
+
for (const result of results) {
|
|
151
|
+
const content = result.content;
|
|
152
|
+
if (!content) continue;
|
|
153
|
+
|
|
154
|
+
if (result.type === "OCR") {
|
|
155
|
+
formattedResults.push(
|
|
156
|
+
`[OCR] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
|
|
157
|
+
`${content.timestamp || ""}\n` +
|
|
158
|
+
`${content.text || ""}`
|
|
159
|
+
);
|
|
160
|
+
} else if (result.type === "Audio") {
|
|
161
|
+
formattedResults.push(
|
|
162
|
+
`[Audio] ${content.device_name || "?"}\n` +
|
|
163
|
+
`${content.timestamp || ""}\n` +
|
|
164
|
+
`${content.transcription || ""}`
|
|
165
|
+
);
|
|
166
|
+
} else if (result.type === "UI") {
|
|
167
|
+
formattedResults.push(
|
|
168
|
+
`[UI] ${content.app_name || "?"} | ${content.window_name || "?"}\n` +
|
|
169
|
+
`${content.timestamp || ""}\n` +
|
|
170
|
+
`${content.text || ""}`
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const header = `Results: ${results.length}/${pagination.total || "?"}` +
|
|
176
|
+
(pagination.total > results.length ? ` (use offset=${(pagination.offset || 0) + results.length} for more)` : "");
|
|
177
|
+
|
|
178
|
+
return {
|
|
179
|
+
content: [
|
|
180
|
+
{
|
|
181
|
+
type: "text",
|
|
182
|
+
text: header + "\n\n" + formattedResults.join("\n---\n"),
|
|
183
|
+
},
|
|
184
|
+
],
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
throw new Error(`Unknown tool: ${name}`);
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
// Create HTTP server with MCP transport
|
|
192
|
+
const transports = new Map<string, StreamableHTTPServerTransport>();
|
|
193
|
+
|
|
194
|
+
const httpServer = createServer(async (req, res) => {
|
|
195
|
+
// CORS headers
|
|
196
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
197
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
198
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization, mcp-session-id");
|
|
199
|
+
|
|
200
|
+
if (req.method === "OPTIONS") {
|
|
201
|
+
res.writeHead(204);
|
|
202
|
+
res.end();
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Health check
|
|
207
|
+
if (req.url === "/health") {
|
|
208
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
209
|
+
res.end(JSON.stringify({ status: "ok" }));
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// MCP endpoint
|
|
214
|
+
if (req.url === "/mcp" || req.url?.startsWith("/mcp?")) {
|
|
215
|
+
const sessionId = req.headers["mcp-session-id"] as string | undefined;
|
|
216
|
+
|
|
217
|
+
let transport = sessionId ? transports.get(sessionId) : undefined;
|
|
218
|
+
|
|
219
|
+
if (!transport) {
|
|
220
|
+
transport = new StreamableHTTPServerTransport({
|
|
221
|
+
sessionIdGenerator: () => crypto.randomUUID(),
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
await server.connect(transport);
|
|
225
|
+
|
|
226
|
+
if (transport.sessionId) {
|
|
227
|
+
transports.set(transport.sessionId, transport);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
await transport.handleRequest(req, res);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
236
|
+
res.end(JSON.stringify({ error: "Not found" }));
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
httpServer.listen(mcpPort, () => {
|
|
240
|
+
console.log(`Screenpipe MCP HTTP server running on http://localhost:${mcpPort}`);
|
|
241
|
+
console.log(`MCP endpoint: http://localhost:${mcpPort}/mcp`);
|
|
242
|
+
console.log(`Health check: http://localhost:${mcpPort}/health`);
|
|
243
|
+
});
|
package/src/index.ts
CHANGED
|
@@ -71,6 +71,10 @@ const BASE_TOOLS: Tool[] = [
|
|
|
71
71
|
"Returns timestamped results with app context. " +
|
|
72
72
|
"Call with no parameters to get recent activity. " +
|
|
73
73
|
"Use the 'screenpipe://context' resource for current time when building time-based queries.",
|
|
74
|
+
annotations: {
|
|
75
|
+
title: "Search Content",
|
|
76
|
+
readOnlyHint: true,
|
|
77
|
+
},
|
|
74
78
|
inputSchema: {
|
|
75
79
|
type: "object",
|
|
76
80
|
properties: {
|
|
@@ -133,6 +137,10 @@ const BASE_TOOLS: Tool[] = [
|
|
|
133
137
|
description:
|
|
134
138
|
"Control mouse and keyboard at the pixel level. This is a cross-platform tool that works on all operating systems. " +
|
|
135
139
|
"Use this to type text, press keys, move the mouse, and click buttons.",
|
|
140
|
+
annotations: {
|
|
141
|
+
title: "Pixel Control",
|
|
142
|
+
destructiveHint: true,
|
|
143
|
+
},
|
|
136
144
|
inputSchema: {
|
|
137
145
|
type: "object",
|
|
138
146
|
properties: {
|
|
@@ -183,6 +191,10 @@ const BASE_TOOLS: Tool[] = [
|
|
|
183
191
|
"EXAMPLES:\n" +
|
|
184
192
|
"- Last 30 minutes: Calculate timestamps from current time\n" +
|
|
185
193
|
"- Specific meeting: Use the meeting's start and end times in UTC",
|
|
194
|
+
annotations: {
|
|
195
|
+
title: "Export Video",
|
|
196
|
+
destructiveHint: true,
|
|
197
|
+
},
|
|
186
198
|
inputSchema: {
|
|
187
199
|
type: "object",
|
|
188
200
|
properties: {
|
|
@@ -223,6 +235,10 @@ const MACOS_TOOLS: Tool[] = [
|
|
|
223
235
|
"- Clickable items: 'AXButton', 'AXMenuItem', 'AXMenuBarItem', 'AXImage', 'AXStaticText'\n" +
|
|
224
236
|
"- Web content may use: 'AXWebArea', 'AXLink', 'AXHeading', 'AXRadioButton'\n\n" +
|
|
225
237
|
"Use MacOS Accessibility Inspector app to identify the exact roles in your target application.",
|
|
238
|
+
annotations: {
|
|
239
|
+
title: "Find Elements",
|
|
240
|
+
readOnlyHint: true,
|
|
241
|
+
},
|
|
226
242
|
inputSchema: {
|
|
227
243
|
type: "object",
|
|
228
244
|
properties: {
|
|
@@ -267,6 +283,10 @@ const MACOS_TOOLS: Tool[] = [
|
|
|
267
283
|
name: "click-element",
|
|
268
284
|
description:
|
|
269
285
|
"Click an element in an application using its id (MacOS only)",
|
|
286
|
+
annotations: {
|
|
287
|
+
title: "Click Element",
|
|
288
|
+
destructiveHint: true,
|
|
289
|
+
},
|
|
270
290
|
inputSchema: {
|
|
271
291
|
type: "object",
|
|
272
292
|
properties: {
|
|
@@ -299,6 +319,10 @@ const MACOS_TOOLS: Tool[] = [
|
|
|
299
319
|
{
|
|
300
320
|
name: "fill-element",
|
|
301
321
|
description: "Type text into an element in an application (MacOS only)",
|
|
322
|
+
annotations: {
|
|
323
|
+
title: "Fill Element",
|
|
324
|
+
destructiveHint: true,
|
|
325
|
+
},
|
|
302
326
|
inputSchema: {
|
|
303
327
|
type: "object",
|
|
304
328
|
properties: {
|
|
@@ -335,6 +359,10 @@ const MACOS_TOOLS: Tool[] = [
|
|
|
335
359
|
{
|
|
336
360
|
name: "scroll-element",
|
|
337
361
|
description: "Scroll an element in a specific direction (MacOS only)",
|
|
362
|
+
annotations: {
|
|
363
|
+
title: "Scroll Element",
|
|
364
|
+
destructiveHint: true,
|
|
365
|
+
},
|
|
338
366
|
inputSchema: {
|
|
339
367
|
type: "object",
|
|
340
368
|
properties: {
|
|
@@ -376,6 +404,10 @@ const MACOS_TOOLS: Tool[] = [
|
|
|
376
404
|
{
|
|
377
405
|
name: "open-application",
|
|
378
406
|
description: "Open an application by name",
|
|
407
|
+
annotations: {
|
|
408
|
+
title: "Open Application",
|
|
409
|
+
destructiveHint: true,
|
|
410
|
+
},
|
|
379
411
|
inputSchema: {
|
|
380
412
|
type: "object",
|
|
381
413
|
properties: {
|
|
@@ -390,6 +422,10 @@ const MACOS_TOOLS: Tool[] = [
|
|
|
390
422
|
{
|
|
391
423
|
name: "open-url",
|
|
392
424
|
description: "Open a URL in a browser",
|
|
425
|
+
annotations: {
|
|
426
|
+
title: "Open URL",
|
|
427
|
+
destructiveHint: true,
|
|
428
|
+
},
|
|
393
429
|
inputSchema: {
|
|
394
430
|
type: "object",
|
|
395
431
|
properties: {
|