scrapeless-mcp-server 0.3.0 → 0.4.0

package/build/tools/utils.js

@@ -0,0 +1,86 @@
+import { customAlphabet } from "nanoid";
+const notRecordRoleList = [
+    'Page Title',
+    'Root Web Area',
+    'RootWebArea',
+    'generic',
+    'none',
+    'paragraph',
+    'InlineTextBox',
+    'image'
+];
+function assembleSnapshot(snap, indent = 0) {
+    let str = '';
+    const spaces = ' '.repeat(indent);
+    for (const child of snap.children || []) {
+        if (child.role === 'Page Title')
+            continue;
+        if (child.role === 'Root Web Area')
+            continue;
+        if (!notRecordRoleList.includes(child.role)) {
+            str += `${spaces}- ${child.role}: ${child.name}\n`;
+        }
+        if (child.role === 'image') {
+            str += `${spaces} - image\n`;
+        }
+        if (child.children && child.children.length > 0) {
+            str += assembleSnapshot(child, !notRecordRoleList.includes(child.role) ? indent + 2 : indent);
+        }
+    }
+    return str;
+}
+export function defineTool(tool) {
+    return tool;
+}
+export async function wrapMcpResponse(request) {
+    try {
+        const data = await request();
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `Response:\n\n${JSON.stringify(data)}`,
+                },
+            ],
+        };
+    }
+    catch (error) {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `Failed to fetch data. Error: ${error.message}`,
+                },
+            ],
+        };
+    }
+}
+export function wrapMcpBrowserResponse(content) {
+    return {
+        content: [
+            {
+                type: "text",
+                text: content,
+            },
+        ],
+    };
+}
+export function uuid() {
+    const nanoid1 = customAlphabet("0123456789abcdefghijklmnopqrstuvwxyz", 4);
+    const nanoid2 = customAlphabet("0123456789abcdefghijklmnopqrstuvwxyz", 4);
+    const nanoid3 = customAlphabet("0123456789abcdefghijklmnopqrstuvwxyz", 4);
+    const nanoid4 = customAlphabet("0123456789abcdefghijklmnopqrstuvwxyz", 6);
+    return `${nanoid1()}-${nanoid2()}-${nanoid3()}-${nanoid4()}`;
+}
+export async function snapshot(page) {
+    const snap = await page.accessibility.snapshot({ interestingOnly: false, includeIframes: true });
+    if (!snap) {
+        return "No accessibility snapshot available.";
+    }
+    const url = page.url();
+    const title = await page.title();
+    let snapshotString = `- Page URL: ${url}\n`;
+    snapshotString += `- Page Title: ${title}\n`;
+    snapshotString += `\`\`\`yaml\n- Page Snapshot\n\`\`\`yaml`;
+    return snapshotString + assembleSnapshot(snap, 0);
+}

package/build/transport.js

@@ -0,0 +1,120 @@
+import http from "node:http";
+import assert from "node:assert";
+import crypto from "node:crypto";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { SSEServerTransport } from "@modelcontextprotocol/sdk/server/sse.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { createMcpServer, ServerList } from "./server.js";
+import { API_KEY_NAME } from "./config.js";
+export async function startStdioTransport() {
+    const serverList = new ServerList(async () => createMcpServer());
+    const server = await serverList.create();
+    await server.connect(new StdioServerTransport());
+}
+async function handleSSE(req, res, url, sessions) {
+    const serverList = createRemoteServerList(req, res);
+    if (!serverList)
+        return;
+    if (req.method === "POST") {
+        const sessionId = url.searchParams.get("sessionId");
+        if (!sessionId) {
+            res.statusCode = 400;
+            return res.end("Missing sessionId");
+        }
+        const transport = sessions.get(sessionId);
+        if (!transport) {
+            res.statusCode = 404;
+            return res.end("Session not found");
+        }
+        return await transport.handlePostMessage(req, res);
+    }
+    else if (req.method === "GET") {
+        const transport = new SSEServerTransport("/sse", res);
+        sessions.set(transport.sessionId, transport);
+        const server = await serverList.create();
+        res.on("close", () => {
+            sessions.delete(transport.sessionId);
+            serverList.close(server).catch((e) => {
+                // eslint-disable-next-line no-console
+                // console.error(e);
+            });
+        });
+        return await server.connect(transport);
+    }
+    res.statusCode = 405;
+    res.end("Method not allowed");
+}
+async function handleStreamable(req, res, sessions) {
+    const serverList = createRemoteServerList(req, res);
+    if (!serverList)
+        return;
+    const sessionId = req.headers["mcp-session-id"];
+    if (sessionId) {
+        const transport = sessions.get(sessionId);
+        if (!transport) {
+            res.statusCode = 404;
+            res.end("Session not found");
+            return;
+        }
+        return await transport.handleRequest(req, res);
+    }
+    if (req.method === "POST") {
+        const transport = new StreamableHTTPServerTransport({
+            sessionIdGenerator: () => crypto.randomUUID(),
+            onsessioninitialized: (sessionId) => {
+                sessions.set(sessionId, transport);
+            },
+        });
+        transport.onclose = () => {
+            if (transport.sessionId)
+                sessions.delete(transport.sessionId);
+        };
+        const server = await serverList.create();
+        await server.connect(transport);
+        return await transport.handleRequest(req, res);
+    }
+    res.statusCode = 400;
+    res.end("Invalid request");
+}
+export function startHttpTransport(port, hostname) {
+    const sseSessions = new Map();
+    const streamableSessions = new Map();
+    const httpServer = http.createServer(async (req, res) => {
+        const url = new URL(`http://localhost${req.url}`);
+        if (url.pathname.startsWith("/mcp"))
+            await handleStreamable(req, res, streamableSessions);
+        else
+            await handleSSE(req, res, url, sseSessions);
+    });
+    httpServer.listen(port, hostname, () => {
+        const address = httpServer.address();
+        assert(address, "Could not bind server socket");
+        let url;
+        if (typeof address === "string") {
+            url = address;
+        }
+        else {
+            const resolvedPort = address.port;
+            let resolvedHost = address.family === "IPv4" ? address.address : `[${address.address}]`;
+            if (resolvedHost === "0.0.0.0" || resolvedHost === "[::]")
+                resolvedHost = "localhost";
+            url = `http://${resolvedHost}:${resolvedPort}`;
+        }
+        const message = [
+            `Listening on ${url}`,
+            "If your client supports streamable HTTP, you can use the /mcp endpoint instead.",
+        ].join("\n");
+        // eslint-disable-next-line no-console
+        console.log(message);
+    });
+}
+function createRemoteServerList(req, res) {
+    const apiKey = req.headers[API_KEY_NAME];
+    if (!apiKey) {
+        res.statusCode = 401;
+        res.end("Unauthorized: Missing x-api-token header");
+        return null;
+    }
+    const serverList = new ServerList(async () => createMcpServer(apiKey));
+    return serverList;
+}

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "scrapeless-mcp-server",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "main": "index.js",
   "type": "module",
   "bin": {
     "scrapeless-mcp-server": "./build/index.js"
   },
   "scripts": {
-    "build": "tsc && chmod 755 build/index.js"
+    "build": "tsc && chmod 755 build/index.js",
+    "start:http": "node build/index.js --port=9593",
+    "dev:http": "tsc && node build/index.js --port=9593"
   },
   "files": [
     "build"
@@ -29,10 +31,17 @@
   "dependencies": {
     "@chatmcp/sdk": "^1.0.5",
     "@modelcontextprotocol/sdk": "^1.8.0",
+    "@scrapeless-ai/sdk": "^1.6.1",
+    "agents": "^0.0.101",
     "axios": "^1.8.4",
+    "express": "^5.1.0",
+    "nanoid": "^5.1.5",
+    "puppeteer-core": "^24.12.0",
+    "yaml": "^2.8.0",
     "zod": "^3.24.2"
   },
   "devDependencies": {
+    "@types/express": "^5.0.3",
     "@types/node": "^22.13.14",
     "typescript": "^5.8.2"
   },

package/build/client.js

@@ -1,40 +0,0 @@
-import axios from "axios";
-export class ScrapelessClient {
-    api;
-    constructor(config) {
-        this.api = axios.create({
-            baseURL: config.baseURL,
-            headers: {
-                "Content-Type": "application/json",
-                "x-api-token": config.token,
-            },
-        });
-    }
-    async sendRequest(endpoint, actor, inputData) {
-        try {
-            const response = await this.api.post(endpoint, {
-                actor,
-                input: inputData,
-            });
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: `Response:\n\n${JSON.stringify(response.data)}`,
-                    },
-                ],
-            };
-        }
-        catch (error) {
-            console.error("Scrapeless API Error:", error);
-            return {
-                content: [
-                    {
-                        type: "text",
-                        text: `Failed to fetch data. Error: ${error.message}`,
-                    },
-                ],
-            };
-        }
-    }
-}