scrape-do-mcp 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -1
- package/dist/index.js +46 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -31,7 +31,22 @@ Get your free API token at: https://app.scrape.do
|
|
|
31
31
|
|
|
32
32
|
### Smithery.ai
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
Published on [Smithery.ai](https://smithery.ai) - Search for "scrape-do" to install.
|
|
35
|
+
|
|
36
|
+
### HTTP Server Mode
|
|
37
|
+
|
|
38
|
+
The server supports both STDIO and HTTP modes:
|
|
39
|
+
|
|
40
|
+
- **STDIO mode** (default): For local Claude Code / Claude Desktop usage
|
|
41
|
+
- **HTTP mode**: For Smithery托管或 custom HTTP deployment
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
# HTTP mode
|
|
45
|
+
TRANSPORT=http PORT=3000 SCRAPE_DO_TOKEN=your_token npm start
|
|
46
|
+
|
|
47
|
+
# Health check
|
|
48
|
+
curl http://localhost:3000/health
|
|
49
|
+
```
|
|
35
50
|
|
|
36
51
|
## Usage
|
|
37
52
|
|
package/dist/index.js
CHANGED
|
@@ -6,13 +6,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
7
|
const mcp_js_1 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
8
8
|
const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
9
|
+
const streamableHttp_js_1 = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
|
|
9
10
|
const zod_1 = require("zod");
|
|
10
11
|
const axios_1 = __importDefault(require("axios"));
|
|
12
|
+
const http_1 = __importDefault(require("http"));
|
|
11
13
|
const SCRAPE_DO_TOKEN = process.env.SCRAPE_DO_TOKEN || "";
|
|
12
14
|
const SCRAPE_API_BASE = "https://api.scrape.do";
|
|
15
|
+
const HTTP_PORT = process.env.PORT || process.env.HTTP_PORT || 3000;
|
|
13
16
|
const server = new mcp_js_1.McpServer({
|
|
14
17
|
name: "scrape-do-mcp",
|
|
15
|
-
version: "0.1.
|
|
18
|
+
version: "0.1.1",
|
|
16
19
|
});
|
|
17
20
|
// ─── Tool 1: scrape_url ──────────────────────────────────────────────────────
|
|
18
21
|
server.tool("scrape_url", "Scrape any webpage and return its content as Markdown. Automatically bypasses Cloudflare, WAFs, CAPTCHAs, and anti-bot protection. Supports JavaScript-rendered pages.", {
|
|
@@ -94,7 +97,47 @@ server.tool("google_search", "Search Google and return structured SERP results a
|
|
|
94
97
|
});
|
|
95
98
|
// ─── Start Server ────────────────────────────────────────────────────────────
|
|
96
99
|
async function main() {
|
|
97
|
-
const
|
|
98
|
-
|
|
100
|
+
const transportMode = process.env.TRANSPORT || "stdio";
|
|
101
|
+
if (transportMode === "http" || transportMode === "streamable-http") {
|
|
102
|
+
console.error(`Starting Streamable HTTP server on port ${HTTP_PORT}...`);
|
|
103
|
+
const transport = new streamableHttp_js_1.StreamableHTTPServerTransport({
|
|
104
|
+
sessionIdGenerator: () => Math.random().toString(36).substring(2, 15),
|
|
105
|
+
});
|
|
106
|
+
await server.connect(transport);
|
|
107
|
+
const serverInstance = http_1.default.createServer();
|
|
108
|
+
serverInstance.on("request", async (req, res) => {
|
|
109
|
+
// Handle CORS
|
|
110
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
111
|
+
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
112
|
+
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
|
|
113
|
+
if (req.method === "OPTIONS") {
|
|
114
|
+
res.writeHead(204);
|
|
115
|
+
res.end();
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
// Health check
|
|
119
|
+
if (req.url === "/health") {
|
|
120
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
121
|
+
res.end(JSON.stringify({ status: "ok", name: "scrape-do-mcp", version: "0.1.1" }));
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
// MCP endpoint
|
|
125
|
+
if (req.url === "/" || req.url?.startsWith("/mcp")) {
|
|
126
|
+
await transport.handleRequest(req, res);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
res.writeHead(404);
|
|
130
|
+
res.end("Not found");
|
|
131
|
+
});
|
|
132
|
+
serverInstance.listen(parseInt(String(HTTP_PORT), 10), () => {
|
|
133
|
+
console.error(`MCP server running on http://localhost:${HTTP_PORT}`);
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
// Default to stdio mode
|
|
138
|
+
console.error("Starting STDIO server...");
|
|
139
|
+
const transport = new stdio_js_1.StdioServerTransport();
|
|
140
|
+
await server.connect(transport);
|
|
141
|
+
}
|
|
99
142
|
}
|
|
100
143
|
main().catch(console.error);
|