scrape-do-mcp 0.1.0 → 0.1.2

package/README.md

@@ -0,0 +1,99 @@
+# scrape-do-mcp
+
+MCP Server for Scrape.do - Web Scraping & Google Search with anti-bot bypass
+
+## Features
+
+- **scrape_url**: Scrape any webpage and return content as Markdown. Automatically bypasses Cloudflare, WAFs, CAPTCHAs, and anti-bot protection. Supports JavaScript-rendered pages.
+- **google_search**: Search Google and return structured SERP results as JSON. Returns organic results, knowledge graph, local businesses, news stories, and more.
+
+## Installation
+
+### Claude Code / Claude Desktop
+
+Add to your `~/.claude.json`:
+
+```json
+{
+  "mcpServers": {
+    "scrape-do": {
+      "command": "npx",
+      "args": ["-y", "scrape-do-mcp"],
+      "env": {
+        "SCRAPE_DO_TOKEN": "your_token_here"
+      }
+    }
+  }
+}
+```
+
+Get your free API token at: https://app.scrape.do
+
+### Smithery.ai
+
+Published on [Smithery.ai](https://smithery.ai) - Search for "scrape-do" to install.
+
+### HTTP Server Mode
+
+The server supports both STDIO and HTTP modes:
+
+- **STDIO mode** (default): For local Claude Code / Claude Desktop usage
+- **HTTP mode**: For Smithery托管或 custom HTTP deployment
+
+```bash
+# HTTP mode
+TRANSPORT=http PORT=3000 SCRAPE_DO_TOKEN=your_token npm start
+
+# Health check
+curl http://localhost:3000/health
+```
+
+## Usage
+
+### scrape_url
+
+```typescript
+// Parameters
+{
+  url: string,           // Target URL to scrape
+  render_js?: boolean,   // Render JavaScript (default: false)
+  super_proxy?: boolean, // Use residential proxies (costs 10 credits, default: false)
+  output?: "markdown" | "raw"  // Output format (default: markdown)
+}
+```
+
+### google_search
+
+```typescript
+// Parameters
+{
+  query: string,                      // Search query
+  country?: string,                    // Country code (default: "us")
+  language?: string,                   // Interface language (default: "en")
+  page?: number,                       // Page number (default: 1)
+  time_period?: "" | "last_hour" | "last_day" | "last_week" | "last_month" | "last_year",
+  device?: "desktop" | "mobile"        // Device type (default: desktop)
+}
+```
+
+## Credit Usage
+
+| Tool | Credit Cost |
+|------|-------------|
+| scrape_url (regular) | 1 credit/request |
+| scrape_url (super_proxy) | 10 credits/request |
+| google_search | 1 credit/request |
+
+Free registration includes **1,000 credits**: https://app.scrape.do
+
+## Development
+
+```bash
+npm install
+npm run build
+npm run dev  # Run in development mode
+```
+
+## License
+
+MIT

package/dist/index.js

@@ -6,13 +6,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const mcp_js_1 = require("@modelcontextprotocol/sdk/server/mcp.js");
 const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
+const streamableHttp_js_1 = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
 const zod_1 = require("zod");
 const axios_1 = __importDefault(require("axios"));
+const http_1 = __importDefault(require("http"));
 const SCRAPE_DO_TOKEN = process.env.SCRAPE_DO_TOKEN || "";
 const SCRAPE_API_BASE = "https://api.scrape.do";
+const HTTP_PORT = process.env.PORT || process.env.HTTP_PORT || 3000;
 const server = new mcp_js_1.McpServer({
     name: "scrape-do-mcp",
-    version: "0.1.0",
+    version: "0.1.1",
 });
 // ─── Tool 1: scrape_url ──────────────────────────────────────────────────────
 server.tool("scrape_url", "Scrape any webpage and return its content as Markdown. Automatically bypasses Cloudflare, WAFs, CAPTCHAs, and anti-bot protection. Supports JavaScript-rendered pages.", {
@@ -94,7 +97,47 @@ server.tool("google_search", "Search Google and return structured SERP results a
 });
 // ─── Start Server ────────────────────────────────────────────────────────────
 async function main() {
-    const transport = new stdio_js_1.StdioServerTransport();
-    await server.connect(transport);
+    const transportMode = process.env.TRANSPORT || "stdio";
+    if (transportMode === "http" || transportMode === "streamable-http") {
+        console.error(`Starting Streamable HTTP server on port ${HTTP_PORT}...`);
+        const transport = new streamableHttp_js_1.StreamableHTTPServerTransport({
+            sessionIdGenerator: () => Math.random().toString(36).substring(2, 15),
+        });
+        await server.connect(transport);
+        const serverInstance = http_1.default.createServer();
+        serverInstance.on("request", async (req, res) => {
+            // Handle CORS
+            res.setHeader("Access-Control-Allow-Origin", "*");
+            res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+            res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+            if (req.method === "OPTIONS") {
+                res.writeHead(204);
+                res.end();
+                return;
+            }
+            // Health check
+            if (req.url === "/health") {
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify({ status: "ok", name: "scrape-do-mcp", version: "0.1.1" }));
+                return;
+            }
+            // MCP endpoint
+            if (req.url === "/" || req.url?.startsWith("/mcp")) {
+                await transport.handleRequest(req, res);
+                return;
+            }
+            res.writeHead(404);
+            res.end("Not found");
+        });
+        serverInstance.listen(parseInt(String(HTTP_PORT), 10), () => {
+            console.error(`MCP server running on http://localhost:${HTTP_PORT}`);
+        });
+    }
+    else {
+        // Default to stdio mode
+        console.error("Starting STDIO server...");
+        const transport = new stdio_js_1.StdioServerTransport();
+        await server.connect(transport);
+    }
 }
 main().catch(console.error);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrape-do-mcp",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "MCP Server for Scrape.do - Web Scraping & Google Search with anti-bot bypass",
   "main": "dist/index.js",
   "bin": {
@@ -19,8 +19,13 @@
     "firecrawl-alternative"
   ],
   "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dztabel-happy/scrape-do-mcp.git"
+  },
   "files": [
-    "dist"
+    "dist",
+    "README.md"
   ],
   "engines": {
     "node": ">=18.0.0"