scrapeless-mcp-server 0.3.0 → 0.4.0

package/README.md

@@ -6,7 +6,8 @@ Model Context Protocol (MCP) is an open protocol that enables seamless integrati
 
 Seamlessly integrate real-time Google SERP(Google Search, Google Flight, Google Map, Google Jobs....) results into your LLM applications using the Scrapeless MCP server. This server acts as a bridge between LLMs (like ChatGPT, Claude, etc.) and Scrapeless's Google SERP, enabling dynamic context retrieval for AI workflows, chatbots, and research tools.
 
-👉 Live MCP Endpoint: 
+👉 Live MCP Endpoint:
+
 - [mcp.so](https://mcp.so/server/scrapelessMcpServer/scrapeless-ai)
 - [glama.ai](https://glama.ai/mcp/servers/@scrapeless-ai/scrapeless-mcp-server)
 
@@ -18,31 +19,90 @@ This project provides several MCP servers that enable AI assistants like Claude
 
 - Google Search
 
+## Supported Transport Protocols
+
+- **Stdio** `default`: The transport runs locally on your machine and communicates via standard input/output streams.
+- **Streamable HTTP** `--mode=streamable_http`: The new MCP transport protocol that simplifies client-server communication through a single HTTP endpoint
+
 ## Tools
 
-### 1. Search Tool
-- Name: `google-search`
-- Description: Search the web using Scrapeless
-- Parameters:
-    * `query` (required): Parameter defines the query you want to search. You can use anything that you would use in a regular Google search. e.g. inurl:, site:, intitle:.
-    * `gl` (optional, default: "us"): Parameter defines the country to use for the Google search. It's a two-letter country code. (e.g., us for the United States, uk for United Kingdom, or fr for France).
-    * `hl` (optional, default: "en"): Parameter defines the language to use for the Google search. It's a two-letter language code. (e.g., en for English, es for Spanish, or fr for French).
+### Browser Tools
+
+- `browser_goto`
+  Navigate browser to a specified URL.
+
+- `browser_go_back`
+  Go back one step in browser history.
+
+- `browser_go_forward`
+  Go forward one step in browser history.
+
+- `browser_click`
+  Click a specific element on the page.
+
+- `browser_type`
+  Type text into a specified input field.
+
+- `browser_wait_for`
+  Wait for a specific page element to appear.
+
+- `browser_wait`
+  Pause execution for a fixed duration.
+
+- `browser_screenshot`
+  Capture a screenshot of the current page.
 
+- `browser_get_html`
+  Get the full HTML of the current page.
+
+- `browser_get_text`
+  Get all visible text from the current page.
+
+- `browser_scroll`
+  Scroll to the bottom of the page.
+
+- `browser_scroll_to`
+  Scroll a specific element into view.
+
+### Universal Tools
+
+- `scrape_html`
+  Scrape a URL and return its full HTML content.
+- `scrape_markdown`
+  Scrape a URL and return its content as Markdown.
+
+- `scrape_screenshot`
+  Capture a high-quality screenshot of any webpage.
+
+### Deepserp Tools
+
+- `google_search`
+  universal information search engine.retrieves any data information.explanatory queries (why, how).comparative analysis requests.
+- `google_scholar`
+  Search for academic papers on Google Scholar.
+
+- `google_flights`
+  Exclusive flight information query tool.
+
+- `google_trends`
+  Get trending search data from Google Trends.
 
 ## Setup Guide
 
 ### 1. Get Scrapeless Key
+
 1. Register at [Scrapeless](https://app.scrapeless.com/passport/register?utm_source=github&utm_medium=mcp)
 2. [Get your free trial](https://app.scrapeless.com/landing/guide?utm_source=github&utm_medium=mcp)
 3. [Generate API Key](https://app.scrapeless.com/dashboard/settings/api-key?utm_source=github&utm_medium=mcp)
 
-
 ### 2. Configure
 
+#### Stdio
+
 ```json
 {
   "mcpServers": {
-    "scrapelessMcpServer": {
+    "ScrapelessMcpServer": {
       "command": "npx",
       "args": ["-y", "scrapeless-mcp-server"],
       "env": {
@@ -53,17 +113,38 @@ This project provides several MCP servers that enable AI assistants like Claude
 }
 ```
 
+#### Streamable
+
+1. Run locally
+
+```shell
+npx scrapeless-mcp-server --mode=streamable_http --SCRAPELESS_KEY=YOUR_SCRAPELESS_KEY
+```
+
+2. Add mcp configuration
+
+```json
+{
+  "mcpServers": {
+    "ScrapelessMcpServer": {
+      "type": "streamable-http",
+      "url": "http://127.0.0.1:9593/mcp",
+      "disabled": false
+    }
+  }
+}
+```
 
 ## Example Queries
 
 Here are some examples of how to use these servers with Claude Desktop:
 
 ### Google Search
+
 ```
 Please search for "climate change solutions" and summarize the top results.
 ```
 
-
 ## Installation
 
 ### Prerequisites
@@ -74,22 +155,24 @@ Please search for "climate change solutions" and summarize the top results.
 ### Install from Source
 
 1. Clone the repository:
+
 ```bash
 git clone https://github.com/scrapeless-ai/scrapeless-mcp-server.git
 cd scrapeless-mcp-server
 ```
 
 2. Install dependencies:
+
 ```bash
 npm install
 ```
 
-
 3. Build the server:
+
 ```bash
 npm run build
 ```
 
-
 ## Community
-- [MCP Server Discord](https://backend.scrapeless.com/app/api/v1/public/links/discord)
+
+- [MCP Server Discord](https://backend.scrapeless.com/app/api/v1/public/links/discord)

package/build/cf.js

@@ -0,0 +1,27 @@
+import { McpAgent } from "agents/mcp";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { initMcpTools, serverOptions } from "./server.js";
+import { API_KEY_NAME } from "./config.js";
+export class CfMcpServer extends McpAgent {
+    server = new McpServer(serverOptions);
+    async init() {
+        initMcpTools(this.server, this.props.apiKey);
+    }
+}
+export default {
+    fetch(request, env, ctx) {
+        const url = new URL(request.url);
+        const apiKeyHeader = request.headers.get(API_KEY_NAME);
+        if (!apiKeyHeader) {
+            return new Response(`Unauthorized: Missing ${API_KEY_NAME} header`, { status: 401 });
+        }
+        ctx.props = { apiKey: apiKeyHeader };
+        if (url.pathname === "/sse" || url.pathname === "/sse/message") {
+            return CfMcpServer.serveSSE("/sse").fetch(request, env, ctx);
+        }
+        if (url.pathname === "/mcp") {
+            return CfMcpServer.serve("/mcp").fetch(request, env, ctx);
+        }
+        return new Response("Not found", { status: 404 });
+    },
+};

package/build/config.js

@@ -1,16 +1,11 @@
 import { getParamValue } from "@chatmcp/sdk/utils/index.js";
-const API_KEY = process.env.SCRAPELESS_KEY?.trim() || getParamValue("SCRAPELESS_KEY");
-const BASE_URL = process.env.SCRAPELESS_BASE_URL?.trim() || "https://api.scrapeless.com";
+export const API_KEY = process.env.SCRAPELESS_KEY?.trim() || getParamValue("SCRAPELESS_KEY");
+export const BASE_URL = process.env.SCRAPELESS_BASE_URL?.trim() || "https://api.scrapeless.com";
+export const API_KEY_NAME = "x-api-token";
 export const ServerMode = getParamValue("mode") || "stdio";
-export const ServerPort = getParamValue("port") || 9593;
-export const ServerEndpoint = getParamValue("endpoint") || "/rest";
-if (!API_KEY) {
-    throw new Error("❌ Missing environment variable: SCRAPELESS_KEY");
-}
-export const TOOL_ENDPOINTS = {
-    SCRAPER: "/api/v1/scraper/request",
-};
+export const ServerPort = getParamValue("port");
+export const ServerHostname = getParamValue("hostname") || "0.0.0.0";
 export const SCRAPELESS_CONFIG = {
-    baseURL: BASE_URL,
-    token: API_KEY,
+    baseApiUrl: BASE_URL,
+    apiKey: API_KEY,
 };

package/build/context.js

@@ -0,0 +1,120 @@
+import { SessionManager } from "./session-manager.js";
+import { uuid } from "./tools/utils.js";
+export class Context {
+    sessionManager;
+    currentSessionId = "default-session-id";
+    apiKey;
+    constructor(apiKey) {
+        this.sessionManager = new SessionManager();
+        this.apiKey = apiKey;
+    }
+    getSession(id) {
+        return this.sessionManager.getSession(`${id ?? this.currentSessionId}-${this.apiKey}`);
+    }
+    async run(tool, params) {
+        if (!this.apiKey) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `API key is missing.`,
+                    },
+                ],
+            };
+        }
+        if (params.sessionId && params.sessionId !== this.currentSessionId) {
+            this.currentSessionId = params.sessionId;
+        }
+        const toolName = tool.name;
+        if (toolName === "browser_create") {
+            const newSessionId = uuid();
+            try {
+                await this.sessionManager.createSession(`${newSessionId}-${this.apiKey}`, this.apiKey);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `New browser session created with ID: ${newSessionId}`,
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Failed to create browser session. Please try again later.`,
+                        },
+                    ],
+                };
+            }
+        }
+        if (toolName === "browser_close") {
+            const sessionId = params?.sessionId;
+            if (!sessionId) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Session ID is missing.`,
+                        },
+                    ],
+                };
+            }
+            try {
+                await this.sessionManager.closeSession(`${sessionId}-${this.apiKey}`);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Browser session with ID: ${sessionId} has been closed.`,
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Failed to close browser session with ID: ${sessionId}. Please try again later.`,
+                        },
+                    ],
+                };
+            }
+        }
+        let session = null;
+        session = this.sessionManager.getSession(`${this.currentSessionId}-${this.apiKey}`);
+        if (!session) {
+            const newSessionId = uuid();
+            await this.sessionManager.createSession(`${newSessionId}-${this.apiKey}`, this.apiKey);
+            this.currentSessionId = newSessionId;
+        }
+        let toolActionOutput = undefined;
+        let actionSucceeded = false;
+        try {
+            let action = undefined;
+            action = tool.handle;
+            if (action) {
+                toolActionOutput = await action(this, params);
+                actionSucceeded = true;
+            }
+        }
+        catch (e) {
+        }
+        finally {
+            if (actionSucceeded && toolActionOutput !== undefined) {
+                return toolActionOutput;
+            }
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `${toolName} action completed successfully.`,
+                    },
+                ],
+            };
+        }
+    }
+}

package/build/index.js

@@ -1,21 +1,18 @@
 #!/usr/bin/env node
-import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
-import { server } from "./server.js";
-import { RestServerTransport } from "@chatmcp/sdk/server/rest.js";
-import { ServerMode, ServerPort, ServerEndpoint } from "./config.js";
+import { ServerPort, API_KEY, ServerHostname } from "./config.js";
+import { startHttpTransport, startStdioTransport } from "./transport.js";
+process.env.SCRAPELESS_IS_ONLINE = "true";
 async function main() {
     try {
-        if (ServerMode === "rest") {
-            const transport = new RestServerTransport({
-                port: ServerPort,
-                endpoint: ServerEndpoint,
-            });
-            await server.connect(transport);
-            await transport.startServer();
+        if (ServerPort) {
+            // See use the /see endpoint
+            // Streamable HTTP use the /mcp endpoint
+            startHttpTransport(Number(ServerPort), ServerHostname);
         }
         else {
-            const transport = new StdioServerTransport();
-            await server.connect(transport);
+            if (!API_KEY)
+                throw new Error("❌ Missing environment variable: SCRAPELESS_KEY");
+            startStdioTransport();
         }
     }
     catch (error) {

package/build/server.js

@@ -1,102 +1,60 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { z } from "zod";
-import { ScrapelessClient } from "./client.js";
-import { SCRAPELESS_CONFIG, TOOL_ENDPOINTS } from "./config.js";
-const scrapelessClient = new ScrapelessClient(SCRAPELESS_CONFIG);
-export const server = new McpServer({
+import { ScrapelessClient } from "@scrapeless-ai/sdk";
+import { SCRAPELESS_CONFIG, API_KEY } from "./config.js";
+import * as toolsList from './tools/index.js';
+import * as browserTools from './tools/browser/browser.js';
+import { Context } from './context.js';
+export const serverOptions = {
     name: "scrapeless-mcp-server",
     version: "0.2.0",
     capabilities: { resources: {}, tools: {} },
-});
-server.tool("google-search", "Fetch Google Search Results", {
-    query: z
-        .string()
-        .describe("Parameter defines the query you want to search. You can use anything that you would use in a regular Google search. e.g. inurl:, site:, intitle:. We also support advanced search query parameters such as as_dt and as_eq."),
-    gl: z
-        .string()
-        .optional()
-        .describe("Parameter defines the country to use for the Google search. It's a two-letter country code. (e.g., us for the United States, uk for United Kingdom, or fr for France)."),
-    hl: z
-        .string()
-        .optional()
-        .describe("Parameter defines the language to use for the Google search. It's a two-letter language code. (e.g., en for English, es for Spanish, or fr for French)."),
-}, async ({ query, gl = "us", hl = "en" }) => {
-    const response = await scrapelessClient.sendRequest(TOOL_ENDPOINTS.SCRAPER, "scraper.google.search", {
-        q: query,
-        gl,
-        hl,
-        location: "",
-    });
-    return {
-        content: response.content.map((item) => ({
-            type: "text",
-            text: item.text,
-        })),
-    };
-});
-server.tool("google-flights-search", "Search for flights using Google Flights via Scrapeless API", {
-    departure_id: z
-        .string()
-        .describe("Airport code for departure (e.g., CDG for Paris Charles de Gaulle)"),
-    arrival_id: z
-        .string()
-        .describe("Airport code for arrival (e.g., BCN for Barcelona)"),
-    outbound_date: z.string().describe("Departure date in YYYY-MM-DD format"),
-    return_date: z
-        .string()
-        .optional()
-        .describe("Return date in YYYY-MM-DD format for round trips"),
-    gl: z
-        .string()
-        .optional()
-        .describe("Country code (e.g., us for United States, fr for France)"),
-    hl: z
-        .string()
-        .optional()
-        .describe("Language code (e.g., en for English, fr for French)"),
-    currency: z.string().optional().describe("Currency code (e.g., USD, EUR)"),
-    travel_class: z
-        .string()
-        .optional()
-        .describe("Travel class (1 for Economy, 2 for Business, 3 for First Class)"),
-    adults: z.string().optional().describe("Number of adult passengers"),
-    children: z.string().optional().describe("Number of children passengers"),
-    max_price: z
-        .string()
-        .optional()
-        .describe("Maximum price to filter results"),
-    stops: z.string().optional().describe("Filter by number of stops"),
-    emissions: z
-        .string()
-        .optional()
-        .describe("Filter for emissions (e.g., 'Less emissions only')"),
-    data_type: z
-        .string()
-        .optional()
-        .default("1")
-        .describe("Type of flight (1 for Round trip, 2 for One-way)"),
-}, async (params) => {
-    const searchParams = {
-        departure_id: params.departure_id,
-        arrival_id: params.arrival_id,
-        outbound_date: params.outbound_date,
-        return_date: params.return_date,
-        data_type: params.data_type || "1",
-        gl: params.gl || "us",
-        hl: params.hl || "en",
-        currency: params.currency,
-        travel_class: params.travel_class,
-        adults: params.adults,
-        children: params.children,
-        max_price: params.max_price,
-        stops: params.stops,
-        emissions: params.emissions,
-    };
-    const response = await scrapelessClient.sendRequest(TOOL_ENDPOINTS.SCRAPER, "scraper.google.flights", searchParams);
-    return {
-        content: response.content.map((item) => ({
-            type: "text",
-            text: item.text,
-        })),
+};
+export const createMcpServer = (apiKey) => {
+    const server = new McpServer(serverOptions);
+    initMcpTools(server, apiKey);
+    return server.server;
+};
+export const initMcpTools = (server, apiKey) => {
+    const getScrapelessClient = () => {
+        if (apiKey) {
+            return new ScrapelessClient({
+                apiKey: apiKey,
+                baseApiUrl: SCRAPELESS_CONFIG.baseApiUrl,
+            });
+        }
+        // Fallback for Stdio mode or when no API key is provided
+        return new ScrapelessClient(SCRAPELESS_CONFIG);
     };
-});
+    // tools registration
+    Object.values(toolsList).forEach((tool) => {
+        server.tool(tool.name, tool.description, tool.inputSchema, (params) => tool.handle(params, getScrapelessClient()));
+    });
+    const context = new Context(apiKey ?? API_KEY);
+    Object.values(browserTools).forEach((tool) => {
+        server.tool(tool.name, tool.description, tool.inputSchema, async (params) => {
+            const result = await context.run(tool, params);
+            return result;
+        });
+    });
+};
+export class ServerList {
+    _servers = [];
+    _serverFactory;
+    constructor(serverFactory) {
+        this._serverFactory = serverFactory;
+    }
+    async create() {
+        const server = await this._serverFactory();
+        this._servers.push(server);
+        return server;
+    }
+    async close(server) {
+        const index = this._servers.indexOf(server);
+        if (index !== -1)
+            this._servers.splice(index, 1);
+        await server.close();
+    }
+    async closeAll() {
+        await Promise.all(this._servers.map(server => server.close()));
+    }
+}

package/build/session-manager.js

@@ -0,0 +1,55 @@
+import { Scrapeless } from "@scrapeless-ai/sdk";
+import puppeteer from "puppeteer-core";
+export class SessionManager {
+    sessions;
+    constructor() {
+        this.sessions = new Map();
+    }
+    async createSession(id, apiKey) {
+        if (!this.sessions.has(id)) {
+            this.sessions.set(id, { browser: null, page: null, closed: true });
+        }
+        const scrapelessClient = new Scrapeless({ apiKey });
+        const session = this.sessions.get(id);
+        const { browserWSEndpoint } = scrapelessClient.browser.create();
+        const browser = await puppeteer.connect({
+            browserWSEndpoint,
+            defaultViewport: null,
+        });
+        const pages = await browser.pages();
+        const page = pages.length > 0 ? pages[0] : await browser.newPage();
+        browser.on("disconnected", () => {
+            session.closed = true;
+            session.browser = null;
+            session.page = null;
+        });
+        session.browser = browser;
+        session.page = page;
+        session.closed = false;
+        return session;
+    }
+    getSession(id) {
+        if (!this.sessions.has(id))
+            return null;
+        const session = this.sessions.get(id);
+        if (!session.browser || !session.browser?.connected) {
+            session.browser = null;
+            session.page = null;
+            session.closed = true;
+            this.sessions.delete(id);
+            return null;
+        }
+        return session;
+    }
+    async closeSession(id) {
+        const session = this.sessions.get(id);
+        if (session) {
+            session.page?.close().catch(() => { });
+            session.browser?.disconnect().catch(() => { });
+            session.closed = true;
+            session.browser = null;
+            session.page = null;
+            this.sessions.delete(id);
+        }
+    }
+}