scrapeless-mcp-server 0.5.0 → 0.6.1

package/README.md

@@ -10,6 +10,7 @@ Built on the open MCP standard, Scrapeless MCP Server seamlessly connects models
 - **Browser automation** for page-level navigation and interaction
 - **Scrape** dynamic, JS-heavy sites—export as HTML, Markdown, or screenshots
 - **Crawl** entire websites by following links and capture each page in multiple formats
+- **AI Scraper** Create an AI Scraper task for ChatGPT, Gemini, Perplexity, Copilot, Google AI Mode, Google AI Overview, Grok, or Alexa
 
 Whether you're building an AI research assistant, a coding copilot, or autonomous web agents, this server provides the dynamic context and real-world data your workflows need—**without getting blocked**.
 
@@ -72,7 +73,7 @@ Scrapeless MCP Server supports both **Stdio** and **Streamable HTTP** transport
       "command": "npx",
       "args": ["-y", "scrapeless-mcp-server"],
       "env": {
-        "SCRAPELESS_KEY": "YOUR_SCRAPELESS_KEY"
+        "SCRAPELESS_API_KEY": "YOUR_SCRAPELESS_KEY"
       }
     }
   }
@@ -129,31 +130,32 @@ Customize browser session behavior with optional parameters. These can be set vi
 
 ## Supported MCP Tools
 
-| Name               | Description                                                    |
-| ------------------ | -------------------------------------------------------------- |
-| google_search      | Universal information search engine.                           |
-| google_trends      | Get trending search data from Google Trends.                   |
-| browser_create     | Create or reuse a cloud browser session using Scrapeless.      |
-| browser_close      | Closes the current session by disconnecting the cloud browser. |
-| browser_goto       | Navigate browser to a specified URL.                           |
-| browser_go_back    | Go back one step in browser history.                           |
-| browser_go_forward | Go forward one step in browser history.                        |
-| browser_click      | Click a specific element on the page.                          |
-| browser_type       | Type text into a specified input field.                        |
-| browser_press_key  | Simulate a key press.                                          |
-| browser_wait_for   | Wait for a specific page element to appear.                    |
-| browser_wait       | Pause execution for a fixed duration.                          |
-| browser_screenshot | Capture a screenshot of the current page.                      |
-| browser_get_html   | Get the full HTML of the current page.                         |
-| browser_get_text   | Get all visible text from the current page.                    |
-| browser_scroll     | Scroll to the bottom of the page.                              |
-| browser_scroll_to  | Scroll a specific element into view.                           |
-| scrape_html        | Scrape a URL and return its full HTML content.                 |
-| scrape_markdown    | Scrape a URL and return its content as Markdown.               |
-| scrape_screenshot  | Capture a high-quality screenshot of any webpage.              |
-| crawl_start        | Start an asynchronous crawl job from a base URL and return its job id. |
-| crawl_cancel       | Cancel an in-progress crawl job by its id.                     |
-| crawl_result       | Poll a crawl job by its id until it completes and return the crawled data. |
+| Name               | Description                                                                                                             |
+|--------------------|-------------------------------------------------------------------------------------------------------------------------|
+| google_search      | Universal information search engine.                                                                                    |
+| google_trends      | Get trending search data from Google Trends.                                                                            |
+| browser_create     | Create or reuse a cloud browser session using Scrapeless.                                                               |
+| browser_close      | Closes the current session by disconnecting the cloud browser.                                                          |
+| browser_goto       | Navigate browser to a specified URL.                                                                                    |
+| browser_go_back    | Go back one step in browser history.                                                                                    |
+| browser_go_forward | Go forward one step in browser history.                                                                                 |
+| browser_click      | Click a specific element on the page.                                                                                   |
+| browser_type       | Type text into a specified input field.                                                                                 |
+| browser_press_key  | Simulate a key press.                                                                                                   |
+| browser_wait_for   | Wait for a specific page element to appear.                                                                             |
+| browser_wait       | Pause execution for a fixed duration.                                                                                   |
+| browser_screenshot | Capture a screenshot of the current page.                                                                               |
+| browser_get_html   | Get the full HTML of the current page.                                                                                  |
+| browser_get_text   | Get all visible text from the current page.                                                                             |
+| browser_scroll     | Scroll to the bottom of the page.                                                                                       |
+| browser_scroll_to  | Scroll a specific element into view.                                                                                    |
+| scrape_html        | Scrape a URL and return its full HTML content.                                                                          |
+| scrape_markdown    | Scrape a URL and return its content as Markdown.                                                                        |
+| scrape_screenshot  | Capture a high-quality screenshot of any webpage.                                                                       |
+| crawl_start        | Start an asynchronous crawl job from a base URL and return its job id.                                                  |
+| crawl_cancel       | Cancel an in-progress crawl job by its id.                                                                              |
+| crawl_result       | Poll a crawl job by its id until it completes and return the crawled data.                                              |
+| ai_scraper         | Create an AI Scraper task for ChatGPT, Gemini, Perplexity, Copilot, Google AI Mode, Google AI Overview, Grok, or Alexa. |
 
 ## Security Best Practices
 
@@ -163,7 +165,7 @@ When using Scrapeless MCP Server with LLMs (like ChatGPT, Claude, or Cursor), it
 
 - **Never pass raw scraped content directly into LLM prompts.** Raw HTML, JavaScript, or user-generated text may contain hidden injection payloads.
 - **Sanitize and validate all extracted content.** Strip or escape potentially harmful tags and scripts before using content in downstream logic or AI models.
-- **Prefer structured extraction over free-form text.** Use tools like `scrape_html`, `scrape_markdown`, or targeted `browser_get_text` with known-safe selectors to extract only the content you trust.
+- **Prefer structured extraction to free-form text.** Use tools like `scrape_html`, `scrape_markdown`, or targeted `browser_get_text` with known-safe selectors to extract only the content you trust.
 - **Apply domain or selector whitelisting** when scraping dynamically generated pages, to restrict data flow to known and trusted sources.
 - **Log and monitor all outbound requests** made via browser or scraping tools, especially if you're handling sensitive data, tokens, or internal network access.
 

package/build/cf.js

@@ -0,0 +1,27 @@
+import { McpAgent } from "agents/mcp";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { initMcpTools, serverOptions } from "./server.js";
+import { API_KEY_NAME } from "./config.js";
+export class CfMcpServer extends McpAgent {
+    server = new McpServer(serverOptions);
+    async init() {
+        initMcpTools(this.server, this.props.headers, this.props.apiKey);
+    }
+}
+export default {
+    fetch(request, env, ctx) {
+        const url = new URL(request.url);
+        const apiKeyHeader = request.headers.get(API_KEY_NAME);
+        if (!apiKeyHeader) {
+            return new Response(`Unauthorized: Missing ${API_KEY_NAME} header`, { status: 401 });
+        }
+        ctx.props = { apiKey: apiKeyHeader, headers: request.headers };
+        if (url.pathname === "/sse" || url.pathname === "/sse/message") {
+            return CfMcpServer.serveSSE("/sse").fetch(request, env, ctx);
+        }
+        if (url.pathname === "/mcp") {
+            return CfMcpServer.serve("/mcp").fetch(request, env, ctx);
+        }
+        return new Response("Not found", { status: 404 });
+    },
+};

package/build/config.js

@@ -0,0 +1,14 @@
+import { getParamValue } from "@chatmcp/sdk/utils/index.js";
+export const API_KEY = process.env.SCRAPELESS_API_KEY?.trim() ||
+    getParamValue("SCRAPELESS_API_KEY") ||
+    process.env.SCRAPELESS_KEY?.trim() ||
+    getParamValue("SCRAPELESS_KEY");
+export const BASE_URL = process.env.SCRAPELESS_BASE_URL?.trim() || "https://api.scrapeless.com";
+export const API_KEY_NAME = "x-api-token";
+export const ServerMode = getParamValue("mode") || "stdio";
+export const ServerPort = getParamValue("port");
+export const ServerHostname = getParamValue("hostname") || "0.0.0.0";
+export const SCRAPELESS_CONFIG = {
+    baseApiUrl: BASE_URL,
+    apiKey: API_KEY,
+};

package/build/context-manager.js

@@ -0,0 +1,40 @@
+import { Context } from "./context.js";
+import { API_KEY } from "./config.js";
+export class ContextManager {
+    static instance;
+    contexts;
+    constructor() {
+        this.contexts = new Map();
+    }
+    static getInstance() {
+        if (!ContextManager.instance) {
+            ContextManager.instance = new ContextManager();
+        }
+        return ContextManager.instance;
+    }
+    getContext(apiKey) {
+        const key = apiKey ?? API_KEY;
+        if (!this.contexts.has(key)) {
+            console.log(`Creating new Context for API key: ${key?.substring(0, 8)}...`);
+            this.contexts.set(key, new Context(key));
+        }
+        else {
+            console.log(`Reusing existing Context for API key: ${key?.substring(0, 8)}...`);
+        }
+        return this.contexts.get(key);
+    }
+    clearContext(apiKey) {
+        const key = apiKey ?? API_KEY;
+        if (this.contexts.has(key)) {
+            this.contexts.delete(key);
+            console.log(`Cleared Context for API key: ${key?.substring(0, 8)}...`);
+        }
+    }
+    clearAllContexts() {
+        this.contexts.clear();
+        console.log("Cleared all Contexts");
+    }
+    getContextCount() {
+        return this.contexts.size;
+    }
+}

package/build/context.js

@@ -0,0 +1,120 @@
+import { SessionManager } from "./session-manager.js";
+import { uuid } from "./tools/utils.js";
+export class Context {
+    sessionManager;
+    currentSessionId = "default-session-id";
+    apiKey;
+    constructor(apiKey) {
+        this.sessionManager = SessionManager.getInstance();
+        this.apiKey = apiKey;
+    }
+    getSession(id) {
+        return this.sessionManager.getSession(`${id ?? this.currentSessionId}-${this.apiKey}`);
+    }
+    async run(tool, params, headers) {
+        if (!this.apiKey) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `API key is missing.`,
+                    },
+                ],
+            };
+        }
+        if (params.sessionId && params.sessionId !== this.currentSessionId) {
+            this.currentSessionId = params.sessionId;
+        }
+        const toolName = tool.name;
+        if (toolName === "browser_create") {
+            const newSessionId = uuid();
+            try {
+                await this.sessionManager.createSession(`${newSessionId}-${this.apiKey}`, this.apiKey, headers);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `New browser session created with ID: ${newSessionId}`,
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Failed to create browser session. Please try again later.`,
+                        },
+                    ],
+                };
+            }
+        }
+        if (toolName === "browser_close") {
+            const sessionId = params?.sessionId;
+            if (!sessionId) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Session ID is missing.`,
+                        },
+                    ],
+                };
+            }
+            try {
+                await this.sessionManager.closeSession(`${sessionId}-${this.apiKey}`);
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Browser session with ID: ${sessionId} has been closed.`,
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Failed to close browser session with ID: ${sessionId}. Please try again later.`,
+                        },
+                    ],
+                };
+            }
+        }
+        let session = null;
+        session = this.sessionManager.getSession(`${this.currentSessionId}-${this.apiKey}`);
+        if (!session) {
+            const newSessionId = uuid();
+            await this.sessionManager.createSession(`${newSessionId}-${this.apiKey}`, this.apiKey);
+            this.currentSessionId = newSessionId;
+        }
+        let toolActionOutput = undefined;
+        let actionSucceeded = false;
+        try {
+            let action = undefined;
+            action = tool.handle;
+            if (action) {
+                toolActionOutput = await action(this, params);
+                actionSucceeded = true;
+            }
+        }
+        catch (e) {
+        }
+        finally {
+            if (actionSucceeded && toolActionOutput !== undefined) {
+                return toolActionOutput;
+            }
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `${toolName} action completed successfully.`,
+                    },
+                ],
+            };
+        }
+    }
+}

package/build/index.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+import { ServerPort, API_KEY, ServerHostname } from "./config.js";
+import { startHttpTransport, startStdioTransport } from "./transport.js";
+process.env.SCRAPELESS_IS_ONLINE = "true";
+async function main() {
+    try {
+        if (ServerPort) {
+            // See use the /see endpoint
+            // Streamable HTTP use the /mcp endpoint
+            startHttpTransport(Number(ServerPort), ServerHostname);
+        }
+        else {
+            if (!API_KEY)
+                throw new Error("❌ Missing environment variable: SCRAPELESS_API_KEY");
+            startStdioTransport();
+        }
+    }
+    catch (error) {
+        console.error("Fatal error in main():", error);
+        process.exit(1);
+    }
+}
+main();

package/build/server.js

@@ -0,0 +1,60 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { ScrapelessClient } from "@scrapeless-ai/sdk";
+import { SCRAPELESS_CONFIG, API_KEY } from "./config.js";
+import * as toolsList from "./tools/index.js";
+import * as browserTools from "./tools/browser/browser.js";
+import { ContextManager } from "./context-manager.js";
+export const serverOptions = {
+    name: "scrapeless-mcp-server",
+    version: "0.2.0",
+    capabilities: { resources: {}, tools: {} },
+};
+export const createMcpServer = (options) => {
+    const server = new McpServer(serverOptions);
+    initMcpTools(server, options?.headers, options?.apiKey);
+    return server.server;
+};
+export const initMcpTools = (server, headers, apiKey) => {
+    const getScrapelessClient = () => {
+        if (apiKey) {
+            return new ScrapelessClient({
+                apiKey: apiKey,
+                baseApiUrl: SCRAPELESS_CONFIG.baseApiUrl,
+            });
+        }
+        // Fallback for Stdio mode or when no API key is provided
+        return new ScrapelessClient(SCRAPELESS_CONFIG);
+    };
+    // tools registration
+    Object.values(toolsList).forEach((tool) => {
+        server.tool(tool.name, tool.description, tool.inputSchema, (params) => tool.handle(params, getScrapelessClient(), headers));
+    });
+    const context = ContextManager.getInstance().getContext(apiKey ?? API_KEY);
+    Object.values(browserTools).forEach((tool) => {
+        server.tool(tool.name, tool.description, tool.inputSchema, async (params) => {
+            const result = await context.run(tool, params, headers);
+            return result;
+        });
+    });
+};
+export class ServerList {
+    _servers = [];
+    _serverFactory;
+    constructor(serverFactory) {
+        this._serverFactory = serverFactory;
+    }
+    async create() {
+        const server = await this._serverFactory();
+        this._servers.push(server);
+        return server;
+    }
+    async close(server) {
+        const index = this._servers.indexOf(server);
+        if (index !== -1)
+            this._servers.splice(index, 1);
+        await server.close();
+    }
+    async closeAll() {
+        await Promise.all(this._servers.map((server) => server.close()));
+    }
+}

package/build/session-manager.js

@@ -0,0 +1,76 @@
+import { Scrapeless } from "@scrapeless-ai/sdk";
+import { getParamValue } from "@chatmcp/sdk/utils/index.js";
+import puppeteer from "puppeteer-core";
+export class SessionManager {
+    static instance;
+    sessions;
+    constructor() {
+        this.sessions = new Map();
+    }
+    static getInstance() {
+        if (!SessionManager.instance) {
+            SessionManager.instance = new SessionManager();
+        }
+        return SessionManager.instance;
+    }
+    async createSession(id, apiKey, headers) {
+        if (!this.sessions.has(id)) {
+            this.sessions.set(id, { browser: null, page: null, closed: true });
+        }
+        const scrapelessClient = new Scrapeless({ apiKey });
+        const session = this.sessions.get(id);
+        const { browserWSEndpoint } = scrapelessClient.browser.create({
+            session_ttl: Number(process.env.BROWSER_SESSION_TTL ||
+                getParamValue("BROWSER_SESSION_TTL") ||
+                headers?.["x-browser-session-ttl"] ||
+                30000),
+            profile_id: process.env.BROWSER_PROFILE_ID ||
+                getParamValue("BROWSER_PROFILE_ID") ||
+                headers?.["x-browser-profile-id"] ||
+                "",
+            profile_persist: Boolean(process.env.BROWSER_PROFILE_PERSIST ||
+                getParamValue("BROWSER_PROFILE_PERSIST") ||
+                headers?.["x-browser-profile-persist"]),
+        });
+        const browser = await puppeteer.connect({
+            browserWSEndpoint,
+            defaultViewport: null,
+        });
+        const pages = await browser.pages();
+        const page = pages.length > 0 ? pages[0] : await browser.newPage();
+        browser.on("disconnected", () => {
+            session.closed = true;
+            session.browser = null;
+            session.page = null;
+            this.sessions.delete(id);
+        });
+        session.browser = browser;
+        session.page = page;
+        session.closed = false;
+        return session;
+    }
+    getSession(id) {
+        if (!this.sessions.has(id))
+            return null;
+        const session = this.sessions.get(id);
+        if (!session.browser || !session.browser?.connected) {
+            session.browser = null;
+            session.page = null;
+            session.closed = true;
+            this.sessions.delete(id);
+            return null;
+        }
+        return session;
+    }
+    async closeSession(id) {
+        const session = this.sessions.get(id);
+        if (session) {
+            session.page?.close().catch(() => { });
+            session.browser?.disconnect().catch(() => { });
+            session.closed = true;
+            session.browser = null;
+            session.page = null;
+            this.sessions.delete(id);
+        }
+    }
+}