xcrawl-mcp 1.0.0

package/src/core/search.ts

@@ -0,0 +1,81 @@
+import { z } from "zod";
+import type { XCrawlSearchRequest, XCrawlSearchResponse } from "../types.js";
+
+/**
+ * Zod schema for xcrawl_search tool parameters
+ */
+export const searchToolSchema = z.object({
+	query: z.string().min(1).describe("Search keywords"),
+	location: z.string().optional().describe("Search location (country/city name or ISO code)"),
+	language: z.string().optional().describe("Search language (ISO 639-1 code)"),
+	limit: z.number().int().min(1).max(100).optional().describe("Number of results to return (1-100)"),
+	serp_options: z
+		.object({
+			q: z.string().optional(),
+			location: z.string().optional(),
+			uule: z.string().optional(),
+			google_domain: z.string().optional(),
+			gl: z.string().optional(),
+			hl: z.string().optional(),
+			cr: z.string().optional(),
+			lr: z.string().optional(),
+			safe: z.number().int().optional(),
+			nfpr: z.boolean().optional(),
+			filter: z.boolean().optional(),
+			tbs: z.string().optional(),
+			start: z.number().int().optional(),
+			num: z.number().int().optional(),
+			ludocid: z.string().optional(),
+			lsig: z.string().optional(),
+			kgmid: z.string().optional(),
+			si: z.string().optional(),
+			ibp: z.string().optional(),
+			uds: z.string().optional(),
+			no_cache: z.boolean().optional(),
+		})
+		.optional()
+		.describe("Advanced Google SERP parameters"),
+});
+
+export type SearchToolParams = z.infer<typeof searchToolSchema>;
+
+/**
+ * Call xCrawl Search API to get Google SERP results
+ */
+export async function callXCrawlSearchAPI(apiKey: string, params: XCrawlSearchRequest): Promise<XCrawlSearchResponse> {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+
+	try {
+		const response = await fetch("https://run.xcrawl.com/v1/search", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify(params),
+			signal: controller.signal,
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new Error(`xCrawl Search API error: ${response.status} ${response.statusText} - ${errorText}`);
+		}
+
+		return (await response.json()) as XCrawlSearchResponse;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") {
+			throw new Error("Request timeout after 300 seconds");
+		}
+		throw error;
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+
+/**
+ * Format search response for MCP tool output
+ */
+export function formatSearchResponse(response: XCrawlSearchResponse): string {
+	return JSON.stringify(response, null, 2);
+}

package/src/index.ts

@@ -0,0 +1,565 @@
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import { callXCrawlAPI, checkScrapeStatus, formatScrapeResponse, scrapeToolSchema } from "./core/scrape.js";
+import { callXCrawlSearchAPI, formatSearchResponse, searchToolSchema } from "./core/search.js";
+import { callXCrawlMapAPI, formatMapResponse, mapToolSchema } from "./core/map.js";
+import { callXCrawlCrawlAPI, checkCrawlStatus, formatCrawlResponse, crawlToolSchema } from "./core/crawl.js";
+import type { XCrawlScrapeRequest, XCrawlSearchRequest, XCrawlMapRequest, XCrawlCrawlRequest } from "./types.js";
+import {
+	XCRAWL_SCRAPE_TOOL,
+	XCRAWL_CHECK_STATUS_TOOL,
+	XCRAWL_SEARCH_TOOL,
+	XCRAWL_MAP_TOOL,
+	XCRAWL_CRAWL_TOOL,
+	XCRAWL_CHECK_CRAWL_STATUS_TOOL,
+} from "./tools.js";
+
+/**
+ * Extract API key from request headers
+ */
+function extractApiKey(request: Request): string | undefined {
+	const authHeader = request.headers.get("Authorization");
+	if (authHeader?.startsWith("Bearer ")) {
+		return authHeader.slice(7);
+	}
+	return request.headers.get("x-api-key") || request.headers.get("x-xcrawl-api-key") || undefined;
+}
+
+/**
+ * Create MCP server instance
+ */
+function createMCPServer(apiKey: string): Server {
+	const server = new Server(
+		{
+			name: "xCrawl MCP Server",
+			version: "1.0.0",
+		},
+		{
+			capabilities: {
+				tools: {},
+			},
+		}
+	);
+
+	// Register tools/list handler
+	server.setRequestHandler(ListToolsRequestSchema, async () => ({
+		tools: [
+			XCRAWL_SCRAPE_TOOL,
+			XCRAWL_CHECK_STATUS_TOOL,
+			XCRAWL_SEARCH_TOOL,
+			XCRAWL_MAP_TOOL,
+			XCRAWL_CRAWL_TOOL,
+			XCRAWL_CHECK_CRAWL_STATUS_TOOL,
+		],
+	}));
+
+	// Register tools/call handler
+	server.setRequestHandler(CallToolRequestSchema, async (request) => {
+		if (request.params.name === "xcrawl_scrape") {
+			try {
+				const validatedArgs = scrapeToolSchema.parse(request.params.arguments);
+				const response = await callXCrawlAPI(apiKey, validatedArgs as XCrawlScrapeRequest);
+
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: formatScrapeResponse(response),
+						},
+					],
+				};
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error);
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: `Error: ${errorMessage}`,
+						},
+					],
+					isError: true,
+				};
+			}
+		}
+
+		if (request.params.name === "xcrawl_check_status") {
+			try {
+				const args = request.params.arguments as { scrape_id: string };
+
+				if (!args.scrape_id || typeof args.scrape_id !== "string") {
+					throw new Error("scrape_id is required and must be a string");
+				}
+
+				const response = await checkScrapeStatus(apiKey, args.scrape_id);
+
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: formatScrapeResponse(response),
+						},
+					],
+				};
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error);
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: `Error: ${errorMessage}`,
+						},
+					],
+					isError: true,
+				};
+			}
+		}
+
+		if (request.params.name === "xcrawl_search") {
+			try {
+				const validatedArgs = searchToolSchema.parse(request.params.arguments);
+				const response = await callXCrawlSearchAPI(apiKey, validatedArgs as XCrawlSearchRequest);
+
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: formatSearchResponse(response),
+						},
+					],
+				};
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error);
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: `Error: ${errorMessage}`,
+						},
+					],
+					isError: true,
+				};
+			}
+		}
+
+		if (request.params.name === "xcrawl_map") {
+			try {
+				const validatedArgs = mapToolSchema.parse(request.params.arguments);
+				const response = await callXCrawlMapAPI(apiKey, validatedArgs as XCrawlMapRequest);
+
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: formatMapResponse(response),
+						},
+					],
+				};
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error);
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: `Error: ${errorMessage}`,
+						},
+					],
+					isError: true,
+				};
+			}
+		}
+
+		if (request.params.name === "xcrawl_crawl") {
+			try {
+				const validatedArgs = crawlToolSchema.parse(request.params.arguments);
+				const response = await callXCrawlCrawlAPI(apiKey, validatedArgs as XCrawlCrawlRequest);
+
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: formatCrawlResponse(response),
+						},
+					],
+				};
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error);
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: `Error: ${errorMessage}`,
+						},
+					],
+					isError: true,
+				};
+			}
+		}
+
+		if (request.params.name === "xcrawl_check_crawl_status") {
+			try {
+				const args = request.params.arguments as { crawl_id: string };
+				if (!args.crawl_id || typeof args.crawl_id !== "string") {
+					throw new Error("crawl_id is required and must be a string");
+				}
+
+				const response = await checkCrawlStatus(apiKey, args.crawl_id);
+
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: formatCrawlResponse(response),
+						},
+					],
+				};
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error);
+				return {
+					content: [
+						{
+							type: "text" as const,
+							text: `Error: ${errorMessage}`,
+						},
+					],
+					isError: true,
+				};
+			}
+		}
+
+		throw new Error(`Unknown tool: ${request.params.name}`);
+	});
+
+	return server;
+}
+
+/**
+ * Cloudflare Workers fetch handler
+ */
+export default {
+	async fetch(request: Request, env: Record<string, string>, ctx: any): Promise<Response> {
+		const url = new URL(request.url);
+
+		// Health check endpoint
+		if (url.pathname === "/health") {
+			return new Response("OK", { status: 200 });
+		}
+
+		// Extract API key from request headers only
+		const apiKey = extractApiKey(request);
+		if (!apiKey) {
+			return new Response(
+				JSON.stringify({
+					error: "API key required",
+					message: "Please provide xCrawl API key via one of these headers:\n- Authorization: Bearer <your-api-key>\n- x-api-key: <your-api-key>\n- x-xcrawl-api-key: <your-api-key>",
+				}),
+				{
+					status: 401,
+					headers: { "Content-Type": "application/json" },
+				}
+			);
+		}
+
+		// SSE endpoint
+		if (url.pathname === "/sse" || url.pathname.startsWith("/sse/")) {
+			// SSE transport requires streaming support which is complex in Workers
+			// For now, return a message directing to use stdio mode
+			return new Response(
+				JSON.stringify({
+					message: "SSE endpoint not yet implemented in Workers mode. Please use stdio mode for local development.",
+				}),
+				{
+					status: 501,
+					headers: { "Content-Type": "application/json" },
+				}
+			);
+		}
+
+		// MCP endpoint (streamable HTTP / JSON-RPC)
+		if (url.pathname === "/mcp") {
+			try {
+				// Parse JSON-RPC request
+				const body = (await request.json()) as any;
+				const { jsonrpc, id, method, params } = body;
+
+				// Validate JSON-RPC format
+				if (jsonrpc !== "2.0") {
+					return new Response(
+						JSON.stringify({
+							jsonrpc: "2.0",
+							id: id || null,
+							error: { code: -32600, message: "Invalid Request - jsonrpc must be '2.0'" },
+						}),
+						{
+							status: 400,
+							headers: { "Content-Type": "application/json" },
+						}
+					);
+				}
+
+				let result: any;
+
+				// Handle initialize
+				if (method === "initialize") {
+					result = {
+						protocolVersion: "2024-11-05",
+						capabilities: {
+							tools: {},
+						},
+						serverInfo: {
+							name: "xCrawl MCP Server",
+							version: "1.0.0",
+						},
+					};
+				}
+				// Handle notifications/initialized (no response needed for notifications)
+				else if (method === "notifications/initialized") {
+					// This is a notification, return empty success response
+					return new Response(
+						JSON.stringify({
+							jsonrpc: "2.0",
+							id,
+							result: {},
+						}),
+						{
+							status: 200,
+							headers: { "Content-Type": "application/json" },
+						}
+					);
+				}
+				// Handle tools/list
+				else if (method === "tools/list") {
+					result = {
+						tools: [
+							XCRAWL_SCRAPE_TOOL,
+							XCRAWL_CHECK_STATUS_TOOL,
+							XCRAWL_SEARCH_TOOL,
+							XCRAWL_MAP_TOOL,
+							XCRAWL_CRAWL_TOOL,
+							XCRAWL_CHECK_CRAWL_STATUS_TOOL,
+						],
+					};
+				}
+				// Handle tools/call
+				else if (method === "tools/call") {
+					const toolName = params?.name;
+					const toolArgs = params?.arguments || {};
+
+					if (toolName === "xcrawl_scrape") {
+						try {
+							const validatedArgs = scrapeToolSchema.parse(toolArgs);
+							const response = await callXCrawlAPI(apiKey, validatedArgs as XCrawlScrapeRequest);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: formatScrapeResponse(response),
+									},
+								],
+							};
+						} catch (error) {
+							const errorMessage = error instanceof Error ? error.message : String(error);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: `Error: ${errorMessage}`,
+									},
+								],
+								isError: true,
+							};
+						}
+					} else if (toolName === "xcrawl_check_status") {
+						try {
+							const scrapeId = toolArgs.scrape_id;
+							if (!scrapeId || typeof scrapeId !== "string") {
+								throw new Error("scrape_id is required and must be a string");
+							}
+							const response = await checkScrapeStatus(apiKey, scrapeId);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: formatScrapeResponse(response),
+									},
+								],
+							};
+						} catch (error) {
+							const errorMessage = error instanceof Error ? error.message : String(error);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: `Error: ${errorMessage}`,
+									},
+								],
+								isError: true,
+							};
+						}
+					} else if (toolName === "xcrawl_search") {
+						try {
+							const validatedArgs = searchToolSchema.parse(toolArgs);
+							const response = await callXCrawlSearchAPI(apiKey, validatedArgs as XCrawlSearchRequest);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: formatSearchResponse(response),
+									},
+								],
+							};
+						} catch (error) {
+							const errorMessage = error instanceof Error ? error.message : String(error);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: `Error: ${errorMessage}`,
+									},
+								],
+								isError: true,
+							};
+						}
+					} else if (toolName === "xcrawl_map") {
+						try {
+							const validatedArgs = mapToolSchema.parse(toolArgs);
+							const response = await callXCrawlMapAPI(apiKey, validatedArgs as XCrawlMapRequest);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: formatMapResponse(response),
+									},
+								],
+							};
+						} catch (error) {
+							const errorMessage = error instanceof Error ? error.message : String(error);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: `Error: ${errorMessage}`,
+									},
+								],
+								isError: true,
+							};
+						}
+					} else if (toolName === "xcrawl_crawl") {
+						try {
+							const validatedArgs = crawlToolSchema.parse(toolArgs);
+							const response = await callXCrawlCrawlAPI(apiKey, validatedArgs as XCrawlCrawlRequest);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: formatCrawlResponse(response),
+									},
+								],
+							};
+						} catch (error) {
+							const errorMessage = error instanceof Error ? error.message : String(error);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: `Error: ${errorMessage}`,
+									},
+								],
+								isError: true,
+							};
+						}
+					} else if (toolName === "xcrawl_check_crawl_status") {
+						try {
+							const crawlId = toolArgs.crawl_id;
+							if (!crawlId || typeof crawlId !== "string") {
+								throw new Error("crawl_id is required and must be a string");
+							}
+							const response = await checkCrawlStatus(apiKey, crawlId);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: formatCrawlResponse(response),
+									},
+								],
+							};
+						} catch (error) {
+							const errorMessage = error instanceof Error ? error.message : String(error);
+							result = {
+								content: [
+									{
+										type: "text",
+										text: `Error: ${errorMessage}`,
+									},
+								],
+								isError: true,
+							};
+						}
+					} else {
+						return new Response(
+							JSON.stringify({
+								jsonrpc: "2.0",
+								id,
+								error: { code: -32601, message: `Unknown tool: ${toolName}` },
+							}),
+							{
+								status: 404,
+								headers: { "Content-Type": "application/json" },
+							}
+						);
+					}
+				}
+				// Method not found
+				else {
+					return new Response(
+						JSON.stringify({
+							jsonrpc: "2.0",
+							id,
+							error: { code: -32601, message: `Method not found: ${method}` },
+						}),
+						{
+							status: 404,
+							headers: { "Content-Type": "application/json" },
+						}
+					);
+				}
+
+				return new Response(
+					JSON.stringify({
+						jsonrpc: "2.0",
+						id,
+						result,
+					}),
+					{
+						status: 200,
+						headers: { "Content-Type": "application/json" },
+					}
+				);
+			} catch (error) {
+				const errorMessage = error instanceof Error ? error.message : String(error);
+				return new Response(
+					JSON.stringify({
+						jsonrpc: "2.0",
+						id: null,
+						error: { code: -32700, message: `Parse error: ${errorMessage}` },
+					}),
+					{
+						status: 400,
+						headers: { "Content-Type": "application/json" },
+					}
+				);
+			}
+		}
+
+		// Default 404 response
+		return new Response(
+			JSON.stringify({
+				error: "Not found",
+				message: "Available endpoints: /mcp, /sse, /health",
+			}),
+			{
+				status: 404,
+				headers: { "Content-Type": "application/json" },
+			}
+		);
+	},
+};