xcrawl-mcp 1.0.0

package/dist/types.d.ts

@@ -0,0 +1,214 @@
+/**
+ * xCrawl API Request and Response Types
+ */
+export interface XCrawlScrapeRequest {
+    url: string;
+    mode?: "sync" | "async";
+    proxy?: {
+        location?: string;
+        sticky_session?: string;
+    };
+    request?: {
+        locale?: string;
+        device?: "desktop" | "mobile";
+        cookies?: Record<string, unknown>;
+        headers?: Record<string, unknown>;
+        only_main_content?: boolean;
+        block_ads?: boolean;
+        skip_tls_verification?: boolean;
+    };
+    js_render?: {
+        enabled?: boolean;
+        wait_until?: "load" | "domcontentloaded" | "networkidle";
+        viewport?: {
+            width?: number;
+            height?: number;
+        };
+    };
+    output?: {
+        formats?: Array<"html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json">;
+        screenshot?: "full_page" | "viewport";
+        json?: {
+            prompt?: string;
+            json_schema?: Record<string, any>;
+        };
+    };
+    webhook?: {
+        url?: string;
+        headers?: Record<string, string>;
+        events?: Array<"started" | "completed" | "failed">;
+    };
+}
+export interface XCrawlScrapeResponse {
+    scrape_id: string;
+    endpoint: string;
+    version: string;
+    status: string;
+    url?: string;
+    data?: {
+        html?: string;
+        raw_html?: string;
+        markdown?: string;
+        links?: string[];
+        metadata?: Record<string, any>;
+        screenshot?: string;
+        summary?: string;
+        json?: Record<string, any>;
+        traffic_bytes?: number;
+        credits_used?: number;
+        credits_detail?: Record<string, any>;
+    };
+    started_at?: string;
+    ended_at?: string;
+    total_credits_used?: number;
+    message?: string;
+}
+/**
+ * Search API Request and Response Types
+ */
+export interface XCrawlSearchRequest {
+    query: string;
+    location?: string;
+    language?: string;
+    limit?: number;
+    serp_options?: {
+        q?: string;
+        location?: string;
+        uule?: string;
+        google_domain?: string;
+        gl?: string;
+        hl?: string;
+        cr?: string;
+        lr?: string;
+        safe?: number;
+        nfpr?: boolean;
+        filter?: boolean;
+        tbs?: string;
+        start?: number;
+        num?: number;
+        ludocid?: string;
+        lsig?: string;
+        kgmid?: string;
+        si?: string;
+        ibp?: string;
+        uds?: string;
+        no_cache?: boolean;
+    };
+}
+export interface XCrawlSearchResponse {
+    search_id: string;
+    endpoint: string;
+    version: string;
+    status: string;
+    query: string;
+    data?: {
+        results?: Record<string, any>;
+        credits_used?: number;
+        credits_detail?: Record<string, any>;
+    };
+    started_at?: string;
+    ended_at?: string;
+    total_credits_used?: number;
+}
+/**
+ * Map API Request and Response Types
+ */
+export interface XCrawlMapRequest {
+    url: string;
+    filter?: string;
+    limit?: number;
+    include_subdomains?: boolean;
+    ignore_query_parameters?: boolean;
+}
+export interface XCrawlMapResponse {
+    map_id: string;
+    endpoint: string;
+    version: string;
+    status: string;
+    url: string;
+    data?: {
+        links?: string[];
+        total_links?: number;
+        credits_used?: number;
+        credits_detail?: Record<string, any>;
+    };
+    started_at?: string;
+    ended_at?: string;
+    total_credits_used?: number;
+}
+/**
+ * Crawl API Request and Response Types
+ */
+export interface XCrawlCrawlRequest {
+    url: string;
+    crawler?: {
+        limit?: number;
+        include?: string[];
+        exclude?: string[];
+        max_depth?: number;
+        include_entire_domain?: boolean;
+        include_subdomains?: boolean;
+        include_external_links?: boolean;
+        sitemaps?: boolean;
+    };
+    proxy?: {
+        location?: string;
+        sticky_session?: string;
+    };
+    request?: {
+        locale?: string;
+        device?: "desktop" | "mobile";
+        cookies?: Record<string, unknown>;
+        headers?: Record<string, unknown>;
+        only_main_content?: boolean;
+        block_ads?: boolean;
+        skip_tls_verification?: boolean;
+    };
+    js_render?: {
+        enabled?: boolean;
+        wait_until?: "load" | "domcontentloaded" | "networkidle";
+        viewport?: {
+            width?: number;
+            height?: number;
+        };
+    };
+    output?: {
+        formats?: Array<"html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json">;
+        screenshot?: "full_page" | "viewport";
+        json?: {
+            prompt?: string;
+            json_schema?: Record<string, any>;
+        };
+    };
+    webhook?: {
+        url?: string;
+        headers?: Record<string, string>;
+        events?: Array<"started" | "completed" | "failed">;
+    };
+}
+export interface XCrawlCrawlResponse {
+    crawl_id: string;
+    endpoint: string;
+    version: string;
+    status: string;
+    url?: string;
+    data?: Array<{
+        url: string;
+        html?: string;
+        raw_html?: string;
+        markdown?: string;
+        links?: string[];
+        metadata?: Record<string, any>;
+        screenshot?: string;
+        summary?: string;
+        json?: Record<string, any>;
+        traffic_bytes?: number;
+        credits_used?: number;
+        credits_detail?: Record<string, any>;
+    }>;
+    started_at?: string;
+    ended_at?: string;
+    total_credits_used?: number;
+    message?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,SAAS,GAAG,QAAQ,CAAC;QAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAClC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;KAChC,CAAC;IACF,SAAS,CAAC,EAAE;QACX,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACzD,QAAQ,CAAC,EAAE;YACV,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,MAAM,CAAC,EAAE,MAAM,CAAC;SAChB,CAAC;KACF,CAAC;IACF,MAAM,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,YAAY,GAAG,MAAM,CAAC,CAAC;QAChG,UAAU,CAAC,EAAE,WAAW,GAAG,UAAU,CAAC;QACtC,IAAI,CAAC,EAAE;YACN,MAAM,CAAC,EAAE,MAAM,CAAC;YAChB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAClC,CAAC;KACF,CAAC;IACF,OAAO,CAAC,EAAE;QACT,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,KAAK,CAAC,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;KACnD,CAAC;CACF;AAED,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KACrC,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE;QACd,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;CACF;AAED,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KACrC,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,uBAAuB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE;QACN,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KACrC,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE;QACT,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAChC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,sBAAsB,CAAC,EAAE,OAAO,CAAC;QACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,SAAS,GAAG,QAAQ,CAAC;QAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAClC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;KAChC,CAAC;IACF,SAAS,CAAC,EAAE;QACX,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACzD,QAAQ,CAAC,EAAE;YACV,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,MAAM,CAAC,EAAE,MAAM,CAAC;SAChB,CAAC;KACF,CAAC;IACF,MAAM,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,YAAY,GAAG,MAAM,CAAC,CAAC;QAChG,UAAU,CAAC,EAAE,WAAW,GAAG,UAAU,CAAC;QACtC,IAAI,CAAC,EAAE;YACN,MAAM,CAAC,EAAE,MAAM,CAAC;YAChB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAClC,CAAC;KACF,CAAC;IACF,OAAO,CAAC,EAAE;QACT,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,KAAK,CAAC,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;KACnD,CAAC;CACF;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,KAAK,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KACrC,CAAC,CAAC;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB"}

package/dist/types.js

@@ -0,0 +1,5 @@
+/**
+ * xCrawl API Request and Response Types
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/package.json

@@ -0,0 +1,33 @@
+{
+	"name": "xcrawl-mcp",
+	"version": "1.0.0",
+	"license": "UNLICENSED",
+	"type": "module",
+	"main": "dist/stdio.js",
+	"bin": {
+		"xcrawl-mcp": "./dist/stdio.js"
+	},
+	"scripts": {
+		"deploy": "wrangler deploy",
+		"dev": "wrangler dev",
+		"start": "wrangler dev",
+		"build": "tsc -p tsconfig.build.json",
+		"start:stdio": "node dist/stdio.js",
+		"start:stdio:dev": "tsx src/stdio.ts",
+		"test": "vitest",
+		"cf-typegen": "wrangler types"
+	},
+	"dependencies": {
+		"@modelcontextprotocol/sdk": "^1.0.4",
+		"zod": "^3.23.8"
+	},
+	"devDependencies": {
+		"@cloudflare/vitest-pool-workers": "^0.8.19",
+		"@cloudflare/workers-types": "^4.20241218.0",
+		"@types/node": "^25.0.3",
+		"tsx": "^4.19.2",
+		"typescript": "^5.5.2",
+		"vitest": "~3.2.0",
+		"wrangler": "^4.56.0"
+	}
+}

package/src/core/crawl.ts

@@ -0,0 +1,149 @@
+import { z } from "zod";
+import type { XCrawlCrawlRequest, XCrawlCrawlResponse } from "../types.js";
+
+/**
+ * Zod schema for xcrawl_crawl tool parameters
+ */
+export const crawlToolSchema = z.object({
+	url: z.string().url().describe("The entry URL to start crawling from"),
+	crawler: z
+		.object({
+			limit: z.number().int().optional().describe("Maximum number of pages to crawl"),
+			include: z.array(z.string()).optional().describe("Regex patterns for URLs to include"),
+			exclude: z.array(z.string()).optional().describe("Regex patterns for URLs to exclude"),
+			max_depth: z.number().int().optional().describe("Maximum crawl depth"),
+			include_entire_domain: z.boolean().optional().describe("Whether to crawl the entire domain"),
+			include_subdomains: z.boolean().optional().describe("Whether to include subdomains"),
+			include_external_links: z.boolean().optional().describe("Whether to follow external links"),
+			sitemaps: z.boolean().optional().describe("Whether to use sitemap"),
+		})
+		.optional()
+		.describe("Crawler configuration"),
+	proxy: z
+		.object({
+			location: z.string().optional(),
+			sticky_session: z.string().optional(),
+		})
+		.optional(),
+	request: z
+		.object({
+			locale: z.string().optional(),
+			device: z.enum(["desktop", "mobile"]).optional(),
+			cookies: z.record(z.unknown()).optional(),
+			headers: z.record(z.unknown()).optional(),
+			only_main_content: z.boolean().optional(),
+			block_ads: z.boolean().optional(),
+			skip_tls_verification: z.boolean().optional(),
+		})
+		.optional(),
+	js_render: z
+		.object({
+			enabled: z.boolean().optional(),
+			wait_until: z.enum(["load", "domcontentloaded", "networkidle"]).default("load"),
+			viewport: z
+				.object({
+					width: z.number().int().optional(),
+					height: z.number().int().optional(),
+				})
+				.optional(),
+		})
+		.optional(),
+	output: z
+		.object({
+			formats: z.array(z.enum(["html", "raw_html", "markdown", "links", "summary", "screenshot", "json"]))
+				.optional()
+				.default(["markdown"])
+				.describe("Output formats. Default: ['markdown']. Set to [] to get only metadata."),
+			screenshot: z.enum(["full_page", "viewport"]).optional(),
+			json: z
+				.object({
+					prompt: z.string().optional(),
+					json_schema: z.record(z.any()).optional(),
+				})
+				.optional(),
+		})
+		.optional(),
+	webhook: z
+		.object({
+			url: z.string().optional(),
+			headers: z.record(z.string()).optional(),
+			events: z.array(z.enum(["started", "completed", "failed"])).optional(),
+		})
+		.optional(),
+});
+
+export type CrawlToolParams = z.infer<typeof crawlToolSchema>;
+
+/**
+ * Call xCrawl Crawl API to start a batch crawling task (async)
+ */
+export async function callXCrawlCrawlAPI(apiKey: string, params: XCrawlCrawlRequest): Promise<XCrawlCrawlResponse> {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+
+	try {
+		const response = await fetch("https://run.xcrawl.com/v1/crawl", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify(params),
+			signal: controller.signal,
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new Error(`xCrawl Crawl API error: ${response.status} ${response.statusText} - ${errorText}`);
+		}
+
+		return (await response.json()) as XCrawlCrawlResponse;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") {
+			throw new Error("Request timeout after 300 seconds");
+		}
+		throw error;
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+
+/**
+ * Check crawl task status and retrieve results
+ */
+export async function checkCrawlStatus(apiKey: string, crawlId: string): Promise<XCrawlCrawlResponse> {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+
+	try {
+		const response = await fetch(`https://run.xcrawl.com/v1/crawl/${crawlId}`, {
+			method: "GET",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`,
+			},
+			signal: controller.signal,
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new Error(`xCrawl check crawl status error: ${response.status} ${response.statusText} - ${errorText}`);
+		}
+
+		return (await response.json()) as XCrawlCrawlResponse;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") {
+			throw new Error("Request timeout after 300 seconds");
+		}
+		throw error;
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+
+/**
+ * Format crawl response for MCP tool output
+ */
+export function formatCrawlResponse(response: XCrawlCrawlResponse): string {
+	return JSON.stringify(response, null, 2);
+}

package/src/core/map.ts

@@ -0,0 +1,56 @@
+import { z } from "zod";
+import type { XCrawlMapRequest, XCrawlMapResponse } from "../types.js";
+
+/**
+ * Zod schema for xcrawl_map tool parameters
+ */
+export const mapToolSchema = z.object({
+	url: z.string().url().describe("The website URL to map"),
+	filter: z.string().optional().describe("Regex pattern to filter URLs"),
+	limit: z.number().int().min(1).max(100000).optional().describe("Maximum number of URLs to return (1-100000)"),
+	include_subdomains: z.boolean().optional().describe("Whether to include URLs from subdomains"),
+	ignore_query_parameters: z.boolean().optional().describe("Whether to ignore URLs with query parameters"),
+});
+
+export type MapToolParams = z.infer<typeof mapToolSchema>;
+
+/**
+ * Call xCrawl Map API to get all URLs from a website
+ */
+export async function callXCrawlMapAPI(apiKey: string, params: XCrawlMapRequest): Promise<XCrawlMapResponse> {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+
+	try {
+		const response = await fetch("https://run.xcrawl.com/v1/map", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify(params),
+			signal: controller.signal,
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new Error(`xCrawl Map API error: ${response.status} ${response.statusText} - ${errorText}`);
+		}
+
+		return (await response.json()) as XCrawlMapResponse;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") {
+			throw new Error("Request timeout after 300 seconds");
+		}
+		throw error;
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+
+/**
+ * Format map response for MCP tool output
+ */
+export function formatMapResponse(response: XCrawlMapResponse): string {
+	return JSON.stringify(response, null, 2);
+}

package/src/core/scrape.ts

@@ -0,0 +1,156 @@
+import { z } from "zod";
+import type { XCrawlScrapeRequest, XCrawlScrapeResponse } from "../types.js";
+
+/**
+ * Zod schema for xcrawl_scrape tool parameters
+ */
+export const scrapeToolSchema = z.object({
+	url: z.string().url().describe("The URL to scrape"),
+	mode: z
+		.enum(["sync", "async"])
+		.default("sync")
+		.describe("Sync mode returns results immediately, async mode returns a scrape_id for later retrieval"),
+	proxy: z
+		.object({
+			location: z.string().optional().describe("ISO-3166-1 alpha-2 country code (e.g., US, JP, SG)"),
+			sticky_session: z.string().optional().describe("Sticky session ID to reuse the same proxy exit"),
+		})
+		.optional()
+		.describe("Proxy configuration"),
+	request: z
+		.object({
+			locale: z.string().optional().describe("Accept-Language header value"),
+			device: z.enum(["desktop", "mobile"]).optional().describe("Device type for user agent and viewport"),
+			cookies: z.record(z.string()).optional().describe("Cookies to send with the request"),
+			headers: z.record(z.string()).optional().describe("Custom HTTP headers"),
+			only_main_content: z.boolean().optional().describe("Only return main content (default: true)"),
+			block_ads: z.boolean().optional().describe("Attempt to block ads (default: true)"),
+			skip_tls_verification: z.boolean().optional().describe("Skip TLS certificate verification (default: true)"),
+		})
+		.optional()
+		.describe("Request configuration"),
+	js_render: z
+		.object({
+			enabled: z.boolean().default(true).describe("Enable JavaScript rendering"),
+			wait_until: z
+				.enum(["load", "domcontentloaded", "networkidle"])
+				.default("load")
+				.describe("Wait condition for page load"),
+			viewport: z
+				.object({
+					width: z.number().optional().describe("Viewport width"),
+					height: z.number().optional().describe("Viewport height"),
+				})
+				.optional()
+				.describe("Viewport dimensions"),
+		})
+		.optional()
+		.describe("JavaScript rendering configuration"),
+	output: z
+		.object({
+			formats: z
+				.array(z.enum(["html", "raw_html", "markdown", "links", "summary", "screenshot", "json"]))
+				.optional()
+				.default(["markdown"])
+				.describe("Output formats: 'html' (cleaned HTML without scripts), 'raw_html' (original HTML with all scripts and styles), 'markdown' (Markdown format), 'links' (all page links), 'summary' (AI-generated summary), 'screenshot' (page screenshot), 'json' (structured data extraction). Set to [] to get only metadata."),
+			screenshot: z
+				.enum(["full_page", "viewport"])
+				.optional()
+				.describe("Screenshot type: 'viewport' (default, captures visible area) or 'full_page' (captures entire page). Only effective when 'screenshot' is in formats array."),
+			json: z
+				.object({
+					prompt: z.string().optional().describe("Natural language description of what data to extract. The AI engine automatically structures the output based on this prompt."),
+					json_schema: z.record(z.any()).optional().describe("Optional JSON Schema for strict output validation. Rarely needed - the prompt field is usually sufficient."),
+				})
+				.optional()
+				.describe("JSON extraction configuration. Use 'prompt' to describe what to extract - json_schema is optional."),
+		})
+		.optional()
+		.describe("Output configuration"),
+	webhook: z
+		.object({
+			url: z.string().optional().describe("Webhook callback URL for async task completion"),
+			headers: z.record(z.string()).optional().describe("Custom headers for webhook callback request"),
+			events: z
+				.array(z.enum(["started", "completed", "failed"]))
+				.optional()
+				.describe("Events to receive webhook callbacks for (default: ['started', 'completed', 'failed'])"),
+		})
+		.optional()
+		.describe("Webhook configuration for async mode callbacks"),
+});
+
+export type ScrapeToolParams = z.infer<typeof scrapeToolSchema>;
+
+/**
+ * Call xCrawl API to scrape a URL
+ */
+export async function callXCrawlAPI(apiKey: string, params: XCrawlScrapeRequest): Promise<XCrawlScrapeResponse> {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+
+	try {
+		const response = await fetch("https://run.xcrawl.com/v1/scrape", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify(params),
+			signal: controller.signal,
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new Error(`xCrawl API error: ${response.status} ${response.statusText} - ${errorText}`);
+		}
+
+		return (await response.json()) as XCrawlScrapeResponse;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") {
+			throw new Error("Request timeout after 300 seconds");
+		}
+		throw error;
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}
+
+/**
+ * Format scrape response for MCP tool output
+ */
+export function formatScrapeResponse(response: XCrawlScrapeResponse): string {
+	return JSON.stringify(response, null, 2);
+}
+
+/**
+ * Check status and get result of an async scrape task
+ */
+export async function checkScrapeStatus(apiKey: string, scrapeId: string): Promise<XCrawlScrapeResponse> {
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+
+	try {
+		const response = await fetch(`https://run.xcrawl.com/v1/scrape/${scrapeId}`, {
+			method: "GET",
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+			},
+			signal: controller.signal,
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new Error(`xCrawl API error: ${response.status} ${response.statusText} - ${errorText}`);
+		}
+
+		return (await response.json()) as XCrawlScrapeResponse;
+	} catch (error) {
+		if (error instanceof Error && error.name === "AbortError") {
+			throw new Error("Request timeout after 300 seconds");
+		}
+		throw error;
+	} finally {
+		clearTimeout(timeoutId);
+	}
+}