xcrawl-mcp 1.0.0

package/dist/core/crawl.d.ts

@@ -0,0 +1,246 @@
+import { z } from "zod";
+import type { XCrawlCrawlRequest, XCrawlCrawlResponse } from "../types.js";
+/**
+ * Zod schema for xcrawl_crawl tool parameters
+ */
+export declare const crawlToolSchema: z.ZodObject<{
+    url: z.ZodString;
+    crawler: z.ZodOptional<z.ZodObject<{
+        limit: z.ZodOptional<z.ZodNumber>;
+        include: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        exclude: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        max_depth: z.ZodOptional<z.ZodNumber>;
+        include_entire_domain: z.ZodOptional<z.ZodBoolean>;
+        include_subdomains: z.ZodOptional<z.ZodBoolean>;
+        include_external_links: z.ZodOptional<z.ZodBoolean>;
+        sitemaps: z.ZodOptional<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        limit?: number | undefined;
+        include_subdomains?: boolean | undefined;
+        include?: string[] | undefined;
+        exclude?: string[] | undefined;
+        max_depth?: number | undefined;
+        include_entire_domain?: boolean | undefined;
+        include_external_links?: boolean | undefined;
+        sitemaps?: boolean | undefined;
+    }, {
+        limit?: number | undefined;
+        include_subdomains?: boolean | undefined;
+        include?: string[] | undefined;
+        exclude?: string[] | undefined;
+        max_depth?: number | undefined;
+        include_entire_domain?: boolean | undefined;
+        include_external_links?: boolean | undefined;
+        sitemaps?: boolean | undefined;
+    }>>;
+    proxy: z.ZodOptional<z.ZodObject<{
+        location: z.ZodOptional<z.ZodString>;
+        sticky_session: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        location?: string | undefined;
+        sticky_session?: string | undefined;
+    }, {
+        location?: string | undefined;
+        sticky_session?: string | undefined;
+    }>>;
+    request: z.ZodOptional<z.ZodObject<{
+        locale: z.ZodOptional<z.ZodString>;
+        device: z.ZodOptional<z.ZodEnum<["desktop", "mobile"]>>;
+        cookies: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        only_main_content: z.ZodOptional<z.ZodBoolean>;
+        block_ads: z.ZodOptional<z.ZodBoolean>;
+        skip_tls_verification: z.ZodOptional<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        locale?: string | undefined;
+        device?: "desktop" | "mobile" | undefined;
+        cookies?: Record<string, unknown> | undefined;
+        headers?: Record<string, unknown> | undefined;
+        only_main_content?: boolean | undefined;
+        block_ads?: boolean | undefined;
+        skip_tls_verification?: boolean | undefined;
+    }, {
+        locale?: string | undefined;
+        device?: "desktop" | "mobile" | undefined;
+        cookies?: Record<string, unknown> | undefined;
+        headers?: Record<string, unknown> | undefined;
+        only_main_content?: boolean | undefined;
+        block_ads?: boolean | undefined;
+        skip_tls_verification?: boolean | undefined;
+    }>>;
+    js_render: z.ZodOptional<z.ZodObject<{
+        enabled: z.ZodOptional<z.ZodBoolean>;
+        wait_until: z.ZodDefault<z.ZodEnum<["load", "domcontentloaded", "networkidle"]>>;
+        viewport: z.ZodOptional<z.ZodObject<{
+            width: z.ZodOptional<z.ZodNumber>;
+            height: z.ZodOptional<z.ZodNumber>;
+        }, "strip", z.ZodTypeAny, {
+            width?: number | undefined;
+            height?: number | undefined;
+        }, {
+            width?: number | undefined;
+            height?: number | undefined;
+        }>>;
+    }, "strip", z.ZodTypeAny, {
+        wait_until: "load" | "domcontentloaded" | "networkidle";
+        viewport?: {
+            width?: number | undefined;
+            height?: number | undefined;
+        } | undefined;
+        enabled?: boolean | undefined;
+    }, {
+        viewport?: {
+            width?: number | undefined;
+            height?: number | undefined;
+        } | undefined;
+        enabled?: boolean | undefined;
+        wait_until?: "load" | "domcontentloaded" | "networkidle" | undefined;
+    }>>;
+    output: z.ZodOptional<z.ZodObject<{
+        formats: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodEnum<["html", "raw_html", "markdown", "links", "summary", "screenshot", "json"]>, "many">>>;
+        screenshot: z.ZodOptional<z.ZodEnum<["full_page", "viewport"]>>;
+        json: z.ZodOptional<z.ZodObject<{
+            prompt: z.ZodOptional<z.ZodString>;
+            json_schema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        }, "strip", z.ZodTypeAny, {
+            prompt?: string | undefined;
+            json_schema?: Record<string, any> | undefined;
+        }, {
+            prompt?: string | undefined;
+            json_schema?: Record<string, any> | undefined;
+        }>>;
+    }, "strip", z.ZodTypeAny, {
+        formats: ("html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json")[];
+        screenshot?: "full_page" | "viewport" | undefined;
+        json?: {
+            prompt?: string | undefined;
+            json_schema?: Record<string, any> | undefined;
+        } | undefined;
+    }, {
+        screenshot?: "full_page" | "viewport" | undefined;
+        json?: {
+            prompt?: string | undefined;
+            json_schema?: Record<string, any> | undefined;
+        } | undefined;
+        formats?: ("html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json")[] | undefined;
+    }>>;
+    webhook: z.ZodOptional<z.ZodObject<{
+        url: z.ZodOptional<z.ZodString>;
+        headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        events: z.ZodOptional<z.ZodArray<z.ZodEnum<["started", "completed", "failed"]>, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        url?: string | undefined;
+        headers?: Record<string, string> | undefined;
+        events?: ("started" | "completed" | "failed")[] | undefined;
+    }, {
+        url?: string | undefined;
+        headers?: Record<string, string> | undefined;
+        events?: ("started" | "completed" | "failed")[] | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    proxy?: {
+        location?: string | undefined;
+        sticky_session?: string | undefined;
+    } | undefined;
+    request?: {
+        locale?: string | undefined;
+        device?: "desktop" | "mobile" | undefined;
+        cookies?: Record<string, unknown> | undefined;
+        headers?: Record<string, unknown> | undefined;
+        only_main_content?: boolean | undefined;
+        block_ads?: boolean | undefined;
+        skip_tls_verification?: boolean | undefined;
+    } | undefined;
+    js_render?: {
+        wait_until: "load" | "domcontentloaded" | "networkidle";
+        viewport?: {
+            width?: number | undefined;
+            height?: number | undefined;
+        } | undefined;
+        enabled?: boolean | undefined;
+    } | undefined;
+    output?: {
+        formats: ("html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json")[];
+        screenshot?: "full_page" | "viewport" | undefined;
+        json?: {
+            prompt?: string | undefined;
+            json_schema?: Record<string, any> | undefined;
+        } | undefined;
+    } | undefined;
+    webhook?: {
+        url?: string | undefined;
+        headers?: Record<string, string> | undefined;
+        events?: ("started" | "completed" | "failed")[] | undefined;
+    } | undefined;
+    crawler?: {
+        limit?: number | undefined;
+        include_subdomains?: boolean | undefined;
+        include?: string[] | undefined;
+        exclude?: string[] | undefined;
+        max_depth?: number | undefined;
+        include_entire_domain?: boolean | undefined;
+        include_external_links?: boolean | undefined;
+        sitemaps?: boolean | undefined;
+    } | undefined;
+}, {
+    url: string;
+    proxy?: {
+        location?: string | undefined;
+        sticky_session?: string | undefined;
+    } | undefined;
+    request?: {
+        locale?: string | undefined;
+        device?: "desktop" | "mobile" | undefined;
+        cookies?: Record<string, unknown> | undefined;
+        headers?: Record<string, unknown> | undefined;
+        only_main_content?: boolean | undefined;
+        block_ads?: boolean | undefined;
+        skip_tls_verification?: boolean | undefined;
+    } | undefined;
+    js_render?: {
+        viewport?: {
+            width?: number | undefined;
+            height?: number | undefined;
+        } | undefined;
+        enabled?: boolean | undefined;
+        wait_until?: "load" | "domcontentloaded" | "networkidle" | undefined;
+    } | undefined;
+    output?: {
+        screenshot?: "full_page" | "viewport" | undefined;
+        json?: {
+            prompt?: string | undefined;
+            json_schema?: Record<string, any> | undefined;
+        } | undefined;
+        formats?: ("html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json")[] | undefined;
+    } | undefined;
+    webhook?: {
+        url?: string | undefined;
+        headers?: Record<string, string> | undefined;
+        events?: ("started" | "completed" | "failed")[] | undefined;
+    } | undefined;
+    crawler?: {
+        limit?: number | undefined;
+        include_subdomains?: boolean | undefined;
+        include?: string[] | undefined;
+        exclude?: string[] | undefined;
+        max_depth?: number | undefined;
+        include_entire_domain?: boolean | undefined;
+        include_external_links?: boolean | undefined;
+        sitemaps?: boolean | undefined;
+    } | undefined;
+}>;
+export type CrawlToolParams = z.infer<typeof crawlToolSchema>;
+/**
+ * Call xCrawl Crawl API to start a batch crawling task (async)
+ */
+export declare function callXCrawlCrawlAPI(apiKey: string, params: XCrawlCrawlRequest): Promise<XCrawlCrawlResponse>;
+/**
+ * Check crawl task status and retrieve results
+ */
+export declare function checkCrawlStatus(apiKey: string, crawlId: string): Promise<XCrawlCrawlResponse>;
+/**
+ * Format crawl response for MCP tool output
+ */
+export declare function formatCrawlResponse(response: XCrawlCrawlResponse): string;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/core/crawl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAE3E;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkE1B,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAE9D;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA6BjH;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA4BpG;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,GAAG,MAAM,CAEzE"}

package/dist/core/crawl.js

@@ -0,0 +1,141 @@
+import { z } from "zod";
+/**
+ * Zod schema for xcrawl_crawl tool parameters
+ */
+export const crawlToolSchema = z.object({
+    url: z.string().url().describe("The entry URL to start crawling from"),
+    crawler: z
+        .object({
+        limit: z.number().int().optional().describe("Maximum number of pages to crawl"),
+        include: z.array(z.string()).optional().describe("Regex patterns for URLs to include"),
+        exclude: z.array(z.string()).optional().describe("Regex patterns for URLs to exclude"),
+        max_depth: z.number().int().optional().describe("Maximum crawl depth"),
+        include_entire_domain: z.boolean().optional().describe("Whether to crawl the entire domain"),
+        include_subdomains: z.boolean().optional().describe("Whether to include subdomains"),
+        include_external_links: z.boolean().optional().describe("Whether to follow external links"),
+        sitemaps: z.boolean().optional().describe("Whether to use sitemap"),
+    })
+        .optional()
+        .describe("Crawler configuration"),
+    proxy: z
+        .object({
+        location: z.string().optional(),
+        sticky_session: z.string().optional(),
+    })
+        .optional(),
+    request: z
+        .object({
+        locale: z.string().optional(),
+        device: z.enum(["desktop", "mobile"]).optional(),
+        cookies: z.record(z.unknown()).optional(),
+        headers: z.record(z.unknown()).optional(),
+        only_main_content: z.boolean().optional(),
+        block_ads: z.boolean().optional(),
+        skip_tls_verification: z.boolean().optional(),
+    })
+        .optional(),
+    js_render: z
+        .object({
+        enabled: z.boolean().optional(),
+        wait_until: z.enum(["load", "domcontentloaded", "networkidle"]).default("load"),
+        viewport: z
+            .object({
+            width: z.number().int().optional(),
+            height: z.number().int().optional(),
+        })
+            .optional(),
+    })
+        .optional(),
+    output: z
+        .object({
+        formats: z.array(z.enum(["html", "raw_html", "markdown", "links", "summary", "screenshot", "json"]))
+            .optional()
+            .default(["markdown"])
+            .describe("Output formats. Default: ['markdown']. Set to [] to get only metadata."),
+        screenshot: z.enum(["full_page", "viewport"]).optional(),
+        json: z
+            .object({
+            prompt: z.string().optional(),
+            json_schema: z.record(z.any()).optional(),
+        })
+            .optional(),
+    })
+        .optional(),
+    webhook: z
+        .object({
+        url: z.string().optional(),
+        headers: z.record(z.string()).optional(),
+        events: z.array(z.enum(["started", "completed", "failed"])).optional(),
+    })
+        .optional(),
+});
+/**
+ * Call xCrawl Crawl API to start a batch crawling task (async)
+ */
+export async function callXCrawlCrawlAPI(apiKey, params) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+    try {
+        const response = await fetch("https://run.xcrawl.com/v1/crawl", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${apiKey}`,
+            },
+            body: JSON.stringify(params),
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`xCrawl Crawl API error: ${response.status} ${response.statusText} - ${errorText}`);
+        }
+        return (await response.json());
+    }
+    catch (error) {
+        if (error instanceof Error && error.name === "AbortError") {
+            throw new Error("Request timeout after 300 seconds");
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+/**
+ * Check crawl task status and retrieve results
+ */
+export async function checkCrawlStatus(apiKey, crawlId) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+    try {
+        const response = await fetch(`https://run.xcrawl.com/v1/crawl/${crawlId}`, {
+            method: "GET",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${apiKey}`,
+            },
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`xCrawl check crawl status error: ${response.status} ${response.statusText} - ${errorText}`);
+        }
+        return (await response.json());
+    }
+    catch (error) {
+        if (error instanceof Error && error.name === "AbortError") {
+            throw new Error("Request timeout after 300 seconds");
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+/**
+ * Format crawl response for MCP tool output
+ */
+export function formatCrawlResponse(response) {
+    return JSON.stringify(response, null, 2);
+}
+//# sourceMappingURL=crawl.js.map

package/dist/core/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;IACtE,OAAO,EAAE,CAAC;SACR,MAAM,CAAC;QACP,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;QAC/E,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;QACtF,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;QACtF,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;QACtE,qBAAqB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;QAC5F,kBAAkB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;QACpF,sBAAsB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;QAC3F,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;KACnE,CAAC;SACD,QAAQ,EAAE;SACV,QAAQ,CAAC,uBAAuB,CAAC;IACnC,KAAK,EAAE,CAAC;SACN,MAAM,CAAC;QACP,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC/B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KACrC,CAAC;SACD,QAAQ,EAAE;IACZ,OAAO,EAAE,CAAC;SACR,MAAM,CAAC;QACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC7B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE;QAChD,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;QACzC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;QACzC,iBAAiB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QACzC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QACjC,qBAAqB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;KAC7C,CAAC;SACD,QAAQ,EAAE;IACZ,SAAS,EAAE,CAAC;SACV,MAAM,CAAC;QACP,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QAC/B,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAC/E,QAAQ,EAAE,CAAC;aACT,MAAM,CAAC;YACP,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACnC,CAAC;aACD,QAAQ,EAAE;KACZ,CAAC;SACD,QAAQ,EAAE;IACZ,MAAM,EAAE,CAAC;SACP,MAAM,CAAC;QACP,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;aAClG,QAAQ,EAAE;aACV,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC;aACrB,QAAQ,CAAC,wEAAwE,CAAC;QACpF,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE;QACxD,IAAI,EAAE,CAAC;aACL,MAAM,CAAC;YACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC7B,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;SACzC,CAAC;aACD,QAAQ,EAAE;KACZ,CAAC;SACD,QAAQ,EAAE;IACZ,OAAO,EAAE,CAAC;SACR,MAAM,CAAC;QACP,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC1B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;QACxC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KACtE,CAAC;SACD,QAAQ,EAAE;CACZ,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc,EAAE,MAA0B;IAClF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,sBAAsB;IAEtF,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,iCAAiC,EAAE;YAC/D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACR,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;aACjC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAC5B,MAAM,EAAE,UAAU,CAAC,MAAM;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,SAAS,EAAE,CAAC,CAAC;QACrG,CAAC;QAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAwB,CAAC;IACvD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,SAAS,CAAC,CAAC;IACzB,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAc,EAAE,OAAe;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,sBAAsB;IAEtF,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,mCAAmC,OAAO,EAAE,EAAE;YAC1E,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACR,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;aACjC;YACD,MAAM,EAAE,UAAU,CAAC,MAAM;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,SAAS,EAAE,CAAC,CAAC;QAC9G,CAAC;QAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAwB,CAAC;IACvD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,SAAS,CAAC,CAAC;IACzB,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAA6B;IAChE,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC1C,CAAC"}

package/dist/core/map.d.ts

@@ -0,0 +1,34 @@
+import { z } from "zod";
+import type { XCrawlMapRequest, XCrawlMapResponse } from "../types.js";
+/**
+ * Zod schema for xcrawl_map tool parameters
+ */
+export declare const mapToolSchema: z.ZodObject<{
+    url: z.ZodString;
+    filter: z.ZodOptional<z.ZodString>;
+    limit: z.ZodOptional<z.ZodNumber>;
+    include_subdomains: z.ZodOptional<z.ZodBoolean>;
+    ignore_query_parameters: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    filter?: string | undefined;
+    limit?: number | undefined;
+    include_subdomains?: boolean | undefined;
+    ignore_query_parameters?: boolean | undefined;
+}, {
+    url: string;
+    filter?: string | undefined;
+    limit?: number | undefined;
+    include_subdomains?: boolean | undefined;
+    ignore_query_parameters?: boolean | undefined;
+}>;
+export type MapToolParams = z.infer<typeof mapToolSchema>;
+/**
+ * Call xCrawl Map API to get all URLs from a website
+ */
+export declare function callXCrawlMapAPI(apiKey: string, params: XCrawlMapRequest): Promise<XCrawlMapResponse>;
+/**
+ * Format map response for MCP tool output
+ */
+export declare function formatMapResponse(response: XCrawlMapResponse): string;
+//# sourceMappingURL=map.d.ts.map

package/dist/core/map.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEvE;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;EAMxB,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;AAE1D;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA6B3G;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,MAAM,CAErE"}

package/dist/core/map.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+/**
+ * Zod schema for xcrawl_map tool parameters
+ */
+export const mapToolSchema = z.object({
+    url: z.string().url().describe("The website URL to map"),
+    filter: z.string().optional().describe("Regex pattern to filter URLs"),
+    limit: z.number().int().min(1).max(100000).optional().describe("Maximum number of URLs to return (1-100000)"),
+    include_subdomains: z.boolean().optional().describe("Whether to include URLs from subdomains"),
+    ignore_query_parameters: z.boolean().optional().describe("Whether to ignore URLs with query parameters"),
+});
+/**
+ * Call xCrawl Map API to get all URLs from a website
+ */
+export async function callXCrawlMapAPI(apiKey, params) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 300000); // 300 seconds timeout
+    try {
+        const response = await fetch("https://run.xcrawl.com/v1/map", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${apiKey}`,
+            },
+            body: JSON.stringify(params),
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`xCrawl Map API error: ${response.status} ${response.statusText} - ${errorText}`);
+        }
+        return (await response.json());
+    }
+    catch (error) {
+        if (error instanceof Error && error.name === "AbortError") {
+            throw new Error("Request timeout after 300 seconds");
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+/**
+ * Format map response for MCP tool output
+ */
+export function formatMapResponse(response) {
+    return JSON.stringify(response, null, 2);
+}
+//# sourceMappingURL=map.js.map

package/dist/core/map.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/core/map.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACxD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACtE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;IAC7G,kBAAkB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IAC9F,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;CACxG,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAc,EAAE,MAAwB;IAC9E,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,sBAAsB;IAEtF,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+BAA+B,EAAE;YAC7D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACR,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;aACjC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAC5B,MAAM,EAAE,UAAU,CAAC,MAAM;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,SAAS,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAsB,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,SAAS,CAAC,CAAC;IACzB,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAA2B;IAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC1C,CAAC"}