xcrawl-mcp 1.0.0

package/src/types.ts

@@ -0,0 +1,221 @@
+/**
+ * xCrawl API Request and Response Types
+ */
+
+export interface XCrawlScrapeRequest {
+	url: string;
+	mode?: "sync" | "async";
+	proxy?: {
+		location?: string;
+		sticky_session?: string;
+	};
+	request?: {
+		locale?: string;
+		device?: "desktop" | "mobile";
+		cookies?: Record<string, unknown>;
+		headers?: Record<string, unknown>;
+		only_main_content?: boolean;
+		block_ads?: boolean;
+		skip_tls_verification?: boolean;
+	};
+	js_render?: {
+		enabled?: boolean;
+		wait_until?: "load" | "domcontentloaded" | "networkidle";
+		viewport?: {
+			width?: number;
+			height?: number;
+		};
+	};
+	output?: {
+		formats?: Array<"html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json">;
+		screenshot?: "full_page" | "viewport";
+		json?: {
+			prompt?: string;
+			json_schema?: Record<string, any>;
+		};
+	};
+	webhook?: {
+		url?: string;
+		headers?: Record<string, string>;
+		events?: Array<"started" | "completed" | "failed">;
+	};
+}
+
+export interface XCrawlScrapeResponse {
+	scrape_id: string;
+	endpoint: string;
+	version: string;
+	status: string;
+	url?: string;
+	data?: {
+		html?: string;
+		raw_html?: string;
+		markdown?: string;
+		links?: string[];
+		metadata?: Record<string, any>;
+		screenshot?: string;
+		summary?: string;
+		json?: Record<string, any>;
+		traffic_bytes?: number;
+		credits_used?: number;
+		credits_detail?: Record<string, any>;
+	};
+	started_at?: string;
+	ended_at?: string;
+	total_credits_used?: number;
+	message?: string;
+}
+
+/**
+ * Search API Request and Response Types
+ */
+export interface XCrawlSearchRequest {
+	query: string;
+	location?: string;
+	language?: string;
+	limit?: number;
+	serp_options?: {
+		q?: string;
+		location?: string;
+		uule?: string;
+		google_domain?: string;
+		gl?: string;
+		hl?: string;
+		cr?: string;
+		lr?: string;
+		safe?: number;
+		nfpr?: boolean;
+		filter?: boolean;
+		tbs?: string;
+		start?: number;
+		num?: number;
+		ludocid?: string;
+		lsig?: string;
+		kgmid?: string;
+		si?: string;
+		ibp?: string;
+		uds?: string;
+		no_cache?: boolean;
+	};
+}
+
+export interface XCrawlSearchResponse {
+	search_id: string;
+	endpoint: string;
+	version: string;
+	status: string;
+	query: string;
+	data?: {
+		results?: Record<string, any>;
+		credits_used?: number;
+		credits_detail?: Record<string, any>;
+	};
+	started_at?: string;
+	ended_at?: string;
+	total_credits_used?: number;
+}
+
+/**
+ * Map API Request and Response Types
+ */
+export interface XCrawlMapRequest {
+	url: string;
+	filter?: string;
+	limit?: number;
+	include_subdomains?: boolean;
+	ignore_query_parameters?: boolean;
+}
+
+export interface XCrawlMapResponse {
+	map_id: string;
+	endpoint: string;
+	version: string;
+	status: string;
+	url: string;
+	data?: {
+		links?: string[];
+		total_links?: number;
+		credits_used?: number;
+		credits_detail?: Record<string, any>;
+	};
+	started_at?: string;
+	ended_at?: string;
+	total_credits_used?: number;
+}
+
+/**
+ * Crawl API Request and Response Types
+ */
+export interface XCrawlCrawlRequest {
+	url: string;
+	crawler?: {
+		limit?: number;
+		include?: string[];
+		exclude?: string[];
+		max_depth?: number;
+		include_entire_domain?: boolean;
+		include_subdomains?: boolean;
+		include_external_links?: boolean;
+		sitemaps?: boolean;
+	};
+	proxy?: {
+		location?: string;
+		sticky_session?: string;
+	};
+	request?: {
+		locale?: string;
+		device?: "desktop" | "mobile";
+		cookies?: Record<string, unknown>;
+		headers?: Record<string, unknown>;
+		only_main_content?: boolean;
+		block_ads?: boolean;
+		skip_tls_verification?: boolean;
+	};
+	js_render?: {
+		enabled?: boolean;
+		wait_until?: "load" | "domcontentloaded" | "networkidle";
+		viewport?: {
+			width?: number;
+			height?: number;
+		};
+	};
+	output?: {
+		formats?: Array<"html" | "raw_html" | "markdown" | "links" | "summary" | "screenshot" | "json">;
+		screenshot?: "full_page" | "viewport";
+		json?: {
+			prompt?: string;
+			json_schema?: Record<string, any>;
+		};
+	};
+	webhook?: {
+		url?: string;
+		headers?: Record<string, string>;
+		events?: Array<"started" | "completed" | "failed">;
+	};
+}
+
+export interface XCrawlCrawlResponse {
+	crawl_id: string;
+	endpoint: string;
+	version: string;
+	status: string;
+	url?: string;
+	data?: Array<{
+		url: string;
+		html?: string;
+		raw_html?: string;
+		markdown?: string;
+		links?: string[];
+		metadata?: Record<string, any>;
+		screenshot?: string;
+		summary?: string;
+		json?: Record<string, any>;
+		traffic_bytes?: number;
+		credits_used?: number;
+		credits_detail?: Record<string, any>;
+	}>;
+	started_at?: string;
+	ended_at?: string;
+	total_credits_used?: number;
+	message?: string;
+}

package/tsconfig.build.json

@@ -0,0 +1,14 @@
+{
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		"noEmit": false,
+		"outDir": "./dist",
+		"rootDir": "./src",
+		"declaration": true,
+		"declarationMap": true,
+		"sourceMap": true,
+		"types": ["node"]
+	},
+	"include": ["src/**/*.ts"],
+	"exclude": ["node_modules", "dist", "test"]
+}

package/tsconfig.json

@@ -0,0 +1,45 @@
+{
+	"compilerOptions": {
+		/* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+    /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+		"target": "es2024",
+    /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+		"lib": ["es2024"],
+    /* Specify what JSX code is generated. */
+		"jsx": "react-jsx",
+
+    /* Specify what module code is generated. */
+		"module": "es2022",
+    /* Specify how TypeScript looks up a file from a given module specifier. */
+		"moduleResolution": "Bundler",
+    /* Enable importing .json files */
+		"resolveJsonModule": true,
+
+    /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
+		"allowJs": true,
+    /* Enable error reporting in type-checked JavaScript files. */
+		"checkJs": false,
+
+    /* Disable emitting files from a compilation. */
+		"noEmit": true,
+
+    /* Ensure that each file can be safely transpiled without relying on other imports. */
+		"isolatedModules": true,
+    /* Allow 'import x from y' when a module doesn't have a default export. */
+		"allowSyntheticDefaultImports": true,
+    /* Ensure that casing is correct in imports. */
+		"forceConsistentCasingInFileNames": true,
+
+    /* Enable all strict type-checking options. */
+		"strict": true,
+
+    /* Skip type checking all .d.ts files. */
+		"skipLibCheck": true,
+		"types": [
+			"./worker-configuration.d.ts"
+		]
+	},
+	"exclude": ["test"],
+	"include": ["worker-configuration.d.ts", "src/**/*.ts"]
+}

package/vitest.config.mts

@@ -0,0 +1,11 @@
+import { defineWorkersConfig } from '@cloudflare/vitest-pool-workers/config';
+
+export default defineWorkersConfig({
+	test: {
+		poolOptions: {
+			workers: {
+				wrangler: { configPath: './wrangler.jsonc' },
+			},
+		},
+	},
+});