crawl-mcp 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,8 @@
1
+ """crawl_mcp: 基于 crawl4ai 和 FastMCP 的 MCP 服务器"""
2
+
3
+ from crawl4ai_mcp.crawler import Crawler
4
+ from crawl4ai_mcp.fastmcp_server import mcp
5
+ from crawl4ai_mcp.llm_config import LLMConfig, get_llm_config
6
+
7
+ __version__ = "0.1.0"
8
+ __all__ = ["Crawler", "mcp", "LLMConfig", "get_llm_config"]
@@ -0,0 +1,469 @@
1
+ """Crawler 类 - 核心爬虫逻辑"""
2
+
3
+ import asyncio
4
+ from typing import List, Dict, Any, Optional
5
+ from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
6
+ from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
7
+ from .llm_config import get_llm_config
8
+
9
+
10
+ def _run_async(coro):
11
+ """
12
+ 运行异步函数的辅助函数,兼容已有事件循环的环境
13
+ 使用 nest_asyncio 允许嵌套事件循环
14
+ """
15
+ try:
16
+ asyncio.get_running_loop()
17
+ import nest_asyncio
18
+
19
+ nest_asyncio.apply()
20
+ return asyncio.get_event_loop().run_until_complete(coro)
21
+ except RuntimeError:
22
+ return asyncio.run(coro)
23
+
24
+
25
+ def _calculate_success_rate(results: List[Dict[str, Any]]) -> str:
26
+ """计算成功率"""
27
+ if not results:
28
+ return "0%"
29
+ successful = sum(1 for r in results if r["success"])
30
+ return f"{successful / len(results) * 100:.1f}%"
31
+
32
+
33
+ def _parse_llm_config(llm_config: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
34
+ """解析 llm_config,兼容字符串和字典两种格式
35
+
36
+ Args:
37
+ llm_config: 可以是 Dict、JSON 字符串、或纯文本字符串
38
+
39
+ Returns:
40
+ 解析后的字典,如果输入是纯文本则当作 instruction
41
+ """
42
+ if llm_config is None:
43
+ return None
44
+
45
+ # 如果已经是字典,直接返回
46
+ if isinstance(llm_config, dict):
47
+ return llm_config
48
+
49
+ # 如果是字符串,尝试解析
50
+ if isinstance(llm_config, str):
51
+ import json
52
+
53
+ try:
54
+ # 尝试解析为 JSON 对象
55
+ return json.loads(llm_config)
56
+ except json.JSONDecodeError:
57
+ # 解析失败,当作纯文本 instruction
58
+ return {"instruction": llm_config}
59
+
60
+ # 其他类型,尝试转换为字符串再处理
61
+ return {"instruction": str(llm_config)}
62
+
63
+
64
+ class Crawler:
65
+ """统一的爬虫类,整合单页、整站、批量爬取功能"""
66
+
67
+ def _create_config(self, enhanced: bool = False) -> CrawlerRunConfig:
68
+ """创建爬虫配置"""
69
+ markdown_generator = DefaultMarkdownGenerator(
70
+ options={
71
+ "citations": False,
72
+ "body_width": None,
73
+ "ignore_links": False,
74
+ }
75
+ )
76
+ return CrawlerRunConfig(
77
+ markdown_generator=markdown_generator,
78
+ page_timeout=60000,
79
+ delay_before_return_html=5.0 if not enhanced else 15.0,
80
+ )
81
+
82
+ def _add_llm_strategy(
83
+ self, config: CrawlerRunConfig, llm_config: Optional[Dict[str, Any]]
84
+ ) -> CrawlerRunConfig:
85
+ """
86
+ 为配置添加 LLM 提取策略
87
+
88
+ Args:
89
+ config: 基础爬虫配置
90
+ llm_config: LLM 配置(可选),支持字典、JSON 字符串或纯文本
91
+
92
+ Returns:
93
+ 添加了 LLM 策略的配置(原地修改或返回原配置)
94
+ """
95
+ # 兼容处理:解析字符串格式的配置
96
+ llm_config = _parse_llm_config(llm_config)
97
+ if not llm_config:
98
+ return config
99
+
100
+ from crawl4ai.extraction_strategy import LLMExtractionStrategy
101
+ from crawl4ai import LLMConfig as Crawl4AILLMConfig
102
+
103
+ llm = get_llm_config(llm_config)
104
+ # crawl4ai 的 LLMConfig 使用 provider 参数,格式为 "openai/model-name"
105
+ provider = f"openai/{llm.model}" if "/" not in llm.model else llm.model
106
+ crawl4ai_llm_config = Crawl4AILLMConfig(
107
+ provider=provider,
108
+ api_token=llm.api_key,
109
+ base_url=llm.base_url,
110
+ )
111
+
112
+ extraction_strategy = LLMExtractionStrategy(
113
+ llm_config=crawl4ai_llm_config,
114
+ instruction=llm.instruction or "Extract and summarize the main content",
115
+ schema=llm.schema,
116
+ )
117
+ config.extraction_strategy = extraction_strategy
118
+ return config
119
+
120
+ async def _crawl(
121
+ self,
122
+ url: str,
123
+ enhanced: bool = False,
124
+ llm_config: Optional[Dict[str, Any]] = None,
125
+ ) -> Dict[str, Any]:
126
+ """
127
+ 内部异步爬取方法
128
+
129
+ Args:
130
+ url: 目标 URL
131
+ enhanced: 是否使用增强模式 (更长等待时间)
132
+ llm_config: LLM 配置字典(可选)
133
+
134
+ Returns:
135
+ 包含 success, markdown, title, (可选) llm_result 的字典
136
+ """
137
+ config = self._create_config(enhanced)
138
+ config = self._add_llm_strategy(config, llm_config)
139
+
140
+ # 重试机制:最多重试 3 次,只对网络错误重试
141
+ max_retries = 3
142
+
143
+ for attempt in range(max_retries + 1): # +1 因为第一次不是重试
144
+ try:
145
+ async with AsyncWebCrawler(verbose=False) as crawler:
146
+ result = await crawler.arun(url=url, config=config)
147
+
148
+ response = {
149
+ "success": result.success,
150
+ "markdown": result.markdown.raw_markdown
151
+ if result.success
152
+ else "",
153
+ "title": result.metadata.get("title", "")
154
+ if result.success
155
+ else "",
156
+ "error": result.error_message if not result.success else None,
157
+ }
158
+
159
+ # 如果使用了 LLM 提取,添加结果
160
+ if llm_config and result.success and result.extracted_content:
161
+ try:
162
+ import json
163
+
164
+ response["llm_result"] = json.loads(
165
+ result.extracted_content
166
+ )
167
+ except (json.JSONDecodeError, TypeError):
168
+ response["llm_result"] = {"raw": result.extracted_content}
169
+
170
+ return response
171
+
172
+ except Exception as e:
173
+ error_msg = str(e)
174
+
175
+ # 只对 ERR_NETWORK_CHANGED 相关错误重试
176
+ is_network_error = (
177
+ "ERR_NETWORK_CHANGED" in error_msg
178
+ or "ERR_INTERNET_DISCONNECTED" in error_msg
179
+ or "ERR_CONNECTION_RESET" in error_msg
180
+ or "ERR_TIMED_OUT" in error_msg
181
+ )
182
+
183
+ # 如果是网络错误且还有重试次数,等待后重试
184
+ if is_network_error and attempt < max_retries:
185
+ await asyncio.sleep(2**attempt) # 指数退避: 1s, 2s, 4s
186
+ continue
187
+
188
+ # 其他错误或重试用尽,抛出异常
189
+ raise
190
+
191
+ def _call_llm(
192
+ self, content: str, instruction: str, schema: Optional[Dict[str, Any]] = None
193
+ ) -> Dict[str, Any]:
194
+ """
195
+ 调用 LLM 处理文本内容
196
+
197
+ Args:
198
+ content: 要处理的内容
199
+ instruction: 处理指令
200
+ schema: 可选的 JSON Schema
201
+
202
+ Returns:
203
+ 处理结果字典
204
+ """
205
+ from openai import OpenAI
206
+ from .llm_config import get_default_llm_config
207
+
208
+ llm_cfg = get_default_llm_config()
209
+ client = OpenAI(api_key=llm_cfg.api_key, base_url=llm_cfg.base_url)
210
+
211
+ messages = [
212
+ {"role": "system", "content": "你是一个专业的文本处理助手。"},
213
+ {"role": "user", "content": f"指令:{instruction}\n\n内容:\n{content}"},
214
+ ]
215
+
216
+ if schema:
217
+ messages[0]["content"] += f"\n\n请按照以下 JSON Schema 返回结果:{schema}"
218
+
219
+ try:
220
+ response = client.chat.completions.create(
221
+ model=llm_cfg.model,
222
+ messages=messages,
223
+ temperature=0.3,
224
+ )
225
+ result_text = response.choices[0].message.content
226
+
227
+ # 尝试解析为 JSON
228
+ if schema:
229
+ try:
230
+ import json
231
+
232
+ return {"success": True, "data": json.loads(result_text)}
233
+ except json.JSONDecodeError:
234
+ return {"success": True, "content": result_text}
235
+ else:
236
+ return {"success": True, "summary": result_text}
237
+
238
+ except Exception as e:
239
+ return {
240
+ "success": False,
241
+ "error": str(e),
242
+ "content": content, # 失败时返回原内容
243
+ }
244
+
245
+ def postprocess_markdown(
246
+ self, markdown: str, instruction: str, schema: Optional[Dict[str, Any]] = None
247
+ ) -> Dict[str, Any]:
248
+ """
249
+ 对 Markdown 内容进行 LLM 后处理
250
+
251
+ Args:
252
+ markdown: Markdown 内容
253
+ instruction: 处理指令
254
+ schema: 可选的 JSON Schema
255
+
256
+ Returns:
257
+ 处理结果,包含 summary 或 data 字段
258
+ """
259
+ if not instruction or not instruction.strip():
260
+ return {
261
+ "success": True,
262
+ "original_markdown": markdown,
263
+ "skipped": "No instruction provided",
264
+ }
265
+
266
+ # 直接调用同步的 _call_llm
267
+ return self._call_llm(markdown, instruction, schema)
268
+
269
+ def crawl_single(
270
+ self,
271
+ url: str,
272
+ enhanced: bool = False,
273
+ llm_config: Optional[Dict[str, Any]] = None,
274
+ ) -> Dict[str, Any]:
275
+ """
276
+ 爬取单个网页 (同步封装)
277
+
278
+ 新设计:先快速爬取 Markdown,然后可选地进行 LLM 后处理
279
+
280
+ Args:
281
+ url: 网页 URL
282
+ enhanced: 是否使用增强 SPA 模式
283
+ llm_config: LLM 配置(可选),支持:
284
+ - Dict: {"instruction": "...", "schema": {...}}
285
+ - JSON 字符串: '{"instruction": "..."}'
286
+ - 纯文本: "总结页面"(自动作为 instruction)
287
+
288
+ Returns:
289
+ 爬取结果字典,包含:
290
+ - success, markdown, title, error(爬取结果)
291
+ - llm_summary 或 llm_data(LLM 处理结果,如果提供 llm_config)
292
+ """
293
+ # 解析 llm_config
294
+ parsed_llm_config = _parse_llm_config(llm_config)
295
+
296
+ # 第一步:快速爬取(不使用 LLM 策略)
297
+ crawl_result = _run_async(self._crawl(url, enhanced, llm_config=None))
298
+
299
+ # 第二步:如果有 LLM 配置,对 Markdown 进行后处理
300
+ if parsed_llm_config and crawl_result["success"]:
301
+ instruction = parsed_llm_config.get("instruction", "")
302
+ schema = parsed_llm_config.get("schema")
303
+
304
+ if instruction: # 只有有 instruction 时才处理
305
+ llm_result = self.postprocess_markdown(
306
+ crawl_result["markdown"], instruction, schema
307
+ )
308
+
309
+ # 将 LLM 结果合并到响应中
310
+ if llm_result.get("success"):
311
+ if "summary" in llm_result:
312
+ crawl_result["llm_summary"] = llm_result["summary"]
313
+ if "data" in llm_result:
314
+ crawl_result["llm_data"] = llm_result["data"]
315
+ if "content" in llm_result:
316
+ crawl_result["llm_content"] = llm_result["content"]
317
+ else:
318
+ crawl_result["llm_error"] = llm_result.get("error", "Unknown error")
319
+
320
+ return crawl_result
321
+
322
+ async def _crawl_site(
323
+ self, url: str, depth: int = 2, pages: int = 10, concurrent: int = 3
324
+ ) -> Dict[str, Any]:
325
+ """
326
+ 内部整站爬取方法
327
+
328
+ Args:
329
+ url: 起始 URL
330
+ depth: 最大爬取深度
331
+ pages: 最大页面数
332
+ concurrent: 并发数
333
+
334
+ Returns:
335
+ 爬取统计信息
336
+ """
337
+ # TODO: 整合 async_crawler.py 的 AsyncParallelCrawler 逻辑
338
+ # 暂时返回模拟结果
339
+ visited = set()
340
+ results = []
341
+
342
+ async def crawl_with_depth(target_url: str, current_depth: int):
343
+ if current_depth > depth or len(results) >= pages:
344
+ return
345
+ if target_url in visited:
346
+ return
347
+
348
+ visited.add(target_url)
349
+ result = await self._crawl(target_url)
350
+ results.append(result)
351
+
352
+ await crawl_with_depth(url, 0)
353
+
354
+ successful = sum(1 for r in results if r["success"])
355
+
356
+ return {
357
+ "successful_pages": successful,
358
+ "total_pages": len(results),
359
+ "success_rate": _calculate_success_rate(results),
360
+ "results": results,
361
+ }
362
+
363
+ def crawl_site(
364
+ self, url: str, depth: int = 2, pages: int = 10, concurrent: int = 3
365
+ ) -> Dict[str, Any]:
366
+ """
367
+ 爬取整个网站 (同步封装)
368
+
369
+ Args:
370
+ url: 起始 URL
371
+ depth: 最大爬取深度
372
+ pages: 最大页面数
373
+ concurrent: 并发数
374
+
375
+ Returns:
376
+ 爬取统计信息
377
+ """
378
+ return _run_async(
379
+ self._crawl_site(url, depth=depth, pages=pages, concurrent=concurrent)
380
+ )
381
+
382
+ async def _crawl_batch(
383
+ self,
384
+ urls: List[str],
385
+ concurrent: int = 3,
386
+ llm_config: Optional[Dict[str, Any]] = None,
387
+ ) -> List[Dict[str, Any]]:
388
+ """
389
+ 内部批量爬取方法 - 使用 arun_many 实现真正的异步并行
390
+
391
+ 新设计:先快速爬取所有页面,然后对成功的页面进行 LLM 后处理
392
+
393
+ Args:
394
+ urls: URL 列表
395
+ concurrent: 并发数
396
+ llm_config: LLM 配置字典(可选)
397
+
398
+ Returns:
399
+ 爬取结果列表
400
+ """
401
+ if not urls:
402
+ return []
403
+
404
+ from crawl4ai.async_dispatcher import SemaphoreDispatcher
405
+
406
+ # 创建配置(不包含 LLM 策略,保持快速爬取)
407
+ config = self._create_config(enhanced=False)
408
+
409
+ # 创建并发控制器
410
+ dispatcher = SemaphoreDispatcher(semaphore_count=concurrent)
411
+
412
+ # 使用 arun_many 实现真正的并行爬取
413
+ async with AsyncWebCrawler(verbose=False) as crawler:
414
+ results = await crawler.arun_many(
415
+ urls=urls, config=config, dispatcher=dispatcher
416
+ )
417
+
418
+ # 将 CrawlResultContainer 转换为我们的格式
419
+ formatted_results = []
420
+ for r in results:
421
+ response = {
422
+ "success": r.success,
423
+ "markdown": r.markdown.raw_markdown if r.success else "",
424
+ "title": r.metadata.get("title", "") if r.success else "",
425
+ "error": r.error_message if not r.success else None,
426
+ }
427
+ formatted_results.append(response)
428
+
429
+ # 第二阶段:如果有 LLM 配置,对成功的页面进行后处理
430
+ parsed_llm_config = _parse_llm_config(llm_config)
431
+ if parsed_llm_config and parsed_llm_config.get("instruction"):
432
+ instruction = parsed_llm_config["instruction"]
433
+ schema = parsed_llm_config.get("schema")
434
+
435
+ for i, result in enumerate(formatted_results):
436
+ if result["success"]:
437
+ llm_result = self._call_llm(result["markdown"], instruction, schema)
438
+ if llm_result.get("success"):
439
+ if "summary" in llm_result:
440
+ result["llm_summary"] = llm_result["summary"]
441
+ if "data" in llm_result:
442
+ result["llm_data"] = llm_result["data"]
443
+ if "content" in llm_result:
444
+ result["llm_content"] = llm_result["content"]
445
+ else:
446
+ result["llm_error"] = llm_result.get("error", "Unknown error")
447
+
448
+ return formatted_results
449
+
450
+ def crawl_batch(
451
+ self,
452
+ urls: List[str],
453
+ concurrent: int = 3,
454
+ llm_config: Optional[Dict[str, Any]] = None,
455
+ ) -> List[Dict[str, Any]]:
456
+ """
457
+ 批量爬取多个网页 (同步封装)
458
+
459
+ Args:
460
+ urls: URL 列表
461
+ concurrent: 并发数
462
+ llm_config: LLM 配置字典(可选)
463
+
464
+ Returns:
465
+ 爬取结果列表
466
+ """
467
+ return _run_async(
468
+ self._crawl_batch(urls, concurrent=concurrent, llm_config=llm_config)
469
+ )
@@ -0,0 +1,97 @@
1
+ """FastMCP 服务器实现"""
2
+
3
+ from typing import List, Dict, Any, Optional, Union
4
+ from fastmcp import FastMCP
5
+ from crawl4ai_mcp.crawler import Crawler
6
+
7
+ # 读取包版本
8
+ try:
9
+ from importlib.metadata import version as get_version
10
+
11
+ __version__ = get_version("crawl_mcp")
12
+ except Exception:
13
+ __version__ = "0.1.0"
14
+
15
+ # 创建 FastMCP 实例
16
+ mcp = FastMCP(name="crawl-mcp", version=__version__)
17
+
18
+ # 创建爬虫实例(单例)
19
+ _crawler = Crawler()
20
+
21
+
22
+ @mcp.tool
23
+ def crawl_single(
24
+ url: str,
25
+ enhanced: bool = False,
26
+ llm_config: Optional[Union[Dict[str, Any], str]] = None,
27
+ ) -> Dict[str, Any]:
28
+ """
29
+ 爬取单个网页,返回 Markdown 格式内容
30
+
31
+ Args:
32
+ url: 要爬取的网页 URL
33
+ enhanced: 是否使用增强模式(适用于 SPA 网站,等待时间更长)
34
+ llm_config: LLM 配置(可选),支持三种格式:
35
+ - 字典: {"instruction": "总结", "schema": {...}}
36
+ - JSON 字符串: '{"instruction": "总结"}'
37
+ - 纯文本: "总结页面内容"(自动作为 instruction)
38
+
39
+ Returns:
40
+ 包含 success, markdown, title, error, (可选) llm_result 的字典
41
+ """
42
+ return _crawler.crawl_single(url, enhanced, llm_config)
43
+
44
+
45
+ @mcp.tool
46
+ def crawl_site(
47
+ url: str,
48
+ depth: int = 2,
49
+ pages: int = 10,
50
+ concurrent: int = 3,
51
+ llm_config: Optional[Union[Dict[str, Any], str]] = None,
52
+ ) -> Dict[str, Any]:
53
+ """
54
+ 递归爬取整个网站
55
+
56
+ Args:
57
+ url: 起始 URL
58
+ depth: 最大爬取深度(默认:2)
59
+ pages: 最大页面数(默认:10)
60
+ concurrent: 并发请求数(默认:3)
61
+ llm_config: LLM 配置(可选),格式同 crawl_single
62
+
63
+ Returns:
64
+ 包含 successful_pages, total_pages, success_rate, results 的字典
65
+ """
66
+ return _crawler.crawl_site(url, depth, pages, concurrent)
67
+
68
+
69
+ @mcp.tool
70
+ def crawl_batch(
71
+ urls: List[str],
72
+ concurrent: int = 3,
73
+ llm_config: Optional[Union[Dict[str, Any], str]] = None,
74
+ ) -> List[Dict[str, Any]]:
75
+ """
76
+ 批量爬取多个网页(异步并行)
77
+
78
+ Args:
79
+ urls: URL 列表
80
+ concurrent: 并发请求数(默认:3)
81
+ llm_config: LLM 配置(可选),格式同 crawl_single
82
+
83
+ Returns:
84
+ 爬取结果列表
85
+ """
86
+ return _crawler.crawl_batch(urls, concurrent, llm_config)
87
+
88
+
89
+ # CLI 入口点
90
+ if __name__ == "__main__":
91
+ import sys
92
+
93
+ # 默认使用 STDIO,但支持通过参数指定 HTTP
94
+ if "--http" in sys.argv:
95
+ mcp.run(transport="http", host="0.0.0.0", port=8001)
96
+ else:
97
+ mcp.run()
@@ -0,0 +1,73 @@
1
+ """LLM 配置模块 - 管理 LLM API 密钥和模型配置"""
2
+
3
+ import os
4
+ from dataclasses import dataclass
5
+ from typing import Any, Dict, Optional
6
+
7
+
8
+ @dataclass
9
+ class LLMConfig:
10
+ """LLM 配置类
11
+
12
+ 支持兼容 OpenAI API 格式的各种服务商
13
+ """
14
+
15
+ api_key: str
16
+ base_url: str = "https://api.openai.com/v1"
17
+ model: str = "gpt-4o-mini"
18
+ instruction: str = ""
19
+ schema: Optional[Dict[str, Any]] = None
20
+
21
+
22
+ def get_default_llm_config() -> LLMConfig:
23
+ """从环境变量获取默认 LLM 配置
24
+
25
+ 环境变量:
26
+ OPENAI_API_KEY: API 密钥(必需)
27
+ OPENAI_BASE_URL: API 基础 URL(可选)
28
+ LLM_MODEL: 模型名称(可选)
29
+
30
+ Returns:
31
+ LLMConfig 实例
32
+
33
+ Raises:
34
+ ValueError: 如果 OPENAI_API_KEY 未设置
35
+ """
36
+ api_key = os.getenv("OPENAI_API_KEY")
37
+ if not api_key:
38
+ raise ValueError("OPENAI_API_KEY environment variable is required")
39
+
40
+ return LLMConfig(
41
+ api_key=api_key,
42
+ base_url=os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1"),
43
+ model=os.getenv("LLM_MODEL", "gpt-4o-mini"),
44
+ )
45
+
46
+
47
+ def get_llm_config(config: Optional[Dict[str, Any]]) -> LLMConfig:
48
+ """从环境变量获取 LLM 配置,合并用户提供的业务参数
49
+
50
+ Args:
51
+ config: 用户提供的业务配置,可选包含:
52
+ - instruction: 提取指令
53
+ - schema: JSON Schema (可选)
54
+
55
+ Returns:
56
+ LLMConfig 实例,api_key/base_url/model 从环境变量读取
57
+
58
+ Raises:
59
+ ValueError: 如果 OPENAI_API_KEY 环境变量未设置
60
+ """
61
+ default = get_default_llm_config()
62
+
63
+ if config is None:
64
+ return default
65
+
66
+ # 只接受业务参数,认证配置从环境变量读取
67
+ return LLMConfig(
68
+ api_key=default.api_key,
69
+ base_url=default.base_url,
70
+ model=default.model,
71
+ instruction=config.get("instruction", ""),
72
+ schema=config.get("schema"),
73
+ )
@@ -0,0 +1,124 @@
1
+ Metadata-Version: 2.4
2
+ Name: crawl_mcp
3
+ Version: 0.1.0
4
+ Summary: 基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能
5
+ Project-URL: Homepage, https://github.com/gqy20/crawl-mcp
6
+ Project-URL: Repository, https://github.com/gqy20/crawl-mcp
7
+ Project-URL: Issues, https://github.com/gqy20/crawl-mcp/issues
8
+ Author: gqy20
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Requires-Python: >=3.12
18
+ Requires-Dist: beautifulsoup4>=4.13.5
19
+ Requires-Dist: crawl4ai>=0.7.8
20
+ Requires-Dist: fastmcp>=0.1.0
21
+ Requires-Dist: nest-asyncio>=1.6.0
22
+ Requires-Dist: openai>=2.0.0
23
+ Description-Content-Type: text/markdown
24
+
25
+ # crawl_mcp
26
+
27
+ 基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能。
28
+
29
+ ## 功能
30
+
31
+ - **crawl_single** - 爬取单个网页,返回 Markdown 格式
32
+ - **crawl_site** - 递归爬取整个网站
33
+ - **crawl_batch** - 批量爬取多个网页(异步并行)
34
+ - **LLM 集成** - AI 驱动的内容提取和摘要(先快速爬取,后可选处理)
35
+ - **自动重试** - 网络错误自动重试(指数退避)
36
+
37
+ ## LLM 处理设计
38
+
39
+ 爬取和 LLM 处理分离,确保快速响应:
40
+
41
+ 1. **快速爬取**(6-10秒)- 始终返回原始 Markdown
42
+ 2. **可选后处理** - 如提供 `llm_config`,对 Markdown 进行 AI 处理
43
+
44
+ ### 性能对比
45
+
46
+ | 场景 | 爬取阶段 | 总耗时 |
47
+ |------|---------|--------|
48
+ | crawl_single(无 LLM) | ~7s | **11s** |
49
+ | crawl_single(有 LLM) | ~6s | **40s** |
50
+ | crawl_batch 2 页(无 LLM) | ~15s | **15s** |
51
+ | crawl_batch(有 LLM) | ~6s/页 | **~20s/页** |
52
+
53
+ **关键优势**:
54
+ - 不使用 LLM 时 11 秒即可获取结果
55
+ - 使用 LLM 时爬取阶段仍然快速,用户可以先看到原始内容
56
+ - 相比旧设计(LLMExtractionStrategy 需要 123 秒),速度提升 3-10 倍
57
+
58
+ ## 安装
59
+
60
+ ```bash
61
+ pip install crawl-mcp
62
+ ```
63
+
64
+ ## MCP 配置
65
+
66
+ ### Claude Desktop
67
+
68
+ ```json
69
+ {
70
+ "mcpServers": {
71
+ "crawl-mcp": {
72
+ "command": "uv",
73
+ "args": ["--directory", "/path/to/crawl4ai", "run", "crawl-mcp", "--http"],
74
+ "env": {
75
+ "OPENAI_API_KEY": "your-api-key",
76
+ "OPENAI_BASE_URL": "https://api.openai.com/v1",
77
+ "LLM_MODEL": "gpt-4o-mini"
78
+ }
79
+ }
80
+ }
81
+ }
82
+ ```
83
+
84
+ ### 环境变量
85
+
86
+ | 变量 | 说明 | 默认值 |
87
+ |------|------|--------|
88
+ | `OPENAI_API_KEY` | API 密钥 | *必填* |
89
+ | `OPENAI_BASE_URL` | API 基础 URL | `https://api.openai.com/v1` |
90
+ | `LLM_MODEL` | 模型名称 | `gpt-4o-mini` |
91
+
92
+ ## LLM 配置
93
+
94
+ 所有工具支持可选的 `llm_config` 参数:
95
+
96
+ ```json
97
+ {
98
+ "instruction": "提取产品信息",
99
+ "schema": {
100
+ "type": "object",
101
+ "properties": {
102
+ "name": {"type": "string"},
103
+ "price": {"type": "number"}
104
+ }
105
+ }
106
+ }
107
+ ```
108
+
109
+ - `instruction`: 提取指令
110
+ - `schema`: 可选的 JSON Schema
111
+
112
+ **注意**: `api_key`、`base_url`、`model` 从环境变量读取。
113
+
114
+ ## 开发
115
+
116
+ ```bash
117
+ uv sync
118
+ uv run pytest
119
+ uv run python -m crawl4ai_mcp.fastmcp_server --http
120
+ ```
121
+
122
+ ## 许可证
123
+
124
+ MIT License
@@ -0,0 +1,9 @@
1
+ crawl4ai_mcp/__init__.py,sha256=9qAXbEImh_4v3RGnkW4LEOtsRzDdFnNUYJWFSycwFuM,294
2
+ crawl4ai_mcp/crawler.py,sha256=a8uWwGRGbLMTALHkgZwAWIF9bU4YSYMgbFJNqxQ0-eg,16141
3
+ crawl4ai_mcp/fastmcp_server.py,sha256=X6NV8pEsqzN4zLvr5jJQ6EvfbRrLGGQp-F_6KSN4odA,2598
4
+ crawl4ai_mcp/llm_config.py,sha256=qAqN9hBZ4TyybzDSTESbO7ytcp-f6MNchcZjaYVMhoc,1946
5
+ crawl_mcp-0.1.0.dist-info/METADATA,sha256=F3hyo99SBPlzlmIeE_v0aeJpbQTi-z013mCAGgzKJeY,3224
6
+ crawl_mcp-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
7
+ crawl_mcp-0.1.0.dist-info/entry_points.txt,sha256=qC9GuAZa--l2dbfCODABSDrLqv-8Df2K-PQV7Uy0pnc,58
8
+ crawl_mcp-0.1.0.dist-info/licenses/LICENSE,sha256=-6vYTCxAL2pghdWfVd0f0WTkHvngrRi_DKbKpbfsiNI,1062
9
+ crawl_mcp-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.28.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [mcp.servers]
2
+ crawl-mcp = crawl4ai_mcp.fastmcp_server:mcp
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 gqy20
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.