crawl-mcp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawl4ai_mcp/__init__.py +8 -0
- crawl4ai_mcp/crawler.py +469 -0
- crawl4ai_mcp/fastmcp_server.py +97 -0
- crawl4ai_mcp/llm_config.py +73 -0
- crawl_mcp-0.1.0.dist-info/METADATA +124 -0
- crawl_mcp-0.1.0.dist-info/RECORD +9 -0
- crawl_mcp-0.1.0.dist-info/WHEEL +4 -0
- crawl_mcp-0.1.0.dist-info/entry_points.txt +2 -0
- crawl_mcp-0.1.0.dist-info/licenses/LICENSE +21 -0
crawl4ai_mcp/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""crawl_mcp: 基于 crawl4ai 和 FastMCP 的 MCP 服务器"""
|
|
2
|
+
|
|
3
|
+
from crawl4ai_mcp.crawler import Crawler
|
|
4
|
+
from crawl4ai_mcp.fastmcp_server import mcp
|
|
5
|
+
from crawl4ai_mcp.llm_config import LLMConfig, get_llm_config
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
__all__ = ["Crawler", "mcp", "LLMConfig", "get_llm_config"]
|
crawl4ai_mcp/crawler.py
ADDED
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
"""Crawler 类 - 核心爬虫逻辑"""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from typing import List, Dict, Any, Optional
|
|
5
|
+
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
|
6
|
+
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
|
7
|
+
from .llm_config import get_llm_config
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _run_async(coro):
|
|
11
|
+
"""
|
|
12
|
+
运行异步函数的辅助函数,兼容已有事件循环的环境
|
|
13
|
+
使用 nest_asyncio 允许嵌套事件循环
|
|
14
|
+
"""
|
|
15
|
+
try:
|
|
16
|
+
asyncio.get_running_loop()
|
|
17
|
+
import nest_asyncio
|
|
18
|
+
|
|
19
|
+
nest_asyncio.apply()
|
|
20
|
+
return asyncio.get_event_loop().run_until_complete(coro)
|
|
21
|
+
except RuntimeError:
|
|
22
|
+
return asyncio.run(coro)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _calculate_success_rate(results: List[Dict[str, Any]]) -> str:
|
|
26
|
+
"""计算成功率"""
|
|
27
|
+
if not results:
|
|
28
|
+
return "0%"
|
|
29
|
+
successful = sum(1 for r in results if r["success"])
|
|
30
|
+
return f"{successful / len(results) * 100:.1f}%"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _parse_llm_config(llm_config: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
|
34
|
+
"""解析 llm_config,兼容字符串和字典两种格式
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
llm_config: 可以是 Dict、JSON 字符串、或纯文本字符串
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
解析后的字典,如果输入是纯文本则当作 instruction
|
|
41
|
+
"""
|
|
42
|
+
if llm_config is None:
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
# 如果已经是字典,直接返回
|
|
46
|
+
if isinstance(llm_config, dict):
|
|
47
|
+
return llm_config
|
|
48
|
+
|
|
49
|
+
# 如果是字符串,尝试解析
|
|
50
|
+
if isinstance(llm_config, str):
|
|
51
|
+
import json
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
# 尝试解析为 JSON 对象
|
|
55
|
+
return json.loads(llm_config)
|
|
56
|
+
except json.JSONDecodeError:
|
|
57
|
+
# 解析失败,当作纯文本 instruction
|
|
58
|
+
return {"instruction": llm_config}
|
|
59
|
+
|
|
60
|
+
# 其他类型,尝试转换为字符串再处理
|
|
61
|
+
return {"instruction": str(llm_config)}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Crawler:
|
|
65
|
+
"""统一的爬虫类,整合单页、整站、批量爬取功能"""
|
|
66
|
+
|
|
67
|
+
def _create_config(self, enhanced: bool = False) -> CrawlerRunConfig:
|
|
68
|
+
"""创建爬虫配置"""
|
|
69
|
+
markdown_generator = DefaultMarkdownGenerator(
|
|
70
|
+
options={
|
|
71
|
+
"citations": False,
|
|
72
|
+
"body_width": None,
|
|
73
|
+
"ignore_links": False,
|
|
74
|
+
}
|
|
75
|
+
)
|
|
76
|
+
return CrawlerRunConfig(
|
|
77
|
+
markdown_generator=markdown_generator,
|
|
78
|
+
page_timeout=60000,
|
|
79
|
+
delay_before_return_html=5.0 if not enhanced else 15.0,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def _add_llm_strategy(
|
|
83
|
+
self, config: CrawlerRunConfig, llm_config: Optional[Dict[str, Any]]
|
|
84
|
+
) -> CrawlerRunConfig:
|
|
85
|
+
"""
|
|
86
|
+
为配置添加 LLM 提取策略
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
config: 基础爬虫配置
|
|
90
|
+
llm_config: LLM 配置(可选),支持字典、JSON 字符串或纯文本
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
添加了 LLM 策略的配置(原地修改或返回原配置)
|
|
94
|
+
"""
|
|
95
|
+
# 兼容处理:解析字符串格式的配置
|
|
96
|
+
llm_config = _parse_llm_config(llm_config)
|
|
97
|
+
if not llm_config:
|
|
98
|
+
return config
|
|
99
|
+
|
|
100
|
+
from crawl4ai.extraction_strategy import LLMExtractionStrategy
|
|
101
|
+
from crawl4ai import LLMConfig as Crawl4AILLMConfig
|
|
102
|
+
|
|
103
|
+
llm = get_llm_config(llm_config)
|
|
104
|
+
# crawl4ai 的 LLMConfig 使用 provider 参数,格式为 "openai/model-name"
|
|
105
|
+
provider = f"openai/{llm.model}" if "/" not in llm.model else llm.model
|
|
106
|
+
crawl4ai_llm_config = Crawl4AILLMConfig(
|
|
107
|
+
provider=provider,
|
|
108
|
+
api_token=llm.api_key,
|
|
109
|
+
base_url=llm.base_url,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
extraction_strategy = LLMExtractionStrategy(
|
|
113
|
+
llm_config=crawl4ai_llm_config,
|
|
114
|
+
instruction=llm.instruction or "Extract and summarize the main content",
|
|
115
|
+
schema=llm.schema,
|
|
116
|
+
)
|
|
117
|
+
config.extraction_strategy = extraction_strategy
|
|
118
|
+
return config
|
|
119
|
+
|
|
120
|
+
async def _crawl(
|
|
121
|
+
self,
|
|
122
|
+
url: str,
|
|
123
|
+
enhanced: bool = False,
|
|
124
|
+
llm_config: Optional[Dict[str, Any]] = None,
|
|
125
|
+
) -> Dict[str, Any]:
|
|
126
|
+
"""
|
|
127
|
+
内部异步爬取方法
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
url: 目标 URL
|
|
131
|
+
enhanced: 是否使用增强模式 (更长等待时间)
|
|
132
|
+
llm_config: LLM 配置字典(可选)
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
包含 success, markdown, title, (可选) llm_result 的字典
|
|
136
|
+
"""
|
|
137
|
+
config = self._create_config(enhanced)
|
|
138
|
+
config = self._add_llm_strategy(config, llm_config)
|
|
139
|
+
|
|
140
|
+
# 重试机制:最多重试 3 次,只对网络错误重试
|
|
141
|
+
max_retries = 3
|
|
142
|
+
|
|
143
|
+
for attempt in range(max_retries + 1): # +1 因为第一次不是重试
|
|
144
|
+
try:
|
|
145
|
+
async with AsyncWebCrawler(verbose=False) as crawler:
|
|
146
|
+
result = await crawler.arun(url=url, config=config)
|
|
147
|
+
|
|
148
|
+
response = {
|
|
149
|
+
"success": result.success,
|
|
150
|
+
"markdown": result.markdown.raw_markdown
|
|
151
|
+
if result.success
|
|
152
|
+
else "",
|
|
153
|
+
"title": result.metadata.get("title", "")
|
|
154
|
+
if result.success
|
|
155
|
+
else "",
|
|
156
|
+
"error": result.error_message if not result.success else None,
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
# 如果使用了 LLM 提取,添加结果
|
|
160
|
+
if llm_config and result.success and result.extracted_content:
|
|
161
|
+
try:
|
|
162
|
+
import json
|
|
163
|
+
|
|
164
|
+
response["llm_result"] = json.loads(
|
|
165
|
+
result.extracted_content
|
|
166
|
+
)
|
|
167
|
+
except (json.JSONDecodeError, TypeError):
|
|
168
|
+
response["llm_result"] = {"raw": result.extracted_content}
|
|
169
|
+
|
|
170
|
+
return response
|
|
171
|
+
|
|
172
|
+
except Exception as e:
|
|
173
|
+
error_msg = str(e)
|
|
174
|
+
|
|
175
|
+
# 只对 ERR_NETWORK_CHANGED 相关错误重试
|
|
176
|
+
is_network_error = (
|
|
177
|
+
"ERR_NETWORK_CHANGED" in error_msg
|
|
178
|
+
or "ERR_INTERNET_DISCONNECTED" in error_msg
|
|
179
|
+
or "ERR_CONNECTION_RESET" in error_msg
|
|
180
|
+
or "ERR_TIMED_OUT" in error_msg
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# 如果是网络错误且还有重试次数,等待后重试
|
|
184
|
+
if is_network_error and attempt < max_retries:
|
|
185
|
+
await asyncio.sleep(2**attempt) # 指数退避: 1s, 2s, 4s
|
|
186
|
+
continue
|
|
187
|
+
|
|
188
|
+
# 其他错误或重试用尽,抛出异常
|
|
189
|
+
raise
|
|
190
|
+
|
|
191
|
+
def _call_llm(
|
|
192
|
+
self, content: str, instruction: str, schema: Optional[Dict[str, Any]] = None
|
|
193
|
+
) -> Dict[str, Any]:
|
|
194
|
+
"""
|
|
195
|
+
调用 LLM 处理文本内容
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
content: 要处理的内容
|
|
199
|
+
instruction: 处理指令
|
|
200
|
+
schema: 可选的 JSON Schema
|
|
201
|
+
|
|
202
|
+
Returns:
|
|
203
|
+
处理结果字典
|
|
204
|
+
"""
|
|
205
|
+
from openai import OpenAI
|
|
206
|
+
from .llm_config import get_default_llm_config
|
|
207
|
+
|
|
208
|
+
llm_cfg = get_default_llm_config()
|
|
209
|
+
client = OpenAI(api_key=llm_cfg.api_key, base_url=llm_cfg.base_url)
|
|
210
|
+
|
|
211
|
+
messages = [
|
|
212
|
+
{"role": "system", "content": "你是一个专业的文本处理助手。"},
|
|
213
|
+
{"role": "user", "content": f"指令:{instruction}\n\n内容:\n{content}"},
|
|
214
|
+
]
|
|
215
|
+
|
|
216
|
+
if schema:
|
|
217
|
+
messages[0]["content"] += f"\n\n请按照以下 JSON Schema 返回结果:{schema}"
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
response = client.chat.completions.create(
|
|
221
|
+
model=llm_cfg.model,
|
|
222
|
+
messages=messages,
|
|
223
|
+
temperature=0.3,
|
|
224
|
+
)
|
|
225
|
+
result_text = response.choices[0].message.content
|
|
226
|
+
|
|
227
|
+
# 尝试解析为 JSON
|
|
228
|
+
if schema:
|
|
229
|
+
try:
|
|
230
|
+
import json
|
|
231
|
+
|
|
232
|
+
return {"success": True, "data": json.loads(result_text)}
|
|
233
|
+
except json.JSONDecodeError:
|
|
234
|
+
return {"success": True, "content": result_text}
|
|
235
|
+
else:
|
|
236
|
+
return {"success": True, "summary": result_text}
|
|
237
|
+
|
|
238
|
+
except Exception as e:
|
|
239
|
+
return {
|
|
240
|
+
"success": False,
|
|
241
|
+
"error": str(e),
|
|
242
|
+
"content": content, # 失败时返回原内容
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
def postprocess_markdown(
|
|
246
|
+
self, markdown: str, instruction: str, schema: Optional[Dict[str, Any]] = None
|
|
247
|
+
) -> Dict[str, Any]:
|
|
248
|
+
"""
|
|
249
|
+
对 Markdown 内容进行 LLM 后处理
|
|
250
|
+
|
|
251
|
+
Args:
|
|
252
|
+
markdown: Markdown 内容
|
|
253
|
+
instruction: 处理指令
|
|
254
|
+
schema: 可选的 JSON Schema
|
|
255
|
+
|
|
256
|
+
Returns:
|
|
257
|
+
处理结果,包含 summary 或 data 字段
|
|
258
|
+
"""
|
|
259
|
+
if not instruction or not instruction.strip():
|
|
260
|
+
return {
|
|
261
|
+
"success": True,
|
|
262
|
+
"original_markdown": markdown,
|
|
263
|
+
"skipped": "No instruction provided",
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
# 直接调用同步的 _call_llm
|
|
267
|
+
return self._call_llm(markdown, instruction, schema)
|
|
268
|
+
|
|
269
|
+
def crawl_single(
|
|
270
|
+
self,
|
|
271
|
+
url: str,
|
|
272
|
+
enhanced: bool = False,
|
|
273
|
+
llm_config: Optional[Dict[str, Any]] = None,
|
|
274
|
+
) -> Dict[str, Any]:
|
|
275
|
+
"""
|
|
276
|
+
爬取单个网页 (同步封装)
|
|
277
|
+
|
|
278
|
+
新设计:先快速爬取 Markdown,然后可选地进行 LLM 后处理
|
|
279
|
+
|
|
280
|
+
Args:
|
|
281
|
+
url: 网页 URL
|
|
282
|
+
enhanced: 是否使用增强 SPA 模式
|
|
283
|
+
llm_config: LLM 配置(可选),支持:
|
|
284
|
+
- Dict: {"instruction": "...", "schema": {...}}
|
|
285
|
+
- JSON 字符串: '{"instruction": "..."}'
|
|
286
|
+
- 纯文本: "总结页面"(自动作为 instruction)
|
|
287
|
+
|
|
288
|
+
Returns:
|
|
289
|
+
爬取结果字典,包含:
|
|
290
|
+
- success, markdown, title, error(爬取结果)
|
|
291
|
+
- llm_summary 或 llm_data(LLM 处理结果,如果提供 llm_config)
|
|
292
|
+
"""
|
|
293
|
+
# 解析 llm_config
|
|
294
|
+
parsed_llm_config = _parse_llm_config(llm_config)
|
|
295
|
+
|
|
296
|
+
# 第一步:快速爬取(不使用 LLM 策略)
|
|
297
|
+
crawl_result = _run_async(self._crawl(url, enhanced, llm_config=None))
|
|
298
|
+
|
|
299
|
+
# 第二步:如果有 LLM 配置,对 Markdown 进行后处理
|
|
300
|
+
if parsed_llm_config and crawl_result["success"]:
|
|
301
|
+
instruction = parsed_llm_config.get("instruction", "")
|
|
302
|
+
schema = parsed_llm_config.get("schema")
|
|
303
|
+
|
|
304
|
+
if instruction: # 只有有 instruction 时才处理
|
|
305
|
+
llm_result = self.postprocess_markdown(
|
|
306
|
+
crawl_result["markdown"], instruction, schema
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
# 将 LLM 结果合并到响应中
|
|
310
|
+
if llm_result.get("success"):
|
|
311
|
+
if "summary" in llm_result:
|
|
312
|
+
crawl_result["llm_summary"] = llm_result["summary"]
|
|
313
|
+
if "data" in llm_result:
|
|
314
|
+
crawl_result["llm_data"] = llm_result["data"]
|
|
315
|
+
if "content" in llm_result:
|
|
316
|
+
crawl_result["llm_content"] = llm_result["content"]
|
|
317
|
+
else:
|
|
318
|
+
crawl_result["llm_error"] = llm_result.get("error", "Unknown error")
|
|
319
|
+
|
|
320
|
+
return crawl_result
|
|
321
|
+
|
|
322
|
+
async def _crawl_site(
|
|
323
|
+
self, url: str, depth: int = 2, pages: int = 10, concurrent: int = 3
|
|
324
|
+
) -> Dict[str, Any]:
|
|
325
|
+
"""
|
|
326
|
+
内部整站爬取方法
|
|
327
|
+
|
|
328
|
+
Args:
|
|
329
|
+
url: 起始 URL
|
|
330
|
+
depth: 最大爬取深度
|
|
331
|
+
pages: 最大页面数
|
|
332
|
+
concurrent: 并发数
|
|
333
|
+
|
|
334
|
+
Returns:
|
|
335
|
+
爬取统计信息
|
|
336
|
+
"""
|
|
337
|
+
# TODO: 整合 async_crawler.py 的 AsyncParallelCrawler 逻辑
|
|
338
|
+
# 暂时返回模拟结果
|
|
339
|
+
visited = set()
|
|
340
|
+
results = []
|
|
341
|
+
|
|
342
|
+
async def crawl_with_depth(target_url: str, current_depth: int):
|
|
343
|
+
if current_depth > depth or len(results) >= pages:
|
|
344
|
+
return
|
|
345
|
+
if target_url in visited:
|
|
346
|
+
return
|
|
347
|
+
|
|
348
|
+
visited.add(target_url)
|
|
349
|
+
result = await self._crawl(target_url)
|
|
350
|
+
results.append(result)
|
|
351
|
+
|
|
352
|
+
await crawl_with_depth(url, 0)
|
|
353
|
+
|
|
354
|
+
successful = sum(1 for r in results if r["success"])
|
|
355
|
+
|
|
356
|
+
return {
|
|
357
|
+
"successful_pages": successful,
|
|
358
|
+
"total_pages": len(results),
|
|
359
|
+
"success_rate": _calculate_success_rate(results),
|
|
360
|
+
"results": results,
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
def crawl_site(
|
|
364
|
+
self, url: str, depth: int = 2, pages: int = 10, concurrent: int = 3
|
|
365
|
+
) -> Dict[str, Any]:
|
|
366
|
+
"""
|
|
367
|
+
爬取整个网站 (同步封装)
|
|
368
|
+
|
|
369
|
+
Args:
|
|
370
|
+
url: 起始 URL
|
|
371
|
+
depth: 最大爬取深度
|
|
372
|
+
pages: 最大页面数
|
|
373
|
+
concurrent: 并发数
|
|
374
|
+
|
|
375
|
+
Returns:
|
|
376
|
+
爬取统计信息
|
|
377
|
+
"""
|
|
378
|
+
return _run_async(
|
|
379
|
+
self._crawl_site(url, depth=depth, pages=pages, concurrent=concurrent)
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
async def _crawl_batch(
|
|
383
|
+
self,
|
|
384
|
+
urls: List[str],
|
|
385
|
+
concurrent: int = 3,
|
|
386
|
+
llm_config: Optional[Dict[str, Any]] = None,
|
|
387
|
+
) -> List[Dict[str, Any]]:
|
|
388
|
+
"""
|
|
389
|
+
内部批量爬取方法 - 使用 arun_many 实现真正的异步并行
|
|
390
|
+
|
|
391
|
+
新设计:先快速爬取所有页面,然后对成功的页面进行 LLM 后处理
|
|
392
|
+
|
|
393
|
+
Args:
|
|
394
|
+
urls: URL 列表
|
|
395
|
+
concurrent: 并发数
|
|
396
|
+
llm_config: LLM 配置字典(可选)
|
|
397
|
+
|
|
398
|
+
Returns:
|
|
399
|
+
爬取结果列表
|
|
400
|
+
"""
|
|
401
|
+
if not urls:
|
|
402
|
+
return []
|
|
403
|
+
|
|
404
|
+
from crawl4ai.async_dispatcher import SemaphoreDispatcher
|
|
405
|
+
|
|
406
|
+
# 创建配置(不包含 LLM 策略,保持快速爬取)
|
|
407
|
+
config = self._create_config(enhanced=False)
|
|
408
|
+
|
|
409
|
+
# 创建并发控制器
|
|
410
|
+
dispatcher = SemaphoreDispatcher(semaphore_count=concurrent)
|
|
411
|
+
|
|
412
|
+
# 使用 arun_many 实现真正的并行爬取
|
|
413
|
+
async with AsyncWebCrawler(verbose=False) as crawler:
|
|
414
|
+
results = await crawler.arun_many(
|
|
415
|
+
urls=urls, config=config, dispatcher=dispatcher
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
# 将 CrawlResultContainer 转换为我们的格式
|
|
419
|
+
formatted_results = []
|
|
420
|
+
for r in results:
|
|
421
|
+
response = {
|
|
422
|
+
"success": r.success,
|
|
423
|
+
"markdown": r.markdown.raw_markdown if r.success else "",
|
|
424
|
+
"title": r.metadata.get("title", "") if r.success else "",
|
|
425
|
+
"error": r.error_message if not r.success else None,
|
|
426
|
+
}
|
|
427
|
+
formatted_results.append(response)
|
|
428
|
+
|
|
429
|
+
# 第二阶段:如果有 LLM 配置,对成功的页面进行后处理
|
|
430
|
+
parsed_llm_config = _parse_llm_config(llm_config)
|
|
431
|
+
if parsed_llm_config and parsed_llm_config.get("instruction"):
|
|
432
|
+
instruction = parsed_llm_config["instruction"]
|
|
433
|
+
schema = parsed_llm_config.get("schema")
|
|
434
|
+
|
|
435
|
+
for i, result in enumerate(formatted_results):
|
|
436
|
+
if result["success"]:
|
|
437
|
+
llm_result = self._call_llm(result["markdown"], instruction, schema)
|
|
438
|
+
if llm_result.get("success"):
|
|
439
|
+
if "summary" in llm_result:
|
|
440
|
+
result["llm_summary"] = llm_result["summary"]
|
|
441
|
+
if "data" in llm_result:
|
|
442
|
+
result["llm_data"] = llm_result["data"]
|
|
443
|
+
if "content" in llm_result:
|
|
444
|
+
result["llm_content"] = llm_result["content"]
|
|
445
|
+
else:
|
|
446
|
+
result["llm_error"] = llm_result.get("error", "Unknown error")
|
|
447
|
+
|
|
448
|
+
return formatted_results
|
|
449
|
+
|
|
450
|
+
def crawl_batch(
|
|
451
|
+
self,
|
|
452
|
+
urls: List[str],
|
|
453
|
+
concurrent: int = 3,
|
|
454
|
+
llm_config: Optional[Dict[str, Any]] = None,
|
|
455
|
+
) -> List[Dict[str, Any]]:
|
|
456
|
+
"""
|
|
457
|
+
批量爬取多个网页 (同步封装)
|
|
458
|
+
|
|
459
|
+
Args:
|
|
460
|
+
urls: URL 列表
|
|
461
|
+
concurrent: 并发数
|
|
462
|
+
llm_config: LLM 配置字典(可选)
|
|
463
|
+
|
|
464
|
+
Returns:
|
|
465
|
+
爬取结果列表
|
|
466
|
+
"""
|
|
467
|
+
return _run_async(
|
|
468
|
+
self._crawl_batch(urls, concurrent=concurrent, llm_config=llm_config)
|
|
469
|
+
)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""FastMCP 服务器实现"""
|
|
2
|
+
|
|
3
|
+
from typing import List, Dict, Any, Optional, Union
|
|
4
|
+
from fastmcp import FastMCP
|
|
5
|
+
from crawl4ai_mcp.crawler import Crawler
|
|
6
|
+
|
|
7
|
+
# 读取包版本
|
|
8
|
+
try:
|
|
9
|
+
from importlib.metadata import version as get_version
|
|
10
|
+
|
|
11
|
+
__version__ = get_version("crawl_mcp")
|
|
12
|
+
except Exception:
|
|
13
|
+
__version__ = "0.1.0"
|
|
14
|
+
|
|
15
|
+
# 创建 FastMCP 实例
|
|
16
|
+
mcp = FastMCP(name="crawl-mcp", version=__version__)
|
|
17
|
+
|
|
18
|
+
# 创建爬虫实例(单例)
|
|
19
|
+
_crawler = Crawler()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@mcp.tool
|
|
23
|
+
def crawl_single(
|
|
24
|
+
url: str,
|
|
25
|
+
enhanced: bool = False,
|
|
26
|
+
llm_config: Optional[Union[Dict[str, Any], str]] = None,
|
|
27
|
+
) -> Dict[str, Any]:
|
|
28
|
+
"""
|
|
29
|
+
爬取单个网页,返回 Markdown 格式内容
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
url: 要爬取的网页 URL
|
|
33
|
+
enhanced: 是否使用增强模式(适用于 SPA 网站,等待时间更长)
|
|
34
|
+
llm_config: LLM 配置(可选),支持三种格式:
|
|
35
|
+
- 字典: {"instruction": "总结", "schema": {...}}
|
|
36
|
+
- JSON 字符串: '{"instruction": "总结"}'
|
|
37
|
+
- 纯文本: "总结页面内容"(自动作为 instruction)
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
包含 success, markdown, title, error, (可选) llm_result 的字典
|
|
41
|
+
"""
|
|
42
|
+
return _crawler.crawl_single(url, enhanced, llm_config)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@mcp.tool
|
|
46
|
+
def crawl_site(
|
|
47
|
+
url: str,
|
|
48
|
+
depth: int = 2,
|
|
49
|
+
pages: int = 10,
|
|
50
|
+
concurrent: int = 3,
|
|
51
|
+
llm_config: Optional[Union[Dict[str, Any], str]] = None,
|
|
52
|
+
) -> Dict[str, Any]:
|
|
53
|
+
"""
|
|
54
|
+
递归爬取整个网站
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
url: 起始 URL
|
|
58
|
+
depth: 最大爬取深度(默认:2)
|
|
59
|
+
pages: 最大页面数(默认:10)
|
|
60
|
+
concurrent: 并发请求数(默认:3)
|
|
61
|
+
llm_config: LLM 配置(可选),格式同 crawl_single
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
包含 successful_pages, total_pages, success_rate, results 的字典
|
|
65
|
+
"""
|
|
66
|
+
return _crawler.crawl_site(url, depth, pages, concurrent)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@mcp.tool
|
|
70
|
+
def crawl_batch(
|
|
71
|
+
urls: List[str],
|
|
72
|
+
concurrent: int = 3,
|
|
73
|
+
llm_config: Optional[Union[Dict[str, Any], str]] = None,
|
|
74
|
+
) -> List[Dict[str, Any]]:
|
|
75
|
+
"""
|
|
76
|
+
批量爬取多个网页(异步并行)
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
urls: URL 列表
|
|
80
|
+
concurrent: 并发请求数(默认:3)
|
|
81
|
+
llm_config: LLM 配置(可选),格式同 crawl_single
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
爬取结果列表
|
|
85
|
+
"""
|
|
86
|
+
return _crawler.crawl_batch(urls, concurrent, llm_config)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# CLI 入口点
|
|
90
|
+
if __name__ == "__main__":
|
|
91
|
+
import sys
|
|
92
|
+
|
|
93
|
+
# 默认使用 STDIO,但支持通过参数指定 HTTP
|
|
94
|
+
if "--http" in sys.argv:
|
|
95
|
+
mcp.run(transport="http", host="0.0.0.0", port=8001)
|
|
96
|
+
else:
|
|
97
|
+
mcp.run()
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""LLM 配置模块 - 管理 LLM API 密钥和模型配置"""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any, Dict, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class LLMConfig:
|
|
10
|
+
"""LLM 配置类
|
|
11
|
+
|
|
12
|
+
支持兼容 OpenAI API 格式的各种服务商
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
api_key: str
|
|
16
|
+
base_url: str = "https://api.openai.com/v1"
|
|
17
|
+
model: str = "gpt-4o-mini"
|
|
18
|
+
instruction: str = ""
|
|
19
|
+
schema: Optional[Dict[str, Any]] = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_default_llm_config() -> LLMConfig:
|
|
23
|
+
"""从环境变量获取默认 LLM 配置
|
|
24
|
+
|
|
25
|
+
环境变量:
|
|
26
|
+
OPENAI_API_KEY: API 密钥(必需)
|
|
27
|
+
OPENAI_BASE_URL: API 基础 URL(可选)
|
|
28
|
+
LLM_MODEL: 模型名称(可选)
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
LLMConfig 实例
|
|
32
|
+
|
|
33
|
+
Raises:
|
|
34
|
+
ValueError: 如果 OPENAI_API_KEY 未设置
|
|
35
|
+
"""
|
|
36
|
+
api_key = os.getenv("OPENAI_API_KEY")
|
|
37
|
+
if not api_key:
|
|
38
|
+
raise ValueError("OPENAI_API_KEY environment variable is required")
|
|
39
|
+
|
|
40
|
+
return LLMConfig(
|
|
41
|
+
api_key=api_key,
|
|
42
|
+
base_url=os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1"),
|
|
43
|
+
model=os.getenv("LLM_MODEL", "gpt-4o-mini"),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def get_llm_config(config: Optional[Dict[str, Any]]) -> LLMConfig:
|
|
48
|
+
"""从环境变量获取 LLM 配置,合并用户提供的业务参数
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
config: 用户提供的业务配置,可选包含:
|
|
52
|
+
- instruction: 提取指令
|
|
53
|
+
- schema: JSON Schema (可选)
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
LLMConfig 实例,api_key/base_url/model 从环境变量读取
|
|
57
|
+
|
|
58
|
+
Raises:
|
|
59
|
+
ValueError: 如果 OPENAI_API_KEY 环境变量未设置
|
|
60
|
+
"""
|
|
61
|
+
default = get_default_llm_config()
|
|
62
|
+
|
|
63
|
+
if config is None:
|
|
64
|
+
return default
|
|
65
|
+
|
|
66
|
+
# 只接受业务参数,认证配置从环境变量读取
|
|
67
|
+
return LLMConfig(
|
|
68
|
+
api_key=default.api_key,
|
|
69
|
+
base_url=default.base_url,
|
|
70
|
+
model=default.model,
|
|
71
|
+
instruction=config.get("instruction", ""),
|
|
72
|
+
schema=config.get("schema"),
|
|
73
|
+
)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crawl_mcp
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: 基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能
|
|
5
|
+
Project-URL: Homepage, https://github.com/gqy20/crawl-mcp
|
|
6
|
+
Project-URL: Repository, https://github.com/gqy20/crawl-mcp
|
|
7
|
+
Project-URL: Issues, https://github.com/gqy20/crawl-mcp/issues
|
|
8
|
+
Author: gqy20
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.13.5
|
|
19
|
+
Requires-Dist: crawl4ai>=0.7.8
|
|
20
|
+
Requires-Dist: fastmcp>=0.1.0
|
|
21
|
+
Requires-Dist: nest-asyncio>=1.6.0
|
|
22
|
+
Requires-Dist: openai>=2.0.0
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# crawl_mcp
|
|
26
|
+
|
|
27
|
+
基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能。
|
|
28
|
+
|
|
29
|
+
## 功能
|
|
30
|
+
|
|
31
|
+
- **crawl_single** - 爬取单个网页,返回 Markdown 格式
|
|
32
|
+
- **crawl_site** - 递归爬取整个网站
|
|
33
|
+
- **crawl_batch** - 批量爬取多个网页(异步并行)
|
|
34
|
+
- **LLM 集成** - AI 驱动的内容提取和摘要(先快速爬取,后可选处理)
|
|
35
|
+
- **自动重试** - 网络错误自动重试(指数退避)
|
|
36
|
+
|
|
37
|
+
## LLM 处理设计
|
|
38
|
+
|
|
39
|
+
爬取和 LLM 处理分离,确保快速响应:
|
|
40
|
+
|
|
41
|
+
1. **快速爬取**(6-10秒)- 始终返回原始 Markdown
|
|
42
|
+
2. **可选后处理** - 如提供 `llm_config`,对 Markdown 进行 AI 处理
|
|
43
|
+
|
|
44
|
+
### 性能对比
|
|
45
|
+
|
|
46
|
+
| 场景 | 爬取阶段 | 总耗时 |
|
|
47
|
+
|------|---------|--------|
|
|
48
|
+
| crawl_single(无 LLM) | ~7s | **11s** |
|
|
49
|
+
| crawl_single(有 LLM) | ~6s | **40s** |
|
|
50
|
+
| crawl_batch 2 页(无 LLM) | ~15s | **15s** |
|
|
51
|
+
| crawl_batch(有 LLM) | ~6s/页 | **~20s/页** |
|
|
52
|
+
|
|
53
|
+
**关键优势**:
|
|
54
|
+
- 不使用 LLM 时 11 秒即可获取结果
|
|
55
|
+
- 使用 LLM 时爬取阶段仍然快速,用户可以先看到原始内容
|
|
56
|
+
- 相比旧设计(LLMExtractionStrategy 需要 123 秒),速度提升 3-10 倍
|
|
57
|
+
|
|
58
|
+
## 安装
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install crawl-mcp
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## MCP 配置
|
|
65
|
+
|
|
66
|
+
### Claude Desktop
|
|
67
|
+
|
|
68
|
+
```json
|
|
69
|
+
{
|
|
70
|
+
"mcpServers": {
|
|
71
|
+
"crawl-mcp": {
|
|
72
|
+
"command": "uv",
|
|
73
|
+
"args": ["--directory", "/path/to/crawl4ai", "run", "crawl-mcp", "--http"],
|
|
74
|
+
"env": {
|
|
75
|
+
"OPENAI_API_KEY": "your-api-key",
|
|
76
|
+
"OPENAI_BASE_URL": "https://api.openai.com/v1",
|
|
77
|
+
"LLM_MODEL": "gpt-4o-mini"
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### 环境变量
|
|
85
|
+
|
|
86
|
+
| 变量 | 说明 | 默认值 |
|
|
87
|
+
|------|------|--------|
|
|
88
|
+
| `OPENAI_API_KEY` | API 密钥 | *必填* |
|
|
89
|
+
| `OPENAI_BASE_URL` | API 基础 URL | `https://api.openai.com/v1` |
|
|
90
|
+
| `LLM_MODEL` | 模型名称 | `gpt-4o-mini` |
|
|
91
|
+
|
|
92
|
+
## LLM 配置
|
|
93
|
+
|
|
94
|
+
所有工具支持可选的 `llm_config` 参数:
|
|
95
|
+
|
|
96
|
+
```json
|
|
97
|
+
{
|
|
98
|
+
"instruction": "提取产品信息",
|
|
99
|
+
"schema": {
|
|
100
|
+
"type": "object",
|
|
101
|
+
"properties": {
|
|
102
|
+
"name": {"type": "string"},
|
|
103
|
+
"price": {"type": "number"}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
- `instruction`: 提取指令
|
|
110
|
+
- `schema`: 可选的 JSON Schema
|
|
111
|
+
|
|
112
|
+
**注意**: `api_key`、`base_url`、`model` 从环境变量读取。
|
|
113
|
+
|
|
114
|
+
## 开发
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
uv sync
|
|
118
|
+
uv run pytest
|
|
119
|
+
uv run python -m crawl4ai_mcp.fastmcp_server --http
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## 许可证
|
|
123
|
+
|
|
124
|
+
MIT License
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
crawl4ai_mcp/__init__.py,sha256=9qAXbEImh_4v3RGnkW4LEOtsRzDdFnNUYJWFSycwFuM,294
|
|
2
|
+
crawl4ai_mcp/crawler.py,sha256=a8uWwGRGbLMTALHkgZwAWIF9bU4YSYMgbFJNqxQ0-eg,16141
|
|
3
|
+
crawl4ai_mcp/fastmcp_server.py,sha256=X6NV8pEsqzN4zLvr5jJQ6EvfbRrLGGQp-F_6KSN4odA,2598
|
|
4
|
+
crawl4ai_mcp/llm_config.py,sha256=qAqN9hBZ4TyybzDSTESbO7ytcp-f6MNchcZjaYVMhoc,1946
|
|
5
|
+
crawl_mcp-0.1.0.dist-info/METADATA,sha256=F3hyo99SBPlzlmIeE_v0aeJpbQTi-z013mCAGgzKJeY,3224
|
|
6
|
+
crawl_mcp-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
7
|
+
crawl_mcp-0.1.0.dist-info/entry_points.txt,sha256=qC9GuAZa--l2dbfCODABSDrLqv-8Df2K-PQV7Uy0pnc,58
|
|
8
|
+
crawl_mcp-0.1.0.dist-info/licenses/LICENSE,sha256=-6vYTCxAL2pghdWfVd0f0WTkHvngrRi_DKbKpbfsiNI,1062
|
|
9
|
+
crawl_mcp-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 gqy20
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|