crawl-mcp 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,110 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - 'v*.*.*'
7
+ workflow_dispatch:
8
+ inputs:
9
+ tag:
10
+ description: 'Tag to publish (e.g., v0.1.0)'
11
+ required: true
12
+ type: string
13
+
14
+ permissions:
15
+ contents: read
16
+
17
+ jobs:
18
+ validate:
19
+ name: Validate Release
20
+ runs-on: ubuntu-latest
21
+
22
+ steps:
23
+ - name: Checkout code
24
+ uses: actions/checkout@v4
25
+ with:
26
+ ref: ${{ github.event.inputs.tag || github.ref }}
27
+
28
+ - name: Install uv
29
+ uses: astral-sh/setup-uv@v2
30
+ with:
31
+ version: "latest"
32
+
33
+ - name: Install dependencies
34
+ run: uv sync --dev
35
+
36
+ - name: Run full test suite
37
+ run: uv run pytest -v --cov=src/crawl4ai_mcp --cov-report=xml --cov-report=term-missing
38
+
39
+ - name: Run linting
40
+ run: |
41
+ uv run ruff check .
42
+ uv run ruff format --check .
43
+
44
+ - name: Verify version matches tag
45
+ run: |
46
+ TAG_VERSION="${{ github.event.inputs.tag || github.ref_name }}"
47
+ TAG_VERSION="${TAG_VERSION#v}"
48
+ PACKAGE_VERSION=$(uv run python -c "from crawl4ai_mcp import __version__; print(__version__)")
49
+
50
+ echo "Tag version: $TAG_VERSION"
51
+ echo "Package version: $PACKAGE_VERSION"
52
+
53
+ if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
54
+ echo "Error: Tag version ($TAG_VERSION) does not match package version ($PACKAGE_VERSION)"
55
+ exit 1
56
+ fi
57
+
58
+ build:
59
+ name: Build Distribution
60
+ needs: validate
61
+ runs-on: ubuntu-latest
62
+
63
+ steps:
64
+ - name: Checkout code
65
+ uses: actions/checkout@v4
66
+ with:
67
+ ref: ${{ github.event.inputs.tag || github.ref }}
68
+
69
+ - name: Install uv
70
+ uses: astral-sh/setup-uv@v2
71
+ with:
72
+ version: "latest"
73
+
74
+ - name: Build distribution packages
75
+ run: uv build
76
+
77
+ - name: Check distribution packages
78
+ run: uv sync --dev && uv run twine check dist/*
79
+
80
+ - name: Upload distribution artifacts
81
+ uses: actions/upload-artifact@v4
82
+ with:
83
+ name: python-package-distributions
84
+ path: dist/
85
+
86
+ publish-pypi:
87
+ name: Publish to PyPI
88
+ needs: build
89
+ runs-on: ubuntu-latest
90
+
91
+ steps:
92
+ - name: Checkout code
93
+ uses: actions/checkout@v4
94
+
95
+ - name: Download distribution artifacts
96
+ uses: actions/download-artifact@v4
97
+ with:
98
+ name: python-package-distributions
99
+ path: dist/
100
+
101
+ - name: Install uv
102
+ uses: astral-sh/setup-uv@v2
103
+ with:
104
+ version: "latest"
105
+
106
+ - name: Publish to PyPI
107
+ run: uv sync --dev && uv run twine upload dist/*
108
+ env:
109
+ TWINE_USERNAME: __token__
110
+ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
@@ -0,0 +1,38 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info/
8
+ .pytest_cache/
9
+ .coverage
10
+ htmlcov/
11
+
12
+ # Virtual environments
13
+ .venv/
14
+ venv/
15
+ ENV/
16
+
17
+ # uv
18
+ uv.lock
19
+
20
+ # IDE
21
+ .vscode/
22
+ .idea/
23
+ *.swp
24
+ *.swo
25
+ *~
26
+
27
+ # OS
28
+ .DS_Store
29
+ Thumbs.db
30
+
31
+ # Project specific
32
+ output/
33
+ *.log
34
+
35
+ # Temporary files
36
+ *.tmp
37
+ *.bak
38
+ .cache/
@@ -0,0 +1,22 @@
1
+ repos:
2
+ - repo: local
3
+ hooks:
4
+ - id: ruff-format
5
+ name: ruff format
6
+ entry: uv run ruff format .
7
+ language: system
8
+ types: [python]
9
+
10
+ - id: ruff-check
11
+ name: ruff check
12
+ entry: uv run ruff check . --fix
13
+ language: system
14
+ types: [python]
15
+ pass_filenames: false
16
+
17
+ - id: pytest
18
+ name: pytest
19
+ entry: uv run pytest -q
20
+ language: system
21
+ types: [python]
22
+ pass_filenames: false
@@ -0,0 +1 @@
1
+ 3.12
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 gqy20
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,124 @@
1
+ Metadata-Version: 2.4
2
+ Name: crawl_mcp
3
+ Version: 0.1.0
4
+ Summary: 基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能
5
+ Project-URL: Homepage, https://github.com/gqy20/crawl-mcp
6
+ Project-URL: Repository, https://github.com/gqy20/crawl-mcp
7
+ Project-URL: Issues, https://github.com/gqy20/crawl-mcp/issues
8
+ Author: gqy20
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Requires-Python: >=3.12
18
+ Requires-Dist: beautifulsoup4>=4.13.5
19
+ Requires-Dist: crawl4ai>=0.7.8
20
+ Requires-Dist: fastmcp>=0.1.0
21
+ Requires-Dist: nest-asyncio>=1.6.0
22
+ Requires-Dist: openai>=2.0.0
23
+ Description-Content-Type: text/markdown
24
+
25
+ # crawl_mcp
26
+
27
+ 基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能。
28
+
29
+ ## 功能
30
+
31
+ - **crawl_single** - 爬取单个网页,返回 Markdown 格式
32
+ - **crawl_site** - 递归爬取整个网站
33
+ - **crawl_batch** - 批量爬取多个网页(异步并行)
34
+ - **LLM 集成** - AI 驱动的内容提取和摘要(先快速爬取,后可选处理)
35
+ - **自动重试** - 网络错误自动重试(指数退避)
36
+
37
+ ## LLM 处理设计
38
+
39
+ 爬取和 LLM 处理分离,确保快速响应:
40
+
41
+ 1. **快速爬取**(6-10秒)- 始终返回原始 Markdown
42
+ 2. **可选后处理** - 如提供 `llm_config`,对 Markdown 进行 AI 处理
43
+
44
+ ### 性能对比
45
+
46
+ | 场景 | 爬取阶段 | 总耗时 |
47
+ |------|---------|--------|
48
+ | crawl_single(无 LLM) | ~7s | **11s** |
49
+ | crawl_single(有 LLM) | ~6s | **40s** |
50
+ | crawl_batch 2 页(无 LLM) | ~15s | **15s** |
51
+ | crawl_batch(有 LLM) | ~6s/页 | **~20s/页** |
52
+
53
+ **关键优势**:
54
+ - 不使用 LLM 时 11 秒即可获取结果
55
+ - 使用 LLM 时爬取阶段仍然快速,用户可以先看到原始内容
56
+ - 相比旧设计(LLMExtractionStrategy 需要 123 秒),速度提升 3-10 倍
57
+
58
+ ## 安装
59
+
60
+ ```bash
61
+ pip install crawl-mcp
62
+ ```
63
+
64
+ ## MCP 配置
65
+
66
+ ### Claude Desktop
67
+
68
+ ```json
69
+ {
70
+ "mcpServers": {
71
+ "crawl-mcp": {
72
+ "command": "uv",
73
+ "args": ["--directory", "/path/to/crawl4ai", "run", "crawl-mcp", "--http"],
74
+ "env": {
75
+ "OPENAI_API_KEY": "your-api-key",
76
+ "OPENAI_BASE_URL": "https://api.openai.com/v1",
77
+ "LLM_MODEL": "gpt-4o-mini"
78
+ }
79
+ }
80
+ }
81
+ }
82
+ ```
83
+
84
+ ### 环境变量
85
+
86
+ | 变量 | 说明 | 默认值 |
87
+ |------|------|--------|
88
+ | `OPENAI_API_KEY` | API 密钥 | *必填* |
89
+ | `OPENAI_BASE_URL` | API 基础 URL | `https://api.openai.com/v1` |
90
+ | `LLM_MODEL` | 模型名称 | `gpt-4o-mini` |
91
+
92
+ ## LLM 配置
93
+
94
+ 所有工具支持可选的 `llm_config` 参数:
95
+
96
+ ```json
97
+ {
98
+ "instruction": "提取产品信息",
99
+ "schema": {
100
+ "type": "object",
101
+ "properties": {
102
+ "name": {"type": "string"},
103
+ "price": {"type": "number"}
104
+ }
105
+ }
106
+ }
107
+ ```
108
+
109
+ - `instruction`: 提取指令
110
+ - `schema`: 可选的 JSON Schema
111
+
112
+ **注意**: `api_key`、`base_url`、`model` 从环境变量读取。
113
+
114
+ ## 开发
115
+
116
+ ```bash
117
+ uv sync
118
+ uv run pytest
119
+ uv run python -m crawl4ai_mcp.fastmcp_server --http
120
+ ```
121
+
122
+ ## 许可证
123
+
124
+ MIT License
@@ -0,0 +1,100 @@
1
+ # crawl_mcp
2
+
3
+ 基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能。
4
+
5
+ ## 功能
6
+
7
+ - **crawl_single** - 爬取单个网页,返回 Markdown 格式
8
+ - **crawl_site** - 递归爬取整个网站
9
+ - **crawl_batch** - 批量爬取多个网页(异步并行)
10
+ - **LLM 集成** - AI 驱动的内容提取和摘要(先快速爬取,后可选处理)
11
+ - **自动重试** - 网络错误自动重试(指数退避)
12
+
13
+ ## LLM 处理设计
14
+
15
+ 爬取和 LLM 处理分离,确保快速响应:
16
+
17
+ 1. **快速爬取**(6-10秒)- 始终返回原始 Markdown
18
+ 2. **可选后处理** - 如提供 `llm_config`,对 Markdown 进行 AI 处理
19
+
20
+ ### 性能对比
21
+
22
+ | 场景 | 爬取阶段 | 总耗时 |
23
+ |------|---------|--------|
24
+ | crawl_single(无 LLM) | ~7s | **11s** |
25
+ | crawl_single(有 LLM) | ~6s | **40s** |
26
+ | crawl_batch 2 页(无 LLM) | ~15s | **15s** |
27
+ | crawl_batch(有 LLM) | ~6s/页 | **~20s/页** |
28
+
29
+ **关键优势**:
30
+ - 不使用 LLM 时 11 秒即可获取结果
31
+ - 使用 LLM 时爬取阶段仍然快速,用户可以先看到原始内容
32
+ - 相比旧设计(LLMExtractionStrategy 需要 123 秒),速度提升 3-10 倍
33
+
34
+ ## 安装
35
+
36
+ ```bash
37
+ pip install crawl-mcp
38
+ ```
39
+
40
+ ## MCP 配置
41
+
42
+ ### Claude Desktop
43
+
44
+ ```json
45
+ {
46
+ "mcpServers": {
47
+ "crawl-mcp": {
48
+ "command": "uv",
49
+ "args": ["--directory", "/path/to/crawl4ai", "run", "crawl-mcp", "--http"],
50
+ "env": {
51
+ "OPENAI_API_KEY": "your-api-key",
52
+ "OPENAI_BASE_URL": "https://api.openai.com/v1",
53
+ "LLM_MODEL": "gpt-4o-mini"
54
+ }
55
+ }
56
+ }
57
+ }
58
+ ```
59
+
60
+ ### 环境变量
61
+
62
+ | 变量 | 说明 | 默认值 |
63
+ |------|------|--------|
64
+ | `OPENAI_API_KEY` | API 密钥 | *必填* |
65
+ | `OPENAI_BASE_URL` | API 基础 URL | `https://api.openai.com/v1` |
66
+ | `LLM_MODEL` | 模型名称 | `gpt-4o-mini` |
67
+
68
+ ## LLM 配置
69
+
70
+ 所有工具支持可选的 `llm_config` 参数:
71
+
72
+ ```json
73
+ {
74
+ "instruction": "提取产品信息",
75
+ "schema": {
76
+ "type": "object",
77
+ "properties": {
78
+ "name": {"type": "string"},
79
+ "price": {"type": "number"}
80
+ }
81
+ }
82
+ }
83
+ ```
84
+
85
+ - `instruction`: 提取指令
86
+ - `schema`: 可选的 JSON Schema
87
+
88
+ **注意**: `api_key`、`base_url`、`model` 从环境变量读取。
89
+
90
+ ## 开发
91
+
92
+ ```bash
93
+ uv sync
94
+ uv run pytest
95
+ uv run python -m crawl4ai_mcp.fastmcp_server --http
96
+ ```
97
+
98
+ ## 许可证
99
+
100
+ MIT License
@@ -0,0 +1,50 @@
1
+ [project]
2
+ name = "crawl_mcp"
3
+ version = "0.1.0"
4
+ description = "基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能"
5
+ readme = "README.md"
6
+ requires-python = ">=3.12"
7
+ dependencies = [
8
+ "beautifulsoup4>=4.13.5",
9
+ "crawl4ai>=0.7.8",
10
+ "fastmcp>=0.1.0",
11
+ "nest-asyncio>=1.6.0",
12
+ "openai>=2.0.0",
13
+ ]
14
+ authors = [
15
+ {name = "gqy20"}
16
+ ]
17
+ license = {text = "MIT"}
18
+ classifiers = [
19
+ "Development Status :: 3 - Alpha",
20
+ "Intended Audience :: Developers",
21
+ "License :: OSI Approved :: MIT License",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Programming Language :: Python :: 3.13",
25
+ ]
26
+
27
+ [project.urls]
28
+ Homepage = "https://github.com/gqy20/crawl-mcp"
29
+ Repository = "https://github.com/gqy20/crawl-mcp"
30
+ Issues = "https://github.com/gqy20/crawl-mcp/issues"
31
+
32
+ [project.entry-points."mcp.servers"]
33
+ crawl-mcp = "crawl4ai_mcp.fastmcp_server:mcp"
34
+
35
+ [build-system]
36
+ requires = ["hatchling"]
37
+ build-backend = "hatchling.build"
38
+
39
+ [tool.hatch.build.targets.wheel]
40
+ packages = ["src/crawl4ai_mcp"]
41
+
42
+ [dependency-groups]
43
+ dev = [
44
+ "pytest>=8.0.0",
45
+ "pytest-asyncio>=0.23.0",
46
+ "pytest-cov>=4.0.0",
47
+ "ruff>=0.8.0",
48
+ "twine>=5.0.0",
49
+ "pre-commit>=3.0.0"
50
+ ]
@@ -0,0 +1,26 @@
1
+ """测试 MCP HTTP 客户端"""
2
+
3
+ import asyncio
4
+ from fastmcp.client.transports import StreamableHttpTransport
5
+ from fastmcp import Client
6
+
7
+
8
+ async def main():
9
+ # 连接 HTTP 服务器
10
+ transport = StreamableHttpTransport(url="http://127.0.0.1:8001/mcp")
11
+ async with Client(transport) as client:
12
+ # 列出工具
13
+ tools = await client.list_tools()
14
+ print("=== 可用工具 ===")
15
+ for tool in tools:
16
+ print(f" - {tool.name}: {tool.description}")
17
+ if tool.inputSchema:
18
+ print(
19
+ f" 参数: {list(tool.inputSchema.get('properties', {}).keys())}"
20
+ )
21
+
22
+ print("\n✅ MCP 服务器连接成功!")
23
+
24
+
25
+ if __name__ == "__main__":
26
+ asyncio.run(main())
@@ -0,0 +1,8 @@
1
+ """crawl_mcp: 基于 crawl4ai 和 FastMCP 的 MCP 服务器"""
2
+
3
+ from crawl4ai_mcp.crawler import Crawler
4
+ from crawl4ai_mcp.fastmcp_server import mcp
5
+ from crawl4ai_mcp.llm_config import LLMConfig, get_llm_config
6
+
7
+ __version__ = "0.1.0"
8
+ __all__ = ["Crawler", "mcp", "LLMConfig", "get_llm_config"]