crawl-mcp 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawl_mcp-0.1.0/.github/workflows/publish.yml +110 -0
- crawl_mcp-0.1.0/.gitignore +38 -0
- crawl_mcp-0.1.0/.pre-commit-config.yaml +22 -0
- crawl_mcp-0.1.0/.python-version +1 -0
- crawl_mcp-0.1.0/LICENSE +21 -0
- crawl_mcp-0.1.0/PKG-INFO +124 -0
- crawl_mcp-0.1.0/README.md +100 -0
- crawl_mcp-0.1.0/pyproject.toml +50 -0
- crawl_mcp-0.1.0/scripts/test_mcp_client.py +26 -0
- crawl_mcp-0.1.0/src/crawl4ai_mcp/__init__.py +8 -0
- crawl_mcp-0.1.0/src/crawl4ai_mcp/crawler.py +469 -0
- crawl_mcp-0.1.0/src/crawl4ai_mcp/fastmcp_server.py +97 -0
- crawl_mcp-0.1.0/src/crawl4ai_mcp/llm_config.py +73 -0
- crawl_mcp-0.1.0/tests/conftest.py +8 -0
- crawl_mcp-0.1.0/tests/unit/test_crawler.py +211 -0
- crawl_mcp-0.1.0/tests/unit/test_fastmcp_server.py +138 -0
- crawl_mcp-0.1.0/tests/unit/test_llm_config.py +98 -0
- crawl_mcp-0.1.0/tests/unit/test_llm_postprocess.py +149 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*.*.*'
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
inputs:
|
|
9
|
+
tag:
|
|
10
|
+
description: 'Tag to publish (e.g., v0.1.0)'
|
|
11
|
+
required: true
|
|
12
|
+
type: string
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
validate:
|
|
19
|
+
name: Validate Release
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- name: Checkout code
|
|
24
|
+
uses: actions/checkout@v4
|
|
25
|
+
with:
|
|
26
|
+
ref: ${{ github.event.inputs.tag || github.ref }}
|
|
27
|
+
|
|
28
|
+
- name: Install uv
|
|
29
|
+
uses: astral-sh/setup-uv@v2
|
|
30
|
+
with:
|
|
31
|
+
version: "latest"
|
|
32
|
+
|
|
33
|
+
- name: Install dependencies
|
|
34
|
+
run: uv sync --dev
|
|
35
|
+
|
|
36
|
+
- name: Run full test suite
|
|
37
|
+
run: uv run pytest -v --cov=src/crawl4ai_mcp --cov-report=xml --cov-report=term-missing
|
|
38
|
+
|
|
39
|
+
- name: Run linting
|
|
40
|
+
run: |
|
|
41
|
+
uv run ruff check .
|
|
42
|
+
uv run ruff format --check .
|
|
43
|
+
|
|
44
|
+
- name: Verify version matches tag
|
|
45
|
+
run: |
|
|
46
|
+
TAG_VERSION="${{ github.event.inputs.tag || github.ref_name }}"
|
|
47
|
+
TAG_VERSION="${TAG_VERSION#v}"
|
|
48
|
+
PACKAGE_VERSION=$(uv run python -c "from crawl4ai_mcp import __version__; print(__version__)")
|
|
49
|
+
|
|
50
|
+
echo "Tag version: $TAG_VERSION"
|
|
51
|
+
echo "Package version: $PACKAGE_VERSION"
|
|
52
|
+
|
|
53
|
+
if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
|
|
54
|
+
echo "Error: Tag version ($TAG_VERSION) does not match package version ($PACKAGE_VERSION)"
|
|
55
|
+
exit 1
|
|
56
|
+
fi
|
|
57
|
+
|
|
58
|
+
build:
|
|
59
|
+
name: Build Distribution
|
|
60
|
+
needs: validate
|
|
61
|
+
runs-on: ubuntu-latest
|
|
62
|
+
|
|
63
|
+
steps:
|
|
64
|
+
- name: Checkout code
|
|
65
|
+
uses: actions/checkout@v4
|
|
66
|
+
with:
|
|
67
|
+
ref: ${{ github.event.inputs.tag || github.ref }}
|
|
68
|
+
|
|
69
|
+
- name: Install uv
|
|
70
|
+
uses: astral-sh/setup-uv@v2
|
|
71
|
+
with:
|
|
72
|
+
version: "latest"
|
|
73
|
+
|
|
74
|
+
- name: Build distribution packages
|
|
75
|
+
run: uv build
|
|
76
|
+
|
|
77
|
+
- name: Check distribution packages
|
|
78
|
+
run: uv sync --dev && uv run twine check dist/*
|
|
79
|
+
|
|
80
|
+
- name: Upload distribution artifacts
|
|
81
|
+
uses: actions/upload-artifact@v4
|
|
82
|
+
with:
|
|
83
|
+
name: python-package-distributions
|
|
84
|
+
path: dist/
|
|
85
|
+
|
|
86
|
+
publish-pypi:
|
|
87
|
+
name: Publish to PyPI
|
|
88
|
+
needs: build
|
|
89
|
+
runs-on: ubuntu-latest
|
|
90
|
+
|
|
91
|
+
steps:
|
|
92
|
+
- name: Checkout code
|
|
93
|
+
uses: actions/checkout@v4
|
|
94
|
+
|
|
95
|
+
- name: Download distribution artifacts
|
|
96
|
+
uses: actions/download-artifact@v4
|
|
97
|
+
with:
|
|
98
|
+
name: python-package-distributions
|
|
99
|
+
path: dist/
|
|
100
|
+
|
|
101
|
+
- name: Install uv
|
|
102
|
+
uses: astral-sh/setup-uv@v2
|
|
103
|
+
with:
|
|
104
|
+
version: "latest"
|
|
105
|
+
|
|
106
|
+
- name: Publish to PyPI
|
|
107
|
+
run: uv sync --dev && uv run twine upload dist/*
|
|
108
|
+
env:
|
|
109
|
+
TWINE_USERNAME: __token__
|
|
110
|
+
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.coverage
|
|
10
|
+
htmlcov/
|
|
11
|
+
|
|
12
|
+
# Virtual environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
ENV/
|
|
16
|
+
|
|
17
|
+
# uv
|
|
18
|
+
uv.lock
|
|
19
|
+
|
|
20
|
+
# IDE
|
|
21
|
+
.vscode/
|
|
22
|
+
.idea/
|
|
23
|
+
*.swp
|
|
24
|
+
*.swo
|
|
25
|
+
*~
|
|
26
|
+
|
|
27
|
+
# OS
|
|
28
|
+
.DS_Store
|
|
29
|
+
Thumbs.db
|
|
30
|
+
|
|
31
|
+
# Project specific
|
|
32
|
+
output/
|
|
33
|
+
*.log
|
|
34
|
+
|
|
35
|
+
# Temporary files
|
|
36
|
+
*.tmp
|
|
37
|
+
*.bak
|
|
38
|
+
.cache/
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: local
|
|
3
|
+
hooks:
|
|
4
|
+
- id: ruff-format
|
|
5
|
+
name: ruff format
|
|
6
|
+
entry: uv run ruff format .
|
|
7
|
+
language: system
|
|
8
|
+
types: [python]
|
|
9
|
+
|
|
10
|
+
- id: ruff-check
|
|
11
|
+
name: ruff check
|
|
12
|
+
entry: uv run ruff check . --fix
|
|
13
|
+
language: system
|
|
14
|
+
types: [python]
|
|
15
|
+
pass_filenames: false
|
|
16
|
+
|
|
17
|
+
- id: pytest
|
|
18
|
+
name: pytest
|
|
19
|
+
entry: uv run pytest -q
|
|
20
|
+
language: system
|
|
21
|
+
types: [python]
|
|
22
|
+
pass_filenames: false
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
crawl_mcp-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 gqy20
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
crawl_mcp-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crawl_mcp
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: 基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能
|
|
5
|
+
Project-URL: Homepage, https://github.com/gqy20/crawl-mcp
|
|
6
|
+
Project-URL: Repository, https://github.com/gqy20/crawl-mcp
|
|
7
|
+
Project-URL: Issues, https://github.com/gqy20/crawl-mcp/issues
|
|
8
|
+
Author: gqy20
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.13.5
|
|
19
|
+
Requires-Dist: crawl4ai>=0.7.8
|
|
20
|
+
Requires-Dist: fastmcp>=0.1.0
|
|
21
|
+
Requires-Dist: nest-asyncio>=1.6.0
|
|
22
|
+
Requires-Dist: openai>=2.0.0
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# crawl_mcp
|
|
26
|
+
|
|
27
|
+
基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能。
|
|
28
|
+
|
|
29
|
+
## 功能
|
|
30
|
+
|
|
31
|
+
- **crawl_single** - 爬取单个网页,返回 Markdown 格式
|
|
32
|
+
- **crawl_site** - 递归爬取整个网站
|
|
33
|
+
- **crawl_batch** - 批量爬取多个网页(异步并行)
|
|
34
|
+
- **LLM 集成** - AI 驱动的内容提取和摘要(先快速爬取,后可选处理)
|
|
35
|
+
- **自动重试** - 网络错误自动重试(指数退避)
|
|
36
|
+
|
|
37
|
+
## LLM 处理设计
|
|
38
|
+
|
|
39
|
+
爬取和 LLM 处理分离,确保快速响应:
|
|
40
|
+
|
|
41
|
+
1. **快速爬取**(6-10秒)- 始终返回原始 Markdown
|
|
42
|
+
2. **可选后处理** - 如提供 `llm_config`,对 Markdown 进行 AI 处理
|
|
43
|
+
|
|
44
|
+
### 性能对比
|
|
45
|
+
|
|
46
|
+
| 场景 | 爬取阶段 | 总耗时 |
|
|
47
|
+
|------|---------|--------|
|
|
48
|
+
| crawl_single(无 LLM) | ~7s | **11s** |
|
|
49
|
+
| crawl_single(有 LLM) | ~6s | **40s** |
|
|
50
|
+
| crawl_batch 2 页(无 LLM) | ~15s | **15s** |
|
|
51
|
+
| crawl_batch(有 LLM) | ~6s/页 | **~20s/页** |
|
|
52
|
+
|
|
53
|
+
**关键优势**:
|
|
54
|
+
- 不使用 LLM 时 11 秒即可获取结果
|
|
55
|
+
- 使用 LLM 时爬取阶段仍然快速,用户可以先看到原始内容
|
|
56
|
+
- 相比旧设计(LLMExtractionStrategy 需要 123 秒),速度提升 3-10 倍
|
|
57
|
+
|
|
58
|
+
## 安装
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install crawl-mcp
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## MCP 配置
|
|
65
|
+
|
|
66
|
+
### Claude Desktop
|
|
67
|
+
|
|
68
|
+
```json
|
|
69
|
+
{
|
|
70
|
+
"mcpServers": {
|
|
71
|
+
"crawl-mcp": {
|
|
72
|
+
"command": "uv",
|
|
73
|
+
"args": ["--directory", "/path/to/crawl4ai", "run", "crawl-mcp", "--http"],
|
|
74
|
+
"env": {
|
|
75
|
+
"OPENAI_API_KEY": "your-api-key",
|
|
76
|
+
"OPENAI_BASE_URL": "https://api.openai.com/v1",
|
|
77
|
+
"LLM_MODEL": "gpt-4o-mini"
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### 环境变量
|
|
85
|
+
|
|
86
|
+
| 变量 | 说明 | 默认值 |
|
|
87
|
+
|------|------|--------|
|
|
88
|
+
| `OPENAI_API_KEY` | API 密钥 | *必填* |
|
|
89
|
+
| `OPENAI_BASE_URL` | API 基础 URL | `https://api.openai.com/v1` |
|
|
90
|
+
| `LLM_MODEL` | 模型名称 | `gpt-4o-mini` |
|
|
91
|
+
|
|
92
|
+
## LLM 配置
|
|
93
|
+
|
|
94
|
+
所有工具支持可选的 `llm_config` 参数:
|
|
95
|
+
|
|
96
|
+
```json
|
|
97
|
+
{
|
|
98
|
+
"instruction": "提取产品信息",
|
|
99
|
+
"schema": {
|
|
100
|
+
"type": "object",
|
|
101
|
+
"properties": {
|
|
102
|
+
"name": {"type": "string"},
|
|
103
|
+
"price": {"type": "number"}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
- `instruction`: 提取指令
|
|
110
|
+
- `schema`: 可选的 JSON Schema
|
|
111
|
+
|
|
112
|
+
**注意**: `api_key`、`base_url`、`model` 从环境变量读取。
|
|
113
|
+
|
|
114
|
+
## 开发
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
uv sync
|
|
118
|
+
uv run pytest
|
|
119
|
+
uv run python -m crawl4ai_mcp.fastmcp_server --http
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## 许可证
|
|
123
|
+
|
|
124
|
+
MIT License
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# crawl_mcp
|
|
2
|
+
|
|
3
|
+
基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能。
|
|
4
|
+
|
|
5
|
+
## 功能
|
|
6
|
+
|
|
7
|
+
- **crawl_single** - 爬取单个网页,返回 Markdown 格式
|
|
8
|
+
- **crawl_site** - 递归爬取整个网站
|
|
9
|
+
- **crawl_batch** - 批量爬取多个网页(异步并行)
|
|
10
|
+
- **LLM 集成** - AI 驱动的内容提取和摘要(先快速爬取,后可选处理)
|
|
11
|
+
- **自动重试** - 网络错误自动重试(指数退避)
|
|
12
|
+
|
|
13
|
+
## LLM 处理设计
|
|
14
|
+
|
|
15
|
+
爬取和 LLM 处理分离,确保快速响应:
|
|
16
|
+
|
|
17
|
+
1. **快速爬取**(6-10秒)- 始终返回原始 Markdown
|
|
18
|
+
2. **可选后处理** - 如提供 `llm_config`,对 Markdown 进行 AI 处理
|
|
19
|
+
|
|
20
|
+
### 性能对比
|
|
21
|
+
|
|
22
|
+
| 场景 | 爬取阶段 | 总耗时 |
|
|
23
|
+
|------|---------|--------|
|
|
24
|
+
| crawl_single(无 LLM) | ~7s | **11s** |
|
|
25
|
+
| crawl_single(有 LLM) | ~6s | **40s** |
|
|
26
|
+
| crawl_batch 2 页(无 LLM) | ~15s | **15s** |
|
|
27
|
+
| crawl_batch(有 LLM) | ~6s/页 | **~20s/页** |
|
|
28
|
+
|
|
29
|
+
**关键优势**:
|
|
30
|
+
- 不使用 LLM 时 11 秒即可获取结果
|
|
31
|
+
- 使用 LLM 时爬取阶段仍然快速,用户可以先看到原始内容
|
|
32
|
+
- 相比旧设计(LLMExtractionStrategy 需要 123 秒),速度提升 3-10 倍
|
|
33
|
+
|
|
34
|
+
## 安装
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install crawl-mcp
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## MCP 配置
|
|
41
|
+
|
|
42
|
+
### Claude Desktop
|
|
43
|
+
|
|
44
|
+
```json
|
|
45
|
+
{
|
|
46
|
+
"mcpServers": {
|
|
47
|
+
"crawl-mcp": {
|
|
48
|
+
"command": "uv",
|
|
49
|
+
"args": ["--directory", "/path/to/crawl4ai", "run", "crawl-mcp", "--http"],
|
|
50
|
+
"env": {
|
|
51
|
+
"OPENAI_API_KEY": "your-api-key",
|
|
52
|
+
"OPENAI_BASE_URL": "https://api.openai.com/v1",
|
|
53
|
+
"LLM_MODEL": "gpt-4o-mini"
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### 环境变量
|
|
61
|
+
|
|
62
|
+
| 变量 | 说明 | 默认值 |
|
|
63
|
+
|------|------|--------|
|
|
64
|
+
| `OPENAI_API_KEY` | API 密钥 | *必填* |
|
|
65
|
+
| `OPENAI_BASE_URL` | API 基础 URL | `https://api.openai.com/v1` |
|
|
66
|
+
| `LLM_MODEL` | 模型名称 | `gpt-4o-mini` |
|
|
67
|
+
|
|
68
|
+
## LLM 配置
|
|
69
|
+
|
|
70
|
+
所有工具支持可选的 `llm_config` 参数:
|
|
71
|
+
|
|
72
|
+
```json
|
|
73
|
+
{
|
|
74
|
+
"instruction": "提取产品信息",
|
|
75
|
+
"schema": {
|
|
76
|
+
"type": "object",
|
|
77
|
+
"properties": {
|
|
78
|
+
"name": {"type": "string"},
|
|
79
|
+
"price": {"type": "number"}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
- `instruction`: 提取指令
|
|
86
|
+
- `schema`: 可选的 JSON Schema
|
|
87
|
+
|
|
88
|
+
**注意**: `api_key`、`base_url`、`model` 从环境变量读取。
|
|
89
|
+
|
|
90
|
+
## 开发
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
uv sync
|
|
94
|
+
uv run pytest
|
|
95
|
+
uv run python -m crawl4ai_mcp.fastmcp_server --http
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## 许可证
|
|
99
|
+
|
|
100
|
+
MIT License
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "crawl_mcp"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "基于 crawl4ai 和 FastMCP 的 MCP 服务器,提供网页爬取和 AI 分析功能"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"beautifulsoup4>=4.13.5",
|
|
9
|
+
"crawl4ai>=0.7.8",
|
|
10
|
+
"fastmcp>=0.1.0",
|
|
11
|
+
"nest-asyncio>=1.6.0",
|
|
12
|
+
"openai>=2.0.0",
|
|
13
|
+
]
|
|
14
|
+
authors = [
|
|
15
|
+
{name = "gqy20"}
|
|
16
|
+
]
|
|
17
|
+
license = {text = "MIT"}
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/gqy20/crawl-mcp"
|
|
29
|
+
Repository = "https://github.com/gqy20/crawl-mcp"
|
|
30
|
+
Issues = "https://github.com/gqy20/crawl-mcp/issues"
|
|
31
|
+
|
|
32
|
+
[project.entry-points."mcp.servers"]
|
|
33
|
+
crawl-mcp = "crawl4ai_mcp.fastmcp_server:mcp"
|
|
34
|
+
|
|
35
|
+
[build-system]
|
|
36
|
+
requires = ["hatchling"]
|
|
37
|
+
build-backend = "hatchling.build"
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.wheel]
|
|
40
|
+
packages = ["src/crawl4ai_mcp"]
|
|
41
|
+
|
|
42
|
+
[dependency-groups]
|
|
43
|
+
dev = [
|
|
44
|
+
"pytest>=8.0.0",
|
|
45
|
+
"pytest-asyncio>=0.23.0",
|
|
46
|
+
"pytest-cov>=4.0.0",
|
|
47
|
+
"ruff>=0.8.0",
|
|
48
|
+
"twine>=5.0.0",
|
|
49
|
+
"pre-commit>=3.0.0"
|
|
50
|
+
]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""测试 MCP HTTP 客户端"""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from fastmcp.client.transports import StreamableHttpTransport
|
|
5
|
+
from fastmcp import Client
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
async def main():
|
|
9
|
+
# 连接 HTTP 服务器
|
|
10
|
+
transport = StreamableHttpTransport(url="http://127.0.0.1:8001/mcp")
|
|
11
|
+
async with Client(transport) as client:
|
|
12
|
+
# 列出工具
|
|
13
|
+
tools = await client.list_tools()
|
|
14
|
+
print("=== 可用工具 ===")
|
|
15
|
+
for tool in tools:
|
|
16
|
+
print(f" - {tool.name}: {tool.description}")
|
|
17
|
+
if tool.inputSchema:
|
|
18
|
+
print(
|
|
19
|
+
f" 参数: {list(tool.inputSchema.get('properties', {}).keys())}"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
print("\n✅ MCP 服务器连接成功!")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
if __name__ == "__main__":
|
|
26
|
+
asyncio.run(main())
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""crawl_mcp: 基于 crawl4ai 和 FastMCP 的 MCP 服务器"""
|
|
2
|
+
|
|
3
|
+
from crawl4ai_mcp.crawler import Crawler
|
|
4
|
+
from crawl4ai_mcp.fastmcp_server import mcp
|
|
5
|
+
from crawl4ai_mcp.llm_config import LLMConfig, get_llm_config
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
__all__ = ["Crawler", "mcp", "LLMConfig", "get_llm_config"]
|