gd-crawler-mcp 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: gd-crawler-mcp
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: 广东信息价爬虫 MCP:抓网页、提链接、联网搜索、PDF提取
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
7
|
+
Requires-Dist: duckduckgo-search>=6.0
|
|
8
|
+
Requires-Dist: httpx>=0.27
|
|
9
|
+
Requires-Dist: lxml>=5.0
|
|
10
|
+
Requires-Dist: mcp[cli]<2,>=1.0
|
|
11
|
+
Requires-Dist: pdfplumber>=0.11
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""广东信息价爬虫 MCP - stdio 模式,部署在阿里云百炼 FC 上。
|
|
3
|
+
|
|
4
|
+
提供工具:
|
|
5
|
+
- fetch_webpage: 抓网页提取正文 Markdown
|
|
6
|
+
- extract_links: 提取网页中所有超链接
|
|
7
|
+
- search_web: 联网搜索(DuckDuckGo)
|
|
8
|
+
- fetch_pdf: 下载 PDF 提取文本
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import io
|
|
14
|
+
import logging
|
|
15
|
+
import re
|
|
16
|
+
from urllib.parse import urljoin, urlsplit
|
|
17
|
+
|
|
18
|
+
import httpx
|
|
19
|
+
from bs4 import BeautifulSoup
|
|
20
|
+
try:
|
|
21
|
+
# mcp 2.x: FastMCP 更名为 MCPServer
|
|
22
|
+
from mcp.server.mcpserver import MCPServer as _ServerCls
|
|
23
|
+
except ModuleNotFoundError:
|
|
24
|
+
# mcp 1.x
|
|
25
|
+
from mcp.server.fastmcp import FastMCP as _ServerCls
|
|
26
|
+
|
|
27
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
|
|
28
|
+
logger = logging.getLogger("gd-crawler-mcp")
|
|
29
|
+
|
|
30
|
+
mcp = _ServerCls(
|
|
31
|
+
"gd-crawler-mcp",
|
|
32
|
+
instructions=(
|
|
33
|
+
"广东信息价爬虫工具集:抓取网页正文、提取链接、联网搜索、下载PDF提取文本。"
|
|
34
|
+
"适用于查询广东省各市工程造价信息价、抓取政府网站通知公告和PDF附件。"
|
|
35
|
+
),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
HEADERS = {
|
|
39
|
+
"User-Agent": (
|
|
40
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
41
|
+
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
|
42
|
+
),
|
|
43
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
44
|
+
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
45
|
+
}
|
|
46
|
+
TIMEOUT = 25.0
|
|
47
|
+
MAX_CHARS = 80_000
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _normalize_url(url: str) -> str:
|
|
51
|
+
url = url.strip()
|
|
52
|
+
if not re.match(r"^https?://", url, re.I):
|
|
53
|
+
url = "http://" + url
|
|
54
|
+
return url
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _html_to_markdown(html: str, base_url: str = "") -> str:
|
|
58
|
+
soup = BeautifulSoup(html, "lxml")
|
|
59
|
+
for tag in soup(["script", "style", "noscript", "template"]):
|
|
60
|
+
tag.decompose()
|
|
61
|
+
|
|
62
|
+
parts = []
|
|
63
|
+
for el in soup.descendants:
|
|
64
|
+
name = getattr(el, "name", None)
|
|
65
|
+
if name is None:
|
|
66
|
+
txt = str(el).strip()
|
|
67
|
+
if txt:
|
|
68
|
+
parts.append(txt)
|
|
69
|
+
elif name == "a":
|
|
70
|
+
href = el.get("href", "")
|
|
71
|
+
txt = el.get_text(" ", strip=True)
|
|
72
|
+
if txt and href:
|
|
73
|
+
if base_url:
|
|
74
|
+
href = urljoin(base_url, href)
|
|
75
|
+
parts.append(f"[{txt}]({href})")
|
|
76
|
+
elif name in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
|
77
|
+
level = int(name[1])
|
|
78
|
+
txt = el.get_text(" ", strip=True)
|
|
79
|
+
if txt:
|
|
80
|
+
parts.append("\n" + "#" * level + " " + txt + "\n")
|
|
81
|
+
elif name in ("p", "div", "li", "tr", "blockquote"):
|
|
82
|
+
txt = el.get_text(" ", strip=True)
|
|
83
|
+
if txt:
|
|
84
|
+
parts.append(txt)
|
|
85
|
+
elif name == "td":
|
|
86
|
+
txt = el.get_text(" ", strip=True)
|
|
87
|
+
if txt:
|
|
88
|
+
parts.append(txt + " | ")
|
|
89
|
+
|
|
90
|
+
text = "\n".join(parts)
|
|
91
|
+
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
92
|
+
return text[:MAX_CHARS]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@mcp.tool()
|
|
96
|
+
async def fetch_webpage(url: str, timeout: float = TIMEOUT) -> str:
|
|
97
|
+
"""抓取一个网页,返回正文 Markdown。
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
url: 网页地址
|
|
101
|
+
timeout: 超时秒数
|
|
102
|
+
"""
|
|
103
|
+
url = _normalize_url(url)
|
|
104
|
+
try:
|
|
105
|
+
async with httpx.AsyncClient(
|
|
106
|
+
follow_redirects=True, timeout=httpx.Timeout(timeout, connect=10.0),
|
|
107
|
+
headers=HEADERS, verify=False,
|
|
108
|
+
) as client:
|
|
109
|
+
resp = await client.get(url)
|
|
110
|
+
resp.raise_for_status()
|
|
111
|
+
html = resp.text
|
|
112
|
+
md = _html_to_markdown(html, base_url=str(resp.url))
|
|
113
|
+
return f"# {str(resp.url)}\n\n{md}"
|
|
114
|
+
except Exception as e:
|
|
115
|
+
return f"抓取失败: {type(e).__name__}: {e}"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@mcp.tool()
|
|
119
|
+
async def extract_links(url: str, timeout: float = TIMEOUT) -> str:
|
|
120
|
+
"""提取网页中所有超链接。
|
|
121
|
+
|
|
122
|
+
Args:
|
|
123
|
+
url: 网页地址
|
|
124
|
+
timeout: 超时秒数
|
|
125
|
+
"""
|
|
126
|
+
url = _normalize_url(url)
|
|
127
|
+
try:
|
|
128
|
+
async with httpx.AsyncClient(
|
|
129
|
+
follow_redirects=True, timeout=httpx.Timeout(timeout, connect=10.0),
|
|
130
|
+
headers=HEADERS, verify=False,
|
|
131
|
+
) as client:
|
|
132
|
+
resp = await client.get(url)
|
|
133
|
+
resp.raise_for_status()
|
|
134
|
+
soup = BeautifulSoup(resp.text, "lxml")
|
|
135
|
+
links = []
|
|
136
|
+
for a in soup.find_all("a", href=True):
|
|
137
|
+
href = urljoin(str(resp.url), a["href"])
|
|
138
|
+
txt = a.get_text(" ", strip=True)
|
|
139
|
+
if href.startswith("http"):
|
|
140
|
+
links.append(f"[{txt}]({href})" if txt else href)
|
|
141
|
+
return "\n".join(links[:200])
|
|
142
|
+
except Exception as e:
|
|
143
|
+
return f"提取链接失败: {type(e).__name__}: {e}"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@mcp.tool()
|
|
147
|
+
async def search_web(query: str, max_results: int = 10) -> str:
|
|
148
|
+
"""联网搜索(DuckDuckGo),返回结果标题+链接+摘要。
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
query: 搜索关键词
|
|
152
|
+
max_results: 最大返回条数
|
|
153
|
+
"""
|
|
154
|
+
try:
|
|
155
|
+
from ddgs import DDGS
|
|
156
|
+
except ImportError:
|
|
157
|
+
try:
|
|
158
|
+
from duckduckgo_search import DDGS
|
|
159
|
+
except ImportError:
|
|
160
|
+
return "搜索依赖未安装"
|
|
161
|
+
|
|
162
|
+
results = []
|
|
163
|
+
try:
|
|
164
|
+
with DDGS() as ddgs:
|
|
165
|
+
for r in ddgs.text(query, max_results=max_results, region="cn-zh"):
|
|
166
|
+
results.append(
|
|
167
|
+
f"## {r.get('title', '')}\n{r.get('body', '')}\n{r.get('href', r.get('link', ''))}\n"
|
|
168
|
+
)
|
|
169
|
+
except Exception as e:
|
|
170
|
+
return f"搜索失败: {type(e).__name__}: {e}"
|
|
171
|
+
return "\n---\n".join(results) if results else "无搜索结果"
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@mcp.tool()
|
|
175
|
+
async def fetch_pdf(url: str, max_chars: int = 50000, timeout: float = 60.0) -> str:
|
|
176
|
+
"""下载一个 PDF 文件并提取文本内容。
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
url: PDF 文件地址
|
|
180
|
+
max_chars: 最多返回字符数
|
|
181
|
+
timeout: 下载超时秒数
|
|
182
|
+
"""
|
|
183
|
+
url = _normalize_url(url)
|
|
184
|
+
try:
|
|
185
|
+
async with httpx.AsyncClient(
|
|
186
|
+
follow_redirects=True, timeout=httpx.Timeout(timeout, connect=10.0),
|
|
187
|
+
headers=HEADERS, verify=False,
|
|
188
|
+
) as client:
|
|
189
|
+
resp = await client.get(url)
|
|
190
|
+
resp.raise_for_status()
|
|
191
|
+
pdf_bytes = resp.content
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
import pdfplumber
|
|
195
|
+
text_parts = []
|
|
196
|
+
with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf:
|
|
197
|
+
for i, page in enumerate(pdf.pages[:50]):
|
|
198
|
+
t = page.extract_text() or ""
|
|
199
|
+
text_parts.append(f"--- 第{i+1}页 ---\n{t}")
|
|
200
|
+
text = "\n".join(text_parts)
|
|
201
|
+
except ImportError:
|
|
202
|
+
try:
|
|
203
|
+
import fitz
|
|
204
|
+
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
|
205
|
+
text = "\n".join(
|
|
206
|
+
f"--- 第{i+1}页 ---\n{page.get_text()}"
|
|
207
|
+
for i, page in enumerate(doc)
|
|
208
|
+
)
|
|
209
|
+
doc.close()
|
|
210
|
+
except ImportError:
|
|
211
|
+
return "PDF解析库未安装"
|
|
212
|
+
|
|
213
|
+
return f"# PDF: {url}\n\n{text[:max_chars]}"
|
|
214
|
+
except Exception as e:
|
|
215
|
+
return f"PDF抓取失败: {type(e).__name__}: {e}"
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def main():
|
|
219
|
+
mcp.run(transport="stdio")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
if __name__ == "__main__":
|
|
223
|
+
main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ZnJvbSBtY3Auc2VydmVyLmZhc3RtY3AgaW1wb3J0IEZhc3RNQ1AKaW1wb3J0IGh0dHB4LCByZSwgaW8KZnJvbSBiczQgaW1wb3J0IEJlYXV0aWZ1bFNvdXAKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybGpvaW4KCm1jcCA9IEZhc3RNQ1AoImdkLWNyYXdsZXIiKQoKSCA9IHsiVXNlci1BZ2VudCI6ICJNb3ppbGxhLzUuMCAoV2luZG93cyBOVCAxMC4wOyBXaW42NDsgeDY0KSBDaHJvbWUvMTMxLjAuMC4wIFNhZmFyaS81MzcuMzYiLAogICAgICJBY2NlcHQtTGFuZ3VhZ2UiOiAiemgtQ04semg7cT0wLjkifQoKZGVmIF9ub3JtKHUpOgogICAgdT11LnN0cmlwKCkKICAgIGlmIG5vdCByZS5tYXRjaChyJ15odHRwcz86Ly8nLHUscmUuSSk6IHU9J2h0dHA6Ly8nK3UKICAgIHJldHVybiB1CgpAbWNwLnRvb2woKQphc3luYyBkZWYgZmV0Y2hfd2VicGFnZSh1cmw6IHN0cikgLT4gc3RyOgogICAgIiIi5oqT572R6aG15q2j5paH77yM6L+U5ZueIE1hcmtkb3duIOagvOW8j+aWh+acrCIiIgogICAgdXJsPV9ub3JtKHVybCkKICAgIHRyeToKICAgICAgICBhc3luYyB3aXRoIGh0dHB4LkFzeW5jQ2xpZW50KGZvbGxvd19yZWRpcmVjdHM9VHJ1ZSx0aW1lb3V0PTI1LHZlcmlmeT1GYWxzZSxoZWFkZXJzPUgpIGFzIGM6CiAgICAgICAgICAgIHI9YXdhaXQgYy5nZXQodXJsKTsgci5yYWlzZV9mb3Jfc3RhdHVzKCkKICAgICAgICAgICAgc291cD1CZWF1dGlmdWxTb3VwKHIudGV4dCwnbHhtbCcpCiAgICAgICAgICAgIGZvciB0IGluIHNvdXAoWydzY3JpcHQnLCdzdHlsZScsJ25vc2NyaXB0J10pOiB0LmRlY29tcG9zZSgpCiAgICAgICAgICAgIG91dD1bXQogICAgICAgICAgICBmb3IgZWwgaW4gc291cC5kZXNjZW5kYW50czoKICAgICAgICAgICAgICAgIG49Z2V0YXR0cihlbCwnbmFtZScsTm9uZSkKICAgICAgICAgICAgICAgIGlmIG4gaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICB4PXN0cihlbCkuc3RyaXAoKQogICAgICAgICAgICAgICAgICAgIGlmIHg6IG91dC5hcHBlbmQoeCkKICAgICAgICAgICAgICAgIGVsaWYgbj09J2EnOgogICAgICAgICAgICAgICAgICAgIGg9ZWwuZ2V0KCdocmVmJywnJyk7IHg9ZWwuZ2V0X3RleHQoJyAnLHN0cmlwPVRydWUpCiAgICAgICAgICAgICAgICAgICAgaWYgeCBhbmQgaDogb3V0LmFwcGVuZChmJ1t7eH1dKHt1cmxqb2luKHN0cihyLnVybCksaCl9KScpCiAgICAgICAgICAgICAgICBlbGlmIG4gaW4gKCdoMScsJ2gyJywnaDMnKToKICAgICAgICAgICAgICAgICAgICB4PWVsLmdldF90ZXh0KCcgJyxzdHJpcD1UcnVlKQogICAgICAgICAgICAgICAgICAgIGlmIHg6IG91dC5hcHBlbmQoJ1xuJysnIycqaW50KG5bMV0pKycgJyt4KydcbicpCiAgICAgICAgICAgICAgICBlbGlmIG4gaW4gKCdwJywnZGl2JywnbGknLCd0cicpOgogICAgICAgICAgICAgICAgICAgIHg9ZWwuZ2V0X3RleHQoJyAnLHN0cmlwPVRydWUpCiAgICAgICAgICAgICAgICAgICAgaWYgeDogb3V0LmFwcGVuZCh4KQogICAgICAgICAgICByZXR1cm4gZicjIHtyLnVybH1cblxuJytyZS5zdWIocidcbnszLH0nLCdcblxuJywnXG4nLmpvaW4ob3V0KSlbOjgwMDAwXQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIHJldHVybiBmJ+aKk+WPluWksei0pToge2V9JwoKQG1jcC50b29sKCkKYXN5bmMgZGVmIGV4dHJhY3RfbGlua3ModXJsOiBzdHIpIC0+IHN0cjoKICAgICIiIuaPkOWPlue9kemhteS4reaJgOaciei2hemTvuaOpSIiIgogICAgdXJsPV9ub3JtKHVybCkKICAgIHRyeToKICAgICAgICBhc3luYyB3aXRoIGh0dHB4LkFzeW5jQ2xpZW50KGZvbGxvd19yZWRpcmVjdHM9VHJ1ZSx0aW1lb3V0PTI1LHZlcmlmeT1GYWxzZSxoZWFkZXJzPUgpIGFzIGM6CiAgICAgICAgICAgIHI9YXdhaXQgYy5nZXQodXJsKTsgci5yYWlzZV9mb3Jfc3RhdHVzKCkKICAgICAgICAgICAgc291cD1CZWF1dGlmdWxTb3VwKHIudGV4dCwnbHhtbCcpCiAgICAgICAgICAgIGxpbmtzPVtdCiAgICAgICAgICAgIGZvciBhIGluIHNvdXAuZmluZF9hbGwoJ2EnLGhyZWY9VHJ1ZSk6CiAgICAgICAgICAgICAgICBoPXVybGpvaW4oc3RyKHIudXJsKSxhWydocmVmJ10pOyB4PWEuZ2V0X3RleHQoJyAnLHN0cmlwPVRydWUpCiAgICAgICAgICAgICAgICBpZiBoLnN0YXJ0c3dpdGgoJ2h0dHAnKToKICAgICAgICAgICAgICAgICAgICBsaW5rcy5hcHBlbmQoZidbe3h9XSh7aH0pJyBpZiB4IGVsc2UgaCkKICAgICAgICAgICAgcmV0dXJuICdcbicuam9pbihsaW5rc1s6MjAwXSkKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICByZXR1cm4gZifmj5Dlj5blpLHotKU6IHtlfScKCkBtY3AudG9vbCgpCmFzeW5jIGRlZiBzZWFyY2hfd2ViKHF1ZXJ5OiBzdHIsIG1heF9yZXN1bHRzOiBpbnQgPSAxMCkgLT4gc3RyOgogICAgIiIi6IGU572R5pCc57Si77yIRHVja0R1Y2tHb++8ie+8jOi/lOWbnuagh+mimCvmkZjopoEr6ZO+5o6lIiIiCiAgICB0cnk6CiAgICAgICAgZnJvbSBkZGdzIGltcG9ydCBEREdTCiAgICBleGNlcHQgSW1wb3J0RXJyb3I6CiAgICAgICAgZnJvbSBkdWNrZHVja2dvX3NlYXJjaCBpbXBvcnQgRERHUwogICAgcmVzPVtdCiAgICB0cnk6CiAgICAgICAgd2l0aCBEREdTKCkgYXMgZDoKICAgICAgICAgICAgZm9yIHIgaW4gZC50ZXh0KHF1ZXJ5LG1heF9yZXN1bHRzPW1heF9yZXN1bHRzLHJlZ2lvbj0nY24temgnKToKICAgICAgICAgICAgICAgIHJlcy5hcHBlbmQoZiIjIyB7ci5nZXQoJ3RpdGxlJywnJyl9XG57ci5nZXQoJ2JvZHknLCcnKX1cbntyLmdldCgnaHJlZicsJycpfVxuIikKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICByZXR1cm4gZifmkJzntKLlpLHotKU6IHtlfScKICAgIHJldHVybiAnXG4tLS1cbicuam9pbihyZXMpIGlmIHJlcyBlbHNlICfml6Dnu5PmnpwnCgpAbWNwLnRvb2woKQphc3luYyBkZWYgZmV0Y2hfcGRmKHVybDogc3RyLCBtYXhfY2hhcnM6IGludCA9IDUwMDAwKSAtPiBzdHI6CiAgICAiIiLkuIvovb1QREblubbmj5Dlj5bmlofmnKwiIiIKICAgIHVybD1fbm9ybSh1cmwpCiAgICB0cnk6CiAgICAgICAgYXN5bmMgd2l0aCBodHRweC5Bc3luY0NsaWVudChmb2xsb3dfcmVkaXJlY3RzPVRydWUsdGltZW91dD02MCx2ZXJpZnk9RmFsc2UsaGVhZGVycz1IKSBhcyBjOgogICAgICAgICAgICByPWF3YWl0IGMuZ2V0KHVybCk7IHIucmFpc2VfZm9yX3N0YXR1cygpCiAgICAgICAgICAgIGRhdGE9ci5jb250ZW50CiAgICAgICAgdHJ5OgogICAgICAgICAgICBpbXBvcnQgcGRmcGx1bWJlcgogICAgICAgICAgICBwYXJ0cz1bXQogICAgICAgICAgICB3aXRoIHBkZnBsdW1iZXIub3Blbihpby5CeXRlc0lPKGRhdGEpKSBhcyBwZGY6CiAgICAgICAgICAgICAgICBmb3IgaSxwIGluIGVudW1lcmF0ZShwZGYucGFnZXNbOjUwXSk6CiAgICAgICAgICAgICAgICAgICAgcGFydHMuYXBwZW5kKGYnLS0tIOesrHtpKzF96aG1IC0tLVxuJysocC5leHRyYWN0X3RleHQoKSBvciAnJykpCiAgICAgICAgICAgIHRleHQ9J1xuJy5qb2luKHBhcnRzKQogICAgICAgIGV4Y2VwdCBJbXBvcnRFcnJvcjoKICAgICAgICAgICAgaW1wb3J0IGZpdHoKICAgICAgICAgICAgZG9jPWZpdHoub3BlbihzdHJlYW09ZGF0YSxmaWxldHlwZT0ncGRmJykKICAgICAgICAgICAgdGV4dD0nXG4nLmpvaW4oZictLS0g56yse2krMX3pobUgLS0tXG57cC5nZXRfdGV4dCgpfScgZm9yIGkscCBpbiBlbnVtZXJhdGUoZG9jKSkKICAgICAgICAgICAgZG9jLmNsb3NlKCkKICAgICAgICByZXR1cm4gZicjIFBERjoge3VybH1cblxuJyt0ZXh0WzptYXhfY2hhcnNdCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgcmV0dXJuIGYnUERG5aSx6LSlOiB7ZX0nCgptY3AucnVuKHRyYW5zcG9ydD0nc3RkaW8nKQo=
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
from mcp.server.fastmcp import FastMCP
|
|
2
|
+
import httpx, re, io
|
|
3
|
+
from bs4 import BeautifulSoup
|
|
4
|
+
from urllib.parse import urljoin
|
|
5
|
+
|
|
6
|
+
mcp = FastMCP("gd-crawler")
|
|
7
|
+
|
|
8
|
+
H = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/131.0.0.0 Safari/537.36",
|
|
9
|
+
"Accept-Language": "zh-CN,zh;q=0.9"}
|
|
10
|
+
|
|
11
|
+
def _norm(u):
|
|
12
|
+
u=u.strip()
|
|
13
|
+
if not re.match(r'^https?://',u,re.I): u='http://'+u
|
|
14
|
+
return u
|
|
15
|
+
|
|
16
|
+
@mcp.tool()
|
|
17
|
+
async def fetch_webpage(url: str) -> str:
|
|
18
|
+
"""抓网页正文,返回 Markdown 格式文本"""
|
|
19
|
+
url=_norm(url)
|
|
20
|
+
try:
|
|
21
|
+
async with httpx.AsyncClient(follow_redirects=True,timeout=25,verify=False,headers=H) as c:
|
|
22
|
+
r=await c.get(url); r.raise_for_status()
|
|
23
|
+
soup=BeautifulSoup(r.text,'lxml')
|
|
24
|
+
for t in soup(['script','style','noscript']): t.decompose()
|
|
25
|
+
out=[]
|
|
26
|
+
for el in soup.descendants:
|
|
27
|
+
n=getattr(el,'name',None)
|
|
28
|
+
if n is None:
|
|
29
|
+
x=str(el).strip()
|
|
30
|
+
if x: out.append(x)
|
|
31
|
+
elif n=='a':
|
|
32
|
+
h=el.get('href',''); x=el.get_text(' ',strip=True)
|
|
33
|
+
if x and h: out.append(f'[{x}]({urljoin(str(r.url),h)})')
|
|
34
|
+
elif n in ('h1','h2','h3'):
|
|
35
|
+
x=el.get_text(' ',strip=True)
|
|
36
|
+
if x: out.append('\n'+'#'*int(n[1])+' '+x+'\n')
|
|
37
|
+
elif n in ('p','div','li','tr'):
|
|
38
|
+
x=el.get_text(' ',strip=True)
|
|
39
|
+
if x: out.append(x)
|
|
40
|
+
return f'# {r.url}\n\n'+re.sub(r'\n{3,}','\n\n','\n'.join(out))[:80000]
|
|
41
|
+
except Exception as e:
|
|
42
|
+
return f'抓取失败: {e}'
|
|
43
|
+
|
|
44
|
+
@mcp.tool()
|
|
45
|
+
async def extract_links(url: str) -> str:
|
|
46
|
+
"""提取网页中所有超链接"""
|
|
47
|
+
url=_norm(url)
|
|
48
|
+
try:
|
|
49
|
+
async with httpx.AsyncClient(follow_redirects=True,timeout=25,verify=False,headers=H) as c:
|
|
50
|
+
r=await c.get(url); r.raise_for_status()
|
|
51
|
+
soup=BeautifulSoup(r.text,'lxml')
|
|
52
|
+
links=[]
|
|
53
|
+
for a in soup.find_all('a',href=True):
|
|
54
|
+
h=urljoin(str(r.url),a['href']); x=a.get_text(' ',strip=True)
|
|
55
|
+
if h.startswith('http'):
|
|
56
|
+
links.append(f'[{x}]({h})' if x else h)
|
|
57
|
+
return '\n'.join(links[:200])
|
|
58
|
+
except Exception as e:
|
|
59
|
+
return f'提取失败: {e}'
|
|
60
|
+
|
|
61
|
+
@mcp.tool()
|
|
62
|
+
async def search_web(query: str, max_results: int = 10) -> str:
|
|
63
|
+
"""联网搜索(DuckDuckGo),返回标题+摘要+链接"""
|
|
64
|
+
try:
|
|
65
|
+
from ddgs import DDGS
|
|
66
|
+
except ImportError:
|
|
67
|
+
from duckduckgo_search import DDGS
|
|
68
|
+
res=[]
|
|
69
|
+
try:
|
|
70
|
+
with DDGS() as d:
|
|
71
|
+
for r in d.text(query,max_results=max_results,region='cn-zh'):
|
|
72
|
+
res.append(f"## {r.get('title','')}\n{r.get('body','')}\n{r.get('href','')}\n")
|
|
73
|
+
except Exception as e:
|
|
74
|
+
return f'搜索失败: {e}'
|
|
75
|
+
return '\n---\n'.join(res) if res else '无结果'
|
|
76
|
+
|
|
77
|
+
@mcp.tool()
|
|
78
|
+
async def fetch_pdf(url: str, max_chars: int = 50000) -> str:
|
|
79
|
+
"""下载PDF并提取文本"""
|
|
80
|
+
url=_norm(url)
|
|
81
|
+
try:
|
|
82
|
+
async with httpx.AsyncClient(follow_redirects=True,timeout=60,verify=False,headers=H) as c:
|
|
83
|
+
r=await c.get(url); r.raise_for_status()
|
|
84
|
+
data=r.content
|
|
85
|
+
try:
|
|
86
|
+
import pdfplumber
|
|
87
|
+
parts=[]
|
|
88
|
+
with pdfplumber.open(io.BytesIO(data)) as pdf:
|
|
89
|
+
for i,p in enumerate(pdf.pages[:50]):
|
|
90
|
+
parts.append(f'--- 第{i+1}页 ---\n'+(p.extract_text() or ''))
|
|
91
|
+
text='\n'.join(parts)
|
|
92
|
+
except ImportError:
|
|
93
|
+
import fitz
|
|
94
|
+
doc=fitz.open(stream=data,filetype='pdf')
|
|
95
|
+
text='\n'.join(f'--- 第{i+1}页 ---\n{p.get_text()}' for i,p in enumerate(doc))
|
|
96
|
+
doc.close()
|
|
97
|
+
return f'# PDF: {url}\n\n'+text[:max_chars]
|
|
98
|
+
except Exception as e:
|
|
99
|
+
return f'PDF失败: {e}'
|
|
100
|
+
|
|
101
|
+
mcp.run(transport='stdio')
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "gd-crawler-mcp"
|
|
7
|
+
version = "0.1.2"
|
|
8
|
+
description = "广东信息价爬虫 MCP:抓网页、提链接、联网搜索、PDF提取"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"mcp[cli]>=1.0,<2",
|
|
12
|
+
"httpx>=0.27",
|
|
13
|
+
"beautifulsoup4>=4.12",
|
|
14
|
+
"lxml>=5.0",
|
|
15
|
+
"duckduckgo-search>=6.0",
|
|
16
|
+
"pdfplumber>=0.11",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.scripts]
|
|
20
|
+
gd-crawler-mcp = "gd_crawler_mcp.__main__:main"
|
|
21
|
+
|
|
22
|
+
[tool.hatch.build.targets.wheel]
|
|
23
|
+
packages = ["gd_crawler_mcp"]
|