web-forager 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +112 -0
- package/CHANGELOG.md +454 -0
- package/LICENSE +21 -0
- package/README.md +477 -3
- package/README_zh.md +477 -0
- package/SKILL.md +351 -0
- package/THIRD_PARTY_LICENSES.txt +35 -0
- package/browser/bridges/playwright-bridge.js +375 -0
- package/browser/launch_chrome.py +144 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +79 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +115 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +116 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +56 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +272 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +105 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +432 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +308 -0
- package/docs/REFERENCE_zh.md +309 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +97 -0
- package/mcp_server.py +597 -0
- package/package.json +80 -6
- package/requirements.txt +12 -0
- package/run.py +852 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +874 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +4220 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +135 -0
- package/scripts/scholar.py +354 -0
- package/scripts/sitemap.py +180 -0
- package/scripts/untrusted.py +95 -0
package/mcp_server.py
ADDED
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Minimal MCP (Model Context Protocol) stdio server for web-forager.
|
|
3
|
+
|
|
4
|
+
No SDK is required: this speaks newline-delimited JSON-RPC 2.0 on stdin/stdout,
|
|
5
|
+
implements ``initialize``, ``tools/list`` and ``tools/call``, and exposes the
|
|
6
|
+
engine as eight tools. Diagnostics go to stderr only, never to stdout.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Callable, Optional
|
|
15
|
+
|
|
16
|
+
_PLUGIN_ROOT = Path(__file__).resolve().parent
|
|
17
|
+
if str(_PLUGIN_ROOT) not in sys.path:
|
|
18
|
+
sys.path.insert(0, str(_PLUGIN_ROOT))
|
|
19
|
+
|
|
20
|
+
from scripts import bilibili, documents, engine, pdf, scholar, sitemap # noqa: E402
|
|
21
|
+
|
|
22
|
+
SERVER_NAME = "web-forager"
|
|
23
|
+
PROTOCOL_VERSION = "2024-11-05"
|
|
24
|
+
|
|
25
|
+
_TOOLS: list[dict[str, Any]] = [
|
|
26
|
+
{
|
|
27
|
+
"name": "web_fetch",
|
|
28
|
+
"description": "Fetch one URL and return clean Markdown, metadata, tables and optional chunks.",
|
|
29
|
+
"inputSchema": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"properties": {
|
|
32
|
+
"url": {"type": "string"},
|
|
33
|
+
"max_chars": {"type": "integer", "minimum": 0},
|
|
34
|
+
"chunk_size": {"type": "integer", "minimum": 0},
|
|
35
|
+
"render": {"type": "string", "enum": ["auto", "never", "always"]},
|
|
36
|
+
"include_tables": {"type": "boolean"},
|
|
37
|
+
"cache": {"type": "boolean"},
|
|
38
|
+
"respect_robots": {
|
|
39
|
+
"type": "boolean",
|
|
40
|
+
"description": "Check robots.txt before fetching (default true). Set false only for resources you are authorised to read.",
|
|
41
|
+
},
|
|
42
|
+
"robots_policy": {
|
|
43
|
+
"type": "string",
|
|
44
|
+
"enum": ["respect", "warn", "ignore"],
|
|
45
|
+
"description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
|
|
46
|
+
},
|
|
47
|
+
"input_gate": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"enum": ["off", "mark", "strict"],
|
|
50
|
+
"description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
|
|
51
|
+
},
|
|
52
|
+
"fullResults": {
|
|
53
|
+
"type": "boolean",
|
|
54
|
+
"description": "Repeat every page record inside results as well as pages (1.0.1 behaviour; doubles the response).",
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
"required": ["url"],
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"name": "web_search",
|
|
62
|
+
"description": "Search 360/Bing/Sogou/WeChat and optionally fetch the top result pages.",
|
|
63
|
+
"inputSchema": {
|
|
64
|
+
"type": "object",
|
|
65
|
+
"properties": {
|
|
66
|
+
"query": {"type": "string"},
|
|
67
|
+
"engine": {
|
|
68
|
+
"type": "string",
|
|
69
|
+
"enum": ["auto", "all", "360", "bing", "sogou", "weixin", "duckduckgo"],
|
|
70
|
+
},
|
|
71
|
+
"num_results": {"type": "integer", "minimum": 1, "maximum": 50},
|
|
72
|
+
"max_pages": {"type": "integer", "minimum": 0, "maximum": 20},
|
|
73
|
+
"search_pages": {"type": "integer", "minimum": 1, "maximum": 5},
|
|
74
|
+
"respect_robots": {
|
|
75
|
+
"type": "boolean",
|
|
76
|
+
"description": "Check robots.txt before fetching a result page (default true). Set false only for resources you are authorised to read.",
|
|
77
|
+
},
|
|
78
|
+
"robots_policy": {
|
|
79
|
+
"type": "string",
|
|
80
|
+
"enum": ["respect", "warn", "ignore"],
|
|
81
|
+
"description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
|
|
82
|
+
},
|
|
83
|
+
"input_gate": {
|
|
84
|
+
"type": "string",
|
|
85
|
+
"enum": ["off", "mark", "strict"],
|
|
86
|
+
"description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
|
|
87
|
+
},
|
|
88
|
+
"fullResults": {
|
|
89
|
+
"type": "boolean",
|
|
90
|
+
"description": "Repeat every page record inside results as well as pages (1.0.1 behaviour; doubles the response).",
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
"required": ["query"],
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"name": "web_surf",
|
|
98
|
+
"description": "Search or open a page, then follow ranked in-page links like a person browsing.",
|
|
99
|
+
"inputSchema": {
|
|
100
|
+
"type": "object",
|
|
101
|
+
"properties": {
|
|
102
|
+
"query": {"type": "string"},
|
|
103
|
+
"url": {"type": "string"},
|
|
104
|
+
"depth": {"type": "integer", "minimum": 0, "maximum": 5},
|
|
105
|
+
"max_pages": {"type": "integer", "minimum": 1, "maximum": 50},
|
|
106
|
+
"follow": {"type": "string", "enum": ["same-site", "all", "none"]},
|
|
107
|
+
"session": {"type": "string"},
|
|
108
|
+
"engine": {"type": "string"},
|
|
109
|
+
"respect_robots": {
|
|
110
|
+
"type": "boolean",
|
|
111
|
+
"description": "Check robots.txt before fetching a page (default true). Set false only for resources you are authorised to read.",
|
|
112
|
+
},
|
|
113
|
+
"robots_policy": {
|
|
114
|
+
"type": "string",
|
|
115
|
+
"enum": ["respect", "warn", "ignore"],
|
|
116
|
+
"description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
|
|
117
|
+
},
|
|
118
|
+
"input_gate": {
|
|
119
|
+
"type": "string",
|
|
120
|
+
"enum": ["off", "mark", "strict"],
|
|
121
|
+
"description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
|
|
122
|
+
},
|
|
123
|
+
"fullResults": {
|
|
124
|
+
"type": "boolean",
|
|
125
|
+
"description": "Repeat every page record inside results as well as pages (1.0.1 behaviour; doubles the response).",
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
},
|
|
129
|
+
},
|
|
130
|
+
{
|
|
131
|
+
"name": "web_scholar",
|
|
132
|
+
"description": "Query arXiv, Crossref, PubMed or resolve a DOI through public APIs.",
|
|
133
|
+
"inputSchema": {
|
|
134
|
+
"type": "object",
|
|
135
|
+
"properties": {
|
|
136
|
+
"source": {"type": "string", "enum": ["arxiv", "crossref", "pubmed", "doi"]},
|
|
137
|
+
"query": {"type": "string"},
|
|
138
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 50},
|
|
139
|
+
},
|
|
140
|
+
"required": ["source", "query"],
|
|
141
|
+
},
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
"name": "web_map",
|
|
145
|
+
"description": "Discover page URLs from robots.txt Sitemap lines and sitemap XML files.",
|
|
146
|
+
"inputSchema": {
|
|
147
|
+
"type": "object",
|
|
148
|
+
"properties": {
|
|
149
|
+
"url": {"type": "string"},
|
|
150
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 5000},
|
|
151
|
+
},
|
|
152
|
+
"required": ["url"],
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
"name": "web_bilibili",
|
|
157
|
+
"description": "Browse Bilibili public pages: feed, video, related, search or surf.",
|
|
158
|
+
"inputSchema": {
|
|
159
|
+
"type": "object",
|
|
160
|
+
"properties": {
|
|
161
|
+
"mode": {
|
|
162
|
+
"type": "string",
|
|
163
|
+
"enum": ["feed", "video", "search", "related", "surf"],
|
|
164
|
+
},
|
|
165
|
+
"url": {"type": "string"},
|
|
166
|
+
"query": {"type": "string"},
|
|
167
|
+
"num_results": {"type": "integer", "minimum": 1, "maximum": 50},
|
|
168
|
+
"depth": {"type": "integer", "minimum": 0, "maximum": 5},
|
|
169
|
+
},
|
|
170
|
+
},
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
"name": "web_doctor",
|
|
174
|
+
"description": "Report which optional capabilities this installation can serve, with the install command for each missing one.",
|
|
175
|
+
"inputSchema": {
|
|
176
|
+
"type": "object",
|
|
177
|
+
"properties": {
|
|
178
|
+
"probe": {
|
|
179
|
+
"type": "boolean",
|
|
180
|
+
"description": "Ask each search engine once and report which ones answer from this network.",
|
|
181
|
+
},
|
|
182
|
+
"proxy": {"type": "string", "description": "Proxy URL to report as configured."},
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
"name": "web_find",
|
|
188
|
+
"description": "Search inside one site through the site's own search page, then fetch the best matching pages.",
|
|
189
|
+
"inputSchema": {
|
|
190
|
+
"type": "object",
|
|
191
|
+
"properties": {
|
|
192
|
+
"site": {"type": "string", "description": "Domain or base URL, e.g. the-decoder.com."},
|
|
193
|
+
"query": {"type": "string"},
|
|
194
|
+
"max_pages": {"type": "integer", "minimum": 0, "maximum": 20},
|
|
195
|
+
"respect_robots": {
|
|
196
|
+
"type": "boolean",
|
|
197
|
+
"description": "Check robots.txt before fetching (default true).",
|
|
198
|
+
},
|
|
199
|
+
"robots_policy": {
|
|
200
|
+
"type": "string",
|
|
201
|
+
"enum": ["respect", "warn", "ignore"],
|
|
202
|
+
"description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
|
|
203
|
+
},
|
|
204
|
+
"input_gate": {
|
|
205
|
+
"type": "string",
|
|
206
|
+
"enum": ["off", "mark", "strict"],
|
|
207
|
+
"description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
|
|
208
|
+
},
|
|
209
|
+
"proxy": {"type": "string"},
|
|
210
|
+
"fullResults": {
|
|
211
|
+
"type": "boolean",
|
|
212
|
+
"description": "Repeat every page record inside results as well as pages.",
|
|
213
|
+
},
|
|
214
|
+
},
|
|
215
|
+
"required": ["site", "query"],
|
|
216
|
+
"additionalProperties": False,
|
|
217
|
+
},
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
"name": "web_feed",
|
|
221
|
+
"description": "Read a site's RSS/Atom feed, discovering it from the page when given the site URL.",
|
|
222
|
+
"inputSchema": {
|
|
223
|
+
"type": "object",
|
|
224
|
+
"properties": {
|
|
225
|
+
"url": {"type": "string"},
|
|
226
|
+
"num_results": {"type": "integer", "minimum": 1, "maximum": 50},
|
|
227
|
+
"respect_robots": {"type": "boolean"},
|
|
228
|
+
"proxy": {"type": "string"},
|
|
229
|
+
"fullResults": {"type": "boolean"},
|
|
230
|
+
},
|
|
231
|
+
"required": ["url"],
|
|
232
|
+
"additionalProperties": False,
|
|
233
|
+
},
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
"name": "web_clean",
|
|
237
|
+
"description": "Clean a local HTML, PDF, DOCX, XLSX, EPUB or CSV file into Markdown.",
|
|
238
|
+
"inputSchema": {
|
|
239
|
+
"type": "object",
|
|
240
|
+
"properties": {
|
|
241
|
+
"path": {"type": "string"},
|
|
242
|
+
"base_url": {"type": "string"},
|
|
243
|
+
"include_tables": {"type": "boolean"},
|
|
244
|
+
},
|
|
245
|
+
"required": ["path"],
|
|
246
|
+
},
|
|
247
|
+
},
|
|
248
|
+
]
|
|
249
|
+
|
|
250
|
+
_TOOL_NAMES = {tool["name"] for tool in _TOOLS}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _full_results(arguments: dict[str, Any]) -> bool:
|
|
254
|
+
"""Whether to repeat every page body inside ``results`` as well as ``pages``."""
|
|
255
|
+
|
|
256
|
+
return bool(arguments.get("fullResults", False))
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _config(arguments: dict[str, Any]) -> engine.FetchConfig:
|
|
260
|
+
config = engine.FetchConfig(
|
|
261
|
+
max_chars=int(arguments.get("max_chars", 20_000)),
|
|
262
|
+
chunk_size=int(arguments.get("chunk_size", 0)),
|
|
263
|
+
render=str(arguments.get("render", "auto")),
|
|
264
|
+
include_tables=bool(arguments.get("include_tables", True)),
|
|
265
|
+
cache=bool(arguments.get("cache", False)),
|
|
266
|
+
num_results=int(arguments.get("num_results", 8)),
|
|
267
|
+
max_pages=max(0, int(arguments.get("max_pages", 3))),
|
|
268
|
+
search_pages=max(1, int(arguments.get("search_pages", 1))),
|
|
269
|
+
search_engine=str(arguments.get("engine", "auto")),
|
|
270
|
+
follow=str(arguments.get("follow", "same-site")),
|
|
271
|
+
# robots.txt is checked for every fetched page unless the caller turns
|
|
272
|
+
# it off explicitly, which is the same switch the CLI spells
|
|
273
|
+
# --ignore-robots.
|
|
274
|
+
respect_robots=bool(arguments.get("respect_robots", True)),
|
|
275
|
+
robots_policy=(
|
|
276
|
+
"ignore"
|
|
277
|
+
if not bool(arguments.get("respect_robots", True))
|
|
278
|
+
else str(arguments.get("robots_policy", "respect"))
|
|
279
|
+
),
|
|
280
|
+
proxy=str(arguments.get("proxy", "")),
|
|
281
|
+
input_gate=str(arguments.get("input_gate", "mark")),
|
|
282
|
+
verbose=False,
|
|
283
|
+
)
|
|
284
|
+
config.validate()
|
|
285
|
+
return config
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _tool_fetch(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
289
|
+
config = _config(arguments)
|
|
290
|
+
if not arguments.get("url"):
|
|
291
|
+
raise ValueError("url is required")
|
|
292
|
+
page = engine.fetch_page(str(arguments["url"]), config)
|
|
293
|
+
return engine.build_envelope("fetch", page, full_results=_full_results(arguments))
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _tool_search(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
297
|
+
if not arguments.get("query"):
|
|
298
|
+
raise ValueError("query is required")
|
|
299
|
+
config = _config(arguments)
|
|
300
|
+
payload = engine.fetch_by_query(
|
|
301
|
+
str(arguments["query"]),
|
|
302
|
+
config,
|
|
303
|
+
engine=str(arguments.get("engine", "auto")),
|
|
304
|
+
)
|
|
305
|
+
return engine.build_envelope("fetch", payload, full_results=_full_results(arguments))
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _tool_surf(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
309
|
+
config = _config(arguments)
|
|
310
|
+
seed = arguments.get("url") or arguments.get("query")
|
|
311
|
+
if not seed:
|
|
312
|
+
raise ValueError("url or query is required")
|
|
313
|
+
payload = engine.surf_web(
|
|
314
|
+
str(seed),
|
|
315
|
+
config,
|
|
316
|
+
depth=max(0, int(arguments.get("depth", 1))),
|
|
317
|
+
follow=str(arguments.get("follow", "same-site")),
|
|
318
|
+
max_pages=max(1, int(arguments.get("max_pages", 3))),
|
|
319
|
+
engine_name=str(arguments.get("engine", "auto")),
|
|
320
|
+
session=arguments.get("session") or None,
|
|
321
|
+
)
|
|
322
|
+
if arguments.get("url") and arguments.get("query"):
|
|
323
|
+
payload.setdefault("warnings", []).append(
|
|
324
|
+
"query was ignored because url was given; omit url to surf from a search"
|
|
325
|
+
)
|
|
326
|
+
return engine.build_envelope("surf", payload, full_results=_full_results(arguments))
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _tool_scholar(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
330
|
+
source = str(arguments.get("source", "arxiv"))
|
|
331
|
+
query = str(arguments.get("query", ""))
|
|
332
|
+
config = _config(arguments)
|
|
333
|
+
payload = scholar.search(
|
|
334
|
+
source,
|
|
335
|
+
query,
|
|
336
|
+
config,
|
|
337
|
+
limit=int(arguments.get("limit", config.num_results)),
|
|
338
|
+
)
|
|
339
|
+
return engine.build_envelope("scholar", payload)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _tool_map(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
343
|
+
config = _config(arguments)
|
|
344
|
+
payload = sitemap.discover(
|
|
345
|
+
str(arguments.get("url", "")),
|
|
346
|
+
config,
|
|
347
|
+
limit=int(arguments.get("limit", 500)),
|
|
348
|
+
)
|
|
349
|
+
return engine.build_envelope("map", payload)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _tool_bilibili(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
353
|
+
config = _config(arguments)
|
|
354
|
+
mode = str(arguments.get("mode", "auto"))
|
|
355
|
+
url = str(arguments.get("url") or "")
|
|
356
|
+
query = str(arguments.get("query") or "")
|
|
357
|
+
if mode == "auto":
|
|
358
|
+
mode = "search" if query else "video" if url else "feed"
|
|
359
|
+
if mode == "feed":
|
|
360
|
+
payload = bilibili.feed(
|
|
361
|
+
config, limit=config.num_results, enrich=config.max_pages, url=url or bilibili.BILI_HOME
|
|
362
|
+
)
|
|
363
|
+
elif mode == "video":
|
|
364
|
+
payload = {"mode": "video", "videos": [bilibili.fetch_video(url, config)]}
|
|
365
|
+
payload["results"] = payload["videos"]
|
|
366
|
+
elif mode == "search":
|
|
367
|
+
payload = bilibili.search(query, config)
|
|
368
|
+
elif mode == "related":
|
|
369
|
+
payload = bilibili.related(url, config, limit=config.num_results, enrich=config.max_pages)
|
|
370
|
+
elif mode == "surf":
|
|
371
|
+
payload = bilibili.surf(
|
|
372
|
+
url or query,
|
|
373
|
+
config,
|
|
374
|
+
depth=max(0, int(arguments.get("depth", 1))),
|
|
375
|
+
per_level=config.max_pages,
|
|
376
|
+
)
|
|
377
|
+
else:
|
|
378
|
+
raise ValueError(f"unsupported Bilibili mode: {mode}")
|
|
379
|
+
return engine.build_envelope("bili", payload)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _tool_doctor(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
383
|
+
config = _config(arguments)
|
|
384
|
+
payload = engine.doctor_report(config)
|
|
385
|
+
if arguments.get("probe"):
|
|
386
|
+
payload["search_probe"] = engine.probe_engines(config)
|
|
387
|
+
return engine.build_envelope("doctor", payload)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _tool_find(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
391
|
+
site = str(arguments.get("site", "")).strip()
|
|
392
|
+
query = str(arguments.get("query", "")).strip()
|
|
393
|
+
if not site:
|
|
394
|
+
raise ValueError("site is required")
|
|
395
|
+
if not query:
|
|
396
|
+
raise ValueError("query is required")
|
|
397
|
+
config = _config(arguments)
|
|
398
|
+
payload = engine.find_within_site(
|
|
399
|
+
site,
|
|
400
|
+
query,
|
|
401
|
+
config,
|
|
402
|
+
max_pages=int(arguments.get("max_pages", config.max_pages)),
|
|
403
|
+
)
|
|
404
|
+
return engine.build_envelope("find", payload, full_results=_full_results(arguments))
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _tool_feed(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
408
|
+
url = str(arguments.get("url", "")).strip()
|
|
409
|
+
if not url:
|
|
410
|
+
raise ValueError("url is required")
|
|
411
|
+
config = _config(arguments)
|
|
412
|
+
payload = engine.discover_feed(url, config)
|
|
413
|
+
payload["results"] = [
|
|
414
|
+
{
|
|
415
|
+
"url": item.get("url", ""),
|
|
416
|
+
"title": item.get("title", ""),
|
|
417
|
+
"published": item.get("published", ""),
|
|
418
|
+
}
|
|
419
|
+
for item in payload.get("items") or []
|
|
420
|
+
]
|
|
421
|
+
return engine.build_envelope("feed", payload, full_results=_full_results(arguments))
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _tool_clean(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
425
|
+
config = _config(arguments)
|
|
426
|
+
path = Path(str(arguments.get("path", ""))).expanduser()
|
|
427
|
+
if not path.is_file():
|
|
428
|
+
raise ValueError(f"file not found: {path}")
|
|
429
|
+
data = path.read_bytes()
|
|
430
|
+
base_url = str(arguments.get("base_url") or "")
|
|
431
|
+
if pdf.is_pdf(data):
|
|
432
|
+
document = pdf.extract(data)
|
|
433
|
+
markdown = pdf.to_markdown(document, url=base_url)
|
|
434
|
+
metadata = pdf.to_metadata(document, url=base_url)
|
|
435
|
+
tables: list[dict[str, Any]] = []
|
|
436
|
+
kind = "pdf"
|
|
437
|
+
elif (kind := documents.detect(data, url=str(path))) and kind:
|
|
438
|
+
extracted = documents.extract(data, kind, url=str(path))
|
|
439
|
+
markdown = extracted["markdown"]
|
|
440
|
+
metadata = documents.to_metadata(extracted, base_url)
|
|
441
|
+
tables = extracted.get("tables") or []
|
|
442
|
+
else:
|
|
443
|
+
html = engine.decode_bytes(data)
|
|
444
|
+
markdown = engine.html_to_markdown(
|
|
445
|
+
html,
|
|
446
|
+
url=base_url or None,
|
|
447
|
+
include_tables=config.include_tables,
|
|
448
|
+
keep_links=config.keep_links,
|
|
449
|
+
)
|
|
450
|
+
metadata = engine.extract_metadata(html, url=base_url or None)
|
|
451
|
+
tables = engine.extract_tables(html) if config.include_tables else []
|
|
452
|
+
kind = "html"
|
|
453
|
+
if config.max_chars and len(markdown) > config.max_chars:
|
|
454
|
+
markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
|
|
455
|
+
payload = {
|
|
456
|
+
"source": str(path),
|
|
457
|
+
"content_type": kind,
|
|
458
|
+
"metadata": metadata,
|
|
459
|
+
"tables": tables,
|
|
460
|
+
"markdown": markdown,
|
|
461
|
+
"chunks": engine.chunk_markdown(markdown, config.chunk_size) if config.chunk_size else [],
|
|
462
|
+
}
|
|
463
|
+
return engine.build_envelope("clean", payload)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
_DISPATCH: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
|
|
467
|
+
"web_fetch": _tool_fetch,
|
|
468
|
+
"web_search": _tool_search,
|
|
469
|
+
"web_surf": _tool_surf,
|
|
470
|
+
"web_scholar": _tool_scholar,
|
|
471
|
+
"web_map": _tool_map,
|
|
472
|
+
"web_bilibili": _tool_bilibili,
|
|
473
|
+
"web_doctor": _tool_doctor,
|
|
474
|
+
"web_find": _tool_find,
|
|
475
|
+
"web_feed": _tool_feed,
|
|
476
|
+
"web_clean": _tool_clean,
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _result(request_id: Any, result: dict[str, Any]) -> dict[str, Any]:
|
|
481
|
+
return {"jsonrpc": "2.0", "id": request_id, "result": result}
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _error(request_id: Any, code: int, message: str) -> dict[str, Any]:
|
|
485
|
+
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def handle_request(request: dict[str, Any]) -> Optional[dict[str, Any]]:
|
|
489
|
+
"""Handle one JSON-RPC object; return None for notifications."""
|
|
490
|
+
|
|
491
|
+
method = request.get("method", "")
|
|
492
|
+
request_id = request.get("id")
|
|
493
|
+
params = request.get("params") or {}
|
|
494
|
+
|
|
495
|
+
if method == "initialize":
|
|
496
|
+
return _result(
|
|
497
|
+
request_id,
|
|
498
|
+
{
|
|
499
|
+
"protocolVersion": params.get("protocolVersion") or PROTOCOL_VERSION,
|
|
500
|
+
"capabilities": {"tools": {}},
|
|
501
|
+
"serverInfo": {"name": SERVER_NAME, "version": engine.__version__},
|
|
502
|
+
},
|
|
503
|
+
)
|
|
504
|
+
if method in ("notifications/initialized", "initialized", "notifications/cancelled"):
|
|
505
|
+
return None
|
|
506
|
+
if method == "ping":
|
|
507
|
+
return _result(request_id, {})
|
|
508
|
+
if method == "tools/list":
|
|
509
|
+
return _result(request_id, {"tools": _TOOLS})
|
|
510
|
+
if method == "tools/call":
|
|
511
|
+
name = str(params.get("name", ""))
|
|
512
|
+
arguments = params.get("arguments") or {}
|
|
513
|
+
if name not in _TOOL_NAMES:
|
|
514
|
+
return _error(request_id, -32602, f"unknown tool: {name}")
|
|
515
|
+
try:
|
|
516
|
+
payload = _DISPATCH[name](dict(arguments))
|
|
517
|
+
text = json.dumps(payload, ensure_ascii=False, indent=2)
|
|
518
|
+
return _result(
|
|
519
|
+
request_id,
|
|
520
|
+
{"content": [{"type": "text", "text": text}], "isError": False},
|
|
521
|
+
)
|
|
522
|
+
except Exception as exc: # tool failures are data, not protocol failures
|
|
523
|
+
code = exc.__class__.__name__
|
|
524
|
+
if code == "ProhibitedQuery":
|
|
525
|
+
code = "prohibited_query"
|
|
526
|
+
elif code == "UnsafeAddress":
|
|
527
|
+
code = "unsafe_address"
|
|
528
|
+
elif code == "VerificationRequired":
|
|
529
|
+
code = "verification_required"
|
|
530
|
+
error: dict[str, Any] = {"code": code, "message": str(exc)}
|
|
531
|
+
if code == "verification_required":
|
|
532
|
+
error["hint"] = (
|
|
533
|
+
"Retry in browser mode with playbook wechat-dig and a profile that "
|
|
534
|
+
"has passed verification; web-forager does not solve challenges."
|
|
535
|
+
)
|
|
536
|
+
text = json.dumps(
|
|
537
|
+
{
|
|
538
|
+
"ok": False,
|
|
539
|
+
"error": error,
|
|
540
|
+
},
|
|
541
|
+
ensure_ascii=False,
|
|
542
|
+
)
|
|
543
|
+
return _result(
|
|
544
|
+
request_id,
|
|
545
|
+
{"content": [{"type": "text", "text": text}], "isError": True},
|
|
546
|
+
)
|
|
547
|
+
return _error(request_id, -32601, f"method not found: {method}")
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _utf8_stream(stream: Any) -> Any:
|
|
551
|
+
"""Force UTF-8 on a text stream when the host allows it.
|
|
552
|
+
|
|
553
|
+
Windows hosts may launch ``python`` with the ANSI code page (cp1252/cp936)
|
|
554
|
+
as the stdio encoding. Writing a Chinese title then raises
|
|
555
|
+
``UnicodeEncodeError`` and kills the server, so the streams are
|
|
556
|
+
reconfigured before the loop starts. ``serve`` still has an ASCII
|
|
557
|
+
fallback for streams that cannot be reconfigured.
|
|
558
|
+
"""
|
|
559
|
+
|
|
560
|
+
reconfigure = getattr(stream, "reconfigure", None)
|
|
561
|
+
if reconfigure is None:
|
|
562
|
+
return stream
|
|
563
|
+
try:
|
|
564
|
+
reconfigure(encoding="utf-8", errors="strict")
|
|
565
|
+
except (AttributeError, OSError, ValueError):
|
|
566
|
+
return stream
|
|
567
|
+
return stream
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def serve(stdin: Any = None, stdout: Any = None) -> int:
|
|
571
|
+
stdin = _utf8_stream(stdin or sys.stdin)
|
|
572
|
+
stdout = _utf8_stream(stdout or sys.stdout)
|
|
573
|
+
for line in stdin:
|
|
574
|
+
line = line.strip()
|
|
575
|
+
if not line:
|
|
576
|
+
continue
|
|
577
|
+
try:
|
|
578
|
+
request = json.loads(line)
|
|
579
|
+
except json.JSONDecodeError:
|
|
580
|
+
response = _error(None, -32700, "parse error")
|
|
581
|
+
else:
|
|
582
|
+
if not isinstance(request, dict):
|
|
583
|
+
response = _error(None, -32600, "invalid request")
|
|
584
|
+
else:
|
|
585
|
+
response = handle_request(request)
|
|
586
|
+
if response is not None:
|
|
587
|
+
try:
|
|
588
|
+
payload = json.dumps(response, ensure_ascii=False)
|
|
589
|
+
stdout.write(payload + "\n")
|
|
590
|
+
except UnicodeEncodeError:
|
|
591
|
+
stdout.write(json.dumps(response, ensure_ascii=True) + "\n")
|
|
592
|
+
stdout.flush()
|
|
593
|
+
return 0
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
if __name__ == "__main__":
|
|
597
|
+
raise SystemExit(serve())
|
package/package.json
CHANGED
|
@@ -1,6 +1,80 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "web-forager",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "web-forager",
|
|
3
|
+
"version": "1.3.0",
|
|
4
|
+
"description": "Clean-room, robots-aware web information gathering: multi-engine search, HTML/PDF/Office/feed extraction, Bilibili/Zhihu/Xiaohongshu/WeChat reading, sitemaps and resumable surfing, plus a browser mode with eight read-only playbooks. CLI + two MCP servers.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"engines": {
|
|
8
|
+
"node": ">=18"
|
|
9
|
+
},
|
|
10
|
+
"keywords": [
|
|
11
|
+
"dsh",
|
|
12
|
+
"dsh-plugin",
|
|
13
|
+
"deepseek-harness",
|
|
14
|
+
"web-scraping",
|
|
15
|
+
"search",
|
|
16
|
+
"mcp",
|
|
17
|
+
"mcp-server",
|
|
18
|
+
"bilibili",
|
|
19
|
+
"zhihu",
|
|
20
|
+
"crawler",
|
|
21
|
+
"robots-txt",
|
|
22
|
+
"playwright"
|
|
23
|
+
],
|
|
24
|
+
"repository": {
|
|
25
|
+
"type": "git",
|
|
26
|
+
"url": "git+https://github.com/JiamingShi-JsPhys/web-forager.git"
|
|
27
|
+
},
|
|
28
|
+
"homepage": "https://github.com/JiamingShi-JsPhys/web-forager#readme",
|
|
29
|
+
"bugs": {
|
|
30
|
+
"url": "https://github.com/JiamingShi-JsPhys/web-forager/issues"
|
|
31
|
+
},
|
|
32
|
+
"scripts": {
|
|
33
|
+
"test": "python -m pytest",
|
|
34
|
+
"test:browser": "node --test \"browser/tests/*.test.js\"",
|
|
35
|
+
"test:live": "node browser/tests/live-smoke.js",
|
|
36
|
+
"prepublishOnly": "node clean-pycache.mjs"
|
|
37
|
+
},
|
|
38
|
+
"files": [
|
|
39
|
+
"index.js",
|
|
40
|
+
"cordis.patch.yml",
|
|
41
|
+
".claude-plugin/",
|
|
42
|
+
".codex-plugin/",
|
|
43
|
+
".codebuddy-plugin/",
|
|
44
|
+
".dsh-plugin/",
|
|
45
|
+
".cursor/",
|
|
46
|
+
".clinerules/",
|
|
47
|
+
".continue/",
|
|
48
|
+
".roo/",
|
|
49
|
+
".windsurf/",
|
|
50
|
+
".github/copilot-instructions.md",
|
|
51
|
+
".mcp.json",
|
|
52
|
+
"install.py",
|
|
53
|
+
"requirements.txt",
|
|
54
|
+
"run.py",
|
|
55
|
+
"mcp_server.py",
|
|
56
|
+
"browser/run.js",
|
|
57
|
+
"browser/mcp_server.js",
|
|
58
|
+
"browser/launch_chrome.py",
|
|
59
|
+
"browser/package.json",
|
|
60
|
+
"browser/lib/",
|
|
61
|
+
"browser/playbooks/",
|
|
62
|
+
"browser/bridges/",
|
|
63
|
+
"scripts/",
|
|
64
|
+
"commands/",
|
|
65
|
+
"docs/",
|
|
66
|
+
"SKILL.md",
|
|
67
|
+
"AGENTS.md",
|
|
68
|
+
"manifest.json",
|
|
69
|
+
"README.md",
|
|
70
|
+
"README_zh.md",
|
|
71
|
+
"CHANGELOG.md",
|
|
72
|
+
"THIRD_PARTY_LICENSES.txt",
|
|
73
|
+
"LICENSE"
|
|
74
|
+
],
|
|
75
|
+
"dsh": {
|
|
76
|
+
"bundle": {
|
|
77
|
+
"patch": "./cordis.patch.yml"
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|