web-forager 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +112 -0
  18. package/CHANGELOG.md +454 -0
  19. package/LICENSE +21 -0
  20. package/README.md +477 -3
  21. package/README_zh.md +477 -0
  22. package/SKILL.md +351 -0
  23. package/THIRD_PARTY_LICENSES.txt +35 -0
  24. package/browser/bridges/playwright-bridge.js +375 -0
  25. package/browser/launch_chrome.py +144 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +79 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +115 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +116 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +56 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +272 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +105 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +432 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/scholar.md +28 -0
  58. package/commands/surf.md +28 -0
  59. package/cordis.patch.yml +19 -0
  60. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  61. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  62. package/docs/DEPENDENCIES.md +40 -0
  63. package/docs/INSTALL_COMMANDS.md +113 -0
  64. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  65. package/docs/REFERENCE.md +308 -0
  66. package/docs/REFERENCE_zh.md +309 -0
  67. package/index.js +7 -0
  68. package/install.py +254 -0
  69. package/manifest.json +97 -0
  70. package/mcp_server.py +597 -0
  71. package/package.json +80 -6
  72. package/requirements.txt +12 -0
  73. package/run.py +852 -0
  74. package/scripts/__init__.py +1 -0
  75. package/scripts/bilibili.py +874 -0
  76. package/scripts/documents.py +437 -0
  77. package/scripts/engine.py +4220 -0
  78. package/scripts/feeds.py +180 -0
  79. package/scripts/pdf.py +135 -0
  80. package/scripts/scholar.py +354 -0
  81. package/scripts/sitemap.py +180 -0
  82. package/scripts/untrusted.py +95 -0
package/mcp_server.py ADDED
@@ -0,0 +1,597 @@
1
+ #!/usr/bin/env python3
2
+ """Minimal MCP (Model Context Protocol) stdio server for web-forager.
3
+
4
+ No SDK is required: this speaks newline-delimited JSON-RPC 2.0 on stdin/stdout,
5
+ implements ``initialize``, ``tools/list`` and ``tools/call``, and exposes the
6
+ engine as eight tools. Diagnostics go to stderr only, never to stdout.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import sys
13
+ from pathlib import Path
14
+ from typing import Any, Callable, Optional
15
+
16
+ _PLUGIN_ROOT = Path(__file__).resolve().parent
17
+ if str(_PLUGIN_ROOT) not in sys.path:
18
+ sys.path.insert(0, str(_PLUGIN_ROOT))
19
+
20
+ from scripts import bilibili, documents, engine, pdf, scholar, sitemap # noqa: E402
21
+
22
+ SERVER_NAME = "web-forager"
23
+ PROTOCOL_VERSION = "2024-11-05"
24
+
25
+ _TOOLS: list[dict[str, Any]] = [
26
+ {
27
+ "name": "web_fetch",
28
+ "description": "Fetch one URL and return clean Markdown, metadata, tables and optional chunks.",
29
+ "inputSchema": {
30
+ "type": "object",
31
+ "properties": {
32
+ "url": {"type": "string"},
33
+ "max_chars": {"type": "integer", "minimum": 0},
34
+ "chunk_size": {"type": "integer", "minimum": 0},
35
+ "render": {"type": "string", "enum": ["auto", "never", "always"]},
36
+ "include_tables": {"type": "boolean"},
37
+ "cache": {"type": "boolean"},
38
+ "respect_robots": {
39
+ "type": "boolean",
40
+ "description": "Check robots.txt before fetching (default true). Set false only for resources you are authorised to read.",
41
+ },
42
+ "robots_policy": {
43
+ "type": "string",
44
+ "enum": ["respect", "warn", "ignore"],
45
+ "description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
46
+ },
47
+ "input_gate": {
48
+ "type": "string",
49
+ "enum": ["off", "mark", "strict"],
50
+ "description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
51
+ },
52
+ "fullResults": {
53
+ "type": "boolean",
54
+ "description": "Repeat every page record inside results as well as pages (1.0.1 behaviour; doubles the response).",
55
+ },
56
+ },
57
+ "required": ["url"],
58
+ },
59
+ },
60
+ {
61
+ "name": "web_search",
62
+ "description": "Search 360/Bing/Sogou/WeChat and optionally fetch the top result pages.",
63
+ "inputSchema": {
64
+ "type": "object",
65
+ "properties": {
66
+ "query": {"type": "string"},
67
+ "engine": {
68
+ "type": "string",
69
+ "enum": ["auto", "all", "360", "bing", "sogou", "weixin", "duckduckgo"],
70
+ },
71
+ "num_results": {"type": "integer", "minimum": 1, "maximum": 50},
72
+ "max_pages": {"type": "integer", "minimum": 0, "maximum": 20},
73
+ "search_pages": {"type": "integer", "minimum": 1, "maximum": 5},
74
+ "respect_robots": {
75
+ "type": "boolean",
76
+ "description": "Check robots.txt before fetching a result page (default true). Set false only for resources you are authorised to read.",
77
+ },
78
+ "robots_policy": {
79
+ "type": "string",
80
+ "enum": ["respect", "warn", "ignore"],
81
+ "description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
82
+ },
83
+ "input_gate": {
84
+ "type": "string",
85
+ "enum": ["off", "mark", "strict"],
86
+ "description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
87
+ },
88
+ "fullResults": {
89
+ "type": "boolean",
90
+ "description": "Repeat every page record inside results as well as pages (1.0.1 behaviour; doubles the response).",
91
+ },
92
+ },
93
+ "required": ["query"],
94
+ },
95
+ },
96
+ {
97
+ "name": "web_surf",
98
+ "description": "Search or open a page, then follow ranked in-page links like a person browsing.",
99
+ "inputSchema": {
100
+ "type": "object",
101
+ "properties": {
102
+ "query": {"type": "string"},
103
+ "url": {"type": "string"},
104
+ "depth": {"type": "integer", "minimum": 0, "maximum": 5},
105
+ "max_pages": {"type": "integer", "minimum": 1, "maximum": 50},
106
+ "follow": {"type": "string", "enum": ["same-site", "all", "none"]},
107
+ "session": {"type": "string"},
108
+ "engine": {"type": "string"},
109
+ "respect_robots": {
110
+ "type": "boolean",
111
+ "description": "Check robots.txt before fetching a page (default true). Set false only for resources you are authorised to read.",
112
+ },
113
+ "robots_policy": {
114
+ "type": "string",
115
+ "enum": ["respect", "warn", "ignore"],
116
+ "description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
117
+ },
118
+ "input_gate": {
119
+ "type": "string",
120
+ "enum": ["off", "mark", "strict"],
121
+ "description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
122
+ },
123
+ "fullResults": {
124
+ "type": "boolean",
125
+ "description": "Repeat every page record inside results as well as pages (1.0.1 behaviour; doubles the response).",
126
+ },
127
+ },
128
+ },
129
+ },
130
+ {
131
+ "name": "web_scholar",
132
+ "description": "Query arXiv, Crossref, PubMed or resolve a DOI through public APIs.",
133
+ "inputSchema": {
134
+ "type": "object",
135
+ "properties": {
136
+ "source": {"type": "string", "enum": ["arxiv", "crossref", "pubmed", "doi"]},
137
+ "query": {"type": "string"},
138
+ "limit": {"type": "integer", "minimum": 1, "maximum": 50},
139
+ },
140
+ "required": ["source", "query"],
141
+ },
142
+ },
143
+ {
144
+ "name": "web_map",
145
+ "description": "Discover page URLs from robots.txt Sitemap lines and sitemap XML files.",
146
+ "inputSchema": {
147
+ "type": "object",
148
+ "properties": {
149
+ "url": {"type": "string"},
150
+ "limit": {"type": "integer", "minimum": 1, "maximum": 5000},
151
+ },
152
+ "required": ["url"],
153
+ },
154
+ },
155
+ {
156
+ "name": "web_bilibili",
157
+ "description": "Browse Bilibili public pages: feed, video, related, search or surf.",
158
+ "inputSchema": {
159
+ "type": "object",
160
+ "properties": {
161
+ "mode": {
162
+ "type": "string",
163
+ "enum": ["feed", "video", "search", "related", "surf"],
164
+ },
165
+ "url": {"type": "string"},
166
+ "query": {"type": "string"},
167
+ "num_results": {"type": "integer", "minimum": 1, "maximum": 50},
168
+ "depth": {"type": "integer", "minimum": 0, "maximum": 5},
169
+ },
170
+ },
171
+ },
172
+ {
173
+ "name": "web_doctor",
174
+ "description": "Report which optional capabilities this installation can serve, with the install command for each missing one.",
175
+ "inputSchema": {
176
+ "type": "object",
177
+ "properties": {
178
+ "probe": {
179
+ "type": "boolean",
180
+ "description": "Ask each search engine once and report which ones answer from this network.",
181
+ },
182
+ "proxy": {"type": "string", "description": "Proxy URL to report as configured."},
183
+ },
184
+ },
185
+ },
186
+ {
187
+ "name": "web_find",
188
+ "description": "Search inside one site through the site's own search page, then fetch the best matching pages.",
189
+ "inputSchema": {
190
+ "type": "object",
191
+ "properties": {
192
+ "site": {"type": "string", "description": "Domain or base URL, e.g. the-decoder.com."},
193
+ "query": {"type": "string"},
194
+ "max_pages": {"type": "integer", "minimum": 0, "maximum": 20},
195
+ "respect_robots": {
196
+ "type": "boolean",
197
+ "description": "Check robots.txt before fetching (default true).",
198
+ },
199
+ "robots_policy": {
200
+ "type": "string",
201
+ "enum": ["respect", "warn", "ignore"],
202
+ "description": "respect: refuse a disallowed path (default); warn: read it and mark compliance.robots_overridden; ignore: do not consult robots.txt.",
203
+ },
204
+ "input_gate": {
205
+ "type": "string",
206
+ "enum": ["off", "mark", "strict"],
207
+ "description": "Treat fetched text as untrusted: mark instruction-shaped content (default), withhold the body (strict), or skip the scan (off).",
208
+ },
209
+ "proxy": {"type": "string"},
210
+ "fullResults": {
211
+ "type": "boolean",
212
+ "description": "Repeat every page record inside results as well as pages.",
213
+ },
214
+ },
215
+ "required": ["site", "query"],
216
+ "additionalProperties": False,
217
+ },
218
+ },
219
+ {
220
+ "name": "web_feed",
221
+ "description": "Read a site's RSS/Atom feed, discovering it from the page when given the site URL.",
222
+ "inputSchema": {
223
+ "type": "object",
224
+ "properties": {
225
+ "url": {"type": "string"},
226
+ "num_results": {"type": "integer", "minimum": 1, "maximum": 50},
227
+ "respect_robots": {"type": "boolean"},
228
+ "proxy": {"type": "string"},
229
+ "fullResults": {"type": "boolean"},
230
+ },
231
+ "required": ["url"],
232
+ "additionalProperties": False,
233
+ },
234
+ },
235
+ {
236
+ "name": "web_clean",
237
+ "description": "Clean a local HTML, PDF, DOCX, XLSX, EPUB or CSV file into Markdown.",
238
+ "inputSchema": {
239
+ "type": "object",
240
+ "properties": {
241
+ "path": {"type": "string"},
242
+ "base_url": {"type": "string"},
243
+ "include_tables": {"type": "boolean"},
244
+ },
245
+ "required": ["path"],
246
+ },
247
+ },
248
+ ]
249
+
250
+ _TOOL_NAMES = {tool["name"] for tool in _TOOLS}
251
+
252
+
253
+ def _full_results(arguments: dict[str, Any]) -> bool:
254
+ """Whether to repeat every page body inside ``results`` as well as ``pages``."""
255
+
256
+ return bool(arguments.get("fullResults", False))
257
+
258
+
259
+ def _config(arguments: dict[str, Any]) -> engine.FetchConfig:
260
+ config = engine.FetchConfig(
261
+ max_chars=int(arguments.get("max_chars", 20_000)),
262
+ chunk_size=int(arguments.get("chunk_size", 0)),
263
+ render=str(arguments.get("render", "auto")),
264
+ include_tables=bool(arguments.get("include_tables", True)),
265
+ cache=bool(arguments.get("cache", False)),
266
+ num_results=int(arguments.get("num_results", 8)),
267
+ max_pages=max(0, int(arguments.get("max_pages", 3))),
268
+ search_pages=max(1, int(arguments.get("search_pages", 1))),
269
+ search_engine=str(arguments.get("engine", "auto")),
270
+ follow=str(arguments.get("follow", "same-site")),
271
+ # robots.txt is checked for every fetched page unless the caller turns
272
+ # it off explicitly, which is the same switch the CLI spells
273
+ # --ignore-robots.
274
+ respect_robots=bool(arguments.get("respect_robots", True)),
275
+ robots_policy=(
276
+ "ignore"
277
+ if not bool(arguments.get("respect_robots", True))
278
+ else str(arguments.get("robots_policy", "respect"))
279
+ ),
280
+ proxy=str(arguments.get("proxy", "")),
281
+ input_gate=str(arguments.get("input_gate", "mark")),
282
+ verbose=False,
283
+ )
284
+ config.validate()
285
+ return config
286
+
287
+
288
+ def _tool_fetch(arguments: dict[str, Any]) -> dict[str, Any]:
289
+ config = _config(arguments)
290
+ if not arguments.get("url"):
291
+ raise ValueError("url is required")
292
+ page = engine.fetch_page(str(arguments["url"]), config)
293
+ return engine.build_envelope("fetch", page, full_results=_full_results(arguments))
294
+
295
+
296
+ def _tool_search(arguments: dict[str, Any]) -> dict[str, Any]:
297
+ if not arguments.get("query"):
298
+ raise ValueError("query is required")
299
+ config = _config(arguments)
300
+ payload = engine.fetch_by_query(
301
+ str(arguments["query"]),
302
+ config,
303
+ engine=str(arguments.get("engine", "auto")),
304
+ )
305
+ return engine.build_envelope("fetch", payload, full_results=_full_results(arguments))
306
+
307
+
308
+ def _tool_surf(arguments: dict[str, Any]) -> dict[str, Any]:
309
+ config = _config(arguments)
310
+ seed = arguments.get("url") or arguments.get("query")
311
+ if not seed:
312
+ raise ValueError("url or query is required")
313
+ payload = engine.surf_web(
314
+ str(seed),
315
+ config,
316
+ depth=max(0, int(arguments.get("depth", 1))),
317
+ follow=str(arguments.get("follow", "same-site")),
318
+ max_pages=max(1, int(arguments.get("max_pages", 3))),
319
+ engine_name=str(arguments.get("engine", "auto")),
320
+ session=arguments.get("session") or None,
321
+ )
322
+ if arguments.get("url") and arguments.get("query"):
323
+ payload.setdefault("warnings", []).append(
324
+ "query was ignored because url was given; omit url to surf from a search"
325
+ )
326
+ return engine.build_envelope("surf", payload, full_results=_full_results(arguments))
327
+
328
+
329
+ def _tool_scholar(arguments: dict[str, Any]) -> dict[str, Any]:
330
+ source = str(arguments.get("source", "arxiv"))
331
+ query = str(arguments.get("query", ""))
332
+ config = _config(arguments)
333
+ payload = scholar.search(
334
+ source,
335
+ query,
336
+ config,
337
+ limit=int(arguments.get("limit", config.num_results)),
338
+ )
339
+ return engine.build_envelope("scholar", payload)
340
+
341
+
342
+ def _tool_map(arguments: dict[str, Any]) -> dict[str, Any]:
343
+ config = _config(arguments)
344
+ payload = sitemap.discover(
345
+ str(arguments.get("url", "")),
346
+ config,
347
+ limit=int(arguments.get("limit", 500)),
348
+ )
349
+ return engine.build_envelope("map", payload)
350
+
351
+
352
+ def _tool_bilibili(arguments: dict[str, Any]) -> dict[str, Any]:
353
+ config = _config(arguments)
354
+ mode = str(arguments.get("mode", "auto"))
355
+ url = str(arguments.get("url") or "")
356
+ query = str(arguments.get("query") or "")
357
+ if mode == "auto":
358
+ mode = "search" if query else "video" if url else "feed"
359
+ if mode == "feed":
360
+ payload = bilibili.feed(
361
+ config, limit=config.num_results, enrich=config.max_pages, url=url or bilibili.BILI_HOME
362
+ )
363
+ elif mode == "video":
364
+ payload = {"mode": "video", "videos": [bilibili.fetch_video(url, config)]}
365
+ payload["results"] = payload["videos"]
366
+ elif mode == "search":
367
+ payload = bilibili.search(query, config)
368
+ elif mode == "related":
369
+ payload = bilibili.related(url, config, limit=config.num_results, enrich=config.max_pages)
370
+ elif mode == "surf":
371
+ payload = bilibili.surf(
372
+ url or query,
373
+ config,
374
+ depth=max(0, int(arguments.get("depth", 1))),
375
+ per_level=config.max_pages,
376
+ )
377
+ else:
378
+ raise ValueError(f"unsupported Bilibili mode: {mode}")
379
+ return engine.build_envelope("bili", payload)
380
+
381
+
382
+ def _tool_doctor(arguments: dict[str, Any]) -> dict[str, Any]:
383
+ config = _config(arguments)
384
+ payload = engine.doctor_report(config)
385
+ if arguments.get("probe"):
386
+ payload["search_probe"] = engine.probe_engines(config)
387
+ return engine.build_envelope("doctor", payload)
388
+
389
+
390
+ def _tool_find(arguments: dict[str, Any]) -> dict[str, Any]:
391
+ site = str(arguments.get("site", "")).strip()
392
+ query = str(arguments.get("query", "")).strip()
393
+ if not site:
394
+ raise ValueError("site is required")
395
+ if not query:
396
+ raise ValueError("query is required")
397
+ config = _config(arguments)
398
+ payload = engine.find_within_site(
399
+ site,
400
+ query,
401
+ config,
402
+ max_pages=int(arguments.get("max_pages", config.max_pages)),
403
+ )
404
+ return engine.build_envelope("find", payload, full_results=_full_results(arguments))
405
+
406
+
407
+ def _tool_feed(arguments: dict[str, Any]) -> dict[str, Any]:
408
+ url = str(arguments.get("url", "")).strip()
409
+ if not url:
410
+ raise ValueError("url is required")
411
+ config = _config(arguments)
412
+ payload = engine.discover_feed(url, config)
413
+ payload["results"] = [
414
+ {
415
+ "url": item.get("url", ""),
416
+ "title": item.get("title", ""),
417
+ "published": item.get("published", ""),
418
+ }
419
+ for item in payload.get("items") or []
420
+ ]
421
+ return engine.build_envelope("feed", payload, full_results=_full_results(arguments))
422
+
423
+
424
+ def _tool_clean(arguments: dict[str, Any]) -> dict[str, Any]:
425
+ config = _config(arguments)
426
+ path = Path(str(arguments.get("path", ""))).expanduser()
427
+ if not path.is_file():
428
+ raise ValueError(f"file not found: {path}")
429
+ data = path.read_bytes()
430
+ base_url = str(arguments.get("base_url") or "")
431
+ if pdf.is_pdf(data):
432
+ document = pdf.extract(data)
433
+ markdown = pdf.to_markdown(document, url=base_url)
434
+ metadata = pdf.to_metadata(document, url=base_url)
435
+ tables: list[dict[str, Any]] = []
436
+ kind = "pdf"
437
+ elif (kind := documents.detect(data, url=str(path))) and kind:
438
+ extracted = documents.extract(data, kind, url=str(path))
439
+ markdown = extracted["markdown"]
440
+ metadata = documents.to_metadata(extracted, base_url)
441
+ tables = extracted.get("tables") or []
442
+ else:
443
+ html = engine.decode_bytes(data)
444
+ markdown = engine.html_to_markdown(
445
+ html,
446
+ url=base_url or None,
447
+ include_tables=config.include_tables,
448
+ keep_links=config.keep_links,
449
+ )
450
+ metadata = engine.extract_metadata(html, url=base_url or None)
451
+ tables = engine.extract_tables(html) if config.include_tables else []
452
+ kind = "html"
453
+ if config.max_chars and len(markdown) > config.max_chars:
454
+ markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
455
+ payload = {
456
+ "source": str(path),
457
+ "content_type": kind,
458
+ "metadata": metadata,
459
+ "tables": tables,
460
+ "markdown": markdown,
461
+ "chunks": engine.chunk_markdown(markdown, config.chunk_size) if config.chunk_size else [],
462
+ }
463
+ return engine.build_envelope("clean", payload)
464
+
465
+
466
+ _DISPATCH: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
467
+ "web_fetch": _tool_fetch,
468
+ "web_search": _tool_search,
469
+ "web_surf": _tool_surf,
470
+ "web_scholar": _tool_scholar,
471
+ "web_map": _tool_map,
472
+ "web_bilibili": _tool_bilibili,
473
+ "web_doctor": _tool_doctor,
474
+ "web_find": _tool_find,
475
+ "web_feed": _tool_feed,
476
+ "web_clean": _tool_clean,
477
+ }
478
+
479
+
480
+ def _result(request_id: Any, result: dict[str, Any]) -> dict[str, Any]:
481
+ return {"jsonrpc": "2.0", "id": request_id, "result": result}
482
+
483
+
484
+ def _error(request_id: Any, code: int, message: str) -> dict[str, Any]:
485
+ return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
486
+
487
+
488
+ def handle_request(request: dict[str, Any]) -> Optional[dict[str, Any]]:
489
+ """Handle one JSON-RPC object; return None for notifications."""
490
+
491
+ method = request.get("method", "")
492
+ request_id = request.get("id")
493
+ params = request.get("params") or {}
494
+
495
+ if method == "initialize":
496
+ return _result(
497
+ request_id,
498
+ {
499
+ "protocolVersion": params.get("protocolVersion") or PROTOCOL_VERSION,
500
+ "capabilities": {"tools": {}},
501
+ "serverInfo": {"name": SERVER_NAME, "version": engine.__version__},
502
+ },
503
+ )
504
+ if method in ("notifications/initialized", "initialized", "notifications/cancelled"):
505
+ return None
506
+ if method == "ping":
507
+ return _result(request_id, {})
508
+ if method == "tools/list":
509
+ return _result(request_id, {"tools": _TOOLS})
510
+ if method == "tools/call":
511
+ name = str(params.get("name", ""))
512
+ arguments = params.get("arguments") or {}
513
+ if name not in _TOOL_NAMES:
514
+ return _error(request_id, -32602, f"unknown tool: {name}")
515
+ try:
516
+ payload = _DISPATCH[name](dict(arguments))
517
+ text = json.dumps(payload, ensure_ascii=False, indent=2)
518
+ return _result(
519
+ request_id,
520
+ {"content": [{"type": "text", "text": text}], "isError": False},
521
+ )
522
+ except Exception as exc: # tool failures are data, not protocol failures
523
+ code = exc.__class__.__name__
524
+ if code == "ProhibitedQuery":
525
+ code = "prohibited_query"
526
+ elif code == "UnsafeAddress":
527
+ code = "unsafe_address"
528
+ elif code == "VerificationRequired":
529
+ code = "verification_required"
530
+ error: dict[str, Any] = {"code": code, "message": str(exc)}
531
+ if code == "verification_required":
532
+ error["hint"] = (
533
+ "Retry in browser mode with playbook wechat-dig and a profile that "
534
+ "has passed verification; web-forager does not solve challenges."
535
+ )
536
+ text = json.dumps(
537
+ {
538
+ "ok": False,
539
+ "error": error,
540
+ },
541
+ ensure_ascii=False,
542
+ )
543
+ return _result(
544
+ request_id,
545
+ {"content": [{"type": "text", "text": text}], "isError": True},
546
+ )
547
+ return _error(request_id, -32601, f"method not found: {method}")
548
+
549
+
550
+ def _utf8_stream(stream: Any) -> Any:
551
+ """Force UTF-8 on a text stream when the host allows it.
552
+
553
+ Windows hosts may launch ``python`` with the ANSI code page (cp1252/cp936)
554
+ as the stdio encoding. Writing a Chinese title then raises
555
+ ``UnicodeEncodeError`` and kills the server, so the streams are
556
+ reconfigured before the loop starts. ``serve`` still has an ASCII
557
+ fallback for streams that cannot be reconfigured.
558
+ """
559
+
560
+ reconfigure = getattr(stream, "reconfigure", None)
561
+ if reconfigure is None:
562
+ return stream
563
+ try:
564
+ reconfigure(encoding="utf-8", errors="strict")
565
+ except (AttributeError, OSError, ValueError):
566
+ return stream
567
+ return stream
568
+
569
+
570
+ def serve(stdin: Any = None, stdout: Any = None) -> int:
571
+ stdin = _utf8_stream(stdin or sys.stdin)
572
+ stdout = _utf8_stream(stdout or sys.stdout)
573
+ for line in stdin:
574
+ line = line.strip()
575
+ if not line:
576
+ continue
577
+ try:
578
+ request = json.loads(line)
579
+ except json.JSONDecodeError:
580
+ response = _error(None, -32700, "parse error")
581
+ else:
582
+ if not isinstance(request, dict):
583
+ response = _error(None, -32600, "invalid request")
584
+ else:
585
+ response = handle_request(request)
586
+ if response is not None:
587
+ try:
588
+ payload = json.dumps(response, ensure_ascii=False)
589
+ stdout.write(payload + "\n")
590
+ except UnicodeEncodeError:
591
+ stdout.write(json.dumps(response, ensure_ascii=True) + "\n")
592
+ stdout.flush()
593
+ return 0
594
+
595
+
596
+ if __name__ == "__main__":
597
+ raise SystemExit(serve())
package/package.json CHANGED
@@ -1,6 +1,80 @@
1
- {
2
- "name": "web-forager",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
1
+ {
2
+ "name": "web-forager",
3
+ "version": "1.3.0",
4
+ "description": "Clean-room, robots-aware web information gathering: multi-engine search, HTML/PDF/Office/feed extraction, Bilibili/Zhihu/Xiaohongshu/WeChat reading, sitemaps and resumable surfing, plus a browser mode with eight read-only playbooks. CLI + two MCP servers.",
5
+ "main": "index.js",
6
+ "license": "MIT",
7
+ "engines": {
8
+ "node": ">=18"
9
+ },
10
+ "keywords": [
11
+ "dsh",
12
+ "dsh-plugin",
13
+ "deepseek-harness",
14
+ "web-scraping",
15
+ "search",
16
+ "mcp",
17
+ "mcp-server",
18
+ "bilibili",
19
+ "zhihu",
20
+ "crawler",
21
+ "robots-txt",
22
+ "playwright"
23
+ ],
24
+ "repository": {
25
+ "type": "git",
26
+ "url": "git+https://github.com/JiamingShi-JsPhys/web-forager.git"
27
+ },
28
+ "homepage": "https://github.com/JiamingShi-JsPhys/web-forager#readme",
29
+ "bugs": {
30
+ "url": "https://github.com/JiamingShi-JsPhys/web-forager/issues"
31
+ },
32
+ "scripts": {
33
+ "test": "python -m pytest",
34
+ "test:browser": "node --test \"browser/tests/*.test.js\"",
35
+ "test:live": "node browser/tests/live-smoke.js",
36
+ "prepublishOnly": "node clean-pycache.mjs"
37
+ },
38
+ "files": [
39
+ "index.js",
40
+ "cordis.patch.yml",
41
+ ".claude-plugin/",
42
+ ".codex-plugin/",
43
+ ".codebuddy-plugin/",
44
+ ".dsh-plugin/",
45
+ ".cursor/",
46
+ ".clinerules/",
47
+ ".continue/",
48
+ ".roo/",
49
+ ".windsurf/",
50
+ ".github/copilot-instructions.md",
51
+ ".mcp.json",
52
+ "install.py",
53
+ "requirements.txt",
54
+ "run.py",
55
+ "mcp_server.py",
56
+ "browser/run.js",
57
+ "browser/mcp_server.js",
58
+ "browser/launch_chrome.py",
59
+ "browser/package.json",
60
+ "browser/lib/",
61
+ "browser/playbooks/",
62
+ "browser/bridges/",
63
+ "scripts/",
64
+ "commands/",
65
+ "docs/",
66
+ "SKILL.md",
67
+ "AGENTS.md",
68
+ "manifest.json",
69
+ "README.md",
70
+ "README_zh.md",
71
+ "CHANGELOG.md",
72
+ "THIRD_PARTY_LICENSES.txt",
73
+ "LICENSE"
74
+ ],
75
+ "dsh": {
76
+ "bundle": {
77
+ "patch": "./cordis.patch.yml"
78
+ }
79
+ }
80
+ }