web-forager 0.0.0-stage → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +119 -0
  18. package/CHANGELOG.md +508 -0
  19. package/LICENSE +21 -0
  20. package/README.md +498 -2
  21. package/README_zh.md +498 -0
  22. package/SKILL.md +451 -0
  23. package/THIRD_PARTY_LICENSES.txt +40 -0
  24. package/browser/bridges/playwright-bridge.js +382 -0
  25. package/browser/launch_chrome.py +193 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +90 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +141 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +130 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +215 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +252 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +107 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +441 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/research.md +35 -0
  58. package/commands/scholar.md +28 -0
  59. package/commands/surf.md +28 -0
  60. package/cordis.patch.yml +19 -0
  61. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  62. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  63. package/docs/DEPENDENCIES.md +40 -0
  64. package/docs/INSTALL_COMMANDS.md +113 -0
  65. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  66. package/docs/REFERENCE.md +349 -0
  67. package/docs/REFERENCE_zh.md +342 -0
  68. package/index.js +7 -0
  69. package/install.py +254 -0
  70. package/manifest.json +98 -0
  71. package/mcp_server.py +850 -0
  72. package/package.json +82 -6
  73. package/requirements.txt +15 -0
  74. package/run.py +908 -0
  75. package/scripts/__init__.py +1 -0
  76. package/scripts/bilibili.py +882 -0
  77. package/scripts/documents.py +437 -0
  78. package/scripts/engine.py +5141 -0
  79. package/scripts/feeds.py +180 -0
  80. package/scripts/pdf.py +156 -0
  81. package/scripts/research.py +2087 -0
  82. package/scripts/scholar.py +381 -0
  83. package/scripts/sitemap.py +195 -0
  84. package/scripts/untrusted.py +312 -0
package/run.py ADDED
@@ -0,0 +1,908 @@
1
+ #!/usr/bin/env python3
2
+ """web-forager command-line entry point.
3
+
4
+ Actions
5
+ -------
6
+ fetch
7
+ ``--url URL`` fetch one page and return metadata, Markdown and tables
8
+ ``--query TEXT`` search (360+Bing by default), then fetch the top results
9
+
10
+ research
11
+ ``--query TEXT`` broad recall, then read a few authoritative sources and
12
+ return only the passages that match the query, with citations
13
+
14
+ clean
15
+ ``--input file`` clean a local HTML/PDF/DOCX/XLSX/EPUB/CSV file
16
+ ``--url URL`` fetch a page/document and clean it
17
+ (no input) read HTML/PDF/document bytes from stdin
18
+
19
+ bili
20
+ ``--url BV...`` / ``--query TEXT`` / neither Bilibili video, search or feed
21
+
22
+ surf
23
+ ``--url URL`` or ``--query TEXT`` open/search, then follow ranked links
24
+ ``--depth N --follow same-site|all|none --session NAME``
25
+
26
+ scholar
27
+ ``--source arxiv|crossref|pubmed|doi --query TEXT`` bibliographic records
28
+
29
+ map
30
+ ``--url SITE`` discover page URLs from robots.txt / sitemaps
31
+
32
+ Global
33
+ ``--engine auto|all|360|bing|sogou|weixin|duckduckgo``
34
+ ``--cache --chunk-size N --search-pages N``
35
+ ``--agent`` OpenCLI-style ``{ok,data}`` / ``{ok,error}`` envelope for agents
36
+
37
+ MCP
38
+ ``python mcp_server.py`` stdio JSON-RPC server exposing the same engine.
39
+
40
+ 本插件仅供学习与信息收集使用。使用者需遵守目标网站的 Terms of Service 及相关版权法律。
41
+ 禁止用于恶意并发攻击。严禁收集、聚合或推断特定自然人的个人信息,或将其用于人肉搜索、骚扰、跟踪等侵害他人隐私的活动;此类用法产生的法律责任由使用者自行承担。
42
+ """
43
+
44
+ from __future__ import annotations
45
+
46
+ import argparse
47
+ import hashlib
48
+ import json
49
+ import re
50
+ import sys
51
+ from pathlib import Path
52
+
53
+ _PLUGIN_ROOT = Path(__file__).resolve().parent
54
+ MAX_LOCAL_BYTES = 16 * 1024 * 1024
55
+ if str(_PLUGIN_ROOT) not in sys.path:
56
+ sys.path.insert(0, str(_PLUGIN_ROOT))
57
+
58
+ from scripts import bilibili, documents, engine, pdf as pdf_mod, research, scholar, sitemap # noqa: E402
59
+
60
+
61
+ def _reconfigure_stdio() -> None:
62
+ """Force UTF-8 so Chinese metadata survives Windows code pages."""
63
+
64
+ for stream in (sys.stdout, sys.stderr):
65
+ try:
66
+ stream.reconfigure(encoding="utf-8") # type: ignore[attr-defined]
67
+ except Exception:
68
+ pass
69
+
70
+
71
+ def build_parser() -> argparse.ArgumentParser:
72
+ parser = argparse.ArgumentParser(
73
+ prog="web-forager",
74
+ description=(
75
+ "Polite, robots-aware web information gathering and "
76
+ "HTML-to-Markdown cleaning for public pages."
77
+
78
+ ),
79
+ epilog=engine.DISCLAIMER,
80
+ formatter_class=argparse.RawDescriptionHelpFormatter,
81
+ )
82
+ parser.add_argument(
83
+ "--action",
84
+ required=True,
85
+ choices=["fetch", "research", "find", "feed", "clean", "bili", "surf", "scholar", "map", "doctor"],
86
+ help="operation to run",
87
+ )
88
+ parser.add_argument("--url", help="target URL, or a Bilibili BV/av id (fetch/clean/bili/surf/map)")
89
+ parser.add_argument("--query", help="keyword query (fetch/research/bili/surf/scholar)")
90
+ parser.add_argument("--site", help="domain or base URL to search inside (find)")
91
+ parser.add_argument("--input", dest="input_path", help="local HTML/PDF/DOCX/XLSX/EPUB/CSV file (clean)")
92
+ parser.add_argument(
93
+ "--source",
94
+ choices=list(scholar.SCHOLAR_SOURCES),
95
+ default="arxiv",
96
+ help="scholarly source for --action scholar (default: arxiv)",
97
+ )
98
+ parser.add_argument(
99
+ "--sort",
100
+ choices=["relevance", "submittedDate", "lastUpdatedDate"],
101
+ default="relevance",
102
+ help="arXiv sort order (default: relevance)",
103
+ )
104
+ parser.add_argument("--limit", type=int, default=500, help="map: maximum URLs to return (default: 500)")
105
+ parser.add_argument("--max-sitemaps", type=int, default=5, help="map: sitemap files to read (default: 5)")
106
+ parser.add_argument(
107
+ "--engine",
108
+ choices=list(engine.SEARCH_ENGINES),
109
+ default="auto",
110
+ help="search backend: auto=360+Bing (mostly-CJK queries: 360+Sogou+WeChat); all=every engine (default: auto)",
111
+ )
112
+ parser.add_argument("--num-results", type=int, default=8, help="search results to request (default: 8; research: recall pool)")
113
+ parser.add_argument(
114
+ "--focus",
115
+ type=int,
116
+ default=3,
117
+ help="research: distinct sources to read in full (default: 3)",
118
+ )
119
+ parser.add_argument(
120
+ "--focus-chars",
121
+ type=int,
122
+ default=40_000,
123
+ help="research: maximum Markdown chars read per focused page (default: 40000)",
124
+ )
125
+ parser.add_argument(
126
+ "--serial-engines",
127
+ action="store_true",
128
+ help="ask search engines one after another (default: parallel across different hosts)",
129
+ )
130
+ parser.add_argument(
131
+ "--max-pages",
132
+ type=int,
133
+ default=3,
134
+ help="result pages to fetch (default: 3; 0 = search results only)",
135
+ )
136
+ parser.add_argument("--search-pages", type=int, default=1, help="search result pages to read per engine (default: 1)")
137
+ parser.add_argument(
138
+ "--mode",
139
+ choices=["auto", "feed", "video", "search", "related", "surf"],
140
+ default="auto",
141
+ help="Bilibili mode (default auto: query->search, url->video, neither->feed)",
142
+ )
143
+ parser.add_argument("--depth", type=int, default=1, help="Bilibili/surf depth (default: 1)")
144
+ parser.add_argument(
145
+ "--follow",
146
+ choices=["same-site", "all", "none"],
147
+ default="same-site",
148
+ help="surf link policy: same-site (default), all, none",
149
+ )
150
+ parser.add_argument(
151
+ "--session",
152
+ help="surf session name; visited URLs and the trail are saved under ~/.web-forager/sessions/",
153
+ )
154
+ parser.add_argument(
155
+ "--direct-search",
156
+ action="store_true",
157
+ help=(
158
+ "Bilibili only: query search.bilibili.com directly. That host disallows "
159
+ "generic agents in robots.txt, so this is an explicit opt-in override; "
160
+ "video pages are still fetched with robots checks enabled."
161
+ ),
162
+ )
163
+ parser.add_argument(
164
+ "--render",
165
+ choices=["auto", "never", "always"],
166
+ default="auto",
167
+ help="dynamic rendering with Playwright: auto detects JS shells (default), never, always",
168
+ )
169
+ parser.add_argument("--no-tables", action="store_true", help="do not extract tables")
170
+ parser.add_argument("--keep-links", action="store_true", help="preserve hyperlinks in Markdown")
171
+ parser.add_argument("--images", action="store_true", help="preserve image references in Markdown")
172
+ parser.add_argument("--max-chars", type=int, default=20_000, help="truncate Markdown to N characters (0 = no limit)")
173
+ parser.add_argument(
174
+ "--chunk-size",
175
+ type=int,
176
+ default=0,
177
+ help="also emit Markdown chunks of about N characters (0 = off)",
178
+ )
179
+ parser.add_argument("--cache", action="store_true", help="reuse responses cached under ~/.web-forager/cache")
180
+ parser.add_argument("--cache-ttl", type=int, default=3600, help="cache lifetime in seconds (default: 3600)")
181
+ parser.add_argument("--cache-dir", default="", help="override the cache directory")
182
+ parser.add_argument("--wait-for", default="", help="Playwright: wait for this CSS selector before reading the page")
183
+ parser.add_argument("--scroll", type=int, default=0, help="Playwright: wheel-scroll N times to load lazy content")
184
+ parser.add_argument("--timeout", type=float, default=20.0, help="per-request timeout in seconds")
185
+ parser.add_argument("--delay", type=float, default=1.0, help="politeness delay per host in seconds")
186
+ parser.add_argument(
187
+ "--robots-policy",
188
+ choices=("respect", "warn", "ignore"),
189
+ default="respect",
190
+ help=(
191
+ "respect: refuse a disallowed path (default); warn: read it and mark "
192
+ "the record with compliance.robots_overridden; ignore: do not consult robots.txt"
193
+ ),
194
+ )
195
+ parser.add_argument(
196
+ "--ignore-robots",
197
+ action="store_true",
198
+ help="DANGEROUS: same as --robots-policy ignore (only for resources you are authorised to access)",
199
+ )
200
+ parser.add_argument(
201
+ "--query-robots",
202
+ choices=("respect", "warn", "ignore"),
203
+ default=None,
204
+ help=(
205
+ "policy for search-engine and scholarly query endpoints "
206
+ "(default: WEB_FORAGER_QUERY_ROBOTS or warn). These endpoints "
207
+ "are not target pages; warn reads a disallowed one and records the "
208
+ "override, respect refuses it, ignore skips the check. MCP does not "
209
+ "expose this switch to the model."
210
+ ),
211
+ )
212
+ parser.add_argument(
213
+ "--allow-private",
214
+ action="store_true",
215
+ help=(
216
+ "DANGEROUS: allow loopback, private and link-local targets. "
217
+ "For local fixtures and trusted intranet tools only"
218
+ ),
219
+ )
220
+ parser.add_argument(
221
+ "--input-gate",
222
+ choices=["off", "mark", "strict"],
223
+ default="mark",
224
+ help=(
225
+ "treat fetched text as untrusted: mark instruction-shaped content "
226
+ "(default), withhold the body on a hard hit (strict), or skip the scan"
227
+ ),
228
+ )
229
+ parser.add_argument("--user-agent", default=engine.DEFAULT_USER_AGENT, help="explicit User-Agent string")
230
+ parser.add_argument(
231
+ "--probe",
232
+ action="store_true",
233
+ help="doctor: ask each search engine once and report which ones answer",
234
+ )
235
+ parser.add_argument(
236
+ "--proxy",
237
+ default="",
238
+ help="route every request through this proxy (default: HTTPS_PROXY / ALL_PROXY from the environment)",
239
+ )
240
+ parser.add_argument("--format", choices=["json", "markdown"], default="json", help="stdout format")
241
+ parser.add_argument(
242
+ "--full-results",
243
+ action="store_true",
244
+ help=(
245
+ "repeat every page record inside results as well as pages "
246
+ "(1.0.1 behaviour; off by default because it doubles the response)"
247
+ ),
248
+ )
249
+ parser.add_argument(
250
+ "--core",
251
+ action="store_true",
252
+ help="report results in the shared core-1 schema that browser mode also emits",
253
+ )
254
+ parser.add_argument(
255
+ "--agent",
256
+ action="store_true",
257
+ help="OpenCLI-style envelope: {ok:true,data:...} or {ok:false,error:{code,message,hint}}",
258
+ )
259
+ parser.add_argument("--out", help="write the result to this file instead of stdout")
260
+ parser.add_argument(
261
+ "--root",
262
+ help=(
263
+ "Chelate workspace root; writes <root>/process/web-forager/ and a "
264
+ "<root>/_broadcast/ index entry. Ignored when --out is given."
265
+ ),
266
+ )
267
+ parser.add_argument("--verbose", "-v", action="store_true", help="diagnostics on stderr")
268
+ parser.add_argument("--version", action="version", version=f"web-forager {engine.__version__}")
269
+ return parser
270
+
271
+
272
+ def build_config(args: argparse.Namespace) -> engine.FetchConfig:
273
+ config = engine.FetchConfig(
274
+ user_agent=args.user_agent,
275
+ timeout=args.timeout,
276
+ delay=args.delay,
277
+ max_chars=args.max_chars,
278
+ respect_robots=not args.ignore_robots,
279
+ robots_policy="ignore" if args.ignore_robots else args.robots_policy,
280
+ query_robots=args.query_robots or engine._query_robots_from_env(),
281
+ render=args.render,
282
+ include_tables=not args.no_tables,
283
+ keep_links=args.keep_links,
284
+ include_images=args.images,
285
+ max_pages=max(0, args.max_pages),
286
+ num_results=max(1, args.num_results),
287
+ verbose=args.verbose,
288
+ extract_links=args.action == "surf",
289
+ follow=args.follow,
290
+ search_engine=args.engine,
291
+ parallel_engines=not args.serial_engines,
292
+ proxy=args.proxy,
293
+ search_pages=max(1, args.search_pages),
294
+ cache=args.cache,
295
+ cache_ttl=max(0, args.cache_ttl),
296
+ cache_dir=args.cache_dir,
297
+ chunk_size=max(0, args.chunk_size),
298
+ wait_for=args.wait_for,
299
+ scroll=max(0, args.scroll),
300
+ allow_private=bool(args.allow_private) or engine._allow_private_from_env(),
301
+ input_gate=args.input_gate,
302
+ )
303
+ config.validate()
304
+ return config
305
+
306
+
307
+ # ---------------------------------------------------------------------------
308
+ # Output helpers
309
+ # ---------------------------------------------------------------------------
310
+
311
+
312
+ def document_to_markdown(document: dict) -> str:
313
+ """Render a result envelope as a human-readable Markdown document."""
314
+
315
+ if document.get("action") == "clean":
316
+ return document.get("markdown", "")
317
+ if document.get("action") == "scholar":
318
+ return scholar.to_markdown(document)
319
+ if document.get("action") == "map":
320
+ return sitemap.to_markdown(document)
321
+
322
+ blocks: list[str] = []
323
+ for record in document.get("results", []):
324
+ if record.get("error"):
325
+ blocks.append(f"## {record.get('url', 'unknown')}\n\n**Error:** {record['error']}")
326
+ continue
327
+ title = record.get("title") or record.get("final_url") or record.get("url", "")
328
+ if not record.get("markdown"):
329
+ blocks.append(f"- [{title}]({record.get('url', '')})")
330
+ continue
331
+ lines = [f"# {title}", "", f"> Source: {record.get('final_url') or record.get('url', '')}", ""]
332
+ metadata = record.get("metadata") or {}
333
+ if metadata.get("authors"):
334
+ lines.append(f"**Authors:** {', '.join(metadata['authors'])}")
335
+ if metadata.get("published"):
336
+ lines.append(f"**Published:** {metadata['published']}")
337
+ if metadata.get("doi"):
338
+ lines.append(f"**DOI:** {metadata['doi']}")
339
+ if metadata.get("journal"):
340
+ lines.append(f"**Journal:** {metadata['journal']}")
341
+ if metadata.get("authors") or metadata.get("published") or metadata.get("doi") or metadata.get("journal"):
342
+ lines.append("")
343
+ lines.append(record.get("markdown", "").strip())
344
+ blocks.append("\n".join(lines).strip())
345
+ return "\n\n---\n\n".join(blocks)
346
+
347
+
348
+ def emit(text: str, output_path: str | None) -> None:
349
+ if output_path:
350
+ target = Path(output_path)
351
+ if target.parent and not target.parent.exists():
352
+ target.parent.mkdir(parents=True, exist_ok=True)
353
+ target.write_text(text, encoding="utf-8", newline="\n")
354
+ print(f"[web-forager] wrote {target}", file=sys.stderr)
355
+ else:
356
+ sys.stdout.write(text)
357
+ if not text.endswith("\n"):
358
+ sys.stdout.write("\n")
359
+ sys.stdout.flush()
360
+
361
+
362
+ # ---------------------------------------------------------------------------
363
+ # Chelate workspace integration
364
+ # ---------------------------------------------------------------------------
365
+
366
+
367
+ def _workspace_slug(args: argparse.Namespace) -> str:
368
+ raw = args.query or args.url or (Path(args.input_path).stem if args.input_path else "stdin")
369
+ slug = re.sub(r"[^A-Za-z0-9._-]+", "-", str(raw)).strip("-._")
370
+ if not slug:
371
+ slug = "result-" + hashlib.sha1(str(raw).encode("utf-8")).hexdigest()[:8]
372
+ return slug[:48]
373
+
374
+
375
+ def _workspace_timestamp(document: dict) -> str:
376
+ stamp = str(document.get("generated_at") or engine.utc_now())
377
+ return re.sub(r"[^0-9TZ]", "", stamp) or "00000000T000000Z"
378
+
379
+
380
+ def _document_summary(document: dict) -> str:
381
+ action = document.get("action")
382
+ if action == "clean":
383
+ return f"cleaned {document.get('source', 'input')}"
384
+ if action == "scholar":
385
+ return f"{document.get('count', 0)} record(s) from {document.get('source', 'scholar')}"
386
+ if action == "map":
387
+ return f"{document.get('count', 0)} URL(s) from {document.get('origin', 'sitemap')}"
388
+ results = document.get("results") or []
389
+ if action == "bili":
390
+ first = results[0] if results else {}
391
+ title = first.get("title") or document.get("query") or document.get("seed") or ""
392
+ mode = document.get("mode", "bili")
393
+ return f"bilibili {mode} · {len(results)} item(s)" + (f" · {title}" if title else "")
394
+ first = results[0] if results else {}
395
+ title = first.get("title") or first.get("url") or ""
396
+ return f"{len(results)} page(s)" + (f" · {title}" if title else "")
397
+
398
+
399
+ def write_workspace(
400
+ args: argparse.Namespace,
401
+ document: dict,
402
+ json_text: str,
403
+ markdown_text: str,
404
+ ) -> list[Path]:
405
+ """Write the run into a Chelate four-zone workspace.
406
+
407
+ Layout follows the main library:
408
+
409
+ * ``<root>/process/web-forager/`` - intermediate JSON + Markdown artefacts
410
+ * ``<root>/_broadcast/`` - one read-only index entry per run
411
+ """
412
+
413
+ root = Path(args.root).expanduser().resolve()
414
+ if root.exists() and not root.is_dir():
415
+ raise ValueError(f"--root is not a directory: {root}")
416
+
417
+ process_dir = root / "process" / "web-forager"
418
+ broadcast_dir = root / "_broadcast"
419
+ process_dir.mkdir(parents=True, exist_ok=True)
420
+ broadcast_dir.mkdir(parents=True, exist_ok=True)
421
+
422
+ base = f"web-forager_{_workspace_timestamp(document)}_{_workspace_slug(args)}"
423
+ json_path = process_dir / f"{base}.json"
424
+ markdown_path = process_dir / f"{base}.md"
425
+ index_path = broadcast_dir / f"{base}.json"
426
+
427
+ json_path.write_text(json_text + "\n", encoding="utf-8", newline="\n")
428
+ markdown_path.write_text(markdown_text + "\n", encoding="utf-8", newline="\n")
429
+
430
+ index = {
431
+ "tool": "web-forager",
432
+ "version": engine.__version__,
433
+ "action": document.get("action", ""),
434
+ "generated_at": document.get("generated_at", ""),
435
+ "summary": _document_summary(document),
436
+ "source": document.get("source") or document.get("query") or document.get("seed") or "",
437
+ "files": {
438
+ "json": json_path.relative_to(root).as_posix(),
439
+ "markdown": markdown_path.relative_to(root).as_posix(),
440
+ },
441
+ }
442
+ index_path.write_text(json.dumps(index, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", newline="\n")
443
+
444
+ for path in (json_path, markdown_path, index_path):
445
+ print(f"[web-forager] workspace: {path}", file=sys.stderr)
446
+ return [json_path, markdown_path, index_path]
447
+
448
+
449
+ # ---------------------------------------------------------------------------
450
+ # Actions
451
+ # ---------------------------------------------------------------------------
452
+
453
+
454
+ def _envelope(action: str, payload: dict, args: argparse.Namespace) -> dict:
455
+ """Build the response envelope, honouring --full-results."""
456
+
457
+ return engine.build_envelope(action, payload, full_results=bool(getattr(args, "full_results", False)))
458
+
459
+
460
+ def cmd_research(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
461
+ if not args.query:
462
+ raise ValueError("--action research needs --query")
463
+ payload = research.research(
464
+ args.query,
465
+ config,
466
+ engine_name=args.engine,
467
+ focus=args.focus,
468
+ breadth=args.num_results,
469
+ focus_chars=args.focus_chars,
470
+ )
471
+ return _envelope("research", payload, args)
472
+
473
+
474
+ def cmd_find(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
475
+ if not args.site:
476
+ raise ValueError("--action find needs --site")
477
+ if not args.query:
478
+ raise ValueError("--action find needs --query")
479
+ payload = engine.find_within_site(
480
+ args.site, args.query, config, max_pages=args.max_pages
481
+ )
482
+ return _envelope("find", payload, args)
483
+
484
+
485
+ def cmd_feed(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
486
+ if not args.url:
487
+ raise ValueError("--action feed needs --url")
488
+ payload = engine.discover_feed(args.url, config)
489
+ payload["results"] = [
490
+ {
491
+ "url": item.get("url", ""),
492
+ "title": item.get("title", ""),
493
+ "published": item.get("published", ""),
494
+ }
495
+ for item in payload.get("items") or []
496
+ ]
497
+ return _envelope("feed", payload, args)
498
+
499
+
500
+ def cmd_fetch(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
501
+ if bool(args.url) == bool(args.query):
502
+ raise ValueError("--action fetch needs exactly one of --url or --query")
503
+
504
+ if args.url:
505
+ page = engine.fetch_page(args.url, config)
506
+ return _envelope("fetch", page, args)
507
+
508
+ payload = engine.fetch_by_query(args.query, config, engine=args.engine)
509
+ return _envelope("fetch", payload, args)
510
+
511
+
512
+ def _clean_chunks(payload: dict, config: engine.FetchConfig) -> dict:
513
+ if config.chunk_size:
514
+ payload["chunks"] = engine.chunk_markdown(payload.get("markdown", ""), config.chunk_size)
515
+ return payload
516
+
517
+
518
+ def _pdf_document(
519
+ data: bytes,
520
+ source: str,
521
+ url: str,
522
+ config: engine.FetchConfig,
523
+ ) -> dict:
524
+ """Build a clean-style document from PDF bytes.
525
+
526
+ ``pdf_mod`` raises ``PdfUnavailable`` when pypdf is missing and
527
+ ``PdfExtractionError`` when the file cannot be parsed; ``main`` maps both
528
+ to stable exit codes instead of letting a traceback escape.
529
+ """
530
+
531
+ extracted = pdf_mod.extract(data)
532
+ markdown = pdf_mod.to_markdown(extracted, url=url or "")
533
+ if config.max_chars and len(markdown) > config.max_chars:
534
+ markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
535
+
536
+ return {
537
+ "tool": "web-forager",
538
+ "version": engine.__version__,
539
+ "action": "clean",
540
+ "generated_at": engine.utc_now(),
541
+ "source": source,
542
+ "content_type": "application/pdf",
543
+ "page_count": extracted.get("page_count", 0),
544
+ "empty_pages": extracted.get("empty_pages", 0),
545
+ "metadata": pdf_mod.to_metadata(extracted, url=url or ""),
546
+ "tables": [],
547
+ "markdown": markdown,
548
+ "chunks": engine.chunk_markdown(markdown, config.chunk_size) if config.chunk_size else [],
549
+ }
550
+
551
+
552
+ def _document_document(
553
+ data: bytes,
554
+ kind: str,
555
+ source: str,
556
+ url: str,
557
+ config: engine.FetchConfig,
558
+ ) -> dict:
559
+ extracted = documents.extract(data, kind, url=url or source)
560
+ markdown = extracted["markdown"]
561
+ if config.max_chars and len(markdown) > config.max_chars:
562
+ markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
563
+ payload = {
564
+ "tool": "web-forager",
565
+ "version": engine.__version__,
566
+ "action": "clean",
567
+ "generated_at": engine.utc_now(),
568
+ "source": source,
569
+ "content_type": kind,
570
+ "metadata": documents.to_metadata(extracted, url=url or ""),
571
+ "tables": extracted.get("tables") or [],
572
+ "markdown": markdown,
573
+ }
574
+ return _clean_chunks(payload, config)
575
+
576
+
577
+ def cmd_clean(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
578
+ source = ""
579
+ base_url: str | None = None
580
+ fetcher: engine.HttpFetcher | None = None
581
+
582
+ try:
583
+ if args.input_path and args.url:
584
+ raise ValueError("--action clean accepts --input or --url, not both")
585
+
586
+ if args.input_path:
587
+ path = Path(args.input_path)
588
+ if not path.is_file():
589
+ raise ValueError(f"input file not found: {path}")
590
+ if path.stat().st_size > MAX_LOCAL_BYTES:
591
+ raise ValueError(
592
+ f"input file is larger than {MAX_LOCAL_BYTES // (1024 * 1024)} MiB; "
593
+ "web-forager does not read unbounded local files"
594
+ )
595
+ data = path.read_bytes()
596
+ source = str(path)
597
+ if pdf_mod.is_pdf(data):
598
+ return _pdf_document(data, source, "", config)
599
+ kind = documents.detect(data, url=source)
600
+ if kind:
601
+ return _document_document(data, kind, source, "", config)
602
+ html = engine.decode_bytes(data)
603
+ base_url = args.url if args.url else None
604
+ elif args.url:
605
+ fetcher = engine.HttpFetcher(config)
606
+ response = fetcher.fetch(args.url)
607
+ source = response.final_url
608
+ base_url = response.final_url
609
+ if "application/pdf" in response.content_type or pdf_mod.is_pdf(response.content):
610
+ return _pdf_document(response.content, source, response.final_url, config)
611
+ kind = documents.detect(response.content, response.content_type, response.final_url)
612
+ if kind:
613
+ return _document_document(
614
+ response.content, kind, source, response.final_url, config
615
+ )
616
+ html = response.text
617
+ else:
618
+ if sys.stdin.isatty():
619
+ raise ValueError("--action clean needs --input, --url, or HTML on stdin")
620
+ data = sys.stdin.buffer.read(MAX_LOCAL_BYTES + 1)
621
+ if len(data) > MAX_LOCAL_BYTES:
622
+ raise ValueError(
623
+ f"stdin is larger than {MAX_LOCAL_BYTES // (1024 * 1024)} MiB; "
624
+ "web-forager does not read unbounded input"
625
+ )
626
+ source = "stdin"
627
+ if pdf_mod.is_pdf(data):
628
+ return _pdf_document(data, source, "", config)
629
+ kind = documents.detect(data)
630
+ if kind:
631
+ return _document_document(data, kind, source, "", config)
632
+ html = engine.decode_bytes(data)
633
+
634
+ markdown = engine.html_to_markdown(
635
+ html,
636
+ url=base_url,
637
+ include_tables=config.include_tables,
638
+ keep_links=config.keep_links,
639
+ include_images=config.include_images,
640
+ )
641
+ if not markdown.strip():
642
+ raise engine.ExtractionError("no readable content found in the supplied HTML")
643
+ if config.max_chars and len(markdown) > config.max_chars:
644
+ markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
645
+
646
+ payload = {
647
+ "tool": "web-forager",
648
+ "version": engine.__version__,
649
+ "action": "clean",
650
+ "generated_at": engine.utc_now(),
651
+ "source": source,
652
+ "metadata": engine.extract_metadata(html, url=base_url),
653
+ "tables": engine.extract_tables(html) if config.include_tables else [],
654
+ "markdown": markdown,
655
+ }
656
+ return _clean_chunks(payload, config)
657
+ finally:
658
+ if fetcher is not None:
659
+ fetcher.close()
660
+
661
+
662
+ # ---------------------------------------------------------------------------
663
+ # Entry point
664
+ # ---------------------------------------------------------------------------
665
+
666
+
667
+ def cmd_surf(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
668
+ """Search (or open) a page, then follow ranked links like a human browsing."""
669
+
670
+ if bool(args.url) == bool(args.query):
671
+ raise ValueError("--action surf needs exactly one of --url or --query")
672
+
673
+ seed = args.url or args.query
674
+ payload = engine.surf_web(
675
+ seed,
676
+ config,
677
+ depth=max(0, args.depth),
678
+ follow=args.follow,
679
+ max_pages=max(1, args.max_pages),
680
+ engine_name=args.engine,
681
+ session=args.session or None,
682
+ )
683
+ return _envelope("surf", payload, args)
684
+
685
+
686
+ def cmd_scholar(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
687
+ if not args.query:
688
+ raise ValueError("--action scholar needs --query (or a DOI with --source doi)")
689
+ payload = scholar.search(
690
+ args.source,
691
+ args.query,
692
+ config,
693
+ limit=max(1, args.num_results),
694
+ sort=args.sort,
695
+ )
696
+ return _envelope("scholar", payload, args)
697
+
698
+
699
+ def cmd_map(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
700
+ if not args.url:
701
+ raise ValueError("--action map needs --url (a site, sitemap or robots.txt URL)")
702
+ payload = sitemap.discover(
703
+ args.url,
704
+ config,
705
+ limit=max(1, args.limit),
706
+ max_sitemaps=max(1, args.max_sitemaps),
707
+ )
708
+ return _envelope("map", payload, args)
709
+
710
+
711
+ def cmd_bili(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
712
+ mode = args.mode
713
+ if mode == "auto":
714
+ mode = "search" if args.query else "video" if args.url else "feed"
715
+
716
+ if mode == "feed":
717
+ payload = bilibili.feed(
718
+ config,
719
+ limit=args.num_results,
720
+ enrich=max(0, args.max_pages),
721
+ url=args.url or bilibili.BILI_HOME,
722
+ )
723
+ elif mode == "video":
724
+ if not args.url:
725
+ raise ValueError("--mode video needs --url with a BV id or a video URL")
726
+ record = bilibili.fetch_video(args.url, config)
727
+ payload = {"mode": "video", "videos": [record], "results": [record]}
728
+ elif mode == "search":
729
+ if not args.query:
730
+ raise ValueError("--mode search needs --query")
731
+ payload = bilibili.search(args.query, config, allow_direct=args.direct_search)
732
+ elif mode == "related":
733
+ if not args.url:
734
+ raise ValueError("--mode related needs --url with a BV id or a video URL")
735
+ payload = bilibili.related(
736
+ args.url, config, limit=args.num_results, enrich=max(0, args.max_pages)
737
+ )
738
+ elif mode == "surf":
739
+ seed = args.url or args.query
740
+ if not seed:
741
+ raise ValueError("--mode surf needs --url (a video) or --query (a keyword)")
742
+ payload = bilibili.surf(
743
+ seed,
744
+ config,
745
+ depth=max(0, args.depth),
746
+ per_level=max(1, args.max_pages),
747
+ allow_direct=args.direct_search,
748
+ )
749
+ else: # pragma: no cover - argparse prevents this
750
+ raise ValueError(f"unsupported Bilibili mode: {mode}")
751
+
752
+ return _envelope("bili", payload, args)
753
+
754
+
755
+ def cmd_doctor(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
756
+ """Report optional dependencies so an agent can install what is missing."""
757
+
758
+ payload = engine.doctor_report(config)
759
+ if getattr(args, "probe", False):
760
+ payload["search_probe"] = engine.probe_engines(config)
761
+ return _envelope("doctor", payload, args)
762
+
763
+
764
+ def _fail(
765
+ args: argparse.Namespace,
766
+ code: str,
767
+ message: str,
768
+ hint: str,
769
+ exit_code: int,
770
+ ) -> int:
771
+ """Report a failure either as an OpenCLI-style envelope or on stderr."""
772
+
773
+ if getattr(args, "agent", False):
774
+ payload = {
775
+ "ok": False,
776
+ "error": {"code": code, "message": message, "hint": hint},
777
+ }
778
+ print(json.dumps(payload, ensure_ascii=False))
779
+ else:
780
+ suffix = f" ({hint})" if hint else ""
781
+ print(f"error: {message}{suffix}", file=sys.stderr)
782
+ return exit_code
783
+
784
+
785
+ def main(argv: list[str] | None = None) -> int:
786
+ _reconfigure_stdio()
787
+ parser = build_parser()
788
+ args = parser.parse_args(argv)
789
+
790
+ try:
791
+ config = build_config(args)
792
+ if args.action == "fetch":
793
+ document = cmd_fetch(args, config)
794
+ elif args.action == "research":
795
+ document = cmd_research(args, config)
796
+ elif args.action == "find":
797
+ document = cmd_find(args, config)
798
+ elif args.action == "feed":
799
+ document = cmd_feed(args, config)
800
+ elif args.action == "clean":
801
+ document = cmd_clean(args, config)
802
+ elif args.action == "surf":
803
+ document = cmd_surf(args, config)
804
+ elif args.action == "scholar":
805
+ document = cmd_scholar(args, config)
806
+ elif args.action == "map":
807
+ document = cmd_map(args, config)
808
+ elif args.action == "bili":
809
+ document = cmd_bili(args, config)
810
+ else:
811
+ document = cmd_doctor(args, config)
812
+ except engine.ProhibitedQuery as exc:
813
+ return _fail(
814
+ args,
815
+ "prohibited_query",
816
+ str(exc),
817
+ "search a topic, a paper title or an organisation; person profiling is out of scope",
818
+ 10,
819
+ )
820
+ except engine.RobotsDenied as exc:
821
+ return _fail(
822
+ args,
823
+ "robots_denied",
824
+ f"robots.txt denied this request: {exc}",
825
+ "robots.txt denied this request. Use --robots-policy warn only when "
826
+ "you are authorised to read the path (the record will mark the override), "
827
+ "or choose another source; do not silently switch to a tool that ignores robots",
828
+ 3,
829
+ )
830
+ except engine.UnsafeAddress as exc:
831
+ return _fail(
832
+ args,
833
+ "unsafe_address",
834
+ str(exc),
835
+ "loopback, private, link-local and cloud-metadata addresses are refused",
836
+ 2,
837
+ )
838
+ except engine.InvalidURLError as exc:
839
+ return _fail(args, "invalid_url", f"invalid URL: {exc}", "pass an absolute http(s) URL", 2)
840
+ except engine.PdfUnavailable as exc:
841
+ return _fail(args, "pdf_unavailable", str(exc), "pip install pypdf", 7)
842
+ except engine.PdfExtractionError as exc:
843
+ return _fail(args, "pdf_extraction_failed", str(exc), "check that the file is a valid PDF", 5)
844
+ except engine.VerificationRequired as exc:
845
+ return _fail(
846
+ args,
847
+ "verification_required",
848
+ f"verification required: {exc}",
849
+ "open it in browser mode with the wechat-dig playbook "
850
+ "(node browser/run.js --playbook wechat-dig --url <URL> --backend cdp) "
851
+ "after signing in once with browser/launch_chrome.py; "
852
+ "web-forager does not solve challenges",
853
+ 9,
854
+ )
855
+ except engine.FetchError as exc:
856
+ return _fail(args, "fetch_failed", f"fetch failed: {exc}", "check the URL and network", 4)
857
+ except engine.ExtractionError as exc:
858
+ return _fail(
859
+ args,
860
+ "extraction_failed",
861
+ f"extraction failed: {exc}",
862
+ "try --render always (needs Playwright) for JavaScript-heavy pages",
863
+ 5,
864
+ )
865
+ except engine.RenderUnavailable as exc:
866
+ return _fail(
867
+ args,
868
+ "render_unavailable",
869
+ f"dynamic rendering unavailable: {exc}",
870
+ "pip install playwright && playwright install chromium",
871
+ 6,
872
+ )
873
+ except ValueError as exc:
874
+ return _fail(args, "invalid_argument", str(exc), "", 2)
875
+ except KeyboardInterrupt:
876
+ return _fail(args, "interrupted", "interrupted", "", 130)
877
+ except Exception as exc: # never surface a raw traceback to the user
878
+ if getattr(args, "verbose", False):
879
+ import traceback
880
+
881
+ traceback.print_exc()
882
+ return _fail(args, "unexpected_error", f"unexpected failure: {exc}", "run with --verbose for the traceback", 1)
883
+
884
+ if getattr(args, "core", False):
885
+ document = engine.apply_core(document)
886
+
887
+ json_text = json.dumps(document, ensure_ascii=False, indent=2)
888
+ markdown_text = document_to_markdown(document)
889
+ if args.agent:
890
+ output = json.dumps({"ok": True, "data": document}, ensure_ascii=False, indent=2)
891
+ elif args.format == "markdown":
892
+ output = markdown_text
893
+ else:
894
+ output = json_text
895
+
896
+ if args.root and not args.out:
897
+ try:
898
+ write_workspace(args, document, json_text, markdown_text)
899
+ except (OSError, ValueError) as exc:
900
+ print(f"error: cannot write Chelate workspace: {exc}", file=sys.stderr)
901
+ return 2
902
+
903
+ emit(output, args.out)
904
+ return 0
905
+
906
+
907
+ if __name__ == "__main__":
908
+ raise SystemExit(main())