web-forager 1.3.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/.clinerules/web-forager.md +3 -3
  4. package/.codebuddy-plugin/marketplace.json +1 -1
  5. package/.codebuddy-plugin/plugin.json +1 -1
  6. package/.codex-plugin/marketplace.json +1 -1
  7. package/.codex-plugin/mcp.json +20 -20
  8. package/.codex-plugin/plugin.json +1 -1
  9. package/.continue/rules/web-forager.md +4 -4
  10. package/.cursor/rules/web-forager.mdc +4 -4
  11. package/.dsh-plugin/index.js +1 -1
  12. package/.dsh-plugin/plugin.json +1 -1
  13. package/.github/copilot-instructions.md +3 -3
  14. package/.roo/rules/web-forager.md +3 -3
  15. package/.windsurf/rules/web-forager.md +4 -4
  16. package/AGENTS.md +10 -3
  17. package/CHANGELOG.md +508 -454
  18. package/README.md +499 -477
  19. package/README_zh.md +498 -477
  20. package/SKILL.md +115 -15
  21. package/THIRD_PARTY_LICENSES.txt +5 -0
  22. package/browser/bridges/playwright-bridge.js +8 -1
  23. package/browser/launch_chrome.py +193 -144
  24. package/browser/lib/bili-api.js +12 -1
  25. package/browser/lib/safety.js +27 -1
  26. package/browser/lib/session.js +16 -2
  27. package/browser/lib/untrusted.js +215 -56
  28. package/browser/mcp_server.js +5 -25
  29. package/browser/package.json +1 -1
  30. package/browser/playbooks/search-dig.js +3 -1
  31. package/browser/playbooks/wechat-dig.js +78 -78
  32. package/browser/run.js +19 -10
  33. package/commands/research.md +35 -0
  34. package/cordis.patch.yml +19 -19
  35. package/docs/DEPENDENCIES.md +1 -1
  36. package/docs/INSTALL_COMMANDS.md +4 -4
  37. package/docs/INSTALL_COMMANDS_zh.md +4 -4
  38. package/docs/REFERENCE.md +349 -308
  39. package/docs/REFERENCE_zh.md +342 -309
  40. package/manifest.json +3 -2
  41. package/mcp_server.py +383 -130
  42. package/package.json +4 -2
  43. package/requirements.txt +7 -4
  44. package/run.py +61 -5
  45. package/scripts/bilibili.py +11 -3
  46. package/scripts/engine.py +1039 -118
  47. package/scripts/pdf.py +21 -0
  48. package/scripts/research.py +2087 -0
  49. package/scripts/scholar.py +33 -6
  50. package/scripts/sitemap.py +20 -5
  51. package/scripts/untrusted.py +312 -95
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "web-forager",
3
- "version": "1.3.0",
3
+ "version": "1.3.1",
4
4
  "description": "Clean-room, robots-aware web information gathering: multi-engine search, HTML/PDF/Office/feed extraction, Bilibili/Zhihu/Xiaohongshu/WeChat reading, sitemaps and resumable surfing, plus a browser mode with eight read-only playbooks. CLI + two MCP servers.",
5
5
  "main": "index.js",
6
6
  "license": "MIT",
@@ -33,7 +33,9 @@
33
33
  "test": "python -m pytest",
34
34
  "test:browser": "node --test \"browser/tests/*.test.js\"",
35
35
  "test:live": "node browser/tests/live-smoke.js",
36
- "prepublishOnly": "node clean-pycache.mjs"
36
+ "clean:pycache": "node clean-pycache.mjs",
37
+ "prepack": "npm run clean:pycache",
38
+ "prepublishOnly": "npm run clean:pycache"
37
39
  },
38
40
  "files": [
39
41
  "index.js",
package/requirements.txt CHANGED
@@ -1,11 +1,14 @@
1
1
  # Core runtime (all Apache-2.0 or MIT; no GPL/AGPL anywhere)
2
- requests>=2.31
3
- beautifulsoup4>=4.12
4
- trafilatura>=1.6
2
+ # certifi is a requests dependency; the floor avoids the compromised
3
+ # emSign root CAs fixed in 2026.10.18.
4
+ certifi>=2026.10.18
5
+ requests>=2.33.0
6
+ beautifulsoup4>=4.14.3
7
+ trafilatura>=2.3.1
5
8
 
6
9
  # Optional: PDF text extraction (BSD-3-Clause).
7
10
  # Without it, PDF inputs return exit code 7 with a clear install hint.
8
- # pypdf>=4
11
+ # pypdf>=6.21.4 (or 7.x); earlier versions have seven DoS advisories
9
12
 
10
13
  # Optional: JavaScript rendering (Apache-2.0).
11
14
  # Install the package, then: python -m playwright install chromium
package/run.py CHANGED
@@ -7,6 +7,10 @@ fetch
7
7
  ``--url URL`` fetch one page and return metadata, Markdown and tables
8
8
  ``--query TEXT`` search (360+Bing by default), then fetch the top results
9
9
 
10
+ research
11
+ ``--query TEXT`` broad recall, then read a few authoritative sources and
12
+ return only the passages that match the query, with citations
13
+
10
14
  clean
11
15
  ``--input file`` clean a local HTML/PDF/DOCX/XLSX/EPUB/CSV file
12
16
  ``--url URL`` fetch a page/document and clean it
@@ -47,10 +51,11 @@ import sys
47
51
  from pathlib import Path
48
52
 
49
53
  _PLUGIN_ROOT = Path(__file__).resolve().parent
54
+ MAX_LOCAL_BYTES = 16 * 1024 * 1024
50
55
  if str(_PLUGIN_ROOT) not in sys.path:
51
56
  sys.path.insert(0, str(_PLUGIN_ROOT))
52
57
 
53
- from scripts import bilibili, documents, engine, pdf as pdf_mod, scholar, sitemap # noqa: E402
58
+ from scripts import bilibili, documents, engine, pdf as pdf_mod, research, scholar, sitemap # noqa: E402
54
59
 
55
60
 
56
61
  def _reconfigure_stdio() -> None:
@@ -77,11 +82,11 @@ def build_parser() -> argparse.ArgumentParser:
77
82
  parser.add_argument(
78
83
  "--action",
79
84
  required=True,
80
- choices=["fetch", "find", "feed", "clean", "bili", "surf", "scholar", "map", "doctor"],
85
+ choices=["fetch", "research", "find", "feed", "clean", "bili", "surf", "scholar", "map", "doctor"],
81
86
  help="operation to run",
82
87
  )
83
88
  parser.add_argument("--url", help="target URL, or a Bilibili BV/av id (fetch/clean/bili/surf/map)")
84
- parser.add_argument("--query", help="keyword query (fetch/bili/surf/scholar)")
89
+ parser.add_argument("--query", help="keyword query (fetch/research/bili/surf/scholar)")
85
90
  parser.add_argument("--site", help="domain or base URL to search inside (find)")
86
91
  parser.add_argument("--input", dest="input_path", help="local HTML/PDF/DOCX/XLSX/EPUB/CSV file (clean)")
87
92
  parser.add_argument(
@@ -104,7 +109,19 @@ def build_parser() -> argparse.ArgumentParser:
104
109
  default="auto",
105
110
  help="search backend: auto=360+Bing (mostly-CJK queries: 360+Sogou+WeChat); all=every engine (default: auto)",
106
111
  )
107
- parser.add_argument("--num-results", type=int, default=8, help="search results to request (default: 8)")
112
+ parser.add_argument("--num-results", type=int, default=8, help="search results to request (default: 8; research: recall pool)")
113
+ parser.add_argument(
114
+ "--focus",
115
+ type=int,
116
+ default=3,
117
+ help="research: distinct sources to read in full (default: 3)",
118
+ )
119
+ parser.add_argument(
120
+ "--focus-chars",
121
+ type=int,
122
+ default=40_000,
123
+ help="research: maximum Markdown chars read per focused page (default: 40000)",
124
+ )
108
125
  parser.add_argument(
109
126
  "--serial-engines",
110
127
  action="store_true",
@@ -180,6 +197,18 @@ def build_parser() -> argparse.ArgumentParser:
180
197
  action="store_true",
181
198
  help="DANGEROUS: same as --robots-policy ignore (only for resources you are authorised to access)",
182
199
  )
200
+ parser.add_argument(
201
+ "--query-robots",
202
+ choices=("respect", "warn", "ignore"),
203
+ default=None,
204
+ help=(
205
+ "policy for search-engine and scholarly query endpoints "
206
+ "(default: WEB_FORAGER_QUERY_ROBOTS or warn). These endpoints "
207
+ "are not target pages; warn reads a disallowed one and records the "
208
+ "override, respect refuses it, ignore skips the check. MCP does not "
209
+ "expose this switch to the model."
210
+ ),
211
+ )
183
212
  parser.add_argument(
184
213
  "--allow-private",
185
214
  action="store_true",
@@ -248,6 +277,7 @@ def build_config(args: argparse.Namespace) -> engine.FetchConfig:
248
277
  max_chars=args.max_chars,
249
278
  respect_robots=not args.ignore_robots,
250
279
  robots_policy="ignore" if args.ignore_robots else args.robots_policy,
280
+ query_robots=args.query_robots or engine._query_robots_from_env(),
251
281
  render=args.render,
252
282
  include_tables=not args.no_tables,
253
283
  keep_links=args.keep_links,
@@ -427,6 +457,20 @@ def _envelope(action: str, payload: dict, args: argparse.Namespace) -> dict:
427
457
  return engine.build_envelope(action, payload, full_results=bool(getattr(args, "full_results", False)))
428
458
 
429
459
 
460
+ def cmd_research(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
461
+ if not args.query:
462
+ raise ValueError("--action research needs --query")
463
+ payload = research.research(
464
+ args.query,
465
+ config,
466
+ engine_name=args.engine,
467
+ focus=args.focus,
468
+ breadth=args.num_results,
469
+ focus_chars=args.focus_chars,
470
+ )
471
+ return _envelope("research", payload, args)
472
+
473
+
430
474
  def cmd_find(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
431
475
  if not args.site:
432
476
  raise ValueError("--action find needs --site")
@@ -543,6 +587,11 @@ def cmd_clean(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
543
587
  path = Path(args.input_path)
544
588
  if not path.is_file():
545
589
  raise ValueError(f"input file not found: {path}")
590
+ if path.stat().st_size > MAX_LOCAL_BYTES:
591
+ raise ValueError(
592
+ f"input file is larger than {MAX_LOCAL_BYTES // (1024 * 1024)} MiB; "
593
+ "web-forager does not read unbounded local files"
594
+ )
546
595
  data = path.read_bytes()
547
596
  source = str(path)
548
597
  if pdf_mod.is_pdf(data):
@@ -568,7 +617,12 @@ def cmd_clean(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
568
617
  else:
569
618
  if sys.stdin.isatty():
570
619
  raise ValueError("--action clean needs --input, --url, or HTML on stdin")
571
- data = sys.stdin.buffer.read()
620
+ data = sys.stdin.buffer.read(MAX_LOCAL_BYTES + 1)
621
+ if len(data) > MAX_LOCAL_BYTES:
622
+ raise ValueError(
623
+ f"stdin is larger than {MAX_LOCAL_BYTES // (1024 * 1024)} MiB; "
624
+ "web-forager does not read unbounded input"
625
+ )
572
626
  source = "stdin"
573
627
  if pdf_mod.is_pdf(data):
574
628
  return _pdf_document(data, source, "", config)
@@ -737,6 +791,8 @@ def main(argv: list[str] | None = None) -> int:
737
791
  config = build_config(args)
738
792
  if args.action == "fetch":
739
793
  document = cmd_fetch(args, config)
794
+ elif args.action == "research":
795
+ document = cmd_research(args, config)
740
796
  elif args.action == "find":
741
797
  document = cmd_find(args, config)
742
798
  elif args.action == "feed":
@@ -721,11 +721,18 @@ def search(
721
721
  f"{route} answered"
722
722
  )
723
723
  elif allow_direct:
724
+ if config.query_robots == "respect":
725
+ raise engine.RobotsDenied(
726
+ "--direct-search queries api.bilibili.com and search.bilibili.com, "
727
+ "which disallow generic agents in robots.txt. Pass "
728
+ "--query-robots warn (record the override) or ignore (skip the "
729
+ "check) only after you are authorised to query them."
730
+ )
724
731
  cards, route = _search_direct(query, config, fetcher, check_robots=False)
725
732
  note = (
726
- f"--direct-search asked {route} directly; that host disallows "
727
- "generic agents in robots.txt, and video pages are still fetched "
728
- "with robots checks enabled"
733
+ f"--direct-search asked {route} directly under the user-selected "
734
+ "query-robots policy; video pages are still fetched with robots "
735
+ "checks enabled"
729
736
  )
730
737
  else:
731
738
  cards = _search_via_engines(query, config, fetcher)
@@ -759,6 +766,7 @@ def search(
759
766
  return {
760
767
  "mode": "search",
761
768
  "query": query,
769
+ "query_robots": config.query_robots,
762
770
  "route": route,
763
771
  "note": note,
764
772
  "search_results": cards,