web-forager 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.clinerules/web-forager.md +3 -3
- package/.codebuddy-plugin/marketplace.json +1 -1
- package/.codebuddy-plugin/plugin.json +1 -1
- package/.codex-plugin/marketplace.json +1 -1
- package/.codex-plugin/mcp.json +20 -20
- package/.codex-plugin/plugin.json +1 -1
- package/.continue/rules/web-forager.md +4 -4
- package/.cursor/rules/web-forager.mdc +4 -4
- package/.dsh-plugin/index.js +1 -1
- package/.dsh-plugin/plugin.json +1 -1
- package/.github/copilot-instructions.md +3 -3
- package/.roo/rules/web-forager.md +3 -3
- package/.windsurf/rules/web-forager.md +4 -4
- package/AGENTS.md +10 -3
- package/CHANGELOG.md +508 -454
- package/README.md +499 -477
- package/README_zh.md +498 -477
- package/SKILL.md +115 -15
- package/THIRD_PARTY_LICENSES.txt +5 -0
- package/browser/bridges/playwright-bridge.js +8 -1
- package/browser/launch_chrome.py +193 -144
- package/browser/lib/bili-api.js +12 -1
- package/browser/lib/safety.js +27 -1
- package/browser/lib/session.js +16 -2
- package/browser/lib/untrusted.js +215 -56
- package/browser/mcp_server.js +5 -25
- package/browser/package.json +1 -1
- package/browser/playbooks/search-dig.js +3 -1
- package/browser/playbooks/wechat-dig.js +78 -78
- package/browser/run.js +19 -10
- package/commands/research.md +35 -0
- package/cordis.patch.yml +19 -19
- package/docs/DEPENDENCIES.md +1 -1
- package/docs/INSTALL_COMMANDS.md +4 -4
- package/docs/INSTALL_COMMANDS_zh.md +4 -4
- package/docs/REFERENCE.md +349 -308
- package/docs/REFERENCE_zh.md +342 -309
- package/manifest.json +3 -2
- package/mcp_server.py +383 -130
- package/package.json +4 -2
- package/requirements.txt +7 -4
- package/run.py +61 -5
- package/scripts/bilibili.py +11 -3
- package/scripts/engine.py +1039 -118
- package/scripts/pdf.py +21 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +33 -6
- package/scripts/sitemap.py +20 -5
- package/scripts/untrusted.py +312 -95
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "web-forager",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "Clean-room, robots-aware web information gathering: multi-engine search, HTML/PDF/Office/feed extraction, Bilibili/Zhihu/Xiaohongshu/WeChat reading, sitemaps and resumable surfing, plus a browser mode with eight read-only playbooks. CLI + two MCP servers.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"license": "MIT",
|
|
@@ -33,7 +33,9 @@
|
|
|
33
33
|
"test": "python -m pytest",
|
|
34
34
|
"test:browser": "node --test \"browser/tests/*.test.js\"",
|
|
35
35
|
"test:live": "node browser/tests/live-smoke.js",
|
|
36
|
-
"
|
|
36
|
+
"clean:pycache": "node clean-pycache.mjs",
|
|
37
|
+
"prepack": "npm run clean:pycache",
|
|
38
|
+
"prepublishOnly": "npm run clean:pycache"
|
|
37
39
|
},
|
|
38
40
|
"files": [
|
|
39
41
|
"index.js",
|
package/requirements.txt
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
# Core runtime (all Apache-2.0 or MIT; no GPL/AGPL anywhere)
|
|
2
|
-
requests
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
# certifi is a requests dependency; the floor avoids the compromised
|
|
3
|
+
# emSign root CAs fixed in 2026.10.18.
|
|
4
|
+
certifi>=2026.10.18
|
|
5
|
+
requests>=2.33.0
|
|
6
|
+
beautifulsoup4>=4.14.3
|
|
7
|
+
trafilatura>=2.3.1
|
|
5
8
|
|
|
6
9
|
# Optional: PDF text extraction (BSD-3-Clause).
|
|
7
10
|
# Without it, PDF inputs return exit code 7 with a clear install hint.
|
|
8
|
-
# pypdf>=4
|
|
11
|
+
# pypdf>=6.21.4 (or 7.x); earlier versions have seven DoS advisories
|
|
9
12
|
|
|
10
13
|
# Optional: JavaScript rendering (Apache-2.0).
|
|
11
14
|
# Install the package, then: python -m playwright install chromium
|
package/run.py
CHANGED
|
@@ -7,6 +7,10 @@ fetch
|
|
|
7
7
|
``--url URL`` fetch one page and return metadata, Markdown and tables
|
|
8
8
|
``--query TEXT`` search (360+Bing by default), then fetch the top results
|
|
9
9
|
|
|
10
|
+
research
|
|
11
|
+
``--query TEXT`` broad recall, then read a few authoritative sources and
|
|
12
|
+
return only the passages that match the query, with citations
|
|
13
|
+
|
|
10
14
|
clean
|
|
11
15
|
``--input file`` clean a local HTML/PDF/DOCX/XLSX/EPUB/CSV file
|
|
12
16
|
``--url URL`` fetch a page/document and clean it
|
|
@@ -47,10 +51,11 @@ import sys
|
|
|
47
51
|
from pathlib import Path
|
|
48
52
|
|
|
49
53
|
_PLUGIN_ROOT = Path(__file__).resolve().parent
|
|
54
|
+
MAX_LOCAL_BYTES = 16 * 1024 * 1024
|
|
50
55
|
if str(_PLUGIN_ROOT) not in sys.path:
|
|
51
56
|
sys.path.insert(0, str(_PLUGIN_ROOT))
|
|
52
57
|
|
|
53
|
-
from scripts import bilibili, documents, engine, pdf as pdf_mod, scholar, sitemap # noqa: E402
|
|
58
|
+
from scripts import bilibili, documents, engine, pdf as pdf_mod, research, scholar, sitemap # noqa: E402
|
|
54
59
|
|
|
55
60
|
|
|
56
61
|
def _reconfigure_stdio() -> None:
|
|
@@ -77,11 +82,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
77
82
|
parser.add_argument(
|
|
78
83
|
"--action",
|
|
79
84
|
required=True,
|
|
80
|
-
choices=["fetch", "find", "feed", "clean", "bili", "surf", "scholar", "map", "doctor"],
|
|
85
|
+
choices=["fetch", "research", "find", "feed", "clean", "bili", "surf", "scholar", "map", "doctor"],
|
|
81
86
|
help="operation to run",
|
|
82
87
|
)
|
|
83
88
|
parser.add_argument("--url", help="target URL, or a Bilibili BV/av id (fetch/clean/bili/surf/map)")
|
|
84
|
-
parser.add_argument("--query", help="keyword query (fetch/bili/surf/scholar)")
|
|
89
|
+
parser.add_argument("--query", help="keyword query (fetch/research/bili/surf/scholar)")
|
|
85
90
|
parser.add_argument("--site", help="domain or base URL to search inside (find)")
|
|
86
91
|
parser.add_argument("--input", dest="input_path", help="local HTML/PDF/DOCX/XLSX/EPUB/CSV file (clean)")
|
|
87
92
|
parser.add_argument(
|
|
@@ -104,7 +109,19 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
104
109
|
default="auto",
|
|
105
110
|
help="search backend: auto=360+Bing (mostly-CJK queries: 360+Sogou+WeChat); all=every engine (default: auto)",
|
|
106
111
|
)
|
|
107
|
-
parser.add_argument("--num-results", type=int, default=8, help="search results to request (default: 8)")
|
|
112
|
+
parser.add_argument("--num-results", type=int, default=8, help="search results to request (default: 8; research: recall pool)")
|
|
113
|
+
parser.add_argument(
|
|
114
|
+
"--focus",
|
|
115
|
+
type=int,
|
|
116
|
+
default=3,
|
|
117
|
+
help="research: distinct sources to read in full (default: 3)",
|
|
118
|
+
)
|
|
119
|
+
parser.add_argument(
|
|
120
|
+
"--focus-chars",
|
|
121
|
+
type=int,
|
|
122
|
+
default=40_000,
|
|
123
|
+
help="research: maximum Markdown chars read per focused page (default: 40000)",
|
|
124
|
+
)
|
|
108
125
|
parser.add_argument(
|
|
109
126
|
"--serial-engines",
|
|
110
127
|
action="store_true",
|
|
@@ -180,6 +197,18 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
180
197
|
action="store_true",
|
|
181
198
|
help="DANGEROUS: same as --robots-policy ignore (only for resources you are authorised to access)",
|
|
182
199
|
)
|
|
200
|
+
parser.add_argument(
|
|
201
|
+
"--query-robots",
|
|
202
|
+
choices=("respect", "warn", "ignore"),
|
|
203
|
+
default=None,
|
|
204
|
+
help=(
|
|
205
|
+
"policy for search-engine and scholarly query endpoints "
|
|
206
|
+
"(default: WEB_FORAGER_QUERY_ROBOTS or warn). These endpoints "
|
|
207
|
+
"are not target pages; warn reads a disallowed one and records the "
|
|
208
|
+
"override, respect refuses it, ignore skips the check. MCP does not "
|
|
209
|
+
"expose this switch to the model."
|
|
210
|
+
),
|
|
211
|
+
)
|
|
183
212
|
parser.add_argument(
|
|
184
213
|
"--allow-private",
|
|
185
214
|
action="store_true",
|
|
@@ -248,6 +277,7 @@ def build_config(args: argparse.Namespace) -> engine.FetchConfig:
|
|
|
248
277
|
max_chars=args.max_chars,
|
|
249
278
|
respect_robots=not args.ignore_robots,
|
|
250
279
|
robots_policy="ignore" if args.ignore_robots else args.robots_policy,
|
|
280
|
+
query_robots=args.query_robots or engine._query_robots_from_env(),
|
|
251
281
|
render=args.render,
|
|
252
282
|
include_tables=not args.no_tables,
|
|
253
283
|
keep_links=args.keep_links,
|
|
@@ -427,6 +457,20 @@ def _envelope(action: str, payload: dict, args: argparse.Namespace) -> dict:
|
|
|
427
457
|
return engine.build_envelope(action, payload, full_results=bool(getattr(args, "full_results", False)))
|
|
428
458
|
|
|
429
459
|
|
|
460
|
+
def cmd_research(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
461
|
+
if not args.query:
|
|
462
|
+
raise ValueError("--action research needs --query")
|
|
463
|
+
payload = research.research(
|
|
464
|
+
args.query,
|
|
465
|
+
config,
|
|
466
|
+
engine_name=args.engine,
|
|
467
|
+
focus=args.focus,
|
|
468
|
+
breadth=args.num_results,
|
|
469
|
+
focus_chars=args.focus_chars,
|
|
470
|
+
)
|
|
471
|
+
return _envelope("research", payload, args)
|
|
472
|
+
|
|
473
|
+
|
|
430
474
|
def cmd_find(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
431
475
|
if not args.site:
|
|
432
476
|
raise ValueError("--action find needs --site")
|
|
@@ -543,6 +587,11 @@ def cmd_clean(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
|
543
587
|
path = Path(args.input_path)
|
|
544
588
|
if not path.is_file():
|
|
545
589
|
raise ValueError(f"input file not found: {path}")
|
|
590
|
+
if path.stat().st_size > MAX_LOCAL_BYTES:
|
|
591
|
+
raise ValueError(
|
|
592
|
+
f"input file is larger than {MAX_LOCAL_BYTES // (1024 * 1024)} MiB; "
|
|
593
|
+
"web-forager does not read unbounded local files"
|
|
594
|
+
)
|
|
546
595
|
data = path.read_bytes()
|
|
547
596
|
source = str(path)
|
|
548
597
|
if pdf_mod.is_pdf(data):
|
|
@@ -568,7 +617,12 @@ def cmd_clean(args: argparse.Namespace, config: engine.FetchConfig) -> dict:
|
|
|
568
617
|
else:
|
|
569
618
|
if sys.stdin.isatty():
|
|
570
619
|
raise ValueError("--action clean needs --input, --url, or HTML on stdin")
|
|
571
|
-
data = sys.stdin.buffer.read()
|
|
620
|
+
data = sys.stdin.buffer.read(MAX_LOCAL_BYTES + 1)
|
|
621
|
+
if len(data) > MAX_LOCAL_BYTES:
|
|
622
|
+
raise ValueError(
|
|
623
|
+
f"stdin is larger than {MAX_LOCAL_BYTES // (1024 * 1024)} MiB; "
|
|
624
|
+
"web-forager does not read unbounded input"
|
|
625
|
+
)
|
|
572
626
|
source = "stdin"
|
|
573
627
|
if pdf_mod.is_pdf(data):
|
|
574
628
|
return _pdf_document(data, source, "", config)
|
|
@@ -737,6 +791,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
737
791
|
config = build_config(args)
|
|
738
792
|
if args.action == "fetch":
|
|
739
793
|
document = cmd_fetch(args, config)
|
|
794
|
+
elif args.action == "research":
|
|
795
|
+
document = cmd_research(args, config)
|
|
740
796
|
elif args.action == "find":
|
|
741
797
|
document = cmd_find(args, config)
|
|
742
798
|
elif args.action == "feed":
|
package/scripts/bilibili.py
CHANGED
|
@@ -721,11 +721,18 @@ def search(
|
|
|
721
721
|
f"{route} answered"
|
|
722
722
|
)
|
|
723
723
|
elif allow_direct:
|
|
724
|
+
if config.query_robots == "respect":
|
|
725
|
+
raise engine.RobotsDenied(
|
|
726
|
+
"--direct-search queries api.bilibili.com and search.bilibili.com, "
|
|
727
|
+
"which disallow generic agents in robots.txt. Pass "
|
|
728
|
+
"--query-robots warn (record the override) or ignore (skip the "
|
|
729
|
+
"check) only after you are authorised to query them."
|
|
730
|
+
)
|
|
724
731
|
cards, route = _search_direct(query, config, fetcher, check_robots=False)
|
|
725
732
|
note = (
|
|
726
|
-
f"--direct-search asked {route} directly
|
|
727
|
-
"
|
|
728
|
-
"
|
|
733
|
+
f"--direct-search asked {route} directly under the user-selected "
|
|
734
|
+
"query-robots policy; video pages are still fetched with robots "
|
|
735
|
+
"checks enabled"
|
|
729
736
|
)
|
|
730
737
|
else:
|
|
731
738
|
cards = _search_via_engines(query, config, fetcher)
|
|
@@ -759,6 +766,7 @@ def search(
|
|
|
759
766
|
return {
|
|
760
767
|
"mode": "search",
|
|
761
768
|
"query": query,
|
|
769
|
+
"query_robots": config.query_robots,
|
|
762
770
|
"route": route,
|
|
763
771
|
"note": note,
|
|
764
772
|
"search_results": cards,
|