web-forager 0.0.0-stage → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +119 -0
- package/CHANGELOG.md +508 -0
- package/LICENSE +21 -0
- package/README.md +498 -2
- package/README_zh.md +498 -0
- package/SKILL.md +451 -0
- package/THIRD_PARTY_LICENSES.txt +40 -0
- package/browser/bridges/playwright-bridge.js +382 -0
- package/browser/launch_chrome.py +193 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +90 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +141 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +130 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +215 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +252 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +107 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +441 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/research.md +35 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +349 -0
- package/docs/REFERENCE_zh.md +342 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +98 -0
- package/mcp_server.py +850 -0
- package/package.json +82 -6
- package/requirements.txt +15 -0
- package/run.py +908 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +882 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +5141 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +156 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +381 -0
- package/scripts/sitemap.py +195 -0
- package/scripts/untrusted.py +312 -0
package/manifest.json
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "web-forager",
|
|
3
|
+
"version": "1.3.1",
|
|
4
|
+
"stage": "domain_extensions",
|
|
5
|
+
"input_types": [
|
|
6
|
+
"url",
|
|
7
|
+
"html",
|
|
8
|
+
"pdf",
|
|
9
|
+
"docx",
|
|
10
|
+
"xlsx",
|
|
11
|
+
"epub",
|
|
12
|
+
"csv",
|
|
13
|
+
"rss",
|
|
14
|
+
"atom",
|
|
15
|
+
"sitemap",
|
|
16
|
+
"keyword",
|
|
17
|
+
"doi",
|
|
18
|
+
"bvid"
|
|
19
|
+
],
|
|
20
|
+
"output_type": "markdown",
|
|
21
|
+
"output_types": [
|
|
22
|
+
"markdown",
|
|
23
|
+
"json"
|
|
24
|
+
],
|
|
25
|
+
"dependencies": [
|
|
26
|
+
"requests",
|
|
27
|
+
"beautifulsoup4",
|
|
28
|
+
"trafilatura"
|
|
29
|
+
],
|
|
30
|
+
"optional_dependencies": [
|
|
31
|
+
"playwright",
|
|
32
|
+
"pypdf"
|
|
33
|
+
],
|
|
34
|
+
"sandbox_required": false,
|
|
35
|
+
"entrypoint": "run.py",
|
|
36
|
+
"actions": [
|
|
37
|
+
"fetch",
|
|
38
|
+
"research",
|
|
39
|
+
"find",
|
|
40
|
+
"feed",
|
|
41
|
+
"clean",
|
|
42
|
+
"bili",
|
|
43
|
+
"surf",
|
|
44
|
+
"scholar",
|
|
45
|
+
"map"
|
|
46
|
+
],
|
|
47
|
+
"mcp": {
|
|
48
|
+
"entrypoint": "mcp_server.py",
|
|
49
|
+
"config": ".mcp.json",
|
|
50
|
+
"tools": [
|
|
51
|
+
"web_fetch",
|
|
52
|
+
"web_search",
|
|
53
|
+
"web_research",
|
|
54
|
+
"web_surf",
|
|
55
|
+
"web_scholar",
|
|
56
|
+
"web_map",
|
|
57
|
+
"web_bilibili",
|
|
58
|
+
"web_doctor",
|
|
59
|
+
"web_find",
|
|
60
|
+
"web_feed"
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
"browser": {
|
|
64
|
+
"entrypoint": "browser/run.js",
|
|
65
|
+
"runtime": "node >= 18",
|
|
66
|
+
"playbooks": [
|
|
67
|
+
"bili-feed",
|
|
68
|
+
"bili-video",
|
|
69
|
+
"bili-chain",
|
|
70
|
+
"bili-danmaku",
|
|
71
|
+
"zhihu-dig",
|
|
72
|
+
"xhs-dig",
|
|
73
|
+
"search-dig",
|
|
74
|
+
"wechat-dig"
|
|
75
|
+
],
|
|
76
|
+
"backends": [
|
|
77
|
+
"iab",
|
|
78
|
+
"extension",
|
|
79
|
+
"cdp"
|
|
80
|
+
],
|
|
81
|
+
"mcp": {
|
|
82
|
+
"entrypoint": "browser/mcp_server.js",
|
|
83
|
+
"server": "web-forager-browser",
|
|
84
|
+
"tools": [
|
|
85
|
+
"browser_list",
|
|
86
|
+
"browser_run",
|
|
87
|
+
"browser_session",
|
|
88
|
+
"browser_doctor"
|
|
89
|
+
]
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
"license": "MIT",
|
|
93
|
+
"author": {
|
|
94
|
+
"name": "Jiaming Shi"
|
|
95
|
+
},
|
|
96
|
+
"description": "Clean-room, robots-aware web information gathering for agents and analysts. HTTP mode searches 360/Bing/Sogou/WeChat, searches inside a site through its own search page, reads HTML/PDF/DOCX/XLSX/EPUB/CSV and RSS/Atom, collects public pages from Bilibili through a dedicated adapter and reads other sites as ordinary pages, extracts metadata and tables, discovers sitemaps and caches politely. Browser mode drives a real browser through a pluggable bridge for login-state pages, rendered content, danmaku and eight read-only playbooks covering Bilibili, Zhihu, Xiaohongshu and WeChat. Optional scholarly lookups cover arXiv/Crossref/PubMed/DOI. Ships a CLI, an --agent envelope, and two stdio MCP servers.",
|
|
97
|
+
"disclaimer": "本插件仅供学习与信息收集使用。使用者需遵守目标网站的 Terms of Service 及相关版权法律。禁止用于恶意并发攻击。严禁收集、聚合或推断特定自然人的个人信息,或将其用于人肉搜索、骚扰、跟踪等侵害他人隐私的活动;此类用法产生的法律责任由使用者自行承担。"
|
|
98
|
+
}
|