web-forager 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +112 -0
  18. package/CHANGELOG.md +454 -0
  19. package/LICENSE +21 -0
  20. package/README.md +477 -3
  21. package/README_zh.md +477 -0
  22. package/SKILL.md +351 -0
  23. package/THIRD_PARTY_LICENSES.txt +35 -0
  24. package/browser/bridges/playwright-bridge.js +375 -0
  25. package/browser/launch_chrome.py +144 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +79 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +115 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +116 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +56 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +272 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +105 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +432 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/scholar.md +28 -0
  58. package/commands/surf.md +28 -0
  59. package/cordis.patch.yml +19 -0
  60. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  61. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  62. package/docs/DEPENDENCIES.md +40 -0
  63. package/docs/INSTALL_COMMANDS.md +113 -0
  64. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  65. package/docs/REFERENCE.md +308 -0
  66. package/docs/REFERENCE_zh.md +309 -0
  67. package/index.js +7 -0
  68. package/install.py +254 -0
  69. package/manifest.json +97 -0
  70. package/mcp_server.py +597 -0
  71. package/package.json +80 -6
  72. package/requirements.txt +12 -0
  73. package/run.py +852 -0
  74. package/scripts/__init__.py +1 -0
  75. package/scripts/bilibili.py +874 -0
  76. package/scripts/documents.py +437 -0
  77. package/scripts/engine.py +4220 -0
  78. package/scripts/feeds.py +180 -0
  79. package/scripts/pdf.py +135 -0
  80. package/scripts/scholar.py +354 -0
  81. package/scripts/sitemap.py +180 -0
  82. package/scripts/untrusted.py +95 -0
@@ -0,0 +1,309 @@
1
+ # web-forager 参考手册
2
+
3
+ > [web-forager](../README_zh.md) 的一部分。这里放查询类内容:记录结构、动作、退出码、环境要求、目录结构与测试。
4
+
5
+ [English](REFERENCE.md) · **简体中文**
6
+
7
+ ---
8
+
9
+ ## MCP 服务
10
+
11
+ <details>
12
+ <summary>工具名</summary>
13
+
14
+ HTTP 工具:`web_fetch`、`web_search`、`web_surf`、`web_scholar`、`web_map`、`web_bilibili`、`web_clean`、`web_doctor`、`web_find`、`web_feed`。浏览器工具:`browser_list`、`browser_run`、`browser_session`、`browser_doctor`。
15
+
16
+ </details>
17
+
18
+ 直接测试协议:
19
+
20
+ ```bash
21
+ printf '%s\n' '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
22
+ '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' | python mcp_server.py
23
+ ```
24
+
25
+ ```bash
26
+ printf '%s\n' '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
27
+ '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' | node browser/mcp_server.js
28
+ ```
29
+
30
+ ## 浏览器模式
31
+
32
+ 浏览器层位于 `browser/`,需要 Node.js 18 或更新。它默认只读:点赞、投币、收藏、关注、分享、评论、弹幕与发布一律拒绝,除非用户明确授权某一次动作后传入 `--allow-write`。
33
+
34
+ ```bash
35
+ npm install playwright && npx playwright install chromium
36
+ node browser/run.js --doctor
37
+ node browser/run.js --playbook bili-feed --session bili-01 --max-items 8 --agent
38
+ node browser/run.js --playbook zhihu-dig --query "强化学习" --agent
39
+ node browser/run.js --playbook wechat-dig --url "https://mp.weixin.qq.com/s/..." --agent
40
+ ```
41
+
42
+ | 剧本 | 作用 | 需要 |
43
+ |---|---|---|
44
+ | `bili-feed` | 首页信息流,逐屏捕捉未见卡片 | 无 |
45
+ | `bili-video` | 单个视频与可见评论 | `--url` 或 `--bvid` |
46
+ | `bili-chain` | 跟随「下一话」,并从会话续接链路 | `--url`、`--bvid` 或会话链路 |
47
+ | `bili-danmaku` | 带时间戳的弹幕,优先用桥的 `observe`,DOM 文本兜底(实验性) | `--url` 或 `--bvid` |
48
+ | `zhihu-dig` | 搜知乎、排序链接、捕捉最佳未见页面 | `--query` |
49
+ | `xhs-dig` | 搜小红书公开笔记、排序 `/explore` 链接、捕捉一条 | `--query` |
50
+ | `search-dig` | 搜 B 站、排序卡片、打开最佳未见结果 | `--query` |
51
+ | `wechat-dig` | 单篇公众号文章;遇到环境校验页就如实报告,不把错误页当正文 | `--url` |
52
+
53
+ 公众号是 HTTP 模式唯一走不完的一类:`mp.weixin.qq.com` 在 robots.txt 里禁止 `/s`,即使加 `--robots-policy warn`,文章 URL 也会返回「环境异常」或跳转到验证码页,而不是正文;HTTP 记录会写成 `verification_required`(退出码 9),不会把错误页当成内容交出去。`wechat-dig` 在真实浏览器配置里打开同一个 URL:该配置已通过微信校验时能读到正文,没通过时报告 `blocked`,不会去解验证码。
54
+
55
+ 装好 `playwright` 与它的 Chromium 后,自带的桥会被自动启用;`--bridge` 或 `WEB_FORAGER_BRIDGE` 用于指向别处的桥。命令按空白拆分后直接启动,不经过 shell——路径含空格时加引号,参数照常写;管道、`&&` 与重定向不会被解释。`WEB_FORAGER_BROWSER_HEADFUL=1` 显示窗口;`WEB_FORAGER_BROWSER_PROFILE=/path` 复用持久化配置目录(`extension` 后端的登录态正是靠它保留);`WEB_FORAGER_BROWSER_CDP=http://127.0.0.1:9222` 接管你已有的浏览器(`cdp`)。桥通过可选的 `describe` 调用声明自己实现了什么;不实现 `describe` 的桥会被标记为 `declared: false`,所有能力一律为 false。
56
+
57
+ 登录一次的做法:运行 `python browser/launch_chrome.py`(默认使用 `~/.web-forager/browser/profile`,会打印 `WEB_FORAGER_BROWSER_CDP` 的值,支持 `--dry-run`),然后给剧本加 `--backend cdp`。`--storage-state FILE` 在运行前载入 Playwright 状态文件;`--save-storage-state FILE` 在运行后写出,权限为 `0600`,路径落在 `--root` 内会被拒绝。这个文件包含 Cookie 与登录态:按凭据对待,不要提交进 git,也不要复制到共享机器。
58
+
59
+ `--strict-robots` 在每次导航前读取 `robots.txt`,被禁止的路径以 `robots_denied`(退出码 3)停下。不加这个开关时记录同样说清楚:`compliance.robots_checked` 为 `false`。读到了页面却什么也没找到会报告 `degraded`(退出码 8)并附上记录;反爬或错误页以 `blocked`(退出码 9)停下;站点要求登录是 `login_required`(退出码 6)。
60
+
61
+ 浏览器模式是插件里最贵的一环,默认值因此开得很小:完整记录用 `--out result.json`(MCP 传 `out:`)落盘,默认不截图,`--max-screens 5` 与 `--min-interval 3000` 保持礼貌,长时间刷屏宁可用 `--session` 分几次续跑。
62
+
63
+ ---
64
+
65
+ ## 搜索引擎不可用时
66
+
67
+ 默认搜索走 `auto`。以中日韩文字为主的查询会同时问 360、搜狗与搜狗微信——中文人名或机构名通常只有微信文章索引这条渠道有内容——并且不会交给 Bing:Bing 会把中文短语拆成单字,返回词典页。这类单字词典页无论来自哪个引擎都会被过滤掉;确有需要时可以用 `--engine bing` 明确指定。
68
+
69
+ 另有两条完全不依赖搜索引擎的路:
70
+
71
+ ```bash
72
+ python run.py --action find --site the-decoder.com --query "OpenAI watermark" --max-pages 3
73
+ python run.py --action feed --url https://the-decoder.com/ --num-results 10
74
+ ```
75
+
76
+ `find` 尝试站点自带的搜索地址(`/?s=`、`/search?q=` 等四种),按查询词给页面链接排序并抓取最相关的几篇。`feed` 从页面的 `<link rel=alternate>` 发现 RSS/Atom 并读取。`doctor --probe` 对每个引擎各问一次,报告当前网络下哪些可用;`doctor` 还会显示正在使用的代理(凭据已抹去)。请求会遵循 `HTTPS_PROXY` / `ALL_PROXY`,`--proxy URL` 可为一次运行覆盖它们。
77
+
78
+ ---
79
+
80
+ ## 更多命令行示例
81
+
82
+ 每条命令单独一个代码块,复制时不用删掉相邻的行。
83
+
84
+ **抓一个 B 站视频。**
85
+
86
+ ```bash
87
+ python run.py --action fetch --url "https://www.bilibili.com/video/BV1xx411c7mD"
88
+ ```
89
+
90
+ **抓一个需要 JS 渲染的页面。**
91
+
92
+ ```bash
93
+ python run.py --action fetch --url "https://www.zhihu.com/" --render auto
94
+ ```
95
+
96
+ **冲浪一个站点,之后用会话续游。**
97
+
98
+ ```bash
99
+ python run.py --action surf --url "https://www.zhihu.com/" --depth 1 --max-pages 5 --session zhihu-01
100
+ ```
101
+
102
+ **把本地 PDF 清成 Markdown。**
103
+
104
+ ```bash
105
+ python run.py --action clean --input notes.pdf --format markdown
106
+ ```
107
+
108
+ **从站点地图列出最多 500 个页面 URL。**
109
+
110
+ ```bash
111
+ python run.py --action map --url "https://example.org" --limit 500
112
+ ```
113
+
114
+ **读取 B 站首页信息流。**
115
+
116
+ ```bash
117
+ python run.py --action bili --mode feed --max-items 10
118
+ ```
119
+
120
+ ---
121
+
122
+ ## 记录与输出
123
+
124
+ ### 每页一条记录
125
+
126
+ | 字段 | HTTP 模式 | 浏览器模式 |
127
+ |---|---|---|
128
+ | `metadata`、`markdown`、`tables`、`warnings` | 有 | 有 |
129
+ | `trust`、`injection_flags` | 有 | 有 |
130
+ | `url`、`title` | 有(`url` 与 `final_url`) | 有 |
131
+ | `status`、`content_type`、`document_kind`、`cached`、`rendered`、`render_fallback`、`fetched_at`、`chunks`、`links` | 有 | 无 |
132
+ | `schema_version`、`captured_at`、`playbook`、`backend`、`compliance`、`viewport_evidence`、`interaction_trace` | 无 | 有 |
133
+
134
+ `render_fallback` 为 `true` 表示服务器只返回了极小的脚本壳,HTTP 模式在抽取前用 Playwright 重渲染了一次。`fetch --query` 每次最多允许两次这样的重渲染,每次会多花数秒。
135
+
136
+ 搜索结果按查询覆盖度、标题命中、域名先验、摘要时效与引擎顺序排序;最终列表每个域名最多两条,避免单一站点挤掉其他来源;来源不足时再用溢出的结果补满。首轮结果过少时会用简化查询重试一次(去掉引号与常见停用词,或把带空格的中文查询合并),重试会在 `engine_report` 里标 `query_variant`。
137
+
138
+ 加 `--core` 时返回共享形状:
139
+
140
+ ```json
141
+ {
142
+ "schema_version": "core-1",
143
+ "source": "web-forager",
144
+ "mode": "http",
145
+ "url": "https://example.org/article",
146
+ "title": "An article",
147
+ "collected_at": "2026-11-07T10:00:00Z",
148
+ "metadata": {},
149
+ "markdown": "...",
150
+ "tables": [],
151
+ "warnings": [],
152
+ "extra": { "status": 200, "cached": false }
153
+ }
154
+ ```
155
+
156
+ `mode` 为 `http` 或 `browser`,`source` 标明产出方,`extra` 保留两种模式不共享的全部字段。同样的折叠也提供库调用:Python 侧是 `engine.normalize_record()`,浏览器侧是 `browser/lib/record.js` 的 `normalizeRecord()`。
157
+
158
+ `results` 是给智能体循环用的索引,不是每页正文的第二份拷贝。搜索时它只带 `url`、`title`、`snippet`、`status`、`content_type`、`markdown_chars`、`tables` 以及可能的 `warnings` 或 `error`;Markdown 正文只在 `pages` 里出现一次,单页 `fetch` 则把它放在顶层。要旧行为加 `--full-results`(CLI)或 `fullResults: true`(MCP)。
159
+
160
+ ### 工作区输出
161
+
162
+ ```bash
163
+ python run.py --action fetch --url "https://example.org/article" --root <WS>
164
+ ```
165
+
166
+ 只写这些位置:
167
+
168
+ ```text
169
+ <WS>/process/web-forager/web-forager_<timestamp>_<slug>.json
170
+ <WS>/process/web-forager/web-forager_<timestamp>_<slug>.md
171
+ <WS>/_broadcast/web-forager_<timestamp>_<slug>.json
172
+ ```
173
+
174
+ `material/` 与 `results/` 属于用户与编排器,web-forager 从不写入。浏览器捕捉遵循同一套规则,落在 `<WS>/process/web-forager/browser/` 与 `<WS>/_broadcast/`。
175
+
176
+ ---
177
+
178
+ ---
179
+
180
+ ## 动作与退出码
181
+
182
+ | 动作 | 用途 |
183
+ |---|---|
184
+ | `fetch` | 一个 URL 或关键词查询 → 页面 |
185
+ | `find` | 用站点自己的搜索页在站内检索 |
186
+ | `feed` | 读取站点的 RSS/Atom,自动从页面里发现订阅地址 |
187
+ | `clean` | 本地或远程 HTML/PDF/DOCX/XLSX/EPUB/CSV → Markdown |
188
+ | `bili` | B 站首页 / 视频 / 相关 / 搜索 / 冲浪 |
189
+ | `surf` | 搜索或种子 URL,再按相关度做链接 BFS |
190
+ | `scholar` | arXiv / Crossref / PubMed / DOI 记录 |
191
+ | `map` | 从 robots.txt 与 sitemap 发现页面 URL |
192
+ | `doctor` | 依赖自检:装了什么,该装什么 |
193
+
194
+ | 退出码 | 含义(未在下表改写的,两种模式相同) |
195
+ |---|---|
196
+ | 0 | 成功 |
197
+ | 2 | 参数或 URL 错误 |
198
+ | 3 | robots.txt 拒绝访问,或没有配置浏览器桥 |
199
+ | 4 | 网络 / HTTP / 导航失败 |
200
+ | 5 | 抽取或捕捉失败 |
201
+ | 6 | 请求了 Playwright 但不可用 |
202
+ | 7 | 处理 PDF 但 `pypdf` 不可用 |
203
+ | 9 | 返回的是验证页或反爬页而不是内容(HTTP 模式为 `verification_required`,浏览器模式为 `blocked`) |
204
+ | 130 | 被中断 |
205
+
206
+ 浏览器模式沿用这张表,只改写或新增以下含义:
207
+
208
+ | 退出码 | 浏览器模式含义 |
209
+ |---|---|
210
+ | 2 | 另外表示未知后端 |
211
+ | 6 | `login_required`:站点要求登录 |
212
+ | 7 | `confirmation_required`:写类操作被拒绝 |
213
+ | 8 | `degraded`:页面读到了,但剧本要的东西一个都没找到 |
214
+ | 9 | `blocked`:页面被反爬或错误页替换 |
215
+
216
+ `--agent` 保留这些退出码,并在 stdout 输出 `{"ok":false,"error":{"code","message","hint"}}` 信封。`doctor` 逐项报告依赖的版本或安装命令;缺 Playwright 返回退出码 6,缺 `pypdf` 返回退出码 7。
217
+
218
+ ---
219
+
220
+ ---
221
+
222
+ ## 环境要求
223
+
224
+ Python 3.10+。核心运行时:`requests`、`beautifulsoup4`、`trafilatura`。可选:`playwright`(JS 渲染)、`pypdf`(PDF)。`defusedxml` 若已安装则使用,但从不强制依赖——含内部 DTD 子集或任何 ENTITY 声明的 XML 一律拒绝。浏览器模式另需 Node.js 18+ 与一个桥命令;`browser/bridges/playwright-bridge.js` 还需要 npm 安装 `playwright`。没有内置任何第三方代码;完整的能力-依赖矩阵见 [DEPENDENCIES.md](DEPENDENCIES.md)。
225
+
226
+ ---
227
+
228
+ ---
229
+
230
+ ## 目录结构
231
+
232
+ <details>
233
+ <summary>完整目录树</summary>
234
+
235
+ ```text
236
+ web-forager/
237
+ ├── .claude-plugin/ Claude Code 清单 + marketplace
238
+ ├── .codex-plugin/ Codex 清单 + marketplace
239
+ ├── .codebuddy-plugin/ CodeBuddy 清单 + marketplace
240
+ ├── .dsh-plugin/ DSH 清单 + CJS 入口
241
+ ├── .cursor/rules/ Cursor 规则
242
+ ├── .windsurf/rules/ Windsurf 规则
243
+ ├── .clinerules/ Cline 规则
244
+ ├── .continue/rules/ Continue 规则
245
+ ├── .roo/rules/ Roo Code 规则
246
+ ├── .github/ GitHub Copilot 指令 + Actions CI
247
+ ├── AGENTS.md 通用智能体契约
248
+ ├── install.py 一键安装规则文件
249
+ ├── manifest.json 插件清单
250
+ ├── .mcp.json MCP stdio 注册
251
+ ├── package.json / index.js npm + dsh 打包
252
+ ├── cordis.patch.yml dsh bundle 补丁
253
+ ├── run.py CLI:九个动作
254
+ ├── mcp_server.py MCP stdio 服务(10 个 HTTP 工具)
255
+ ├── browser/ 浏览器模式(Node 18+)
256
+ │ ├── run.js CLI:八个只读剧本
257
+ │ ├── mcp_server.js MCP stdio 服务(4 个浏览器工具)
258
+ │ ├── lib/ 桥后端、会话、记录、安全门禁、选择器、robots、快照
259
+ │ ├── playbooks/ bili-* / zhihu-dig / xhs-dig / search-dig / wechat-dig
260
+ │ ├── bridges/ 可选 Playwright 参考桥
261
+ │ └── tests/ 假后端、真实页面快照、live 冒烟
262
+ ├── SKILL.md Claude Code 技能契约
263
+ ├── commands/ /fetch /surf /scholar /map /browse
264
+ ├── docs/ 安装命令、致谢、依赖矩阵
265
+ ├── scripts/
266
+ │ ├── engine.py robots、抓取、搜索、缓存、分块、Markdown、surf、find、feed
267
+ │ ├── pdf.py PDF 文本(可选 pypdf)
268
+ │ ├── documents.py DOCX / XLSX / EPUB / CSV
269
+ │ ├── feeds.py RSS / Atom / RSS 1.0
270
+ │ ├── scholar.py arXiv / Crossref / PubMed / DOI
271
+ │ ├── sitemap.py sitemap 发现
272
+ │ └── bilibili.py B 站适配器
273
+ ├── tests/ 离线 HTTP 夹具
274
+ ├── requirements.txt
275
+ ├── THIRD_PARTY_LICENSES.txt
276
+ ├── CHANGELOG.md
277
+ ├── README.md · README_zh.md
278
+ └── LICENSE MIT
279
+ ```
280
+
281
+ </details>
282
+
283
+ ---
284
+
285
+ ---
286
+
287
+ ## 测试
288
+
289
+ ```bash
290
+ python -m pytest tests -q
291
+ npm run test:browser
292
+ ```
293
+
294
+ 共 **228 个 Python 测试**加 **75 个浏览器测试**,全部离线通过。
295
+
296
+ <details>
297
+ <summary>测试覆盖了什么</summary>
298
+
299
+ `conftest.py` 会起一个临时本地 HTTP 站点,提供 robots.txt、文章、PDF/Office 文件、订阅源、站点地图、搜索结果页、站点搜索页与多跳 surf 页面。Python 套件覆盖 robots 拒绝、抽取、表格、文档、订阅源、缓存、搜索分页与路由、分块、MCP、surf 会话、安装器行为、宿主规则与免责声明检查、共享的 `core-1` 记录,以及每一个退出码。浏览器套件用假后端与说同一套线协议的假桥,不打开浏览器也能把 CLI 跑通;它还覆盖能力声明、robots.txt 匹配、`--strict-robots` 往返、私有地址拦截、不可信内容筛查,以及基于真实页面快照的 `degraded` 与 `blocked` 路径。
300
+
301
+ </details>
302
+
303
+ CI 在每次推送到 `main`、拉取请求与版本 tag 时运行两套测试:Python 在 Ubuntu 3.10/3.12 与 Windows 3.12,浏览器模式在 Node 20/22。真实站点检查是可选项,CI 从不运行:
304
+
305
+ ```bash
306
+ WEB_FORAGER_LIVE=1 WEB_FORAGER_BRIDGE="node browser/bridges/playwright-bridge.js" npm run test:live
307
+ ```
308
+
309
+ ---
package/index.js ADDED
@@ -0,0 +1,7 @@
1
+ module.exports = {
2
+ name: "web-forager",
3
+ apply: function (ctx) {
4
+ console.log("[web-forager] Plugin mounted via CJS; CLI and MCP entry points stay in the plugin directory.");
5
+ return { status: "active" };
6
+ }
7
+ };
package/install.py ADDED
@@ -0,0 +1,254 @@
1
+ """Cross-host rule-file installer for web-forager.
2
+
3
+ No agent host ships an installer that understands another host's rules, so
4
+ "install everywhere" cannot be delegated to a single vendor command. This
5
+ script copies the agent contract into every supported host's rule location in
6
+ one run.
7
+
8
+ Two scopes:
9
+
10
+ python install.py --project <dir> project-scoped rules inside <dir>
11
+ python install.py --global user-scoped rules for detected hosts
12
+
13
+ `--project` writes Cursor, Windsurf, Cline, Continue, Roo Code and GitHub
14
+ Copilot rule files plus the generic `AGENTS.md` that Codex, Cursor, Copilot
15
+ agents and opencode read. `--global` writes user-level rules for opencode and
16
+ Codex, and appends a marked block to `~/.claude/CLAUDE.md` instead of replacing
17
+ the user's own instructions. A host whose directory is absent is skipped.
18
+
19
+ Plugin hosts (Claude Code, CodeBuddy, Codex, dsh) should install the real
20
+ plugin from its marketplace instead; this script covers hosts that read rule
21
+ files. Re-running is safe: identical files report `unchanged`, and a drifted
22
+ file is left alone unless `--force` is passed.
23
+
24
+ Usage:
25
+ python install.py --list
26
+ python install.py --project . --dry-run
27
+ python install.py --project .
28
+ python install.py --global --only opencode,codex
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import argparse
34
+ import os
35
+ import shutil
36
+ import sys
37
+ import tempfile
38
+
39
+ REPO_ROOT = os.path.dirname(os.path.abspath(__file__))
40
+
41
+ PROJECT_RULES = {
42
+ "cursor": (
43
+ os.path.join(".cursor", "rules", "web-forager.mdc"),
44
+ os.path.join(".cursor", "rules", "web-forager.mdc"),
45
+ ),
46
+ "windsurf": (
47
+ os.path.join(".windsurf", "rules", "web-forager.md"),
48
+ os.path.join(".windsurf", "rules", "web-forager.md"),
49
+ ),
50
+ "cline": (
51
+ os.path.join(".clinerules", "web-forager.md"),
52
+ os.path.join(".clinerules", "web-forager.md"),
53
+ ),
54
+ "continue": (
55
+ os.path.join(".continue", "rules", "web-forager.md"),
56
+ os.path.join(".continue", "rules", "web-forager.md"),
57
+ ),
58
+ "roo": (
59
+ os.path.join(".roo", "rules", "web-forager.md"),
60
+ os.path.join(".roo", "rules", "web-forager.md"),
61
+ ),
62
+ "copilot": (
63
+ os.path.join(".github", "copilot-instructions.md"),
64
+ os.path.join(".github", "copilot-instructions.md"),
65
+ ),
66
+ "agents": ("AGENTS.md", "AGENTS.md"),
67
+ }
68
+
69
+
70
+ def global_rules() -> dict[str, tuple[str, str]]:
71
+ """Return user-scoped rule targets for this machine."""
72
+
73
+ home = os.path.expanduser("~")
74
+ return {
75
+ "opencode": (
76
+ os.path.join(home, ".config", "opencode", "AGENTS.md"),
77
+ "AGENTS.md",
78
+ ),
79
+ "codex": (
80
+ os.path.join(os.environ.get("CODEX_HOME") or os.path.join(home, ".codex"), "AGENTS.md"),
81
+ "AGENTS.md",
82
+ ),
83
+ "claude": (os.path.join(home, ".claude", "CLAUDE.md"), "AGENTS.md"),
84
+ }
85
+
86
+
87
+ def _same(left_path: str, right_path: str) -> bool:
88
+ try:
89
+ with open(left_path, "rb") as left, open(right_path, "rb") as right:
90
+ return left.read() == right.read()
91
+ except OSError:
92
+ return False
93
+
94
+
95
+ def _install(src_rel: str, dest: str, force: bool, dry_run: bool) -> str:
96
+ """Copy one rule file; return 'wrote', 'unchanged' or 'skipped'."""
97
+
98
+ src = os.path.join(REPO_ROOT, src_rel)
99
+ if not os.path.isfile(src):
100
+ raise SystemExit(f"install.py: missing source rule file: {src}")
101
+ if os.path.isfile(dest) and _same(src, dest):
102
+ return "unchanged"
103
+ if os.path.isfile(dest) and not force:
104
+ return "skipped"
105
+ if dry_run:
106
+ return "wrote"
107
+ parent = os.path.dirname(dest) or "."
108
+ os.makedirs(parent, exist_ok=True)
109
+ fd, tmp = tempfile.mkstemp(dir=parent, prefix=".web-forager-", suffix=".tmp")
110
+ os.close(fd)
111
+ try:
112
+ shutil.copyfile(src, tmp)
113
+ os.replace(tmp, dest)
114
+ finally:
115
+ if os.path.exists(tmp):
116
+ os.remove(tmp)
117
+ return "wrote"
118
+
119
+
120
+ BEGIN_MARKER = "<!-- web-forager:begin -->"
121
+ END_MARKER = "<!-- web-forager:end -->"
122
+ # Hosts whose global file also carries the user's own instructions. Copying
123
+ # over it would destroy them, so web-forager owns only a marked block.
124
+ APPEND_HOSTS = {"claude"}
125
+
126
+
127
+ def _append_block(src_rel: str, dest: str, force: bool, dry_run: bool) -> str:
128
+ """Insert or update a marked block; never replace the rest of the file."""
129
+
130
+ src = os.path.join(REPO_ROOT, src_rel)
131
+ if not os.path.isfile(src):
132
+ raise SystemExit(f"install.py: missing source rule file: {src}")
133
+ with open(src, encoding="utf-8") as handle:
134
+ block = f"{BEGIN_MARKER}\n{handle.read().rstrip()}\n{END_MARKER}\n"
135
+
136
+ text = ""
137
+ if os.path.isfile(dest):
138
+ with open(dest, encoding="utf-8", newline="") as handle:
139
+ text = handle.read()
140
+
141
+ if BEGIN_MARKER in text and END_MARKER in text:
142
+ start = text.index(BEGIN_MARKER)
143
+ finish = text.index(END_MARKER, start) + len(END_MARKER)
144
+ current = text[start:finish].replace("\r\n", "\n") + "\n"
145
+ if current == block:
146
+ return "unchanged"
147
+ if not force:
148
+ return "skipped"
149
+ updated = text[:start] + block + text[finish:].lstrip("\r\n")
150
+ elif text.strip():
151
+ updated = text.rstrip("\r\n") + "\n\n" + block
152
+ else:
153
+ updated = block
154
+
155
+ if dry_run:
156
+ return "wrote"
157
+ parent = os.path.dirname(dest) or "."
158
+ os.makedirs(parent, exist_ok=True)
159
+ fd, tmp = tempfile.mkstemp(dir=parent, prefix=".web-forager-", suffix=".tmp")
160
+ os.close(fd)
161
+ try:
162
+ with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
163
+ handle.write(updated)
164
+ os.replace(tmp, dest)
165
+ finally:
166
+ if os.path.exists(tmp):
167
+ os.remove(tmp)
168
+ return "wrote"
169
+
170
+
171
+ def _select(only: str | None, keys) -> list[str]:
172
+ if not only:
173
+ return list(keys)
174
+ wanted = [part.strip() for part in only.split(",") if part.strip()]
175
+ unknown = [name for name in wanted if name not in keys]
176
+ if unknown:
177
+ raise SystemExit(
178
+ "install.py: unknown host(s): "
179
+ + ", ".join(unknown)
180
+ + "\nknown: "
181
+ + ", ".join(keys)
182
+ )
183
+ return wanted
184
+
185
+
186
+ def main(argv: list[str] | None = None) -> int:
187
+ parser = argparse.ArgumentParser(
188
+ prog="install.py",
189
+ description="Copy the web-forager rule file into every supported agent host.",
190
+ )
191
+ parser.add_argument("--project", metavar="DIR", help="install project-scoped rules into DIR")
192
+ parser.add_argument(
193
+ "--global",
194
+ dest="global_scope",
195
+ action="store_true",
196
+ help="install user-scoped rules for detected hosts",
197
+ )
198
+ parser.add_argument("--only", metavar="HOSTS", help="comma-separated subset, e.g. cursor,opencode")
199
+ parser.add_argument("--force", action="store_true", help="replace an existing rule file or marked block that differs")
200
+ parser.add_argument("--dry-run", action="store_true", help="show what would be written, write nothing")
201
+ parser.add_argument("--list", action="store_true", help="show every host and its destination, then exit")
202
+ args = parser.parse_args(argv)
203
+
204
+ if args.list:
205
+ print("project scope (--project DIR):")
206
+ for key, (rel, _src) in PROJECT_RULES.items():
207
+ print(f" {key:<10} {rel}")
208
+ print("global scope (--global), written only when the host is present:")
209
+ for key, (dest, _src) in global_rules().items():
210
+ state = "" if os.path.isdir(os.path.dirname(dest)) else " (absent)"
211
+ mode = " (append)" if key in APPEND_HOSTS else ""
212
+ print(f" {key:<10} {dest}{state}{mode}")
213
+ return 0
214
+
215
+ if not args.project and not args.global_scope:
216
+ parser.error("choose --project DIR and/or --global (or --list)")
217
+
218
+ results: list[tuple[str, str, str]] = []
219
+ if args.project:
220
+ project = os.path.abspath(args.project)
221
+ os.makedirs(project, exist_ok=True)
222
+ for key in _select(args.only, PROJECT_RULES):
223
+ rel, src_rel = PROJECT_RULES[key]
224
+ dest = os.path.join(project, rel)
225
+ results.append((key, dest, _install(src_rel, dest, args.force, args.dry_run)))
226
+
227
+ if args.global_scope:
228
+ targets = global_rules()
229
+ for key in _select(args.only, targets):
230
+ dest, src_rel = targets[key]
231
+ host_dir = os.path.dirname(dest)
232
+ if not os.path.isdir(host_dir):
233
+ results.append((key, dest, "absent"))
234
+ continue
235
+ if key in APPEND_HOSTS:
236
+ results.append((key, dest, _append_block(src_rel, dest, args.force, args.dry_run)))
237
+ else:
238
+ results.append((key, dest, _install(src_rel, dest, args.force, args.dry_run)))
239
+
240
+ width = max(len(key) for key, _dest, _state in results)
241
+ for key, dest, state in results:
242
+ print(f" {key:<{width}} {state:<9} {dest}")
243
+
244
+ wrote = sum(1 for _key, _dest, state in results if state == "wrote")
245
+ print(
246
+ f"\n{wrote} file(s) written"
247
+ + (" (dry run)" if args.dry_run else "")
248
+ + ". Start a new session in the host to pick the rule up."
249
+ )
250
+ return 0
251
+
252
+
253
+ if __name__ == "__main__":
254
+ sys.exit(main())
package/manifest.json ADDED
@@ -0,0 +1,97 @@
1
+ {
2
+ "name": "web-forager",
3
+ "version": "1.3.0",
4
+ "stage": "domain_extensions",
5
+ "input_types": [
6
+ "url",
7
+ "html",
8
+ "pdf",
9
+ "docx",
10
+ "xlsx",
11
+ "epub",
12
+ "csv",
13
+ "rss",
14
+ "atom",
15
+ "sitemap",
16
+ "keyword",
17
+ "doi",
18
+ "bvid"
19
+ ],
20
+ "output_type": "markdown",
21
+ "output_types": [
22
+ "markdown",
23
+ "json"
24
+ ],
25
+ "dependencies": [
26
+ "requests",
27
+ "beautifulsoup4",
28
+ "trafilatura"
29
+ ],
30
+ "optional_dependencies": [
31
+ "playwright",
32
+ "pypdf"
33
+ ],
34
+ "sandbox_required": false,
35
+ "entrypoint": "run.py",
36
+ "actions": [
37
+ "fetch",
38
+ "find",
39
+ "feed",
40
+ "clean",
41
+ "bili",
42
+ "surf",
43
+ "scholar",
44
+ "map"
45
+ ],
46
+ "mcp": {
47
+ "entrypoint": "mcp_server.py",
48
+ "config": ".mcp.json",
49
+ "tools": [
50
+ "web_fetch",
51
+ "web_search",
52
+ "web_surf",
53
+ "web_scholar",
54
+ "web_map",
55
+ "web_bilibili",
56
+ "web_clean",
57
+ "web_doctor",
58
+ "web_find",
59
+ "web_feed"
60
+ ]
61
+ },
62
+ "browser": {
63
+ "entrypoint": "browser/run.js",
64
+ "runtime": "node >= 18",
65
+ "playbooks": [
66
+ "bili-feed",
67
+ "bili-video",
68
+ "bili-chain",
69
+ "bili-danmaku",
70
+ "zhihu-dig",
71
+ "xhs-dig",
72
+ "search-dig",
73
+ "wechat-dig"
74
+ ],
75
+ "backends": [
76
+ "iab",
77
+ "extension",
78
+ "cdp"
79
+ ],
80
+ "mcp": {
81
+ "entrypoint": "browser/mcp_server.js",
82
+ "server": "web-forager-browser",
83
+ "tools": [
84
+ "browser_list",
85
+ "browser_run",
86
+ "browser_session",
87
+ "browser_doctor"
88
+ ]
89
+ }
90
+ },
91
+ "license": "MIT",
92
+ "author": {
93
+ "name": "Jiaming Shi"
94
+ },
95
+ "description": "Clean-room, robots-aware web information gathering for agents and analysts. HTTP mode searches 360/Bing/Sogou/WeChat, searches inside a site through its own search page, reads HTML/PDF/DOCX/XLSX/EPUB/CSV and RSS/Atom, collects public pages from Bilibili through a dedicated adapter and reads other sites as ordinary pages, extracts metadata and tables, discovers sitemaps and caches politely. Browser mode drives a real browser through a pluggable bridge for login-state pages, rendered content, danmaku and eight read-only playbooks covering Bilibili, Zhihu, Xiaohongshu and WeChat. Optional scholarly lookups cover arXiv/Crossref/PubMed/DOI. Ships a CLI, an --agent envelope, and two stdio MCP servers.",
96
+ "disclaimer": "本插件仅供学习与信息收集使用。使用者需遵守目标网站的 Terms of Service 及相关版权法律。禁止用于恶意并发攻击。严禁收集、聚合或推断特定自然人的个人信息,或将其用于人肉搜索、骚扰、跟踪等侵害他人隐私的活动;此类用法产生的法律责任由使用者自行承担。"
97
+ }