web-forager 0.0.0-stage → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +119 -0
- package/CHANGELOG.md +508 -0
- package/LICENSE +21 -0
- package/README.md +498 -2
- package/README_zh.md +498 -0
- package/SKILL.md +451 -0
- package/THIRD_PARTY_LICENSES.txt +40 -0
- package/browser/bridges/playwright-bridge.js +382 -0
- package/browser/launch_chrome.py +193 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +90 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +141 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +130 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +215 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +252 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +107 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +441 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/research.md +35 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +349 -0
- package/docs/REFERENCE_zh.md +342 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +98 -0
- package/mcp_server.py +850 -0
- package/package.json +82 -6
- package/requirements.txt +15 -0
- package/run.py +908 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +882 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +5141 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +156 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +381 -0
- package/scripts/sitemap.py +195 -0
- package/scripts/untrusted.py +312 -0
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
# web-forager 参考手册
|
|
2
|
+
|
|
3
|
+
> [web-forager](../README_zh.md) 的一部分。这里放查询类内容:记录结构、动作、退出码、环境要求、目录结构与测试。
|
|
4
|
+
|
|
5
|
+
[English](REFERENCE.md) · **简体中文**
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## MCP 服务
|
|
10
|
+
|
|
11
|
+
<details>
|
|
12
|
+
<summary>工具名</summary>
|
|
13
|
+
|
|
14
|
+
HTTP 工具:`web_fetch`、`web_search`、`web_research`、`web_surf`、`web_scholar`、`web_map`、`web_bilibili`、`web_doctor`、`web_find`、`web_feed`。浏览器工具:`browser_list`、`browser_run`、`browser_session`、`browser_doctor`。
|
|
15
|
+
|
|
16
|
+
MCP 调用始终只读,并保持目标页边界:没有任何工具参数能改 `respect_robots`、`robots_policy`、`input_gate`、代理、浏览器桥命令或写开关,本地文件 `clean` 动作也不对 MCP 暴露。搜索引擎与学术查询接口有单独的用户级策略(命令行 `--query-robots`,服务进程环境变量 `WEB_FORAGER_QUERY_ROBOTS`),默认 `warn`:会读取被禁止的查询路径,并在引擎报告里写明这次覆盖;`respect` 会拒绝。模型无法通过工具调用改变它。
|
|
17
|
+
|
|
18
|
+
</details>
|
|
19
|
+
|
|
20
|
+
直接测试协议:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
printf '%s\n' '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
|
|
24
|
+
'{"jsonrpc":"2.0","id":2,"method":"tools/list"}' | python mcp_server.py
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
printf '%s\n' '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
|
|
29
|
+
'{"jsonrpc":"2.0","id":2,"method":"tools/list"}' | node browser/mcp_server.js
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## 浏览器模式
|
|
33
|
+
|
|
34
|
+
浏览器层位于 `browser/`,需要 Node.js 18 或更新。它默认只读:点赞、投币、收藏、关注、分享、评论、弹幕与发布一律拒绝,除非用户明确授权某一次动作后传入 `--allow-write`。
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
npm install playwright && npx playwright install chromium
|
|
38
|
+
node browser/run.js --doctor
|
|
39
|
+
node browser/run.js --playbook bili-feed --session bili-01 --max-items 8 --agent
|
|
40
|
+
node browser/run.js --playbook zhihu-dig --query "强化学习" --agent
|
|
41
|
+
node browser/run.js --playbook wechat-dig --url "https://mp.weixin.qq.com/s/..." --agent
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
| 剧本 | 作用 | 需要 |
|
|
45
|
+
|---|---|---|
|
|
46
|
+
| `bili-feed` | 首页信息流,逐屏捕捉未见卡片 | 无 |
|
|
47
|
+
| `bili-video` | 单个视频与可见评论 | `--url` 或 `--bvid` |
|
|
48
|
+
| `bili-chain` | 跟随「下一话」,并从会话续接链路 | `--url`、`--bvid` 或会话链路 |
|
|
49
|
+
| `bili-danmaku` | 带时间戳的弹幕,优先用桥的 `observe`,DOM 文本兜底(实验性) | `--url` 或 `--bvid` |
|
|
50
|
+
| `zhihu-dig` | 搜知乎、排序链接、捕捉最佳未见页面 | `--query` |
|
|
51
|
+
| `xhs-dig` | 搜小红书公开笔记、排序 `/explore` 链接、捕捉一条 | `--query` |
|
|
52
|
+
| `search-dig` | 搜 B 站、排序卡片、打开最佳未见结果 | `--query` |
|
|
53
|
+
| `wechat-dig` | 单篇公众号文章;遇到环境校验页就如实报告,不把错误页当正文 | `--url` |
|
|
54
|
+
|
|
55
|
+
公众号是 HTTP 模式唯一走不完的一类:`mp.weixin.qq.com` 在 robots.txt 里禁止 `/s`,即使加 `--robots-policy warn`,文章 URL 也会返回「环境异常」或跳转到验证码页,而不是正文;HTTP 记录会写成 `verification_required`(退出码 9),不会把错误页当成内容交出去。`wechat-dig` 在真实浏览器配置里打开同一个 URL:该配置已通过微信校验时能读到正文,没通过时报告 `blocked`,不会去解验证码。
|
|
56
|
+
|
|
57
|
+
装好 `playwright` 与它的 Chromium 后,自带的桥会被自动启用;`--bridge` 或 `WEB_FORAGER_BRIDGE` 用于指向别处的桥。命令按空白拆分后直接启动,不经过 shell——路径含空格时加引号,参数照常写;管道、`&&` 与重定向不会被解释。`WEB_FORAGER_BROWSER_HEADFUL=1` 显示窗口;`WEB_FORAGER_BROWSER_PROFILE=/path` 复用持久化配置目录(`extension` 后端的登录态正是靠它保留);`WEB_FORAGER_BROWSER_CDP=http://127.0.0.1:9222` 接管你已有的浏览器(`cdp`)。桥通过可选的 `describe` 调用声明自己实现了什么;不实现 `describe` 的桥会被标记为 `declared: false`,所有能力一律为 false。
|
|
58
|
+
|
|
59
|
+
登录一次的做法:运行 `python browser/launch_chrome.py`(默认使用 `~/.web-forager/browser/profile`,**CDP 默认关闭**;加 `--cdp` 才开启一个环回 CDP 端口,默认随机、`--port 9222` 可固定)。不加 `--cdp` 时,用默认后端运行浏览器即可复用该配置;加了 `--cdp` 才需要给剧本加 `--backend cdp`。CDP 本身没有认证:端口只留在本机,用完关闭浏览器。`--storage-state FILE` 在运行前载入 Playwright 状态文件;`--save-storage-state FILE` 在运行后写出,权限为 `0600`,路径落在 `--root` 内会被拒绝。这个文件包含 Cookie 与登录态:按凭据对待,不要提交进 git,也不要复制到共享机器。
|
|
60
|
+
|
|
61
|
+
`--strict-robots` 在每次导航前读取 `robots.txt`,被禁止的路径以 `robots_denied`(退出码 3)停下。不加这个开关时记录同样说清楚:`compliance.robots_checked` 为 `false`。读到了页面却什么也没找到会报告 `degraded`(退出码 8)并附上记录;反爬或错误页以 `blocked`(退出码 9)停下;站点要求登录是 `login_required`(退出码 6)。
|
|
62
|
+
|
|
63
|
+
浏览器模式是插件里最贵的一环,默认值因此开得很小:完整记录用 `--out result.json`(MCP 传 `out:`)落盘,默认不截图,`--max-screens 5` 与 `--min-interval 3000` 保持礼貌,长时间刷屏宁可用 `--session` 分几次续跑。
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## 搜索引擎不可用时
|
|
68
|
+
|
|
69
|
+
默认搜索走 `auto`。以中日韩文字为主的查询会同时问 360、搜狗与搜狗微信——中文人名或机构名通常只有微信文章索引这条渠道有内容——并且不会交给 Bing:Bing 会把中文短语拆成单字,返回词典页。这类单字词典页无论来自哪个引擎都会被过滤掉;确有需要时可以用 `--engine bing` 明确指定。
|
|
70
|
+
|
|
71
|
+
另有两条完全不依赖搜索引擎的路:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python run.py --action find --site the-decoder.com --query "OpenAI watermark" --max-pages 3
|
|
75
|
+
python run.py --action feed --url https://the-decoder.com/ --num-results 10
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`find` 尝试站点自带的搜索地址(`/?s=`、`/search?q=` 等四种),按查询词给页面链接排序并抓取最相关的几篇。`feed` 从页面的 `<link rel=alternate>` 发现 RSS/Atom 并读取。`doctor --probe` 对每个引擎各问一次,报告当前网络下哪些可用;`doctor` 还会显示正在使用的代理(凭据已抹去)。请求会遵循 `HTTPS_PROXY` / `ALL_PROXY`,`--proxy URL` 可为一次运行覆盖它们。
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## 更多命令行示例
|
|
83
|
+
|
|
84
|
+
每条命令单独一个代码块,复制时不用删掉相邻的行。
|
|
85
|
+
|
|
86
|
+
**抓一个 B 站视频。**
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
python run.py --action fetch --url "https://www.bilibili.com/video/BV1xx411c7mD"
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
**抓一个需要 JS 渲染的页面。**
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
python run.py --action fetch --url "https://www.zhihu.com/" --render auto
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
**冲浪一个站点,之后用会话续游。**
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
python run.py --action surf --url "https://www.zhihu.com/" --depth 1 --max-pages 5 --session zhihu-01
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
**把本地 PDF 清成 Markdown。**
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
python run.py --action clean --input notes.pdf --format markdown
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**从站点地图列出最多 500 个页面 URL。**
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
python run.py --action map --url "https://example.org" --limit 500
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
**读取 B 站首页信息流。**
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
python run.py --action bili --mode feed --max-items 10
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## 记录与输出
|
|
125
|
+
|
|
126
|
+
### 每页一条记录
|
|
127
|
+
|
|
128
|
+
| 字段 | HTTP 模式 | 浏览器模式 |
|
|
129
|
+
|---|---|---|
|
|
130
|
+
| `metadata`、`markdown`、`tables`、`warnings` | 有 | 有 |
|
|
131
|
+
| `trust`、`injection_flags`、`injection_findings` | 有 | 有 |
|
|
132
|
+
| `url`、`title` | 有(`url` 与 `final_url`) | 有 |
|
|
133
|
+
| `status`、`content_type`、`document_kind`、`cached`、`rendered`、`render_fallback`、`fetched_at`、`chunks`、`links` | 有 | 无 |
|
|
134
|
+
| `schema_version`、`captured_at`、`playbook`、`backend`、`compliance`、`viewport_evidence`、`interaction_trace` | 无 | 有 |
|
|
135
|
+
|
|
136
|
+
`render_fallback` 为 `true` 表示服务器只返回了极小的脚本壳,HTTP 模式在抽取前用 Playwright 重渲染了一次。`fetch --query` 每次最多允许两次这样的重渲染,每次会多花数秒。
|
|
137
|
+
|
|
138
|
+
`injection_findings` 是一个列表,每项包含 `{code, severity, confidence, span, snippet, context}`。`span` 是扫描文本里的字符区间,`severity` 为 `hard` 或 `soft`,`context` 为 `direct`、`quote` 或 `example`。硬规则描述具体动作(伪造 system 标签、外传密钥、`curl ... | sh`、`rm -rf /`),只有命中硬规则时 `--input-gate strict` 才扣留正文;软规则是“忽略之前的指令”和角色切换,只报告、不扣留,因为论文与安全文章会合法引用它们。扫描用重叠窗口覆盖全文,发生在 `max_chars` 截断之前;`warnings` 第一条会重复最多三条命中片段。
|
|
139
|
+
|
|
140
|
+
搜索结果按查询覆盖度、短语与标题命中、域名先验、摘要时效与引擎顺序排序,**只有一个引擎应答时同样排序**。每条结果带 `coverage`、`matched_terms`、`relevance` 与 `language_match`。结果包含完整短语,或至少两个有意义的查询词且覆盖一半查询时,才算相关;只重复一个产品名的异语言页面会标 `language_match: false`。跑题结果排在最后,真有至少三条相关结果时直接剔除;否则保留并在 `warnings` 里说明数量。最终列表每个域名最多两条,避免单一站点挤掉其他来源;来源不足时再用溢出的结果补满。首轮结果过少时会用简化查询重试一次(去掉引号与常见停用词,或把带空格的中文查询合并),重试会在 `engine_report` 里标 `query_variant`。搜索标题与摘要也走同一套注入扫描,命中的结果会带自己的 `injection_flags` 与 `injection_findings`。
|
|
141
|
+
|
|
142
|
+
排序前会先规范化 URL:`utm_*`、`fbclid`、片段、`www.`、末尾斜杠与 `/amp` 不再把同一篇文章算成两条结果。被多个引擎同时返回的结果会获得 RRF 加分;标题与查询完全一致时比“标题包含短语”略高。匹配阶段使用轻量英文词形归一,所以 `injection` 能遇到 `injections`,但显示的查询与正文不变。搜索载荷还带 `search_quality: {query, engine, returned, relevant, off_topic, language_mismatch, engines_answered, engine_errors, dropped_off_topic, returned_before_filter}`,供不想解析英文 warnings 的智能体直接读取。
|
|
143
|
+
|
|
144
|
+
加 `--core` 时返回共享形状:
|
|
145
|
+
|
|
146
|
+
```json
|
|
147
|
+
{
|
|
148
|
+
"schema_version": "core-1",
|
|
149
|
+
"source": "web-forager",
|
|
150
|
+
"mode": "http",
|
|
151
|
+
"url": "https://example.org/article",
|
|
152
|
+
"title": "An article",
|
|
153
|
+
"collected_at": "2026-11-07T10:00:00Z",
|
|
154
|
+
"metadata": {},
|
|
155
|
+
"markdown": "...",
|
|
156
|
+
"tables": [],
|
|
157
|
+
"warnings": [],
|
|
158
|
+
"extra": { "status": 200, "cached": false }
|
|
159
|
+
}
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
`mode` 为 `http` 或 `browser`,`source` 标明产出方,`extra` 保留两种模式不共享的全部字段。同样的折叠也提供库调用:Python 侧是 `engine.normalize_record()`,浏览器侧是 `browser/lib/record.js` 的 `normalizeRecord()`。
|
|
163
|
+
|
|
164
|
+
`results` 是给智能体循环用的索引,不是每页正文的第二份拷贝。搜索时它只带 `url`、`title`、`snippet`、`status`、`content_type`、`markdown_chars`、`tables` 以及可能的 `warnings` 或 `error`;Markdown 正文只在 `pages` 里出现一次,单页 `fetch` 则把它放在顶层。要旧行为加 `--full-results`(CLI)或 `fullResults: true`(MCP)。
|
|
165
|
+
|
|
166
|
+
### 工作区输出
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
python run.py --action fetch --url "https://example.org/article" --root <WS>
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
只写这些位置:
|
|
173
|
+
|
|
174
|
+
```text
|
|
175
|
+
<WS>/process/web-forager/web-forager_<timestamp>_<slug>.json
|
|
176
|
+
<WS>/process/web-forager/web-forager_<timestamp>_<slug>.md
|
|
177
|
+
<WS>/_broadcast/web-forager_<timestamp>_<slug>.json
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
`material/` 与 `results/` 属于用户与编排器,web-forager 从不写入。浏览器捕捉遵循同一套规则,落在 `<WS>/process/web-forager/browser/` 与 `<WS>/_broadcast/`。
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## research 输出
|
|
187
|
+
|
|
188
|
+
`research --query` 先宽召回,再集中读取少数不同 host。新增字段:
|
|
189
|
+
|
|
190
|
+
| 字段 | 含义 |
|
|
191
|
+
|---|---|
|
|
192
|
+
| `intent` | 关键词分类器给出的 `{intent, confidence, signals}` |
|
|
193
|
+
| `strategy` | `{breadth, search_pages, targeted_site, targeted_sites, targeted_by_site, official_entries, focus_*}` |
|
|
194
|
+
| `seeds` | 入选结果及 `authority_tier`、`authority_score`、`reason` |
|
|
195
|
+
| `evidence` | `[{url, title, authority_tier, authors, published_at, date_source, passages, partial?}]` |
|
|
196
|
+
| `freshness` | `{newest, oldest, dated_sources, undated_sources, newest_age_days, all_stale}` |
|
|
197
|
+
| `answerability` | `{level, coverage, authority_level, partial_only, source_count, host_count, view_hosts, multi_view, stale, reason}` |
|
|
198
|
+
| `skipped` | 未使用页面及 `code`(`robots_denied`、`browser_required`、`browser_required_low_value`、`no_matching_passage` 等) |
|
|
199
|
+
|
|
200
|
+
宽召回里没有相关的高权威来源时,第二遍按顺序尝试:直接读取已知官方入口页
|
|
201
|
+
(Python asyncio、Go 并发、Rust、MDN、微软 Wi-Fi 支持),再读站点自己的搜索页
|
|
202
|
+
(微软支持、MDN),最后才对非中文问题发一次 `site:` 查询。前两条不依赖搜索引擎
|
|
203
|
+
是否支持 `site:`。新闻题没有任何可用引擎结果时,会回退到中新网财经公开 RSS;
|
|
204
|
+
带日期的标题与摘要标为 `feed_headline`,绝不扩写成伪造的全文;正文没有日期时,
|
|
205
|
+
抓取到的页面继承 feed 的发布日期。每次聚焦抓取最多 8 秒且不重试,robots.txt 最多
|
|
206
|
+
6 秒,避免一个不可达的来源拖垮整次调用。对比题至少需要
|
|
207
|
+
两个独立全文视角;公众号摘要不算一个视角。
|
|
208
|
+
对比题或技术题已经有两个权威全文来源时,纯公众号摘要会被跳过,而不是混进证据。
|
|
209
|
+
HTTP 读不了的公众号文章仍只返回 `partial` 搜索摘要并明确警告,不伪造全文。
|
|
210
|
+
所有句段都是带 URL 的第三方引用文本。
|
|
211
|
+
|
|
212
|
+
## 动作与退出码
|
|
213
|
+
|
|
214
|
+
| 动作 | 用途 |
|
|
215
|
+
|---|---|
|
|
216
|
+
| `fetch` | 一个 URL 或关键词查询 → 页面 |
|
|
217
|
+
| `research` | 宽召回 → 聚焦阅读 → 带引用的证据句段 |
|
|
218
|
+
| `find` | 用站点自己的搜索页在站内检索 |
|
|
219
|
+
| `feed` | 读取站点的 RSS/Atom,自动从页面里发现订阅地址 |
|
|
220
|
+
| `clean` | 本地或远程 HTML/PDF/DOCX/XLSX/EPUB/CSV → Markdown |
|
|
221
|
+
| `bili` | B 站首页 / 视频 / 相关 / 搜索 / 冲浪 |
|
|
222
|
+
| `surf` | 搜索或种子 URL,再按相关度做链接 BFS |
|
|
223
|
+
| `scholar` | arXiv / Crossref / PubMed / DOI 记录 |
|
|
224
|
+
| `map` | 从 robots.txt 与 sitemap 发现页面 URL |
|
|
225
|
+
| `doctor` | 依赖自检:装了什么,该装什么 |
|
|
226
|
+
|
|
227
|
+
| 退出码 | 含义(未在下表改写的,两种模式相同) |
|
|
228
|
+
|---|---|
|
|
229
|
+
| 0 | 成功 |
|
|
230
|
+
| 2 | 参数或 URL 错误 |
|
|
231
|
+
| 3 | robots.txt 拒绝访问,或没有配置浏览器桥 |
|
|
232
|
+
| 4 | 网络 / HTTP / 导航失败 |
|
|
233
|
+
| 5 | 抽取或捕捉失败 |
|
|
234
|
+
| 6 | 请求了 Playwright 但不可用 |
|
|
235
|
+
| 7 | 处理 PDF 但 `pypdf` 不可用 |
|
|
236
|
+
| 9 | 返回的是验证页或反爬页而不是内容(HTTP 模式为 `verification_required`,浏览器模式为 `blocked`) |
|
|
237
|
+
| 130 | 被中断 |
|
|
238
|
+
|
|
239
|
+
浏览器模式沿用这张表,只改写或新增以下含义:
|
|
240
|
+
|
|
241
|
+
| 退出码 | 浏览器模式含义 |
|
|
242
|
+
|---|---|
|
|
243
|
+
| 2 | 另外表示未知后端 |
|
|
244
|
+
| 6 | `login_required`:站点要求登录 |
|
|
245
|
+
| 7 | `confirmation_required`:写类操作被拒绝 |
|
|
246
|
+
| 8 | `degraded`:页面读到了,但剧本要的东西一个都没找到 |
|
|
247
|
+
| 9 | `blocked`:页面被反爬或错误页替换 |
|
|
248
|
+
|
|
249
|
+
`--agent` 保留这些退出码,并在 stdout 输出 `{"ok":false,"error":{"code","message","hint"}}` 信封。`doctor` 逐项报告依赖的版本或安装命令;缺 Playwright 返回退出码 6,缺 `pypdf` 返回退出码 7。
|
|
250
|
+
|
|
251
|
+
---
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
## 环境要求
|
|
256
|
+
|
|
257
|
+
Python 3.10+。核心运行时:`requests`、`beautifulsoup4`、`trafilatura`。可选:`playwright`(JS 渲染)、`pypdf`(PDF)。`defusedxml` 若已安装则使用,但从不强制依赖——含内部 DTD 子集或任何 ENTITY 声明的 XML 一律拒绝。浏览器模式另需 Node.js 18+ 与一个桥命令;`browser/bridges/playwright-bridge.js` 还需要 npm 安装 `playwright`。没有内置任何第三方代码;完整的能力-依赖矩阵见 [DEPENDENCIES.md](DEPENDENCIES.md)。
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
|
|
261
|
+
---
|
|
262
|
+
|
|
263
|
+
## 目录结构
|
|
264
|
+
|
|
265
|
+
<details>
|
|
266
|
+
<summary>完整目录树</summary>
|
|
267
|
+
|
|
268
|
+
```text
|
|
269
|
+
web-forager/
|
|
270
|
+
├── .claude-plugin/ Claude Code 清单 + marketplace
|
|
271
|
+
├── .codex-plugin/ Codex 清单 + marketplace
|
|
272
|
+
├── .codebuddy-plugin/ CodeBuddy 清单 + marketplace
|
|
273
|
+
├── .dsh-plugin/ DSH 清单 + CJS 入口
|
|
274
|
+
├── .cursor/rules/ Cursor 规则
|
|
275
|
+
├── .windsurf/rules/ Windsurf 规则
|
|
276
|
+
├── .clinerules/ Cline 规则
|
|
277
|
+
├── .continue/rules/ Continue 规则
|
|
278
|
+
├── .roo/rules/ Roo Code 规则
|
|
279
|
+
├── .github/ GitHub Copilot 指令 + Actions CI
|
|
280
|
+
├── AGENTS.md 通用智能体契约
|
|
281
|
+
├── install.py 一键安装规则文件
|
|
282
|
+
├── manifest.json 插件清单
|
|
283
|
+
├── .mcp.json MCP stdio 注册
|
|
284
|
+
├── package.json / index.js npm + dsh 打包
|
|
285
|
+
├── cordis.patch.yml dsh bundle 补丁
|
|
286
|
+
├── run.py CLI:九个动作
|
|
287
|
+
├── mcp_server.py MCP stdio 服务(10 个 HTTP 工具)
|
|
288
|
+
├── browser/ 浏览器模式(Node 18+)
|
|
289
|
+
│ ├── run.js CLI:八个只读剧本
|
|
290
|
+
│ ├── mcp_server.js MCP stdio 服务(4 个浏览器工具)
|
|
291
|
+
│ ├── lib/ 桥后端、会话、记录、安全门禁、选择器、robots、快照
|
|
292
|
+
│ ├── playbooks/ bili-* / zhihu-dig / xhs-dig / search-dig / wechat-dig
|
|
293
|
+
│ ├── bridges/ 可选 Playwright 参考桥
|
|
294
|
+
│ └── tests/ 假后端、真实页面快照、live 冒烟
|
|
295
|
+
├── SKILL.md Claude Code 技能契约
|
|
296
|
+
├── commands/ /fetch /surf /scholar /map /browse
|
|
297
|
+
├── docs/ 安装命令、致谢、依赖矩阵
|
|
298
|
+
├── scripts/
|
|
299
|
+
│ ├── engine.py robots、抓取、搜索、缓存、分块、Markdown、surf、find、feed
|
|
300
|
+
│ ├── pdf.py PDF 文本(可选 pypdf)
|
|
301
|
+
│ ├── documents.py DOCX / XLSX / EPUB / CSV
|
|
302
|
+
│ ├── feeds.py RSS / Atom / RSS 1.0
|
|
303
|
+
│ ├── scholar.py arXiv / Crossref / PubMed / DOI
|
|
304
|
+
│ ├── sitemap.py sitemap 发现
|
|
305
|
+
│ └── bilibili.py B 站适配器
|
|
306
|
+
├── tests/ 离线 HTTP 夹具
|
|
307
|
+
├── requirements.txt
|
|
308
|
+
├── THIRD_PARTY_LICENSES.txt
|
|
309
|
+
├── CHANGELOG.md
|
|
310
|
+
├── README.md · README_zh.md
|
|
311
|
+
└── LICENSE MIT
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
</details>
|
|
315
|
+
|
|
316
|
+
---
|
|
317
|
+
|
|
318
|
+
---
|
|
319
|
+
|
|
320
|
+
## 测试
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
python -m pytest tests -q
|
|
324
|
+
npm run test:browser
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
共 **333 个 Python 测试**加 **91 个浏览器测试**,全部离线通过。
|
|
328
|
+
|
|
329
|
+
<details>
|
|
330
|
+
<summary>测试覆盖了什么</summary>
|
|
331
|
+
|
|
332
|
+
`conftest.py` 会起一个临时本地 HTTP 站点,提供 robots.txt、文章、PDF/Office 文件、订阅源、站点地图、搜索结果页、站点搜索页与多跳 surf 页面。Python 套件覆盖 robots 拒绝、抽取、表格、文档、订阅源、缓存、搜索分页与路由、分块、MCP、surf 会话、安装器行为、宿主规则与免责声明检查、共享的 `core-1` 记录,以及每一个退出码。浏览器套件用假后端与说同一套线协议的假桥,不打开浏览器也能把 CLI 跑通;它还覆盖能力声明、robots.txt 匹配、`--strict-robots` 往返、私有地址拦截、不可信内容筛查,以及基于真实页面快照的 `degraded` 与 `blocked` 路径。
|
|
333
|
+
|
|
334
|
+
</details>
|
|
335
|
+
|
|
336
|
+
CI 在每次推送到 `main`、拉取请求与版本 tag 时运行两套测试:Python 在 Ubuntu 3.10/3.12 与 Windows 3.12,浏览器模式在 Node 20/22。真实站点检查是可选项,CI 从不运行:
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
WEB_FORAGER_LIVE=1 WEB_FORAGER_BRIDGE="node browser/bridges/playwright-bridge.js" npm run test:live
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
---
|
package/index.js
ADDED
package/install.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""Cross-host rule-file installer for web-forager.
|
|
2
|
+
|
|
3
|
+
No agent host ships an installer that understands another host's rules, so
|
|
4
|
+
"install everywhere" cannot be delegated to a single vendor command. This
|
|
5
|
+
script copies the agent contract into every supported host's rule location in
|
|
6
|
+
one run.
|
|
7
|
+
|
|
8
|
+
Two scopes:
|
|
9
|
+
|
|
10
|
+
python install.py --project <dir> project-scoped rules inside <dir>
|
|
11
|
+
python install.py --global user-scoped rules for detected hosts
|
|
12
|
+
|
|
13
|
+
`--project` writes Cursor, Windsurf, Cline, Continue, Roo Code and GitHub
|
|
14
|
+
Copilot rule files plus the generic `AGENTS.md` that Codex, Cursor, Copilot
|
|
15
|
+
agents and opencode read. `--global` writes user-level rules for opencode and
|
|
16
|
+
Codex, and appends a marked block to `~/.claude/CLAUDE.md` instead of replacing
|
|
17
|
+
the user's own instructions. A host whose directory is absent is skipped.
|
|
18
|
+
|
|
19
|
+
Plugin hosts (Claude Code, CodeBuddy, Codex, dsh) should install the real
|
|
20
|
+
plugin from its marketplace instead; this script covers hosts that read rule
|
|
21
|
+
files. Re-running is safe: identical files report `unchanged`, and a drifted
|
|
22
|
+
file is left alone unless `--force` is passed.
|
|
23
|
+
|
|
24
|
+
Usage:
|
|
25
|
+
python install.py --list
|
|
26
|
+
python install.py --project . --dry-run
|
|
27
|
+
python install.py --project .
|
|
28
|
+
python install.py --global --only opencode,codex
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import argparse
|
|
34
|
+
import os
|
|
35
|
+
import shutil
|
|
36
|
+
import sys
|
|
37
|
+
import tempfile
|
|
38
|
+
|
|
39
|
+
REPO_ROOT = os.path.dirname(os.path.abspath(__file__))
|
|
40
|
+
|
|
41
|
+
PROJECT_RULES = {
|
|
42
|
+
"cursor": (
|
|
43
|
+
os.path.join(".cursor", "rules", "web-forager.mdc"),
|
|
44
|
+
os.path.join(".cursor", "rules", "web-forager.mdc"),
|
|
45
|
+
),
|
|
46
|
+
"windsurf": (
|
|
47
|
+
os.path.join(".windsurf", "rules", "web-forager.md"),
|
|
48
|
+
os.path.join(".windsurf", "rules", "web-forager.md"),
|
|
49
|
+
),
|
|
50
|
+
"cline": (
|
|
51
|
+
os.path.join(".clinerules", "web-forager.md"),
|
|
52
|
+
os.path.join(".clinerules", "web-forager.md"),
|
|
53
|
+
),
|
|
54
|
+
"continue": (
|
|
55
|
+
os.path.join(".continue", "rules", "web-forager.md"),
|
|
56
|
+
os.path.join(".continue", "rules", "web-forager.md"),
|
|
57
|
+
),
|
|
58
|
+
"roo": (
|
|
59
|
+
os.path.join(".roo", "rules", "web-forager.md"),
|
|
60
|
+
os.path.join(".roo", "rules", "web-forager.md"),
|
|
61
|
+
),
|
|
62
|
+
"copilot": (
|
|
63
|
+
os.path.join(".github", "copilot-instructions.md"),
|
|
64
|
+
os.path.join(".github", "copilot-instructions.md"),
|
|
65
|
+
),
|
|
66
|
+
"agents": ("AGENTS.md", "AGENTS.md"),
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def global_rules() -> dict[str, tuple[str, str]]:
|
|
71
|
+
"""Return user-scoped rule targets for this machine."""
|
|
72
|
+
|
|
73
|
+
home = os.path.expanduser("~")
|
|
74
|
+
return {
|
|
75
|
+
"opencode": (
|
|
76
|
+
os.path.join(home, ".config", "opencode", "AGENTS.md"),
|
|
77
|
+
"AGENTS.md",
|
|
78
|
+
),
|
|
79
|
+
"codex": (
|
|
80
|
+
os.path.join(os.environ.get("CODEX_HOME") or os.path.join(home, ".codex"), "AGENTS.md"),
|
|
81
|
+
"AGENTS.md",
|
|
82
|
+
),
|
|
83
|
+
"claude": (os.path.join(home, ".claude", "CLAUDE.md"), "AGENTS.md"),
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _same(left_path: str, right_path: str) -> bool:
|
|
88
|
+
try:
|
|
89
|
+
with open(left_path, "rb") as left, open(right_path, "rb") as right:
|
|
90
|
+
return left.read() == right.read()
|
|
91
|
+
except OSError:
|
|
92
|
+
return False
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _install(src_rel: str, dest: str, force: bool, dry_run: bool) -> str:
|
|
96
|
+
"""Copy one rule file; return 'wrote', 'unchanged' or 'skipped'."""
|
|
97
|
+
|
|
98
|
+
src = os.path.join(REPO_ROOT, src_rel)
|
|
99
|
+
if not os.path.isfile(src):
|
|
100
|
+
raise SystemExit(f"install.py: missing source rule file: {src}")
|
|
101
|
+
if os.path.isfile(dest) and _same(src, dest):
|
|
102
|
+
return "unchanged"
|
|
103
|
+
if os.path.isfile(dest) and not force:
|
|
104
|
+
return "skipped"
|
|
105
|
+
if dry_run:
|
|
106
|
+
return "wrote"
|
|
107
|
+
parent = os.path.dirname(dest) or "."
|
|
108
|
+
os.makedirs(parent, exist_ok=True)
|
|
109
|
+
fd, tmp = tempfile.mkstemp(dir=parent, prefix=".web-forager-", suffix=".tmp")
|
|
110
|
+
os.close(fd)
|
|
111
|
+
try:
|
|
112
|
+
shutil.copyfile(src, tmp)
|
|
113
|
+
os.replace(tmp, dest)
|
|
114
|
+
finally:
|
|
115
|
+
if os.path.exists(tmp):
|
|
116
|
+
os.remove(tmp)
|
|
117
|
+
return "wrote"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
BEGIN_MARKER = "<!-- web-forager:begin -->"
|
|
121
|
+
END_MARKER = "<!-- web-forager:end -->"
|
|
122
|
+
# Hosts whose global file also carries the user's own instructions. Copying
|
|
123
|
+
# over it would destroy them, so web-forager owns only a marked block.
|
|
124
|
+
APPEND_HOSTS = {"claude"}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _append_block(src_rel: str, dest: str, force: bool, dry_run: bool) -> str:
|
|
128
|
+
"""Insert or update a marked block; never replace the rest of the file."""
|
|
129
|
+
|
|
130
|
+
src = os.path.join(REPO_ROOT, src_rel)
|
|
131
|
+
if not os.path.isfile(src):
|
|
132
|
+
raise SystemExit(f"install.py: missing source rule file: {src}")
|
|
133
|
+
with open(src, encoding="utf-8") as handle:
|
|
134
|
+
block = f"{BEGIN_MARKER}\n{handle.read().rstrip()}\n{END_MARKER}\n"
|
|
135
|
+
|
|
136
|
+
text = ""
|
|
137
|
+
if os.path.isfile(dest):
|
|
138
|
+
with open(dest, encoding="utf-8", newline="") as handle:
|
|
139
|
+
text = handle.read()
|
|
140
|
+
|
|
141
|
+
if BEGIN_MARKER in text and END_MARKER in text:
|
|
142
|
+
start = text.index(BEGIN_MARKER)
|
|
143
|
+
finish = text.index(END_MARKER, start) + len(END_MARKER)
|
|
144
|
+
current = text[start:finish].replace("\r\n", "\n") + "\n"
|
|
145
|
+
if current == block:
|
|
146
|
+
return "unchanged"
|
|
147
|
+
if not force:
|
|
148
|
+
return "skipped"
|
|
149
|
+
updated = text[:start] + block + text[finish:].lstrip("\r\n")
|
|
150
|
+
elif text.strip():
|
|
151
|
+
updated = text.rstrip("\r\n") + "\n\n" + block
|
|
152
|
+
else:
|
|
153
|
+
updated = block
|
|
154
|
+
|
|
155
|
+
if dry_run:
|
|
156
|
+
return "wrote"
|
|
157
|
+
parent = os.path.dirname(dest) or "."
|
|
158
|
+
os.makedirs(parent, exist_ok=True)
|
|
159
|
+
fd, tmp = tempfile.mkstemp(dir=parent, prefix=".web-forager-", suffix=".tmp")
|
|
160
|
+
os.close(fd)
|
|
161
|
+
try:
|
|
162
|
+
with open(tmp, "w", encoding="utf-8", newline="\n") as handle:
|
|
163
|
+
handle.write(updated)
|
|
164
|
+
os.replace(tmp, dest)
|
|
165
|
+
finally:
|
|
166
|
+
if os.path.exists(tmp):
|
|
167
|
+
os.remove(tmp)
|
|
168
|
+
return "wrote"
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _select(only: str | None, keys) -> list[str]:
|
|
172
|
+
if not only:
|
|
173
|
+
return list(keys)
|
|
174
|
+
wanted = [part.strip() for part in only.split(",") if part.strip()]
|
|
175
|
+
unknown = [name for name in wanted if name not in keys]
|
|
176
|
+
if unknown:
|
|
177
|
+
raise SystemExit(
|
|
178
|
+
"install.py: unknown host(s): "
|
|
179
|
+
+ ", ".join(unknown)
|
|
180
|
+
+ "\nknown: "
|
|
181
|
+
+ ", ".join(keys)
|
|
182
|
+
)
|
|
183
|
+
return wanted
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def main(argv: list[str] | None = None) -> int:
|
|
187
|
+
parser = argparse.ArgumentParser(
|
|
188
|
+
prog="install.py",
|
|
189
|
+
description="Copy the web-forager rule file into every supported agent host.",
|
|
190
|
+
)
|
|
191
|
+
parser.add_argument("--project", metavar="DIR", help="install project-scoped rules into DIR")
|
|
192
|
+
parser.add_argument(
|
|
193
|
+
"--global",
|
|
194
|
+
dest="global_scope",
|
|
195
|
+
action="store_true",
|
|
196
|
+
help="install user-scoped rules for detected hosts",
|
|
197
|
+
)
|
|
198
|
+
parser.add_argument("--only", metavar="HOSTS", help="comma-separated subset, e.g. cursor,opencode")
|
|
199
|
+
parser.add_argument("--force", action="store_true", help="replace an existing rule file or marked block that differs")
|
|
200
|
+
parser.add_argument("--dry-run", action="store_true", help="show what would be written, write nothing")
|
|
201
|
+
parser.add_argument("--list", action="store_true", help="show every host and its destination, then exit")
|
|
202
|
+
args = parser.parse_args(argv)
|
|
203
|
+
|
|
204
|
+
if args.list:
|
|
205
|
+
print("project scope (--project DIR):")
|
|
206
|
+
for key, (rel, _src) in PROJECT_RULES.items():
|
|
207
|
+
print(f" {key:<10} {rel}")
|
|
208
|
+
print("global scope (--global), written only when the host is present:")
|
|
209
|
+
for key, (dest, _src) in global_rules().items():
|
|
210
|
+
state = "" if os.path.isdir(os.path.dirname(dest)) else " (absent)"
|
|
211
|
+
mode = " (append)" if key in APPEND_HOSTS else ""
|
|
212
|
+
print(f" {key:<10} {dest}{state}{mode}")
|
|
213
|
+
return 0
|
|
214
|
+
|
|
215
|
+
if not args.project and not args.global_scope:
|
|
216
|
+
parser.error("choose --project DIR and/or --global (or --list)")
|
|
217
|
+
|
|
218
|
+
results: list[tuple[str, str, str]] = []
|
|
219
|
+
if args.project:
|
|
220
|
+
project = os.path.abspath(args.project)
|
|
221
|
+
os.makedirs(project, exist_ok=True)
|
|
222
|
+
for key in _select(args.only, PROJECT_RULES):
|
|
223
|
+
rel, src_rel = PROJECT_RULES[key]
|
|
224
|
+
dest = os.path.join(project, rel)
|
|
225
|
+
results.append((key, dest, _install(src_rel, dest, args.force, args.dry_run)))
|
|
226
|
+
|
|
227
|
+
if args.global_scope:
|
|
228
|
+
targets = global_rules()
|
|
229
|
+
for key in _select(args.only, targets):
|
|
230
|
+
dest, src_rel = targets[key]
|
|
231
|
+
host_dir = os.path.dirname(dest)
|
|
232
|
+
if not os.path.isdir(host_dir):
|
|
233
|
+
results.append((key, dest, "absent"))
|
|
234
|
+
continue
|
|
235
|
+
if key in APPEND_HOSTS:
|
|
236
|
+
results.append((key, dest, _append_block(src_rel, dest, args.force, args.dry_run)))
|
|
237
|
+
else:
|
|
238
|
+
results.append((key, dest, _install(src_rel, dest, args.force, args.dry_run)))
|
|
239
|
+
|
|
240
|
+
width = max(len(key) for key, _dest, _state in results)
|
|
241
|
+
for key, dest, state in results:
|
|
242
|
+
print(f" {key:<{width}} {state:<9} {dest}")
|
|
243
|
+
|
|
244
|
+
wrote = sum(1 for _key, _dest, state in results if state == "wrote")
|
|
245
|
+
print(
|
|
246
|
+
f"\n{wrote} file(s) written"
|
|
247
|
+
+ (" (dry run)" if args.dry_run else "")
|
|
248
|
+
+ ". Start a new session in the host to pick the rule up."
|
|
249
|
+
)
|
|
250
|
+
return 0
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
if __name__ == "__main__":
|
|
254
|
+
sys.exit(main())
|