web-forager 0.0.0-stage → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +119 -0
- package/CHANGELOG.md +508 -0
- package/LICENSE +21 -0
- package/README.md +498 -2
- package/README_zh.md +498 -0
- package/SKILL.md +451 -0
- package/THIRD_PARTY_LICENSES.txt +40 -0
- package/browser/bridges/playwright-bridge.js +382 -0
- package/browser/launch_chrome.py +193 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +90 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +141 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +130 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +215 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +252 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +107 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +441 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/research.md +35 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +349 -0
- package/docs/REFERENCE_zh.md +342 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +98 -0
- package/mcp_server.py +850 -0
- package/package.json +82 -6
- package/requirements.txt +15 -0
- package/run.py +908 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +882 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +5141 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +156 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +381 -0
- package/scripts/sitemap.py +195 -0
- package/scripts/untrusted.py +312 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
|
|
4
|
+
const path = require("node:path");
|
|
5
|
+
const { spawn } = require("node:child_process");
|
|
6
|
+
const { version } = require("./package.json");
|
|
7
|
+
const { listPlaybooks } = require("./lib/playbooks");
|
|
8
|
+
const { loadSession, sessionPath } = require("./lib/session");
|
|
9
|
+
const { BACKENDS } = require("./lib/backend");
|
|
10
|
+
const { doctorReport } = require("./lib/doctor");
|
|
11
|
+
|
|
12
|
+
const SERVER_INFO = { name: "web-forager-browser", version };
|
|
13
|
+
const DEFAULT_PROTOCOL = "2024-11-05";
|
|
14
|
+
const RUN_TIMEOUT_MS = 300000;
|
|
15
|
+
|
|
16
|
+
const TOOLS = [
|
|
17
|
+
{
|
|
18
|
+
name: "browser_list",
|
|
19
|
+
description: "List the read-only web-forager-browser playbooks and the supported browser backends.",
|
|
20
|
+
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: "browser_run",
|
|
24
|
+
description:
|
|
25
|
+
"Run one read-only playbook in a real browser through the configured bridge. Returns the same record shape the CLI writes. MCP runs are always read-only and always check robots.txt; a user who needs a write action or an override must run the CLI directly.",
|
|
26
|
+
inputSchema: {
|
|
27
|
+
type: "object",
|
|
28
|
+
properties: {
|
|
29
|
+
playbook: {
|
|
30
|
+
type: "string",
|
|
31
|
+
enum: ["bili-feed", "bili-video", "bili-chain", "bili-danmaku", "zhihu-dig", "xhs-dig", "search-dig", "wechat-dig"],
|
|
32
|
+
description: "Playbook to run. wechat-dig takes the article URL in `url`.",
|
|
33
|
+
},
|
|
34
|
+
url: { type: "string", description: "Seed or target URL (wechat-dig: the WeChat article URL)." },
|
|
35
|
+
bvid: { type: "string", description: "Bilibili video id (BV...)." },
|
|
36
|
+
query: { type: "string", description: "Search query for *-dig playbooks." },
|
|
37
|
+
session: { type: "string", description: "Resumable session name. Defaults to default." },
|
|
38
|
+
maxItems: { type: "integer", minimum: 1, maximum: 100 },
|
|
39
|
+
maxScreens: { type: "integer", minimum: 1, maximum: 50 },
|
|
40
|
+
commentScrolls: { type: "integer", minimum: 0, maximum: 50 },
|
|
41
|
+
observeMs: { type: "integer", minimum: 0, maximum: 60000 },
|
|
42
|
+
minInterval: { type: "integer", minimum: 0 },
|
|
43
|
+
core: {
|
|
44
|
+
type: "boolean",
|
|
45
|
+
description: "Report records in the shared core-1 schema instead of the full browser record.",
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
required: ["playbook"],
|
|
49
|
+
additionalProperties: false,
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
name: "browser_session",
|
|
54
|
+
description: "Read a resumable session: captured ids, visited pages and the followed chain.",
|
|
55
|
+
inputSchema: {
|
|
56
|
+
type: "object",
|
|
57
|
+
properties: {
|
|
58
|
+
session: { type: "string", description: "Session name. Defaults to default." },
|
|
59
|
+
},
|
|
60
|
+
additionalProperties: false,
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
name: "browser_doctor",
|
|
65
|
+
description:
|
|
66
|
+
"Check the browser stack without opening a page: Node, the bridge command, the playwright package, the Chromium binary, the profile directory and hints for whatever is missing.",
|
|
67
|
+
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
|
68
|
+
},
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
function textResult(value, isError = false) {
|
|
72
|
+
return {
|
|
73
|
+
content: [{ type: "text", text: typeof value === "string" ? value : JSON.stringify(value, null, 2) }],
|
|
74
|
+
isError,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function runPlaybook(args) {
|
|
79
|
+
return new Promise((resolve) => {
|
|
80
|
+
const argv = [path.join(__dirname, "run.js"), "--agent", "--playbook", String(args.playbook)];
|
|
81
|
+
const textual = [
|
|
82
|
+
"url",
|
|
83
|
+
"bvid",
|
|
84
|
+
"query",
|
|
85
|
+
"session",
|
|
86
|
+
"maxItems",
|
|
87
|
+
"maxScreens",
|
|
88
|
+
"commentScrolls",
|
|
89
|
+
"observeMs",
|
|
90
|
+
"minInterval",
|
|
91
|
+
"backend",
|
|
92
|
+
];
|
|
93
|
+
for (const key of textual) {
|
|
94
|
+
if (args[key] === undefined || args[key] === null || args[key] === "") continue;
|
|
95
|
+
const flag = key.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`);
|
|
96
|
+
argv.push(`--${flag}`, String(args[key]));
|
|
97
|
+
}
|
|
98
|
+
if (args.dryRun) argv.push("--dry-run");
|
|
99
|
+
if (args.core === true) argv.push("--core");
|
|
100
|
+
// MCP runs are always read-only and always check robots.txt. The model
|
|
101
|
+
// cannot turn either boundary off; a user who needs an override uses the
|
|
102
|
+
// CLI directly.
|
|
103
|
+
argv.push("--strict-robots");
|
|
104
|
+
|
|
105
|
+
const child = spawn(process.execPath, argv, { cwd: __dirname, stdio: ["ignore", "pipe", "pipe"] });
|
|
106
|
+
let stdout = "";
|
|
107
|
+
let stderr = "";
|
|
108
|
+
const timer = setTimeout(() => {
|
|
109
|
+
child.kill();
|
|
110
|
+
resolve(textResult({ ok: false, error: { code: "timeout", message: "playbook run timed out" } }, true));
|
|
111
|
+
}, RUN_TIMEOUT_MS);
|
|
112
|
+
child.stdout.setEncoding("utf8");
|
|
113
|
+
child.stderr.setEncoding("utf8");
|
|
114
|
+
child.stdout.on("data", (chunk) => {
|
|
115
|
+
stdout += chunk;
|
|
116
|
+
});
|
|
117
|
+
child.stderr.on("data", (chunk) => {
|
|
118
|
+
stderr += chunk;
|
|
119
|
+
});
|
|
120
|
+
child.on("close", (code) => {
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
const line = stdout
|
|
123
|
+
.split("\n")
|
|
124
|
+
.map((item) => item.trim())
|
|
125
|
+
.filter(Boolean)
|
|
126
|
+
.pop();
|
|
127
|
+
let payload;
|
|
128
|
+
try {
|
|
129
|
+
payload = line ? JSON.parse(line) : null;
|
|
130
|
+
} catch {
|
|
131
|
+
payload = null;
|
|
132
|
+
}
|
|
133
|
+
if (!payload) {
|
|
134
|
+
resolve(textResult({ ok: false, error: { code: "protocol", message: stderr || "no JSON envelope" } }, true));
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const isError = payload.ok === false || code !== 0;
|
|
138
|
+
resolve(textResult(payload, isError));
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function sessionSummary(args) {
|
|
144
|
+
const name = args.session || "default";
|
|
145
|
+
const session = loadSession(name);
|
|
146
|
+
return {
|
|
147
|
+
name: session.name,
|
|
148
|
+
file: sessionPath(name),
|
|
149
|
+
playbook: session.playbook,
|
|
150
|
+
created_at: session.created_at,
|
|
151
|
+
updated_at: session.updated_at,
|
|
152
|
+
captured: session.captured,
|
|
153
|
+
visited: session.visited,
|
|
154
|
+
chain: session.chain,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function callTool(name, args = {}) {
|
|
159
|
+
if (name === "browser_list") {
|
|
160
|
+
return textResult({ playbooks: listPlaybooks(), backends: BACKENDS });
|
|
161
|
+
}
|
|
162
|
+
if (name === "browser_run") {
|
|
163
|
+
if (!args.playbook) return textResult({ ok: false, error: { code: "argument", message: "playbook is required" } }, true);
|
|
164
|
+
return runPlaybook(args);
|
|
165
|
+
}
|
|
166
|
+
if (name === "browser_session") {
|
|
167
|
+
try {
|
|
168
|
+
return textResult(sessionSummary(args));
|
|
169
|
+
} catch (error) {
|
|
170
|
+
return textResult({ ok: false, error: { code: "argument", message: error.message } }, true);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (name === "browser_doctor") {
|
|
174
|
+
const report = doctorReport();
|
|
175
|
+
return textResult({ ok: report.ok, data: report });
|
|
176
|
+
}
|
|
177
|
+
return textResult({ ok: false, error: { code: "unknown_tool", message: `unknown tool: ${name}` } }, true);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function send(message) {
|
|
181
|
+
process.stdout.write(`${JSON.stringify(message)}\n`);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async function handle(message) {
|
|
185
|
+
const { id, method, params } = message;
|
|
186
|
+
const isRequest = id !== undefined && id !== null;
|
|
187
|
+
try {
|
|
188
|
+
if (method === "initialize") {
|
|
189
|
+
send({
|
|
190
|
+
jsonrpc: "2.0",
|
|
191
|
+
id,
|
|
192
|
+
result: {
|
|
193
|
+
protocolVersion: (params && params.protocolVersion) || DEFAULT_PROTOCOL,
|
|
194
|
+
capabilities: { tools: { listChanged: false } },
|
|
195
|
+
serverInfo: SERVER_INFO,
|
|
196
|
+
instructions:
|
|
197
|
+
"web-forager-browser drives a real browser read-only. Use browser_list before browser_run and browser_session to resume without repeating captures; browser_doctor names what is missing when the first run fails.",
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
if (method === "notifications/initialized" || method === "initialized") return;
|
|
203
|
+
if (method === "ping") {
|
|
204
|
+
if (isRequest) send({ jsonrpc: "2.0", id, result: {} });
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
if (method === "tools/list") {
|
|
208
|
+
send({ jsonrpc: "2.0", id, result: { tools: TOOLS } });
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
if (method === "tools/call") {
|
|
212
|
+
const name = params && params.name;
|
|
213
|
+
const args = (params && params.arguments) || {};
|
|
214
|
+
const result = await callTool(name, args);
|
|
215
|
+
send({ jsonrpc: "2.0", id, result });
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (isRequest) {
|
|
219
|
+
send({ jsonrpc: "2.0", id, error: { code: -32601, message: `method not found: ${method}` } });
|
|
220
|
+
}
|
|
221
|
+
} catch (error) {
|
|
222
|
+
if (isRequest) send({ jsonrpc: "2.0", id, error: { code: -32603, message: error.message } });
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function main() {
|
|
227
|
+
let buffer = "";
|
|
228
|
+
process.stdin.setEncoding("utf8");
|
|
229
|
+
process.stdin.on("data", (chunk) => {
|
|
230
|
+
buffer += chunk;
|
|
231
|
+
let newline = buffer.indexOf("\n");
|
|
232
|
+
while (newline >= 0) {
|
|
233
|
+
const line = buffer.slice(0, newline).trim();
|
|
234
|
+
buffer = buffer.slice(newline + 1);
|
|
235
|
+
if (line) {
|
|
236
|
+
let message;
|
|
237
|
+
try {
|
|
238
|
+
message = JSON.parse(line);
|
|
239
|
+
} catch {
|
|
240
|
+
send({ jsonrpc: "2.0", id: null, error: { code: -32700, message: "parse error" } });
|
|
241
|
+
message = null;
|
|
242
|
+
}
|
|
243
|
+
if (message) handle(message);
|
|
244
|
+
}
|
|
245
|
+
newline = buffer.indexOf("\n");
|
|
246
|
+
}
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
if (require.main === module) main();
|
|
251
|
+
|
|
252
|
+
module.exports = { TOOLS, callTool, handle, sessionSummary };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "web-forager-browser",
|
|
3
|
+
"version": "1.3.1",
|
|
4
|
+
"description": "Browser mode of web-forager: drive a real browser through a bridge, follow read-only playbooks, and record what it sees in the web-forager workspace shape.",
|
|
5
|
+
"main": "run.js",
|
|
6
|
+
"bin": {
|
|
7
|
+
"web-forager-browser": "run.js",
|
|
8
|
+
"web-forager-browser-mcp": "mcp_server.js"
|
|
9
|
+
},
|
|
10
|
+
"type": "commonjs",
|
|
11
|
+
"engines": {
|
|
12
|
+
"node": ">=18"
|
|
13
|
+
},
|
|
14
|
+
"scripts": {
|
|
15
|
+
"test": "node --test \"tests/*.test.js\"",
|
|
16
|
+
"test:live": "node tests/live-smoke.js",
|
|
17
|
+
"mcp": "node mcp_server.js"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"run.js",
|
|
21
|
+
"mcp_server.js",
|
|
22
|
+
"launch_chrome.py",
|
|
23
|
+
"lib/",
|
|
24
|
+
"playbooks/",
|
|
25
|
+
"bridges/"
|
|
26
|
+
],
|
|
27
|
+
"keywords": [
|
|
28
|
+
"browser",
|
|
29
|
+
"bilibili",
|
|
30
|
+
"zhihu",
|
|
31
|
+
"xiaohongshu",
|
|
32
|
+
"danmaku",
|
|
33
|
+
"mcp",
|
|
34
|
+
"session"
|
|
35
|
+
],
|
|
36
|
+
"author": {
|
|
37
|
+
"name": "Jiaming Shi"
|
|
38
|
+
},
|
|
39
|
+
"license": "MIT"
|
|
40
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { ActorError } = require("../lib/errors");
|
|
4
|
+
const { guardedClick, guardedGoto } = require("../lib/safety");
|
|
5
|
+
const { findNextEpisode } = require("../lib/snapshot");
|
|
6
|
+
const { BILIBILI } = require("../lib/selectors");
|
|
7
|
+
const { pushChain } = require("../lib/session");
|
|
8
|
+
const { captureCurrent } = require("./shared");
|
|
9
|
+
|
|
10
|
+
const NAME = "bili-chain";
|
|
11
|
+
|
|
12
|
+
module.exports = {
|
|
13
|
+
name: NAME,
|
|
14
|
+
description: "Follow the next-episode chain from a seed video, capturing each episode and resuming from the session.",
|
|
15
|
+
requires: ["url"],
|
|
16
|
+
|
|
17
|
+
async run(ctx) {
|
|
18
|
+
const maxItems = Number(ctx.options.maxItems || 3);
|
|
19
|
+
let seed = ctx.options.url;
|
|
20
|
+
if (!seed && ctx.options.bvid) seed = `${BILIBILI.homeUrl}video/${ctx.options.bvid}`;
|
|
21
|
+
if (!seed) {
|
|
22
|
+
const last = ctx.session.chain[ctx.session.chain.length - 1];
|
|
23
|
+
if (last) seed = `${BILIBILI.homeUrl}video/${last}`;
|
|
24
|
+
}
|
|
25
|
+
if (!seed) throw new ActorError("argument", "bili-chain needs --url, --bvid, or a session with a chain");
|
|
26
|
+
|
|
27
|
+
await guardedGoto(ctx, seed);
|
|
28
|
+
let captured = 0;
|
|
29
|
+
let stopped = "";
|
|
30
|
+
|
|
31
|
+
for (let step = 0; step < maxItems; step += 1) {
|
|
32
|
+
const { meta, snapshot } = await captureCurrent(ctx, {
|
|
33
|
+
commentScrolls: 1,
|
|
34
|
+
playbook: NAME,
|
|
35
|
+
});
|
|
36
|
+
captured += 1;
|
|
37
|
+
if (meta.bvid) pushChain(ctx.session, meta.bvid);
|
|
38
|
+
if (step + 1 >= maxItems) break;
|
|
39
|
+
|
|
40
|
+
const next = findNextEpisode(snapshot);
|
|
41
|
+
if (!next) {
|
|
42
|
+
stopped = "no next-episode control found";
|
|
43
|
+
break;
|
|
44
|
+
}
|
|
45
|
+
await guardedClick(ctx, next);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
summary: {
|
|
50
|
+
captured,
|
|
51
|
+
chain: ctx.session.chain.slice(),
|
|
52
|
+
stopped: stopped || "max items reached",
|
|
53
|
+
status: ctx.session.chain.length ? "ok" : "degraded",
|
|
54
|
+
reason: ctx.session.chain.length ? "" : "no BV id was read from any page",
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
},
|
|
58
|
+
};
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { ActorError } = require("../lib/errors");
|
|
4
|
+
const { guardedGoto } = require("../lib/safety");
|
|
5
|
+
const { extractVideoMeta, findDanmaku, bvidFrom, hasLoginWall } = require("../lib/snapshot");
|
|
6
|
+
const { BILIBILI } = require("../lib/selectors");
|
|
7
|
+
const { markVisited } = require("../lib/session");
|
|
8
|
+
|
|
9
|
+
const NAME = "bili-danmaku";
|
|
10
|
+
|
|
11
|
+
module.exports = {
|
|
12
|
+
name: NAME,
|
|
13
|
+
description: "Capture visible danmaku for one Bilibili video, preferring bridge observation over DOM scraping (experimental).",
|
|
14
|
+
requires: ["url"],
|
|
15
|
+
|
|
16
|
+
async run(ctx) {
|
|
17
|
+
let url = ctx.options.url;
|
|
18
|
+
if (!url && ctx.options.bvid) url = `${BILIBILI.homeUrl}video/${ctx.options.bvid}`;
|
|
19
|
+
if (!url) throw new ActorError("argument", "bili-danmaku needs --url or --bvid");
|
|
20
|
+
|
|
21
|
+
await guardedGoto(ctx, url);
|
|
22
|
+
markVisited(ctx.session, url);
|
|
23
|
+
|
|
24
|
+
const snapshot = await ctx.backend.snapshot();
|
|
25
|
+
if (hasLoginWall(snapshot) && !extractVideoMeta(snapshot).bvid) {
|
|
26
|
+
throw new ActorError(
|
|
27
|
+
"login_required",
|
|
28
|
+
"Bilibili asked for credentials before the player loaded",
|
|
29
|
+
"Use a logged-in extension bridge; web-forager never supplies credentials itself.",
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const meta = extractVideoMeta(snapshot);
|
|
34
|
+
let entries = [];
|
|
35
|
+
let source = "network";
|
|
36
|
+
let warnings = [];
|
|
37
|
+
|
|
38
|
+
// Observation comes first: a bridge that watches the player sees the real
|
|
39
|
+
// stream regardless of how the page draws it. DOM text is a fallback that
|
|
40
|
+
// assumes one visible "mm:ss text" layout, so it is labelled as such.
|
|
41
|
+
const capabilities = await ctx.backend.capabilities();
|
|
42
|
+
if (capabilities.observe && typeof ctx.backend.observe === "function") {
|
|
43
|
+
try {
|
|
44
|
+
const observed = await ctx.backend.observe("danmaku", Number(ctx.options.observeMs || 3000));
|
|
45
|
+
entries = ((observed && (observed.entries || observed.events)) || []).map((entry) => ({
|
|
46
|
+
time: entry.time || entry.timestamp || "",
|
|
47
|
+
text: String(entry.text || entry.content || "").slice(0, 200),
|
|
48
|
+
}));
|
|
49
|
+
} catch (error) {
|
|
50
|
+
ctx.warn(`network observation unavailable: ${error.message}`);
|
|
51
|
+
}
|
|
52
|
+
} else {
|
|
53
|
+
ctx.warn("the bridge did not declare observe; falling back to DOM text extraction");
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (!entries.length) {
|
|
57
|
+
entries = findDanmaku(snapshot);
|
|
58
|
+
source = "dom";
|
|
59
|
+
if (entries.length) {
|
|
60
|
+
warnings.push(
|
|
61
|
+
"danmaku came from DOM text, which assumes one visible 'mm:ss text' layout; treat the list as partial",
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (!entries.length) ctx.warn("no danmaku found in the observation window or the DOM snapshot");
|
|
67
|
+
|
|
68
|
+
ctx.capture({
|
|
69
|
+
playbook: NAME,
|
|
70
|
+
url: meta.url || url,
|
|
71
|
+
metadata: {
|
|
72
|
+
bvid: meta.bvid || bvidFrom(url),
|
|
73
|
+
title: meta.title,
|
|
74
|
+
site: "bilibili",
|
|
75
|
+
source,
|
|
76
|
+
experimental: true,
|
|
77
|
+
count: entries.length,
|
|
78
|
+
},
|
|
79
|
+
markdown: entries.length
|
|
80
|
+
? entries.map((entry) => `- ${entry.time || ""} ${entry.text}`.trim()).join("\n")
|
|
81
|
+
: "No danmaku captured.",
|
|
82
|
+
tables: [
|
|
83
|
+
{
|
|
84
|
+
name: "danmaku",
|
|
85
|
+
columns: ["time", "text"],
|
|
86
|
+
rows: entries.map((entry) => [entry.time || "", entry.text || ""]),
|
|
87
|
+
},
|
|
88
|
+
],
|
|
89
|
+
warnings,
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
summary: {
|
|
94
|
+
captured: 1,
|
|
95
|
+
danmaku: entries.length,
|
|
96
|
+
source,
|
|
97
|
+
experimental: true,
|
|
98
|
+
status: entries.length ? "ok" : "degraded",
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
},
|
|
102
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { ActorError } = require("../lib/errors");
|
|
4
|
+
const { guardedGoto, guardedScroll } = require("../lib/safety");
|
|
5
|
+
const { findVideoCards, isBlankPage, hasBlockedPage, hasLoginWall } = require("../lib/snapshot");
|
|
6
|
+
const { BILIBILI } = require("../lib/selectors");
|
|
7
|
+
const { hasCaptured, markCaptured, markVisited } = require("../lib/session");
|
|
8
|
+
|
|
9
|
+
const NAME = "bili-feed";
|
|
10
|
+
|
|
11
|
+
module.exports = {
|
|
12
|
+
name: NAME,
|
|
13
|
+
description: "Open the Bilibili home feed, scroll screen by screen, and capture each unseen video card.",
|
|
14
|
+
requires: [],
|
|
15
|
+
|
|
16
|
+
async run(ctx) {
|
|
17
|
+
const url = ctx.options.url || BILIBILI.homeUrl;
|
|
18
|
+
const maxItems = Number(ctx.options.maxItems || 10);
|
|
19
|
+
const maxScreens = Number(ctx.options.maxScreens || 3);
|
|
20
|
+
|
|
21
|
+
await guardedGoto(ctx, url);
|
|
22
|
+
markVisited(ctx.session, url);
|
|
23
|
+
|
|
24
|
+
let captured = 0;
|
|
25
|
+
let screens = 0;
|
|
26
|
+
let lastSnapshot = null;
|
|
27
|
+
for (let screen = 0; screen < maxScreens && captured < maxItems; screen += 1) {
|
|
28
|
+
const snapshot = await ctx.backend.snapshot();
|
|
29
|
+
lastSnapshot = snapshot;
|
|
30
|
+
screens += 1;
|
|
31
|
+
for (const card of findVideoCards(snapshot)) {
|
|
32
|
+
if (captured >= maxItems) break;
|
|
33
|
+
if (hasCaptured(ctx.session, card.bvid)) continue;
|
|
34
|
+
ctx.capture({
|
|
35
|
+
playbook: NAME,
|
|
36
|
+
url: card.url,
|
|
37
|
+
metadata: { bvid: card.bvid, title: card.title },
|
|
38
|
+
markdown: `- ${card.title}\n- ${card.url}`,
|
|
39
|
+
});
|
|
40
|
+
markCaptured(ctx.session, {
|
|
41
|
+
id: card.bvid,
|
|
42
|
+
url: card.url,
|
|
43
|
+
title: card.title,
|
|
44
|
+
playbook: NAME,
|
|
45
|
+
});
|
|
46
|
+
captured += 1;
|
|
47
|
+
}
|
|
48
|
+
if (screen + 1 < maxScreens && captured < maxItems) await guardedScroll(ctx, 900);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (captured === 0 && lastSnapshot) {
|
|
52
|
+
if (hasLoginWall(lastSnapshot)) {
|
|
53
|
+
throw new ActorError(
|
|
54
|
+
"login_required",
|
|
55
|
+
"Bilibili home feed showed a login wall instead of video cards",
|
|
56
|
+
"Use a logged-in extension bridge, or rely on the HTTP mode adapter which reads public pages.",
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
const reason = hasBlockedPage(lastSnapshot) ? "blocked" : isBlankPage(lastSnapshot) ? "empty" : "no-match";
|
|
60
|
+
ctx.warn(`degraded: no video cards found after ${screens} screen(s) (${reason})`);
|
|
61
|
+
return { summary: { captured: 0, screens, max_items: maxItems, status: "degraded", reason } };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return { summary: { captured, screens, max_items: maxItems, status: "ok" } };
|
|
65
|
+
},
|
|
66
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { ActorError } = require("../lib/errors");
|
|
4
|
+
const { openAndCapture } = require("./shared");
|
|
5
|
+
const { bvidFrom } = require("../lib/snapshot");
|
|
6
|
+
const { BILIBILI } = require("../lib/selectors");
|
|
7
|
+
|
|
8
|
+
const NAME = "bili-video";
|
|
9
|
+
|
|
10
|
+
module.exports = {
|
|
11
|
+
name: NAME,
|
|
12
|
+
description: "Open one Bilibili video, read the player state, scroll the visible comments, and capture the page.",
|
|
13
|
+
requires: ["url"],
|
|
14
|
+
|
|
15
|
+
async run(ctx) {
|
|
16
|
+
let url = ctx.options.url;
|
|
17
|
+
if (!url && ctx.options.bvid) url = `${BILIBILI.homeUrl}video/${ctx.options.bvid}`;
|
|
18
|
+
if (!url) throw new ActorError("argument", "bili-video needs --url or --bvid");
|
|
19
|
+
|
|
20
|
+
const result = await openAndCapture(ctx, url, {
|
|
21
|
+
commentScrolls: Number(ctx.options.commentScrolls === undefined ? 3 : ctx.options.commentScrolls),
|
|
22
|
+
playbook: NAME,
|
|
23
|
+
});
|
|
24
|
+
const bvid = result.meta.bvid || bvidFrom(url);
|
|
25
|
+
const status = bvid && result.comments.length ? "ok" : "degraded";
|
|
26
|
+
|
|
27
|
+
return {
|
|
28
|
+
summary: {
|
|
29
|
+
captured: 1,
|
|
30
|
+
bvid,
|
|
31
|
+
title: result.meta.title,
|
|
32
|
+
comments: result.comments.length,
|
|
33
|
+
status,
|
|
34
|
+
reason: status === "ok" ? "" : bvid ? "no visible comments" : "no BV id found on the page",
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
},
|
|
38
|
+
};
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const { ActorError } = require("../lib/errors");
|
|
4
|
+
const { guardedGoto } = require("../lib/safety");
|
|
5
|
+
const { findVideoCards, rankByQuery, diagnoseEmpty } = require("../lib/snapshot");
|
|
6
|
+
const { BILIBILI } = require("../lib/selectors");
|
|
7
|
+
const { searchVideos } = require("../lib/bili-api");
|
|
8
|
+
const { hasCaptured, markCaptured, markVisited } = require("../lib/session");
|
|
9
|
+
const { openAndCapture, snapshotSafe } = require("./shared");
|
|
10
|
+
|
|
11
|
+
const NAME = "search-dig";
|
|
12
|
+
|
|
13
|
+
module.exports = {
|
|
14
|
+
name: NAME,
|
|
15
|
+
description: "Search Bilibili, rank the result cards by query overlap, and capture the best unseen result.",
|
|
16
|
+
requires: ["query"],
|
|
17
|
+
|
|
18
|
+
async run(ctx) {
|
|
19
|
+
const query = ctx.options.query;
|
|
20
|
+
if (!query) throw new ActorError("argument", "search-dig needs --query");
|
|
21
|
+
|
|
22
|
+
const base = ctx.options.url || BILIBILI.searchUrl;
|
|
23
|
+
const searchUrl = `${base}?keyword=${encodeURIComponent(query)}`;
|
|
24
|
+
|
|
25
|
+
// The search JSON is stable where the result page's markup is not, so ask
|
|
26
|
+
// it first and fall back to the page only when it cannot answer.
|
|
27
|
+
let ranked = [];
|
|
28
|
+
let source = "page";
|
|
29
|
+
try {
|
|
30
|
+
const cards = await searchVideos(query, 20, {
|
|
31
|
+
strictRobots: Boolean(ctx.options.strictRobots),
|
|
32
|
+
});
|
|
33
|
+
if (cards.length) {
|
|
34
|
+
ranked = rankByQuery(cards, query);
|
|
35
|
+
source = "api";
|
|
36
|
+
}
|
|
37
|
+
} catch (error) {
|
|
38
|
+
if (String(error.message) !== "disabled") {
|
|
39
|
+
ctx.warn(`bilibili search API unavailable (${error.message}); reading the result page instead`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (source === "page") {
|
|
44
|
+
await guardedGoto(ctx, searchUrl);
|
|
45
|
+
markVisited(ctx.session, searchUrl);
|
|
46
|
+
const snapshot = await snapshotSafe(ctx, "the search page");
|
|
47
|
+
ranked = rankByQuery(findVideoCards(snapshot), query);
|
|
48
|
+
if (!ranked.length) {
|
|
49
|
+
const reason = diagnoseEmpty(snapshot);
|
|
50
|
+
if (reason === "blocked") {
|
|
51
|
+
throw new ActorError(
|
|
52
|
+
"blocked",
|
|
53
|
+
"Bilibili search returned an anti-bot or error page instead of result cards",
|
|
54
|
+
"Wait and retry later. web-forager does not solve challenges.",
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
ctx.warn(`degraded: no result cards found (${reason})`);
|
|
58
|
+
return { summary: { query, results: 0, captured: 0, status: "degraded", reason } };
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
ctx.capture({
|
|
63
|
+
playbook: NAME,
|
|
64
|
+
url: searchUrl,
|
|
65
|
+
metadata: { query, results: ranked.length, source },
|
|
66
|
+
markdown: ranked
|
|
67
|
+
.slice(0, 10)
|
|
68
|
+
.map((card, index) => `${index + 1}. ${card.title} - ${card.url}`)
|
|
69
|
+
.join("\n"),
|
|
70
|
+
tables: [
|
|
71
|
+
{
|
|
72
|
+
name: "search results",
|
|
73
|
+
columns: ["#", "title", "bvid", "url"],
|
|
74
|
+
rows: ranked.slice(0, 20).map((card, index) => [index + 1, card.title, card.bvid, card.url]),
|
|
75
|
+
},
|
|
76
|
+
],
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
const target = ranked.find((card) => !hasCaptured(ctx.session, card.bvid));
|
|
80
|
+
if (!target) {
|
|
81
|
+
return { summary: { query, results: ranked.length, captured: 0, status: "ok", reason: "all-captured" }, warnings: ["all results already captured"] };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const result = await openAndCapture(ctx, target.url, {
|
|
85
|
+
commentScrolls: Number(ctx.options.commentScrolls === undefined ? 1 : ctx.options.commentScrolls),
|
|
86
|
+
playbook: NAME,
|
|
87
|
+
});
|
|
88
|
+
markCaptured(ctx.session, {
|
|
89
|
+
id: target.bvid,
|
|
90
|
+
url: target.url,
|
|
91
|
+
title: target.title,
|
|
92
|
+
playbook: NAME,
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
return {
|
|
96
|
+
summary: {
|
|
97
|
+
query,
|
|
98
|
+
results: ranked.length,
|
|
99
|
+
captured: 1,
|
|
100
|
+
opened: result.meta.bvid || target.bvid,
|
|
101
|
+
title: result.meta.title || target.title,
|
|
102
|
+
status: "ok",
|
|
103
|
+
source,
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
},
|
|
107
|
+
};
|