web-forager 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +112 -0
  18. package/CHANGELOG.md +454 -0
  19. package/LICENSE +21 -0
  20. package/README.md +477 -3
  21. package/README_zh.md +477 -0
  22. package/SKILL.md +351 -0
  23. package/THIRD_PARTY_LICENSES.txt +35 -0
  24. package/browser/bridges/playwright-bridge.js +375 -0
  25. package/browser/launch_chrome.py +144 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +79 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +115 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +116 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +56 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +272 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +105 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +432 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/scholar.md +28 -0
  58. package/commands/surf.md +28 -0
  59. package/cordis.patch.yml +19 -0
  60. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  61. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  62. package/docs/DEPENDENCIES.md +40 -0
  63. package/docs/INSTALL_COMMANDS.md +113 -0
  64. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  65. package/docs/REFERENCE.md +308 -0
  66. package/docs/REFERENCE_zh.md +309 -0
  67. package/index.js +7 -0
  68. package/install.py +254 -0
  69. package/manifest.json +97 -0
  70. package/mcp_server.py +597 -0
  71. package/package.json +80 -6
  72. package/requirements.txt +12 -0
  73. package/run.py +852 -0
  74. package/scripts/__init__.py +1 -0
  75. package/scripts/bilibili.py +874 -0
  76. package/scripts/documents.py +437 -0
  77. package/scripts/engine.py +4220 -0
  78. package/scripts/feeds.py +180 -0
  79. package/scripts/pdf.py +135 -0
  80. package/scripts/scholar.py +354 -0
  81. package/scripts/sitemap.py +180 -0
  82. package/scripts/untrusted.py +95 -0
@@ -0,0 +1,149 @@
1
+ "use strict";
2
+
3
+ // Refuse browser navigation to this machine, the LAN or a cloud metadata
4
+ // service. The HTTP side has the same guard (scripts/engine.py); this one is
5
+ // used for page navigations and for every subresource Chromium asks for, so a
6
+ // hostile page cannot probe http://127.0.0.1:... from inside the browser.
7
+ //
8
+ // A host that does not resolve is left alone: the navigation fails on its own,
9
+ // and offline runs (and the test fixtures) must keep working.
10
+
11
+ const dns = require("node:dns").promises;
12
+ const net = require("node:net");
13
+
14
+ const { ActorError } = require("./errors");
15
+
16
+ const V4_RANGES = [
17
+ "0.0.0.0/8",
18
+ "10.0.0.0/8",
19
+ "100.64.0.0/10",
20
+ "127.0.0.0/8",
21
+ "169.254.0.0/16",
22
+ "172.16.0.0/12",
23
+ "192.0.0.0/24",
24
+ "192.0.2.0/24",
25
+ "192.168.0.0/16",
26
+ "198.18.0.0/15",
27
+ "198.51.100.0/24",
28
+ "203.0.113.0/24",
29
+ "224.0.0.0/4",
30
+ "240.0.0.0/4",
31
+ "255.255.255.255/32",
32
+ ].map((cidr) => {
33
+ const [base, bits] = cidr.split("/");
34
+ return { base: v4ToInt(base), mask: bitsToMask(Number(bits)) };
35
+ });
36
+
37
+ const DNS_TTL_MS = 60_000;
38
+ const dnsCache = new Map();
39
+
40
+ function v4ToInt(ip) {
41
+ const parts = ip.split(".").map(Number);
42
+ return (((parts[0] << 24) >>> 0) + (parts[1] << 16) + (parts[2] << 8) + parts[3]) >>> 0;
43
+ }
44
+
45
+ function bitsToMask(bits) {
46
+ return bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0;
47
+ }
48
+
49
+ function isBlockedIpv4(ip) {
50
+ const value = v4ToInt(ip);
51
+ return V4_RANGES.some((range) => ((value & range.mask) >>> 0) === ((range.base & range.mask) >>> 0));
52
+ }
53
+
54
+ function isBlockedIpv6(raw) {
55
+ let ip = raw.toLowerCase().split("%")[0];
56
+ if (ip === "::1" || ip === "::") return true;
57
+ const mapped = ip.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
58
+ if (mapped) return isBlockedIpv4(mapped[1]);
59
+ if (/^fe[89ab][0-9a-f]:/.test(ip)) return true; // fe80::/10
60
+ if (/^f[cd][0-9a-f]{2}:/.test(ip)) return true; // fc00::/7
61
+ if (/^ff[0-9a-f]{2}:/.test(ip)) return true; // ff00::/8
62
+ if (ip.startsWith("2001:db8")) return true;
63
+ if (ip.startsWith("64:ff9b")) return true;
64
+ return false;
65
+ }
66
+
67
+ function isBlockedIp(ip) {
68
+ if (net.isIPv4(ip)) return isBlockedIpv4(ip);
69
+ if (net.isIPv6(ip)) return isBlockedIpv6(ip);
70
+ return false;
71
+ }
72
+
73
+ function isLocalName(host) {
74
+ const name = host.replace(/\.$/, "").toLowerCase();
75
+ return (
76
+ name === "localhost" ||
77
+ name.endsWith(".localhost") ||
78
+ name.endsWith(".local") ||
79
+ name.endsWith(".internal") ||
80
+ name === "metadata.google.internal"
81
+ );
82
+ }
83
+
84
+ async function resolveHost(host) {
85
+ const now = Date.now();
86
+ const hit = dnsCache.get(host);
87
+ if (hit && hit.until > now) return hit.addresses;
88
+ const addresses = [];
89
+ try {
90
+ for (const entry of await dns.lookup(host, { all: true })) addresses.push(entry.address);
91
+ } catch (error) {
92
+ return []; // unresolvable: the request fails on its own
93
+ }
94
+ dnsCache.set(host, { addresses, until: now + DNS_TTL_MS });
95
+ return addresses;
96
+ }
97
+
98
+ function allowPrivate() {
99
+ return ["1", "true", "yes", "on"].includes(
100
+ String(process.env.WEB_FORAGER_ALLOW_PRIVATE || "").trim().toLowerCase(),
101
+ );
102
+ }
103
+
104
+ /** True when the URL is http(s) and points at a public address. */
105
+ async function isPublicRequestUrl(rawUrl, options = {}) {
106
+ const allow = options.allowPrivate === undefined ? allowPrivate() : Boolean(options.allowPrivate);
107
+ if (allow) return true;
108
+ let parsed;
109
+ try {
110
+ parsed = new URL(String(rawUrl));
111
+ } catch (error) {
112
+ return false;
113
+ }
114
+ if (!/^https?:$/.test(parsed.protocol)) return true; // data:, blob:, about:
115
+ const host = parsed.hostname.replace(/^\[|\]$/g, "");
116
+ if (!host) return false;
117
+ if (isLocalName(host)) return false;
118
+ if (net.isIP(host)) return !isBlockedIp(host);
119
+ const addresses = await resolveHost(host);
120
+ return !addresses.some((address) => isBlockedIp(address));
121
+ }
122
+
123
+ /** Navigation entry point: throws an ActorError instead of returning a boolean. */
124
+ async function assertPublicUrl(rawUrl, options = {}) {
125
+ let parsed;
126
+ try {
127
+ parsed = new URL(String(rawUrl));
128
+ } catch (error) {
129
+ throw new ActorError("unsafe_url", `invalid URL: ${rawUrl}`, "pass an absolute http(s) URL");
130
+ }
131
+ if (!/^https?:$/.test(parsed.protocol)) {
132
+ throw new ActorError(
133
+ "unsafe_url",
134
+ `refusing non-http(s) URL: ${parsed.protocol}`,
135
+ "only http and https pages are opened",
136
+ );
137
+ }
138
+ const allow = options.allowPrivate === undefined ? allowPrivate() : Boolean(options.allowPrivate);
139
+ if (allow) return;
140
+ if (!(await isPublicRequestUrl(rawUrl, { allowPrivate: false }))) {
141
+ throw new ActorError(
142
+ "unsafe_url",
143
+ `refusing ${parsed.hostname}: loopback, private, link-local and metadata addresses are blocked`,
144
+ "use --allow-private only for a target you control; it is off by default",
145
+ );
146
+ }
147
+ }
148
+
149
+ module.exports = { assertPublicUrl, isPublicRequestUrl, isBlockedIp, isLocalName };
@@ -0,0 +1,30 @@
1
+ "use strict";
2
+
3
+ const playbooks = {
4
+ "bili-feed": require("../playbooks/bili-feed"),
5
+ "bili-video": require("../playbooks/bili-video"),
6
+ "bili-chain": require("../playbooks/bili-chain"),
7
+ "bili-danmaku": require("../playbooks/bili-danmaku"),
8
+ "zhihu-dig": require("../playbooks/zhihu-dig"),
9
+ "xhs-dig": require("../playbooks/xhs-dig"),
10
+ "search-dig": require("../playbooks/search-dig"),
11
+ "wechat-dig": require("../playbooks/wechat-dig"),
12
+ };
13
+
14
+ function getPlaybook(name) {
15
+ const playbook = playbooks[name];
16
+ if (!playbook) {
17
+ throw new Error(`unknown playbook: ${name}. Known: ${Object.keys(playbooks).join(", ")}`);
18
+ }
19
+ return playbook;
20
+ }
21
+
22
+ function listPlaybooks() {
23
+ return Object.values(playbooks).map((playbook) => ({
24
+ name: playbook.name,
25
+ description: playbook.description,
26
+ requires: playbook.requires,
27
+ }));
28
+ }
29
+
30
+ module.exports = { playbooks, getPlaybook, listPlaybooks };
@@ -0,0 +1,109 @@
1
+ "use strict";
2
+
3
+ const { nowIso } = require("./session");
4
+
5
+ function makeRecord(fields) {
6
+ return {
7
+ schema_version: "2",
8
+ source: "web-forager-browser",
9
+ mode: "browser",
10
+ playbook: fields.playbook || "",
11
+ url: fields.url || "",
12
+ captured_at: nowIso(),
13
+ backend: fields.backend || "unknown",
14
+ metadata: fields.metadata || {},
15
+ markdown: fields.markdown || "",
16
+ tables: Array.isArray(fields.tables) ? fields.tables : [],
17
+ warnings: Array.isArray(fields.warnings) ? fields.warnings : [],
18
+ compliance: fields.compliance || { strict_robots: false, robots_checked: false, checks: [] },
19
+ viewport_evidence: fields.viewport_evidence || null,
20
+ interaction_trace: Array.isArray(fields.interaction_trace) ? fields.interaction_trace : [],
21
+ };
22
+ }
23
+
24
+ // Fold a browser record and an HTTP record onto the same core, so a caller can
25
+ // read one list without branching on the mode. Everything the two modes do not
26
+ // share is preserved under `extra` rather than dropped.
27
+ function normalizeRecord(record) {
28
+ const value = record || {};
29
+ const metadata = value.metadata || {};
30
+ const browser = value.source === "web-forager-browser" || Boolean(value.captured_at);
31
+ const collectedAt = value.collected_at || (browser ? value.captured_at : value.fetched_at) || "";
32
+ const url = value.url || value.final_url || "";
33
+ const title = value.title || metadata.title || "";
34
+ const shared = [
35
+ "metadata",
36
+ "markdown",
37
+ "tables",
38
+ "warnings",
39
+ "url",
40
+ "final_url",
41
+ "title",
42
+ "captured_at",
43
+ "fetched_at",
44
+ "collected_at",
45
+ "source",
46
+ "mode",
47
+ "schema_version",
48
+ ];
49
+ const extra = {};
50
+ for (const [key, item] of Object.entries(value)) {
51
+ if (!shared.includes(key)) extra[key] = item;
52
+ }
53
+ if (value.source && value.source !== "web-forager" && value.source !== "web-forager-browser") {
54
+ extra.record_source = value.source;
55
+ }
56
+ return {
57
+ schema_version: "core-1",
58
+ source: browser ? "web-forager-browser" : "web-forager",
59
+ mode: browser ? "browser" : "http",
60
+ url,
61
+ title,
62
+ collected_at: collectedAt,
63
+ metadata,
64
+ markdown: value.markdown || "",
65
+ tables: Array.isArray(value.tables) ? value.tables : [],
66
+ warnings: Array.isArray(value.warnings) ? value.warnings : [],
67
+ extra,
68
+ };
69
+ }
70
+
71
+ function slugify(text, fallback = "capture") {
72
+ const value = String(text || "")
73
+ .toLowerCase()
74
+ .replace(/[^a-z0-9一-鿿]+/g, "-")
75
+ .replace(/^-+|-+$/g, "")
76
+ .slice(0, 60);
77
+ return value || fallback;
78
+ }
79
+
80
+ function recordToMarkdown(record) {
81
+ const title = record.metadata.title || record.metadata.bvid || record.url || "capture";
82
+ const lines = [`# ${title}`, ""];
83
+ if (record.url) lines.push(`Source: ${record.url}`);
84
+ lines.push(`Playbook: ${record.playbook} | Backend: ${record.backend} | Captured: ${record.captured_at}`);
85
+ const meta = record.metadata || {};
86
+ const scalar = Object.entries(meta).filter(([, value]) => value !== null && value !== undefined && typeof value !== "object");
87
+ if (scalar.length) {
88
+ lines.push("", "## Metadata", "");
89
+ for (const [key, value] of scalar) lines.push(`- ${key}: ${value}`);
90
+ }
91
+ if (record.markdown) lines.push("", "## Capture", "", record.markdown.trim());
92
+ if (record.tables && record.tables.length) {
93
+ lines.push("", "## Tables", "");
94
+ for (const table of record.tables) {
95
+ lines.push(`### ${table.name || "table"}`, "");
96
+ lines.push(`| ${table.columns.join(" | ")} |`);
97
+ lines.push(`| ${table.columns.map(() => "---").join(" | ")} |`);
98
+ for (const row of table.rows) lines.push(`| ${row.join(" | ")} |`);
99
+ lines.push("");
100
+ }
101
+ }
102
+ if (record.warnings && record.warnings.length) {
103
+ lines.push("", "## Warnings", "");
104
+ for (const warning of record.warnings) lines.push(`- ${warning}`);
105
+ }
106
+ return `${lines.join("\n").trim()}\n`;
107
+ }
108
+
109
+ module.exports = { makeRecord, normalizeRecord, recordToMarkdown, slugify };
@@ -0,0 +1,195 @@
1
+ "use strict";
2
+
3
+ // A small, dependency-free robots.txt reader for browser mode.
4
+ //
5
+ // Browser mode is read-only but it still navigates real sites, so the record
6
+ // has to be able to say whether robots.txt was consulted. It is consulted only
7
+ // with --strict-robots; without it the record states robots_checked: false and
8
+ // no network request is made. The parsing rules mirror the Python engine
9
+ // (RFC 9309 subset): longest matching pattern wins, ties go to Allow, an
10
+ // unreachable or 5xx robots.txt is denied conservatively.
11
+
12
+ const http = require("node:http");
13
+ const https = require("node:https");
14
+ const { URL } = require("node:url");
15
+
16
+ const { isPublicRequestUrl } = require("./netguard");
17
+
18
+ const USER_AGENT = "WebForager-Browser (Information Gathering; +https://github.com/JiamingShi-JsPhys/web-forager)";
19
+ const MAX_BYTES = 512_000;
20
+ const TIMEOUT_MS = 8000;
21
+ const MAX_REDIRECTS = 3;
22
+
23
+ function originOf(url) {
24
+ const parsed = new URL(url);
25
+ return `${parsed.protocol}//${parsed.host}`;
26
+ }
27
+
28
+ function robotsPath(url) {
29
+ const parsed = new URL(url);
30
+ return `${parsed.pathname || "/"}${parsed.search || ""}`;
31
+ }
32
+
33
+ function parseRobots(text, userAgent = USER_AGENT) {
34
+ const groups = [];
35
+ let current = null;
36
+
37
+ for (const rawLine of String(text || "").split(/\r?\n/)) {
38
+ const line = rawLine.split("#", 1)[0].trim();
39
+ if (!line || !line.includes(":")) continue;
40
+ const index = line.indexOf(":");
41
+ const key = line.slice(0, index).trim().toLowerCase();
42
+ const value = line.slice(index + 1).trim();
43
+
44
+ if (key === "user-agent") {
45
+ if (current === null || current.rules.length) {
46
+ current = { agents: [], rules: [] };
47
+ groups.push(current);
48
+ }
49
+ current.agents.push(value.toLowerCase());
50
+ } else if (key === "allow" || key === "disallow") {
51
+ if (current === null) continue;
52
+ if (key === "disallow" && value === "") continue; // empty Disallow = no restriction
53
+ current.rules.push({ allow: key === "allow", pattern: value });
54
+ }
55
+ }
56
+
57
+ const score = (agent) => {
58
+ const token = agent.trim().toLowerCase();
59
+ if (!token) return -1;
60
+ if (token === "*") return 0;
61
+ const product = token.split("/", 1)[0].trim();
62
+ return product && userAgent.toLowerCase().includes(product) ? product.length : -1;
63
+ };
64
+
65
+ let best = -1;
66
+ let selected = [];
67
+ for (const group of groups) {
68
+ const value = group.agents.reduce((top, agent) => Math.max(top, score(agent)), -1);
69
+ if (value < 0) continue;
70
+ if (value > best) {
71
+ best = value;
72
+ selected = [group];
73
+ } else if (value === best) {
74
+ selected.push(group);
75
+ }
76
+ }
77
+
78
+ const rules = [];
79
+ for (const group of selected) {
80
+ for (const rule of group.rules) rules.push({ ...rule, regex: compileRule(rule.pattern), specificity: rule.pattern.replace(/\*/g, "").length });
81
+ }
82
+ return { rules, source: "", note: selected.length ? "" : "no matching user-agent group", allowAll: false, denyAll: false };
83
+ }
84
+
85
+ function compileRule(pattern) {
86
+ const anchored = pattern.endsWith("$");
87
+ let core = anchored ? pattern.slice(0, -1) : pattern;
88
+ if (!core.startsWith("/") && !core.startsWith("*")) core = `/${core}`;
89
+ const escaped = core.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
90
+ return new RegExp(`^${escaped}${anchored ? "$" : ""}`, "s");
91
+ }
92
+
93
+ function robotsAllows(policy, url) {
94
+ if (policy.denyAll) return false;
95
+ if (policy.allowAll || !policy.rules.length) return true;
96
+ const path = robotsPath(url);
97
+ let winner = null;
98
+ for (const rule of policy.rules) {
99
+ if (!rule.regex.test(path)) continue;
100
+ if (winner === null || rule.specificity > winner.specificity) winner = rule;
101
+ else if (rule.specificity === winner.specificity && rule.allow && !winner.allow) winner = rule;
102
+ }
103
+ return winner === null ? true : winner.allow;
104
+ }
105
+
106
+ async function fetchText(url, redirects = 0) {
107
+ const parsed = new URL(url);
108
+ if (!(await isPublicRequestUrl(url))) {
109
+ // A robots.txt redirect can point anywhere; refuse the private hop instead
110
+ // of fetching it. checkRobots() treats this as unreachable and denies.
111
+ throw new Error(`refusing private robots.txt host: ${parsed.hostname}`);
112
+ }
113
+ return new Promise((resolve, reject) => {
114
+ const client = parsed.protocol === "http:" ? http : https;
115
+ const request = client.get(
116
+ parsed,
117
+ { headers: { "User-Agent": USER_AGENT, Accept: "text/plain,*/*;q=0.5" } },
118
+ (response) => {
119
+ const status = response.statusCode || 0;
120
+ if (status >= 300 && status < 400 && response.headers.location) {
121
+ response.resume();
122
+ if (redirects >= MAX_REDIRECTS) {
123
+ reject(new Error("too many robots.txt redirects"));
124
+ return;
125
+ }
126
+ resolve(fetchText(new URL(response.headers.location, parsed).toString(), redirects + 1));
127
+ return;
128
+ }
129
+ let body = "";
130
+ let size = 0;
131
+ response.setEncoding("utf8");
132
+ response.on("data", (chunk) => {
133
+ size += chunk.length;
134
+ if (size > MAX_BYTES) {
135
+ response.destroy();
136
+ return;
137
+ }
138
+ body += chunk;
139
+ });
140
+ response.on("end", () => resolve({ status, body }));
141
+ response.on("error", reject);
142
+ },
143
+ );
144
+ request.setTimeout(TIMEOUT_MS, () => {
145
+ request.destroy(new Error("robots.txt request timed out"));
146
+ });
147
+ request.on("error", reject);
148
+ });
149
+ }
150
+
151
+ const cache = new Map();
152
+
153
+ async function checkRobots(url, options = {}) {
154
+ const origin = originOf(url);
155
+ const robotsUrl = `${origin}/robots.txt`;
156
+ const agent = options.userAgent || USER_AGENT;
157
+ const cached = cache.get(origin);
158
+ if (cached) return { ...cached, allowed: robotsAllows(cached.policy, url), url };
159
+
160
+ let status = 0;
161
+ let note = "";
162
+ let policy;
163
+ try {
164
+ const response = await fetchText(robotsUrl);
165
+ status = response.status;
166
+ if (status === 200) {
167
+ policy = parseRobots(response.body, agent);
168
+ note = policy.note || `robots.txt read (${policy.rules.length} rule(s))`;
169
+ } else if (status === 401 || status === 403 || (status >= 500 && status < 600)) {
170
+ policy = { rules: [], allowAll: false, denyAll: true, note: `HTTP ${status} on robots.txt` };
171
+ note = policy.note;
172
+ } else {
173
+ policy = { rules: [], allowAll: true, denyAll: false, note: `HTTP ${status} on robots.txt` };
174
+ note = policy.note;
175
+ }
176
+ } catch (error) {
177
+ policy = {
178
+ rules: [],
179
+ allowAll: false,
180
+ denyAll: true,
181
+ note: `robots.txt unreachable (${error.message}); denied conservatively`,
182
+ };
183
+ note = policy.note;
184
+ }
185
+
186
+ const entry = { robots_url: robotsUrl, status, note, policy };
187
+ cache.set(origin, entry);
188
+ return { ...entry, url, allowed: robotsAllows(policy, url) };
189
+ }
190
+
191
+ function clearCache() {
192
+ cache.clear();
193
+ }
194
+
195
+ module.exports = { USER_AGENT, parseRobots, robotsAllows, checkRobots, clearCache, originOf };
@@ -0,0 +1,115 @@
1
+ "use strict";
2
+
3
+ const { ActorError } = require("./errors");
4
+ const { assertPublicUrl } = require("./netguard");
5
+
6
+ const WRITE_PATTERNS = [
7
+ /点赞/,
8
+ /投币/,
9
+ /收藏/,
10
+ /关注/,
11
+ /分享/,
12
+ /转发/,
13
+ /评论/,
14
+ /弹幕/,
15
+ /发布/,
16
+ /三连/,
17
+ /\blike\b/i,
18
+ /\bcoin\b/i,
19
+ /\bfavorite\b/i,
20
+ /\bfollow\b/i,
21
+ /\bshare\b/i,
22
+ /\bcomment\b/i,
23
+ /\bdanmaku\b/i,
24
+ /\bpublish\b/i,
25
+ ];
26
+
27
+ function actionText(node) {
28
+ if (!node) return "";
29
+ return [node.name, node.text, node.label, node.title, node.href]
30
+ .filter(Boolean)
31
+ .join(" ")
32
+ .trim();
33
+ }
34
+
35
+ function isWriteAction(node) {
36
+ const text = actionText(node);
37
+ return WRITE_PATTERNS.some((pattern) => pattern.test(text));
38
+ }
39
+
40
+ async function rateLimit(ctx) {
41
+ const minimum = ctx.options.minInterval === undefined ? 1000 : Number(ctx.options.minInterval);
42
+ if (!Number.isFinite(minimum) || minimum <= 0) {
43
+ ctx.lastActionAt = Date.now();
44
+ return;
45
+ }
46
+ const last = ctx.lastActionAt || 0;
47
+ const wait = last + minimum - Date.now();
48
+ if (wait > 0) await ctx.sleep(wait);
49
+ ctx.lastActionAt = Date.now();
50
+ }
51
+
52
+ async function guardedClick(ctx, node, options = {}) {
53
+ if (!node) throw new ActorError("capture", "click target not found");
54
+ if (isWriteAction(node) && !ctx.options.allowWrite && !options.confirm) {
55
+ throw new ActorError(
56
+ "confirmation_required",
57
+ `refusing write-like action: ${actionText(node).slice(0, 120)}`,
58
+ "M1 is read-only by default. Re-run with --allow-write only after the user explicitly authorises this action.",
59
+ );
60
+ }
61
+ await rateLimit(ctx);
62
+ ctx.trace.push({ action: "click", target: actionText(node).slice(0, 200), at: new Date().toISOString() });
63
+ const locator = node.locator || node.id || node.name;
64
+ return ctx.backend.click(locator);
65
+ }
66
+
67
+ async function guardedGoto(ctx, url) {
68
+ // undefined falls back to WEB_FORAGER_ALLOW_PRIVATE inside the guard.
69
+ const allowPrivate = ctx.options ? ctx.options.allowPrivate : undefined;
70
+ // A page can redirect where it likes; both the requested and the final URL
71
+ // must be public, so a public seed cannot bounce the run into the LAN.
72
+ await assertPublicUrl(url, { allowPrivate });
73
+ if (ctx.options && ctx.options.strictRobots) {
74
+ const { checkRobots } = require("./robots");
75
+ const decision = await checkRobots(url);
76
+ const entry = {
77
+ url,
78
+ robots_url: decision.robots_url,
79
+ status: decision.status,
80
+ allowed: decision.allowed,
81
+ note: decision.note || "",
82
+ };
83
+ ctx.compliance.checks.push(entry);
84
+ ctx.compliance.robots_checked = true;
85
+ if (!decision.allowed) {
86
+ throw new ActorError(
87
+ "robots_denied",
88
+ `robots.txt disallows ${url}`,
89
+ `${decision.robots_url}: ${decision.note || "matched a Disallow rule"}. Drop --strict-robots to browse without the check, or choose another target.`,
90
+ );
91
+ }
92
+ }
93
+ await rateLimit(ctx);
94
+ ctx.trace.push({ action: "goto", url, at: new Date().toISOString() });
95
+ const result = await ctx.backend.goto(url);
96
+ const finalUrl = result && result.url;
97
+ if (finalUrl) await assertPublicUrl(finalUrl, { allowPrivate });
98
+ return result;
99
+ }
100
+
101
+ async function guardedScroll(ctx, dy) {
102
+ await rateLimit(ctx);
103
+ ctx.trace.push({ action: "scroll", dy, at: new Date().toISOString() });
104
+ return ctx.backend.scroll(dy);
105
+ }
106
+
107
+ module.exports = {
108
+ WRITE_PATTERNS,
109
+ actionText,
110
+ isWriteAction,
111
+ rateLimit,
112
+ guardedClick,
113
+ guardedGoto,
114
+ guardedScroll,
115
+ };
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+
3
+ // Every structural assumption about a third-party page lives here, so a site
4
+ // redesign is a one-file change. Each block names the page it was written
5
+ // against and the fixture under tests/fixtures/snapshots/ that keeps it
6
+ // regression-tested. These patterns are heuristics: a redesign can silently
7
+ // stop matching, which is why the playbooks report a degraded run instead of
8
+ // an empty success.
9
+
10
+ const BILIBILI = {
11
+ // https://www.bilibili.com/ and /video/BV...
12
+ videoPath: /\/video\/(BV[0-9A-Za-z]+)/,
13
+ videoId: /\b(BV[0-9A-Za-z]{10})\b/,
14
+ statLabels: /([\d.]+\s*[万亿]?)\s*(播放|弹幕|追番|点赞|评论)/g,
15
+ navWords: /^(首页|推荐|热榜|视频|直播|会员|登录|注册|搜索|关注|收藏|消息|通知|设置|下载|APP|反馈)$/,
16
+ chromeLines: /^(首页|番剧|直播|游戏中心|会员购|漫画|赛事|下载客户端|登录|注册|投稿)$/,
17
+ nextEpisode: /^(下一个|下一话|下一集|下一P|next)/i,
18
+ homeUrl: "https://www.bilibili.com/",
19
+ searchUrl: "https://search.bilibili.com/all",
20
+ };
21
+
22
+ const ZHIHU = {
23
+ // https://www.zhihu.com/search?type=content&q=...
24
+ searchUrl: "https://www.zhihu.com/search?type=content&q=",
25
+ resultLink: /\/(question|answer|p)\/\d+/,
26
+ };
27
+
28
+ const XIAOHONGSHU = {
29
+ // https://www.xiaohongshu.com/search_result?keyword=...
30
+ searchUrl: "https://www.xiaohongshu.com/search_result?keyword=",
31
+ resultLink: /\/explore\/[0-9A-Za-z]+|\/discovery\/item\/[0-9A-Za-z]+/,
32
+ };
33
+
34
+ const WECHAT = {
35
+ // https://mp.weixin.qq.com/s/... — the article body is server-rendered but
36
+ // only after the session passes WeChat's environment check.
37
+ host: /(^|\.)weixin\.qq\.com$/i,
38
+ articleBody: "#js_content",
39
+ title: "#activity-name",
40
+ author: "#js_name",
41
+ published: "#publish_time",
42
+ verification: /环境异常|完成验证后即可继续访问|访问过于频繁|请在微信客户端打开链接/,
43
+ // A blocked article redirects to this handler, whose page body is a canvas
44
+ // and therefore carries no text to match.
45
+ captchaUrl: /(appmsgcaptcha|wappoc|\/captcha|unhuman|antispider|\/verify)/i,
46
+ };
47
+
48
+ // Chinese login interstitials. Read as data: a match only means "the page is
49
+ // asking for credentials", never that we should try to supply them.
50
+ const LOGIN_WALL = /扫码登录|密码登录|登录后查看|登录后继续|请先登录|请登录后/;
51
+
52
+ // Anti-bot, rate-limit and error interstitials. Checked only when a playbook
53
+ // found nothing, so ordinary pages that merely mention these words are safe.
54
+ const BLOCKED = /人机验证|安全验证|访问过于频繁|请求过于频繁|操作过于频繁|请稍后重试|滑块验证|环境异常|完成验证后即可继续访问|请在微信客户端打开链接|captcha|verify you are human|access denied|403 forbidden|404 not found/i;
55
+
56
+ // A page with less readable text than this is treated as blank: either the
57
+ // selector assumptions moved or the site returned a shell.
58
+ const MIN_BODY_CHARS = 40;
59
+
60
+ // Danmaku text layouts seen in DOM snapshots. The bridge may also hand back
61
+ // structured entries ({time, text}), which is preferred because it does not
62
+ // depend on how the player renders text.
63
+ const DANMAKU = {
64
+ textPatterns: [
65
+ /^(\d{1,2}:\d{2}(?::\d{2})?)\s+(.+)$/,
66
+ /^[[(【]?(\d{1,2}:\d{2}(?::\d{2})?)[\])】]?\s*[-—:]?\s*(.+)$/,
67
+ ],
68
+ roles: /^(danmaku|danmu|弹幕)$/i,
69
+ };
70
+
71
+ module.exports = {
72
+ BILIBILI,
73
+ ZHIHU,
74
+ XIAOHONGSHU,
75
+ WECHAT,
76
+ LOGIN_WALL,
77
+ BLOCKED,
78
+ MIN_BODY_CHARS,
79
+ DANMAKU,
80
+ };