web-forager 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +112 -0
  18. package/CHANGELOG.md +454 -0
  19. package/LICENSE +21 -0
  20. package/README.md +477 -3
  21. package/README_zh.md +477 -0
  22. package/SKILL.md +351 -0
  23. package/THIRD_PARTY_LICENSES.txt +35 -0
  24. package/browser/bridges/playwright-bridge.js +375 -0
  25. package/browser/launch_chrome.py +144 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +79 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +115 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +116 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +56 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +272 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +105 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +432 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/scholar.md +28 -0
  58. package/commands/surf.md +28 -0
  59. package/cordis.patch.yml +19 -0
  60. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  61. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  62. package/docs/DEPENDENCIES.md +40 -0
  63. package/docs/INSTALL_COMMANDS.md +113 -0
  64. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  65. package/docs/REFERENCE.md +308 -0
  66. package/docs/REFERENCE_zh.md +309 -0
  67. package/index.js +7 -0
  68. package/install.py +254 -0
  69. package/manifest.json +97 -0
  70. package/mcp_server.py +597 -0
  71. package/package.json +80 -6
  72. package/requirements.txt +12 -0
  73. package/run.py +852 -0
  74. package/scripts/__init__.py +1 -0
  75. package/scripts/bilibili.py +874 -0
  76. package/scripts/documents.py +437 -0
  77. package/scripts/engine.py +4220 -0
  78. package/scripts/feeds.py +180 -0
  79. package/scripts/pdf.py +135 -0
  80. package/scripts/scholar.py +354 -0
  81. package/scripts/sitemap.py +180 -0
  82. package/scripts/untrusted.py +95 -0
@@ -0,0 +1,138 @@
1
+ "use strict";
2
+
3
+ const { ActorError } = require("../lib/errors");
4
+ const { guardedGoto, guardedScroll } = require("../lib/safety");
5
+ const { extractVideoMeta, commentCandidates, pageTitle, bodyText, hasLoginWall, diagnoseEmpty } = require("../lib/snapshot");
6
+ const { markCaptured, markVisited } = require("../lib/session");
7
+
8
+ // A page can swap its document after goto (login walls and verification pages
9
+ // do exactly that), which kills the evaluate that reads it. Classify the failure
10
+ // by where the page actually is, instead of surfacing a bare navigation error.
11
+ const LOGIN_URL = /(account\/unhuman|website-login|need_login|passport|signin|sign_in|\/login)/i;
12
+ const BLOCK_URL = /(captcha|verify|antispider|antibot|unhuman|security-check|\/412)/i;
13
+
14
+ async function currentUrlOf(ctx) {
15
+ try {
16
+ if (ctx && ctx.backend && typeof ctx.backend.currentUrl === "function") {
17
+ return await ctx.backend.currentUrl();
18
+ }
19
+ } catch (error) {
20
+ // The bridge may already be gone; the caller still gets a classified error.
21
+ }
22
+ return "";
23
+ }
24
+
25
+ async function snapshotSafe(ctx, where) {
26
+ try {
27
+ return await ctx.backend.snapshot();
28
+ } catch (error) {
29
+ const url = await currentUrlOf(ctx);
30
+ if (url && LOGIN_URL.test(url)) {
31
+ throw new ActorError(
32
+ "login_required",
33
+ `${where} redirected to a login wall (${url})`,
34
+ "Use a logged-in bridge (CDP or a persistent profile), or open a public URL directly.",
35
+ );
36
+ }
37
+ if (url && BLOCK_URL.test(url)) {
38
+ throw new ActorError(
39
+ "blocked",
40
+ `${where} redirected to a verification page (${url})`,
41
+ "Wait and retry later, or reuse a logged-in profile. web-forager does not solve challenges.",
42
+ );
43
+ }
44
+ throw new ActorError(
45
+ "navigation",
46
+ `the page changed while reading ${where}${url ? ` (now at ${url})` : ""}: ${error.message}`,
47
+ "Retry the playbook; the site redirected during the snapshot.",
48
+ );
49
+ }
50
+ }
51
+
52
+ function videoMarkdown(meta, comments) {
53
+ const lines = [`# ${meta.title}`, ""];
54
+ if (meta.bvid) lines.push(`- BV: ${meta.bvid}`);
55
+ if (meta.url) lines.push(`- URL: ${meta.url}`);
56
+ for (const [key, value] of Object.entries(meta.stats || {})) lines.push(`- ${key}: ${value}`);
57
+ if (comments.length) {
58
+ lines.push("", "## Visible comments", "");
59
+ for (const comment of comments) lines.push(`- ${comment}`);
60
+ }
61
+ return lines.join("\n");
62
+ }
63
+
64
+ async function captureCurrent(ctx, options = {}) {
65
+ const scrolls = Number(options.commentScrolls === undefined ? 3 : options.commentScrolls);
66
+ let snapshot = await ctx.backend.snapshot();
67
+ const meta = extractVideoMeta(snapshot);
68
+ if (!meta.bvid && hasLoginWall(snapshot)) {
69
+ throw new ActorError(
70
+ "login_required",
71
+ "the video page asked for credentials before the player loaded",
72
+ "Use a logged-in extension bridge, or open a public URL with the HTTP mode adapter.",
73
+ );
74
+ }
75
+ const comments = [];
76
+ const seen = new Set();
77
+ for (let index = 0; index <= scrolls; index += 1) {
78
+ for (const comment of commentCandidates(snapshot)) {
79
+ if (!seen.has(comment)) {
80
+ seen.add(comment);
81
+ comments.push(comment);
82
+ }
83
+ }
84
+ if (index < scrolls) {
85
+ await guardedScroll(ctx, 700);
86
+ snapshot = await ctx.backend.snapshot();
87
+ }
88
+ }
89
+
90
+ const record = ctx.capture({
91
+ playbook: options.playbook || "bili-video",
92
+ url: meta.url || (snapshot && snapshot.url) || "",
93
+ metadata: { bvid: meta.bvid, title: meta.title, stats: meta.stats },
94
+ markdown: videoMarkdown(meta, comments),
95
+ warnings: meta.bvid ? [] : ["no BV id found on the page"],
96
+ });
97
+ if (meta.bvid) markCaptured(ctx.session, { id: meta.bvid, url: meta.url, title: meta.title, playbook: options.playbook });
98
+
99
+ return { record, meta, snapshot, comments };
100
+ }
101
+
102
+ async function openAndCapture(ctx, url, options = {}) {
103
+ await guardedGoto(ctx, url);
104
+ markVisited(ctx.session, url);
105
+ return captureCurrent(ctx, options);
106
+ }
107
+
108
+ async function capturePage(ctx, options = {}) {
109
+ const snapshot = await ctx.backend.snapshot();
110
+ const title = pageTitle(snapshot) || options.url || "page";
111
+ const body = bodyText(snapshot);
112
+ if (!body && hasLoginWall(snapshot)) {
113
+ throw new ActorError(
114
+ "login_required",
115
+ `${options.site || "the site"} asked for credentials before showing the page`,
116
+ "Use a logged-in extension bridge, or open a public URL directly.",
117
+ );
118
+ }
119
+ const reason = body ? "" : diagnoseEmpty(snapshot);
120
+ const record = ctx.capture({
121
+ playbook: options.playbook || "page",
122
+ url: (snapshot && snapshot.url) || options.url || "",
123
+ metadata: { site: options.site || "", title },
124
+ markdown: body || "(no readable text extracted from the snapshot)",
125
+ warnings: body ? [] : [`no readable text extracted from the snapshot (${reason})`],
126
+ });
127
+ markCaptured(ctx.session, {
128
+ id: options.url || (snapshot && snapshot.url) || title,
129
+ url: options.url || (snapshot && snapshot.url) || "",
130
+ title,
131
+ playbook: options.playbook,
132
+ });
133
+ return { record, snapshot, title, body, status: body ? "ok" : "degraded", reason };
134
+ }
135
+
136
+ module.exports = { captureCurrent, openAndCapture, capturePage, videoMarkdown };
137
+
138
+ module.exports.snapshotSafe = snapshotSafe;
@@ -0,0 +1,78 @@
1
+ "use strict";
2
+
3
+ const { ActorError } = require("../lib/errors");
4
+ const { guardedGoto } = require("../lib/safety");
5
+ const { WECHAT, MIN_BODY_CHARS } = require("../lib/selectors");
6
+ const { pageTitle, bodyText, diagnoseEmpty } = require("../lib/snapshot");
7
+ const { markVisited } = require("../lib/session");
8
+ const { capturePage, snapshotSafe } = require("./shared");
9
+
10
+ const NAME = "wechat-dig";
11
+
12
+ module.exports = {
13
+ name: NAME,
14
+ description:
15
+ "Open one WeChat article link, report its environment-check wall instead of recycling it, and capture the article text.",
16
+ requires: ["url"],
17
+
18
+ async run(ctx) {
19
+ const url = String(ctx.options.url || "").trim();
20
+ if (!url) {
21
+ throw new ActorError("argument", "wechat-dig needs --url with a WeChat article link");
22
+ }
23
+ let host = "";
24
+ try {
25
+ host = new URL(url).hostname;
26
+ } catch (error) {
27
+ throw new ActorError("argument", `not a URL: ${url}`);
28
+ }
29
+ if (!WECHAT.host.test(host)) {
30
+ throw new ActorError(
31
+ "argument",
32
+ "wechat-dig only opens weixin.qq.com links; use the HTTP fetch action for other sites",
33
+ );
34
+ }
35
+
36
+ await guardedGoto(ctx, url);
37
+ markVisited(ctx.session, url);
38
+ const snapshot = await snapshotSafe(ctx, "the WeChat article");
39
+
40
+ const title = pageTitle(snapshot) || "";
41
+ const text = bodyText(snapshot);
42
+ const finalUrl = (snapshot && snapshot.url) || url;
43
+ if (WECHAT.captchaUrl.test(finalUrl) || WECHAT.verification.test(`${title}\n${text}`)) {
44
+ throw new ActorError(
45
+ "blocked",
46
+ "WeChat returned its environment-check page instead of the article",
47
+ "Open the article once in the profile started by browser/launch_chrome.py and complete the check yourself, then rerun. web-forager does not solve challenges.",
48
+ );
49
+ }
50
+
51
+ const reason = text.trim().length < MIN_BODY_CHARS ? diagnoseEmpty(snapshot) : "";
52
+ if (reason === "login") {
53
+ throw new ActorError(
54
+ "login_required",
55
+ "WeChat asked for credentials before showing the article",
56
+ "Use a profile that is already signed in, or open the article in the WeChat client.",
57
+ );
58
+ }
59
+ if (reason === "blocked") {
60
+ throw new ActorError(
61
+ "blocked",
62
+ "WeChat showed an anti-bot or error page instead of the article",
63
+ "Wait and retry later, or reuse a profile that has passed the check. web-forager does not solve challenges.",
64
+ );
65
+ }
66
+
67
+ const page = await capturePage(ctx, { url, site: "wechat", playbook: NAME });
68
+ return {
69
+ summary: {
70
+ url: page.record.url,
71
+ title: page.title,
72
+ chars: page.body.length,
73
+ status: page.status,
74
+ reason: page.reason || "",
75
+ },
76
+ };
77
+ },
78
+ };
@@ -0,0 +1,89 @@
1
+ "use strict";
2
+
3
+ const { ActorError } = require("../lib/errors");
4
+ const { guardedGoto } = require("../lib/safety");
5
+ const { findLinks, rankByQuery, diagnoseEmpty } = require("../lib/snapshot");
6
+ const { XIAOHONGSHU } = require("../lib/selectors");
7
+ const { hasCaptured, markCaptured, markVisited } = require("../lib/session");
8
+ const { capturePage, snapshotSafe } = require("./shared");
9
+
10
+ const NAME = "xhs-dig";
11
+ const SEARCH = XIAOHONGSHU.searchUrl;
12
+ const RESULT_LINK = XIAOHONGSHU.resultLink;
13
+
14
+ module.exports = {
15
+ name: NAME,
16
+ description: "Search Xiaohongshu public notes, rank note links by query overlap, and capture the best unseen note.",
17
+ requires: ["query"],
18
+
19
+ async run(ctx) {
20
+ const query = ctx.options.query;
21
+ if (!query) throw new ActorError("argument", "xhs-dig needs --query");
22
+
23
+ const searchUrl = `${SEARCH}${encodeURIComponent(query)}`;
24
+ await guardedGoto(ctx, searchUrl);
25
+ markVisited(ctx.session, searchUrl);
26
+
27
+ const snapshot = await snapshotSafe(ctx, "the search page");
28
+ const ranked = rankByQuery(findLinks(snapshot, RESULT_LINK), query);
29
+ if (!ranked.length) {
30
+ const reason = diagnoseEmpty(snapshot);
31
+ if (reason === "login") {
32
+ throw new ActorError(
33
+ "login_required",
34
+ "Xiaohongshu showed a login wall before any note links",
35
+ "Use a logged-in extension bridge, or open a public note URL directly with the page playbook.",
36
+ );
37
+ }
38
+ if (reason === "blocked") {
39
+ throw new ActorError(
40
+ "blocked",
41
+ "Xiaohongshu returned an anti-bot or error page instead of notes",
42
+ "Wait and retry later, or reuse a logged-in profile through the extension backend. web-forager does not solve challenges.",
43
+ );
44
+ }
45
+ ctx.warn(`degraded: no note links found (${reason})`);
46
+ return { summary: { query, notes: 0, captured: 0, status: "degraded", reason } };
47
+ }
48
+
49
+ ctx.capture({
50
+ playbook: NAME,
51
+ url: searchUrl,
52
+ metadata: { query, site: "xiaohongshu", notes: ranked.length },
53
+ markdown: ranked
54
+ .slice(0, 10)
55
+ .map((item, index) => `${index + 1}. ${item.title} - ${item.url}`)
56
+ .join("\n"),
57
+ tables: [
58
+ {
59
+ name: "xiaohongshu notes",
60
+ columns: ["#", "title", "url"],
61
+ rows: ranked.slice(0, 20).map((item, index) => [index + 1, item.title, item.url]),
62
+ },
63
+ ],
64
+ });
65
+
66
+ const target = ranked.find((item) => !hasCaptured(ctx.session, item.url));
67
+ if (!target) {
68
+ ctx.warn("all notes already captured");
69
+ return { summary: { query, notes: ranked.length, captured: 0, status: "ok", reason: "all-captured" } };
70
+ }
71
+
72
+ await guardedGoto(ctx, target.url);
73
+ markVisited(ctx.session, target.url);
74
+ const page = await capturePage(ctx, { url: target.url, site: "xiaohongshu", playbook: NAME });
75
+ markCaptured(ctx.session, { id: target.url, url: target.url, title: page.title, playbook: NAME });
76
+
77
+ return {
78
+ summary: {
79
+ query,
80
+ notes: ranked.length,
81
+ captured: 1,
82
+ opened: target.url,
83
+ title: page.title,
84
+ status: page.status,
85
+ reason: page.reason || "",
86
+ },
87
+ };
88
+ },
89
+ };
@@ -0,0 +1,89 @@
1
+ "use strict";
2
+
3
+ const { ActorError } = require("../lib/errors");
4
+ const { guardedGoto } = require("../lib/safety");
5
+ const { findLinks, rankByQuery, diagnoseEmpty } = require("../lib/snapshot");
6
+ const { ZHIHU } = require("../lib/selectors");
7
+ const { hasCaptured, markCaptured, markVisited } = require("../lib/session");
8
+ const { capturePage, snapshotSafe } = require("./shared");
9
+
10
+ const NAME = "zhihu-dig";
11
+ const SEARCH = ZHIHU.searchUrl;
12
+ const RESULT_LINK = ZHIHU.resultLink;
13
+
14
+ module.exports = {
15
+ name: NAME,
16
+ description: "Search Zhihu, rank question and answer links by query overlap, and capture the best unseen page.",
17
+ requires: ["query"],
18
+
19
+ async run(ctx) {
20
+ const query = ctx.options.query;
21
+ if (!query) throw new ActorError("argument", "zhihu-dig needs --query");
22
+
23
+ const searchUrl = `${SEARCH}${encodeURIComponent(query)}`;
24
+ await guardedGoto(ctx, searchUrl);
25
+ markVisited(ctx.session, searchUrl);
26
+
27
+ const snapshot = await snapshotSafe(ctx, "the search page");
28
+ const ranked = rankByQuery(findLinks(snapshot, RESULT_LINK), query);
29
+ if (!ranked.length) {
30
+ const reason = diagnoseEmpty(snapshot);
31
+ if (reason === "login") {
32
+ throw new ActorError(
33
+ "login_required",
34
+ "Zhihu showed a login wall before any result links",
35
+ "Use a logged-in extension bridge, or open a public question URL directly with the page playbook.",
36
+ );
37
+ }
38
+ if (reason === "blocked") {
39
+ throw new ActorError(
40
+ "blocked",
41
+ "Zhihu returned an anti-bot or error page instead of results",
42
+ "Wait and retry later, or reuse a logged-in profile through the extension backend. web-forager does not solve challenges.",
43
+ );
44
+ }
45
+ ctx.warn(`degraded: no result links found (${reason})`);
46
+ return { summary: { query, results: 0, captured: 0, status: "degraded", reason } };
47
+ }
48
+
49
+ ctx.capture({
50
+ playbook: NAME,
51
+ url: searchUrl,
52
+ metadata: { query, site: "zhihu", results: ranked.length },
53
+ markdown: ranked
54
+ .slice(0, 10)
55
+ .map((item, index) => `${index + 1}. ${item.title} - ${item.url}`)
56
+ .join("\n"),
57
+ tables: [
58
+ {
59
+ name: "zhihu results",
60
+ columns: ["#", "title", "url"],
61
+ rows: ranked.slice(0, 20).map((item, index) => [index + 1, item.title, item.url]),
62
+ },
63
+ ],
64
+ });
65
+
66
+ const target = ranked.find((item) => !hasCaptured(ctx.session, item.url));
67
+ if (!target) {
68
+ ctx.warn("all results already captured");
69
+ return { summary: { query, results: ranked.length, captured: 0, status: "ok", reason: "all-captured" } };
70
+ }
71
+
72
+ await guardedGoto(ctx, target.url);
73
+ markVisited(ctx.session, target.url);
74
+ const page = await capturePage(ctx, { url: target.url, site: "zhihu", playbook: NAME });
75
+ markCaptured(ctx.session, { id: target.url, url: target.url, title: page.title, playbook: NAME });
76
+
77
+ return {
78
+ summary: {
79
+ query,
80
+ results: ranked.length,
81
+ captured: 1,
82
+ opened: target.url,
83
+ title: page.title,
84
+ status: page.status,
85
+ reason: page.reason || "",
86
+ },
87
+ };
88
+ },
89
+ };