web-forager 0.0.0-stage → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +119 -0
  18. package/CHANGELOG.md +508 -0
  19. package/LICENSE +21 -0
  20. package/README.md +498 -2
  21. package/README_zh.md +498 -0
  22. package/SKILL.md +451 -0
  23. package/THIRD_PARTY_LICENSES.txt +40 -0
  24. package/browser/bridges/playwright-bridge.js +382 -0
  25. package/browser/launch_chrome.py +193 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +90 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +141 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +130 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +215 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +252 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +107 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +441 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/research.md +35 -0
  58. package/commands/scholar.md +28 -0
  59. package/commands/surf.md +28 -0
  60. package/cordis.patch.yml +19 -0
  61. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  62. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  63. package/docs/DEPENDENCIES.md +40 -0
  64. package/docs/INSTALL_COMMANDS.md +113 -0
  65. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  66. package/docs/REFERENCE.md +349 -0
  67. package/docs/REFERENCE_zh.md +342 -0
  68. package/index.js +7 -0
  69. package/install.py +254 -0
  70. package/manifest.json +98 -0
  71. package/mcp_server.py +850 -0
  72. package/package.json +82 -6
  73. package/requirements.txt +15 -0
  74. package/run.py +908 -0
  75. package/scripts/__init__.py +1 -0
  76. package/scripts/bilibili.py +882 -0
  77. package/scripts/documents.py +437 -0
  78. package/scripts/engine.py +5141 -0
  79. package/scripts/feeds.py +180 -0
  80. package/scripts/pdf.py +156 -0
  81. package/scripts/research.py +2087 -0
  82. package/scripts/scholar.py +381 -0
  83. package/scripts/sitemap.py +195 -0
  84. package/scripts/untrusted.py +312 -0
package/package.json CHANGED
@@ -1,6 +1,82 @@
1
- {
2
- "name": "web-forager",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
1
+ {
2
+ "name": "web-forager",
3
+ "version": "1.3.1",
4
+ "description": "Clean-room, robots-aware web information gathering: multi-engine search, HTML/PDF/Office/feed extraction, Bilibili/Zhihu/Xiaohongshu/WeChat reading, sitemaps and resumable surfing, plus a browser mode with eight read-only playbooks. CLI + two MCP servers.",
5
+ "main": "index.js",
6
+ "license": "MIT",
7
+ "engines": {
8
+ "node": ">=18"
9
+ },
10
+ "keywords": [
11
+ "dsh",
12
+ "dsh-plugin",
13
+ "deepseek-harness",
14
+ "web-scraping",
15
+ "search",
16
+ "mcp",
17
+ "mcp-server",
18
+ "bilibili",
19
+ "zhihu",
20
+ "crawler",
21
+ "robots-txt",
22
+ "playwright"
23
+ ],
24
+ "repository": {
25
+ "type": "git",
26
+ "url": "git+https://github.com/JiamingShi-JsPhys/web-forager.git"
27
+ },
28
+ "homepage": "https://github.com/JiamingShi-JsPhys/web-forager#readme",
29
+ "bugs": {
30
+ "url": "https://github.com/JiamingShi-JsPhys/web-forager/issues"
31
+ },
32
+ "scripts": {
33
+ "test": "python -m pytest",
34
+ "test:browser": "node --test \"browser/tests/*.test.js\"",
35
+ "test:live": "node browser/tests/live-smoke.js",
36
+ "clean:pycache": "node clean-pycache.mjs",
37
+ "prepack": "npm run clean:pycache",
38
+ "prepublishOnly": "npm run clean:pycache"
39
+ },
40
+ "files": [
41
+ "index.js",
42
+ "cordis.patch.yml",
43
+ ".claude-plugin/",
44
+ ".codex-plugin/",
45
+ ".codebuddy-plugin/",
46
+ ".dsh-plugin/",
47
+ ".cursor/",
48
+ ".clinerules/",
49
+ ".continue/",
50
+ ".roo/",
51
+ ".windsurf/",
52
+ ".github/copilot-instructions.md",
53
+ ".mcp.json",
54
+ "install.py",
55
+ "requirements.txt",
56
+ "run.py",
57
+ "mcp_server.py",
58
+ "browser/run.js",
59
+ "browser/mcp_server.js",
60
+ "browser/launch_chrome.py",
61
+ "browser/package.json",
62
+ "browser/lib/",
63
+ "browser/playbooks/",
64
+ "browser/bridges/",
65
+ "scripts/",
66
+ "commands/",
67
+ "docs/",
68
+ "SKILL.md",
69
+ "AGENTS.md",
70
+ "manifest.json",
71
+ "README.md",
72
+ "README_zh.md",
73
+ "CHANGELOG.md",
74
+ "THIRD_PARTY_LICENSES.txt",
75
+ "LICENSE"
76
+ ],
77
+ "dsh": {
78
+ "bundle": {
79
+ "patch": "./cordis.patch.yml"
80
+ }
81
+ }
82
+ }
@@ -0,0 +1,15 @@
1
+ # Core runtime (all Apache-2.0 or MIT; no GPL/AGPL anywhere)
2
+ # certifi is a requests dependency; the floor avoids the compromised
3
+ # emSign root CAs fixed in 2026.10.18.
4
+ certifi>=2026.10.18
5
+ requests>=2.33.0
6
+ beautifulsoup4>=4.14.3
7
+ trafilatura>=2.3.1
8
+
9
+ # Optional: PDF text extraction (BSD-3-Clause).
10
+ # Without it, PDF inputs return exit code 7 with a clear install hint.
11
+ # pypdf>=6.21.4 (or 7.x); earlier versions have seven DoS advisories
12
+
13
+ # Optional: JavaScript rendering (Apache-2.0).
14
+ # Install the package, then: python -m playwright install chromium
15
+ # playwright>=1.40