web-forager 1.3.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/.clinerules/web-forager.md +3 -3
  4. package/.codebuddy-plugin/marketplace.json +1 -1
  5. package/.codebuddy-plugin/plugin.json +1 -1
  6. package/.codex-plugin/marketplace.json +1 -1
  7. package/.codex-plugin/mcp.json +20 -20
  8. package/.codex-plugin/plugin.json +1 -1
  9. package/.continue/rules/web-forager.md +4 -4
  10. package/.cursor/rules/web-forager.mdc +4 -4
  11. package/.dsh-plugin/index.js +1 -1
  12. package/.dsh-plugin/plugin.json +1 -1
  13. package/.github/copilot-instructions.md +3 -3
  14. package/.roo/rules/web-forager.md +3 -3
  15. package/.windsurf/rules/web-forager.md +4 -4
  16. package/AGENTS.md +10 -3
  17. package/CHANGELOG.md +508 -454
  18. package/README.md +499 -477
  19. package/README_zh.md +498 -477
  20. package/SKILL.md +115 -15
  21. package/THIRD_PARTY_LICENSES.txt +5 -0
  22. package/browser/bridges/playwright-bridge.js +8 -1
  23. package/browser/launch_chrome.py +193 -144
  24. package/browser/lib/bili-api.js +12 -1
  25. package/browser/lib/safety.js +27 -1
  26. package/browser/lib/session.js +16 -2
  27. package/browser/lib/untrusted.js +215 -56
  28. package/browser/mcp_server.js +5 -25
  29. package/browser/package.json +1 -1
  30. package/browser/playbooks/search-dig.js +3 -1
  31. package/browser/playbooks/wechat-dig.js +78 -78
  32. package/browser/run.js +19 -10
  33. package/commands/research.md +35 -0
  34. package/cordis.patch.yml +19 -19
  35. package/docs/DEPENDENCIES.md +1 -1
  36. package/docs/INSTALL_COMMANDS.md +4 -4
  37. package/docs/INSTALL_COMMANDS_zh.md +4 -4
  38. package/docs/REFERENCE.md +349 -308
  39. package/docs/REFERENCE_zh.md +342 -309
  40. package/manifest.json +3 -2
  41. package/mcp_server.py +383 -130
  42. package/package.json +4 -2
  43. package/requirements.txt +7 -4
  44. package/run.py +61 -5
  45. package/scripts/bilibili.py +11 -3
  46. package/scripts/engine.py +1039 -118
  47. package/scripts/pdf.py +21 -0
  48. package/scripts/research.py +2087 -0
  49. package/scripts/scholar.py +33 -6
  50. package/scripts/sitemap.py +20 -5
  51. package/scripts/untrusted.py +312 -95
package/scripts/pdf.py CHANGED
@@ -8,9 +8,15 @@ optional dependency: when it is absent, callers receive a clear
8
8
  from __future__ import annotations
9
9
 
10
10
  import io
11
+ import time
11
12
  from typing import Any
12
13
 
13
14
  PDF_MAGIC = b"%PDF-"
15
+ # pypdf has had denial-of-service advisories around page-tree and xref parsing.
16
+ # Keep the installed package current (>=6.21.4) and put a hard bound on the
17
+ # work this module will do on an untrusted document.
18
+ MAX_PAGES = 500
19
+ MAX_SECONDS = 20.0
14
20
 
15
21
 
16
22
  class PdfError(Exception):
@@ -58,7 +64,12 @@ def extract(data: bytes) -> dict[str, Any]:
58
64
 
59
65
  pages: list[str] = []
60
66
  empty_pages = 0
67
+ truncated = False
68
+ deadline = time.monotonic() + MAX_SECONDS
61
69
  for index, page in enumerate(reader.pages, start=1):
70
+ if index > MAX_PAGES or time.monotonic() > deadline:
71
+ truncated = True
72
+ break
62
73
  try:
63
74
  text = page.extract_text() or ""
64
75
  except Exception as exc: # a single damaged page should not kill the run
@@ -69,10 +80,18 @@ def extract(data: bytes) -> dict[str, Any]:
69
80
  pages.append(text)
70
81
 
71
82
  metadata = reader.metadata
83
+ warnings: list[str] = []
84
+ if truncated:
85
+ warnings.append(
86
+ f"PDF extraction stopped at {len(pages)} page(s) "
87
+ f"(limit: {MAX_PAGES} pages / {MAX_SECONDS:g} s)"
88
+ )
72
89
  return {
73
90
  "pages": pages,
74
91
  "page_count": len(pages),
75
92
  "empty_pages": empty_pages,
93
+ "truncated": truncated,
94
+ "warnings": warnings,
76
95
  "text": "\n\n".join(pages).strip(),
77
96
  "metadata": {
78
97
  "title": _meta_value(metadata, "/Title"),
@@ -100,6 +119,8 @@ def to_markdown(document: dict[str, Any], url: str = "") -> str:
100
119
  )
101
120
  if info:
102
121
  lines.extend([f"> {info}", ""])
122
+ for warning in document.get("warnings") or []:
123
+ lines.extend([f"> 警告:{warning}", ""])
103
124
  pages = document.get("pages") or []
104
125
  for index, text in enumerate(pages, start=1):
105
126
  if not text: