web-forager 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.clinerules/web-forager.md +3 -3
- package/.codebuddy-plugin/marketplace.json +1 -1
- package/.codebuddy-plugin/plugin.json +1 -1
- package/.codex-plugin/marketplace.json +1 -1
- package/.codex-plugin/mcp.json +20 -20
- package/.codex-plugin/plugin.json +1 -1
- package/.continue/rules/web-forager.md +4 -4
- package/.cursor/rules/web-forager.mdc +4 -4
- package/.dsh-plugin/index.js +1 -1
- package/.dsh-plugin/plugin.json +1 -1
- package/.github/copilot-instructions.md +3 -3
- package/.roo/rules/web-forager.md +3 -3
- package/.windsurf/rules/web-forager.md +4 -4
- package/AGENTS.md +10 -3
- package/CHANGELOG.md +508 -454
- package/README.md +499 -477
- package/README_zh.md +498 -477
- package/SKILL.md +115 -15
- package/THIRD_PARTY_LICENSES.txt +5 -0
- package/browser/bridges/playwright-bridge.js +8 -1
- package/browser/launch_chrome.py +193 -144
- package/browser/lib/bili-api.js +12 -1
- package/browser/lib/safety.js +27 -1
- package/browser/lib/session.js +16 -2
- package/browser/lib/untrusted.js +215 -56
- package/browser/mcp_server.js +5 -25
- package/browser/package.json +1 -1
- package/browser/playbooks/search-dig.js +3 -1
- package/browser/playbooks/wechat-dig.js +78 -78
- package/browser/run.js +19 -10
- package/commands/research.md +35 -0
- package/cordis.patch.yml +19 -19
- package/docs/DEPENDENCIES.md +1 -1
- package/docs/INSTALL_COMMANDS.md +4 -4
- package/docs/INSTALL_COMMANDS_zh.md +4 -4
- package/docs/REFERENCE.md +349 -308
- package/docs/REFERENCE_zh.md +342 -309
- package/manifest.json +3 -2
- package/mcp_server.py +383 -130
- package/package.json +4 -2
- package/requirements.txt +7 -4
- package/run.py +61 -5
- package/scripts/bilibili.py +11 -3
- package/scripts/engine.py +1039 -118
- package/scripts/pdf.py +21 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +33 -6
- package/scripts/sitemap.py +20 -5
- package/scripts/untrusted.py +312 -95
package/scripts/pdf.py
CHANGED
|
@@ -8,9 +8,15 @@ optional dependency: when it is absent, callers receive a clear
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import io
|
|
11
|
+
import time
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
13
14
|
PDF_MAGIC = b"%PDF-"
|
|
15
|
+
# pypdf has had denial-of-service advisories around page-tree and xref parsing.
|
|
16
|
+
# Keep the installed package current (>=6.21.4) and put a hard bound on the
|
|
17
|
+
# work this module will do on an untrusted document.
|
|
18
|
+
MAX_PAGES = 500
|
|
19
|
+
MAX_SECONDS = 20.0
|
|
14
20
|
|
|
15
21
|
|
|
16
22
|
class PdfError(Exception):
|
|
@@ -58,7 +64,12 @@ def extract(data: bytes) -> dict[str, Any]:
|
|
|
58
64
|
|
|
59
65
|
pages: list[str] = []
|
|
60
66
|
empty_pages = 0
|
|
67
|
+
truncated = False
|
|
68
|
+
deadline = time.monotonic() + MAX_SECONDS
|
|
61
69
|
for index, page in enumerate(reader.pages, start=1):
|
|
70
|
+
if index > MAX_PAGES or time.monotonic() > deadline:
|
|
71
|
+
truncated = True
|
|
72
|
+
break
|
|
62
73
|
try:
|
|
63
74
|
text = page.extract_text() or ""
|
|
64
75
|
except Exception as exc: # a single damaged page should not kill the run
|
|
@@ -69,10 +80,18 @@ def extract(data: bytes) -> dict[str, Any]:
|
|
|
69
80
|
pages.append(text)
|
|
70
81
|
|
|
71
82
|
metadata = reader.metadata
|
|
83
|
+
warnings: list[str] = []
|
|
84
|
+
if truncated:
|
|
85
|
+
warnings.append(
|
|
86
|
+
f"PDF extraction stopped at {len(pages)} page(s) "
|
|
87
|
+
f"(limit: {MAX_PAGES} pages / {MAX_SECONDS:g} s)"
|
|
88
|
+
)
|
|
72
89
|
return {
|
|
73
90
|
"pages": pages,
|
|
74
91
|
"page_count": len(pages),
|
|
75
92
|
"empty_pages": empty_pages,
|
|
93
|
+
"truncated": truncated,
|
|
94
|
+
"warnings": warnings,
|
|
76
95
|
"text": "\n\n".join(pages).strip(),
|
|
77
96
|
"metadata": {
|
|
78
97
|
"title": _meta_value(metadata, "/Title"),
|
|
@@ -100,6 +119,8 @@ def to_markdown(document: dict[str, Any], url: str = "") -> str:
|
|
|
100
119
|
)
|
|
101
120
|
if info:
|
|
102
121
|
lines.extend([f"> {info}", ""])
|
|
122
|
+
for warning in document.get("warnings") or []:
|
|
123
|
+
lines.extend([f"> 警告:{warning}", ""])
|
|
103
124
|
pages = document.get("pages") or []
|
|
104
125
|
for index, text in enumerate(pages, start=1):
|
|
105
126
|
if not text:
|