web-forager 0.0.0-stage → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +119 -0
- package/CHANGELOG.md +508 -0
- package/LICENSE +21 -0
- package/README.md +498 -2
- package/README_zh.md +498 -0
- package/SKILL.md +451 -0
- package/THIRD_PARTY_LICENSES.txt +40 -0
- package/browser/bridges/playwright-bridge.js +382 -0
- package/browser/launch_chrome.py +193 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +90 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +141 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +130 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +215 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +252 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +107 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +441 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/research.md +35 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +349 -0
- package/docs/REFERENCE_zh.md +342 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +98 -0
- package/mcp_server.py +850 -0
- package/package.json +82 -6
- package/requirements.txt +15 -0
- package/run.py +908 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +882 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +5141 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +156 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +381 -0
- package/scripts/sitemap.py +195 -0
- package/scripts/untrusted.py +312 -0
package/scripts/feeds.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""RSS 2.0 / Atom / RSS 1.0 feed parsing.
|
|
2
|
+
|
|
3
|
+
Feeds are structured XML, so they are far cheaper to read than the rendered
|
|
4
|
+
HTML of a blog index. Parsing goes through :func:`engine.parse_xml`, which
|
|
5
|
+
rejects DOCTYPE/ENTITY and uses defusedxml when available.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import html as html_lib
|
|
11
|
+
import re
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from . import engine
|
|
15
|
+
|
|
16
|
+
_ATOM = "{http://www.w3.org/2005/Atom}"
|
|
17
|
+
_RSS1 = "{http://purl.org/rss/1.0/}"
|
|
18
|
+
_DC = "{http://purl.org/dc/elements/1.1/}"
|
|
19
|
+
_CONTENT = "{http://purl.org/rss/1.0/modules/content/}"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _clean(value: Any, limit: int = 1200) -> str:
|
|
23
|
+
text = html_lib.unescape(str(value or ""))
|
|
24
|
+
if "<" in text:
|
|
25
|
+
text = re.sub(r"<[^>]+>", " ", text)
|
|
26
|
+
text = re.sub(r"\s+", " ", text).strip()
|
|
27
|
+
return text[:limit]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _text(node: Any, path: str = "") -> str:
|
|
31
|
+
target = node.find(path) if path else node
|
|
32
|
+
return (target.text or "").strip() if target is not None else ""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _atom_link(entry: Any) -> str:
|
|
36
|
+
fallback = ""
|
|
37
|
+
for link in entry.findall(f"{_ATOM}link"):
|
|
38
|
+
href = link.get("href", "")
|
|
39
|
+
if link.get("rel", "alternate") == "alternate":
|
|
40
|
+
return href
|
|
41
|
+
fallback = fallback or href
|
|
42
|
+
return fallback
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _atom_entry(entry: Any) -> dict[str, Any]:
|
|
46
|
+
return {
|
|
47
|
+
"title": _clean(_text(entry, f"{_ATOM}title"), 300),
|
|
48
|
+
"url": _atom_link(entry),
|
|
49
|
+
"published": _text(entry, f"{_ATOM}published") or _text(entry, f"{_ATOM}updated"),
|
|
50
|
+
"summary": _clean(
|
|
51
|
+
_text(entry, f"{_ATOM}summary") or _text(entry, f"{_ATOM}content")
|
|
52
|
+
),
|
|
53
|
+
"authors": [
|
|
54
|
+
_clean(_text(author, f"{_ATOM}name"), 120)
|
|
55
|
+
for author in entry.findall(f"{_ATOM}author")
|
|
56
|
+
if _text(author, f"{_ATOM}name")
|
|
57
|
+
],
|
|
58
|
+
"categories": [
|
|
59
|
+
category.get("term", "")
|
|
60
|
+
for category in entry.findall(f"{_ATOM}category")
|
|
61
|
+
if category.get("term")
|
|
62
|
+
],
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _rss_item(item: Any, rss1: bool = False) -> dict[str, Any]:
|
|
67
|
+
ns = _RSS1 if rss1 else ""
|
|
68
|
+
summary = _text(item, f"{ns}description")
|
|
69
|
+
if not summary:
|
|
70
|
+
summary = _text(item, f"{_CONTENT}encoded")
|
|
71
|
+
author = _text(item, f"{_DC}creator") or _text(item, f"{ns}author")
|
|
72
|
+
return {
|
|
73
|
+
"title": _clean(_text(item, f"{ns}title"), 300),
|
|
74
|
+
"url": _text(item, f"{ns}link") or _text(item, f"{ns}guid"),
|
|
75
|
+
"published": _text(item, f"{ns}pubDate") or _text(item, f"{_DC}date"),
|
|
76
|
+
"summary": _clean(summary),
|
|
77
|
+
"authors": [_clean(author, 120)] if author else [],
|
|
78
|
+
"categories": [
|
|
79
|
+
_clean(category.text, 80)
|
|
80
|
+
for category in item.findall(f"{ns}category")
|
|
81
|
+
if _clean(category.text)
|
|
82
|
+
],
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def is_feed(xml_text: str) -> bool:
|
|
87
|
+
"""Cheaply decide whether an XML payload is an RSS/Atom feed."""
|
|
88
|
+
|
|
89
|
+
head = xml_text.lstrip()[:2000].lower()
|
|
90
|
+
if not head.startswith("<?xml") and not head.startswith("<rss") and not head.startswith("<feed"):
|
|
91
|
+
return False
|
|
92
|
+
try:
|
|
93
|
+
root = engine.parse_xml(xml_text)
|
|
94
|
+
except engine.FetchError:
|
|
95
|
+
return False
|
|
96
|
+
tag = root.tag.lower()
|
|
97
|
+
return tag == "rss" or tag.endswith("}feed") or tag.endswith("}rdf")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def extract(xml_text: str, limit: int = 50) -> dict[str, Any]:
|
|
101
|
+
"""Parse a feed and return a Markdown digest plus structured items."""
|
|
102
|
+
|
|
103
|
+
root = engine.parse_xml(xml_text)
|
|
104
|
+
tag = root.tag.lower()
|
|
105
|
+
if tag.endswith("}feed") or tag == "feed":
|
|
106
|
+
feed_format = "atom"
|
|
107
|
+
channel = root
|
|
108
|
+
entries = root.findall(f"{_ATOM}entry")[:limit]
|
|
109
|
+
items = [_atom_entry(entry) for entry in entries]
|
|
110
|
+
title = _clean(_text(channel, f"{_ATOM}title"), 300)
|
|
111
|
+
site_url = _atom_link(channel)
|
|
112
|
+
description = _clean(_text(channel, f"{_ATOM}subtitle"))
|
|
113
|
+
elif tag == "rss":
|
|
114
|
+
feed_format = "rss"
|
|
115
|
+
channel = root.find("channel")
|
|
116
|
+
if channel is None:
|
|
117
|
+
raise engine.ExtractionError("RSS feed has no channel element")
|
|
118
|
+
items = [_rss_item(item) for item in channel.findall("item")[:limit]]
|
|
119
|
+
title = _clean(_text(channel, "title"), 300)
|
|
120
|
+
site_url = _text(channel, "link")
|
|
121
|
+
description = _clean(_text(channel, "description"))
|
|
122
|
+
elif tag.endswith("}rdf"):
|
|
123
|
+
feed_format = "rss1"
|
|
124
|
+
channel = root.find(f"{_RSS1}channel")
|
|
125
|
+
items = [_rss_item(item, rss1=True) for item in root.findall(f"{_RSS1}item")[:limit]]
|
|
126
|
+
title = _clean(_text(channel, f"{_RSS1}title") if channel is not None else "", 300)
|
|
127
|
+
site_url = _text(channel, f"{_RSS1}link") if channel is not None else ""
|
|
128
|
+
description = _clean(_text(channel, f"{_RSS1}description") if channel is not None else "")
|
|
129
|
+
else:
|
|
130
|
+
raise engine.ExtractionError(f"not a feed: unexpected root element {root.tag}")
|
|
131
|
+
|
|
132
|
+
lines = [f"# {title or 'Feed'}", ""]
|
|
133
|
+
if description:
|
|
134
|
+
lines.extend([description, ""])
|
|
135
|
+
if site_url:
|
|
136
|
+
lines.extend([f"> {site_url}", ""])
|
|
137
|
+
for index, item in enumerate(items, start=1):
|
|
138
|
+
lines.append(f"## {index}. {item['title'] or item['url'] or 'Untitled'}")
|
|
139
|
+
if item["published"]:
|
|
140
|
+
lines.append(f"**Published:** {item['published']}")
|
|
141
|
+
if item["authors"]:
|
|
142
|
+
lines.append(f"**Authors:** {', '.join(item['authors'])}")
|
|
143
|
+
if item["url"]:
|
|
144
|
+
lines.append(f"**URL:** {item['url']}")
|
|
145
|
+
if item["summary"]:
|
|
146
|
+
lines.extend(["", item["summary"]])
|
|
147
|
+
lines.append("")
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
"kind": "feed",
|
|
151
|
+
"format": feed_format,
|
|
152
|
+
"title": title,
|
|
153
|
+
"site_url": site_url,
|
|
154
|
+
"description": description,
|
|
155
|
+
"count": len(items),
|
|
156
|
+
"items": items,
|
|
157
|
+
"markdown": "\n".join(lines).strip(),
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
|
|
162
|
+
return {
|
|
163
|
+
"title": document.get("title", ""),
|
|
164
|
+
"authors": [],
|
|
165
|
+
"published": "",
|
|
166
|
+
"modified": "",
|
|
167
|
+
"journal": "",
|
|
168
|
+
"volume": "",
|
|
169
|
+
"issue": "",
|
|
170
|
+
"first_page": "",
|
|
171
|
+
"last_page": "",
|
|
172
|
+
"doi": "",
|
|
173
|
+
"arxiv_id": "",
|
|
174
|
+
"abstract": document.get("description", ""),
|
|
175
|
+
"keywords": [],
|
|
176
|
+
"language": "",
|
|
177
|
+
"site_name": document.get("format", "feed"),
|
|
178
|
+
"canonical_url": url or document.get("site_url", ""),
|
|
179
|
+
"og_type": "feed",
|
|
180
|
+
}
|
package/scripts/pdf.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""PDF text extraction for web-forager.
|
|
2
|
+
|
|
3
|
+
The only third-party dependency here is ``pypdf`` (BSD-3-Clause). It is an
|
|
4
|
+
optional dependency: when it is absent, callers receive a clear
|
|
5
|
+
``PdfUnavailable`` error instead of a traceback.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import io
|
|
11
|
+
import time
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
PDF_MAGIC = b"%PDF-"
|
|
15
|
+
# pypdf has had denial-of-service advisories around page-tree and xref parsing.
|
|
16
|
+
# Keep the installed package current (>=6.21.4) and put a hard bound on the
|
|
17
|
+
# work this module will do on an untrusted document.
|
|
18
|
+
MAX_PAGES = 500
|
|
19
|
+
MAX_SECONDS = 20.0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PdfError(Exception):
|
|
23
|
+
"""Base class for PDF failures."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class PdfUnavailable(PdfError):
|
|
27
|
+
"""pypdf is not installed or could not be imported."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class PdfExtractionError(PdfError):
|
|
31
|
+
"""The file is not a readable PDF."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def is_pdf(data: bytes) -> bool:
|
|
35
|
+
return bool(data) and data[:5] == PDF_MAGIC
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _meta_value(metadata: Any, key: str) -> str:
|
|
39
|
+
try:
|
|
40
|
+
value = metadata.get(key) if metadata else None
|
|
41
|
+
except Exception:
|
|
42
|
+
return ""
|
|
43
|
+
return str(value).strip() if value else ""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def extract(data: bytes) -> dict[str, Any]:
|
|
47
|
+
"""Extract text and document metadata from PDF bytes."""
|
|
48
|
+
|
|
49
|
+
if not is_pdf(data):
|
|
50
|
+
raise PdfExtractionError("the supplied data is not a PDF (missing %PDF- header)")
|
|
51
|
+
try:
|
|
52
|
+
from pypdf import PdfReader # type: ignore
|
|
53
|
+
except ImportError as exc:
|
|
54
|
+
raise PdfUnavailable(
|
|
55
|
+
"pypdf is not installed; enable PDF support with: pip install pypdf"
|
|
56
|
+
) from exc
|
|
57
|
+
except Exception as exc: # pragma: no cover - broken installation
|
|
58
|
+
raise PdfUnavailable(f"pypdf could not be imported ({exc}); reinstall it with: pip install pypdf") from exc
|
|
59
|
+
|
|
60
|
+
try:
|
|
61
|
+
reader = PdfReader(io.BytesIO(data))
|
|
62
|
+
except Exception as exc:
|
|
63
|
+
raise PdfExtractionError(f"cannot open PDF: {exc}") from exc
|
|
64
|
+
|
|
65
|
+
pages: list[str] = []
|
|
66
|
+
empty_pages = 0
|
|
67
|
+
truncated = False
|
|
68
|
+
deadline = time.monotonic() + MAX_SECONDS
|
|
69
|
+
for index, page in enumerate(reader.pages, start=1):
|
|
70
|
+
if index > MAX_PAGES or time.monotonic() > deadline:
|
|
71
|
+
truncated = True
|
|
72
|
+
break
|
|
73
|
+
try:
|
|
74
|
+
text = page.extract_text() or ""
|
|
75
|
+
except Exception as exc: # a single damaged page should not kill the run
|
|
76
|
+
text = f"[page {index}: extraction failed: {exc}]"
|
|
77
|
+
text = text.strip()
|
|
78
|
+
if not text:
|
|
79
|
+
empty_pages += 1
|
|
80
|
+
pages.append(text)
|
|
81
|
+
|
|
82
|
+
metadata = reader.metadata
|
|
83
|
+
warnings: list[str] = []
|
|
84
|
+
if truncated:
|
|
85
|
+
warnings.append(
|
|
86
|
+
f"PDF extraction stopped at {len(pages)} page(s) "
|
|
87
|
+
f"(limit: {MAX_PAGES} pages / {MAX_SECONDS:g} s)"
|
|
88
|
+
)
|
|
89
|
+
return {
|
|
90
|
+
"pages": pages,
|
|
91
|
+
"page_count": len(pages),
|
|
92
|
+
"empty_pages": empty_pages,
|
|
93
|
+
"truncated": truncated,
|
|
94
|
+
"warnings": warnings,
|
|
95
|
+
"text": "\n\n".join(pages).strip(),
|
|
96
|
+
"metadata": {
|
|
97
|
+
"title": _meta_value(metadata, "/Title"),
|
|
98
|
+
"author": _meta_value(metadata, "/Author"),
|
|
99
|
+
"subject": _meta_value(metadata, "/Subject"),
|
|
100
|
+
"creator": _meta_value(metadata, "/Creator"),
|
|
101
|
+
"producer": _meta_value(metadata, "/Producer"),
|
|
102
|
+
"creation_date": _meta_value(metadata, "/CreationDate"),
|
|
103
|
+
},
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def to_markdown(document: dict[str, Any], url: str = "") -> str:
|
|
108
|
+
metadata = document.get("metadata") or {}
|
|
109
|
+
title = metadata.get("title") or (url.rsplit("/", 1)[-1] if url else "PDF document")
|
|
110
|
+
lines = [f"# {title}", ""]
|
|
111
|
+
info = " | ".join(
|
|
112
|
+
part
|
|
113
|
+
for part in (
|
|
114
|
+
f"作者:{metadata.get('author')}" if metadata.get("author") else "",
|
|
115
|
+
f"页数:{document.get('page_count')}" if document.get("page_count") else "",
|
|
116
|
+
f"来源:{url}" if url else "",
|
|
117
|
+
)
|
|
118
|
+
if part
|
|
119
|
+
)
|
|
120
|
+
if info:
|
|
121
|
+
lines.extend([f"> {info}", ""])
|
|
122
|
+
for warning in document.get("warnings") or []:
|
|
123
|
+
lines.extend([f"> 警告:{warning}", ""])
|
|
124
|
+
pages = document.get("pages") or []
|
|
125
|
+
for index, text in enumerate(pages, start=1):
|
|
126
|
+
if not text:
|
|
127
|
+
continue
|
|
128
|
+
if len(pages) > 1:
|
|
129
|
+
lines.extend([f"## 第 {index} 页", ""])
|
|
130
|
+
lines.extend([text, ""])
|
|
131
|
+
return "\n".join(lines).strip()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
|
|
135
|
+
"""Return PDF metadata in the same shape as article metadata."""
|
|
136
|
+
|
|
137
|
+
source = document.get("metadata") or {}
|
|
138
|
+
return {
|
|
139
|
+
"title": source.get("title", ""),
|
|
140
|
+
"authors": [source["author"]] if source.get("author") else [],
|
|
141
|
+
"published": source.get("creation_date", ""),
|
|
142
|
+
"modified": "",
|
|
143
|
+
"journal": "",
|
|
144
|
+
"volume": "",
|
|
145
|
+
"issue": "",
|
|
146
|
+
"first_page": "",
|
|
147
|
+
"last_page": "",
|
|
148
|
+
"doi": "",
|
|
149
|
+
"arxiv_id": "",
|
|
150
|
+
"abstract": source.get("subject", ""),
|
|
151
|
+
"keywords": [],
|
|
152
|
+
"language": "",
|
|
153
|
+
"site_name": "PDF",
|
|
154
|
+
"canonical_url": url,
|
|
155
|
+
"og_type": "pdf",
|
|
156
|
+
}
|