web-forager 0.0.0-stage → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +119 -0
  18. package/CHANGELOG.md +508 -0
  19. package/LICENSE +21 -0
  20. package/README.md +498 -2
  21. package/README_zh.md +498 -0
  22. package/SKILL.md +451 -0
  23. package/THIRD_PARTY_LICENSES.txt +40 -0
  24. package/browser/bridges/playwright-bridge.js +382 -0
  25. package/browser/launch_chrome.py +193 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +90 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +141 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +130 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +215 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +252 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +107 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +441 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/research.md +35 -0
  58. package/commands/scholar.md +28 -0
  59. package/commands/surf.md +28 -0
  60. package/cordis.patch.yml +19 -0
  61. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  62. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  63. package/docs/DEPENDENCIES.md +40 -0
  64. package/docs/INSTALL_COMMANDS.md +113 -0
  65. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  66. package/docs/REFERENCE.md +349 -0
  67. package/docs/REFERENCE_zh.md +342 -0
  68. package/index.js +7 -0
  69. package/install.py +254 -0
  70. package/manifest.json +98 -0
  71. package/mcp_server.py +850 -0
  72. package/package.json +82 -6
  73. package/requirements.txt +15 -0
  74. package/run.py +908 -0
  75. package/scripts/__init__.py +1 -0
  76. package/scripts/bilibili.py +882 -0
  77. package/scripts/documents.py +437 -0
  78. package/scripts/engine.py +5141 -0
  79. package/scripts/feeds.py +180 -0
  80. package/scripts/pdf.py +156 -0
  81. package/scripts/research.py +2087 -0
  82. package/scripts/scholar.py +381 -0
  83. package/scripts/sitemap.py +195 -0
  84. package/scripts/untrusted.py +312 -0
@@ -0,0 +1,180 @@
1
+ """RSS 2.0 / Atom / RSS 1.0 feed parsing.
2
+
3
+ Feeds are structured XML, so they are far cheaper to read than the rendered
4
+ HTML of a blog index. Parsing goes through :func:`engine.parse_xml`, which
5
+ rejects DOCTYPE/ENTITY and uses defusedxml when available.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import html as html_lib
11
+ import re
12
+ from typing import Any
13
+
14
+ from . import engine
15
+
16
+ _ATOM = "{http://www.w3.org/2005/Atom}"
17
+ _RSS1 = "{http://purl.org/rss/1.0/}"
18
+ _DC = "{http://purl.org/dc/elements/1.1/}"
19
+ _CONTENT = "{http://purl.org/rss/1.0/modules/content/}"
20
+
21
+
22
+ def _clean(value: Any, limit: int = 1200) -> str:
23
+ text = html_lib.unescape(str(value or ""))
24
+ if "<" in text:
25
+ text = re.sub(r"<[^>]+>", " ", text)
26
+ text = re.sub(r"\s+", " ", text).strip()
27
+ return text[:limit]
28
+
29
+
30
+ def _text(node: Any, path: str = "") -> str:
31
+ target = node.find(path) if path else node
32
+ return (target.text or "").strip() if target is not None else ""
33
+
34
+
35
+ def _atom_link(entry: Any) -> str:
36
+ fallback = ""
37
+ for link in entry.findall(f"{_ATOM}link"):
38
+ href = link.get("href", "")
39
+ if link.get("rel", "alternate") == "alternate":
40
+ return href
41
+ fallback = fallback or href
42
+ return fallback
43
+
44
+
45
+ def _atom_entry(entry: Any) -> dict[str, Any]:
46
+ return {
47
+ "title": _clean(_text(entry, f"{_ATOM}title"), 300),
48
+ "url": _atom_link(entry),
49
+ "published": _text(entry, f"{_ATOM}published") or _text(entry, f"{_ATOM}updated"),
50
+ "summary": _clean(
51
+ _text(entry, f"{_ATOM}summary") or _text(entry, f"{_ATOM}content")
52
+ ),
53
+ "authors": [
54
+ _clean(_text(author, f"{_ATOM}name"), 120)
55
+ for author in entry.findall(f"{_ATOM}author")
56
+ if _text(author, f"{_ATOM}name")
57
+ ],
58
+ "categories": [
59
+ category.get("term", "")
60
+ for category in entry.findall(f"{_ATOM}category")
61
+ if category.get("term")
62
+ ],
63
+ }
64
+
65
+
66
+ def _rss_item(item: Any, rss1: bool = False) -> dict[str, Any]:
67
+ ns = _RSS1 if rss1 else ""
68
+ summary = _text(item, f"{ns}description")
69
+ if not summary:
70
+ summary = _text(item, f"{_CONTENT}encoded")
71
+ author = _text(item, f"{_DC}creator") or _text(item, f"{ns}author")
72
+ return {
73
+ "title": _clean(_text(item, f"{ns}title"), 300),
74
+ "url": _text(item, f"{ns}link") or _text(item, f"{ns}guid"),
75
+ "published": _text(item, f"{ns}pubDate") or _text(item, f"{_DC}date"),
76
+ "summary": _clean(summary),
77
+ "authors": [_clean(author, 120)] if author else [],
78
+ "categories": [
79
+ _clean(category.text, 80)
80
+ for category in item.findall(f"{ns}category")
81
+ if _clean(category.text)
82
+ ],
83
+ }
84
+
85
+
86
+ def is_feed(xml_text: str) -> bool:
87
+ """Cheaply decide whether an XML payload is an RSS/Atom feed."""
88
+
89
+ head = xml_text.lstrip()[:2000].lower()
90
+ if not head.startswith("<?xml") and not head.startswith("<rss") and not head.startswith("<feed"):
91
+ return False
92
+ try:
93
+ root = engine.parse_xml(xml_text)
94
+ except engine.FetchError:
95
+ return False
96
+ tag = root.tag.lower()
97
+ return tag == "rss" or tag.endswith("}feed") or tag.endswith("}rdf")
98
+
99
+
100
+ def extract(xml_text: str, limit: int = 50) -> dict[str, Any]:
101
+ """Parse a feed and return a Markdown digest plus structured items."""
102
+
103
+ root = engine.parse_xml(xml_text)
104
+ tag = root.tag.lower()
105
+ if tag.endswith("}feed") or tag == "feed":
106
+ feed_format = "atom"
107
+ channel = root
108
+ entries = root.findall(f"{_ATOM}entry")[:limit]
109
+ items = [_atom_entry(entry) for entry in entries]
110
+ title = _clean(_text(channel, f"{_ATOM}title"), 300)
111
+ site_url = _atom_link(channel)
112
+ description = _clean(_text(channel, f"{_ATOM}subtitle"))
113
+ elif tag == "rss":
114
+ feed_format = "rss"
115
+ channel = root.find("channel")
116
+ if channel is None:
117
+ raise engine.ExtractionError("RSS feed has no channel element")
118
+ items = [_rss_item(item) for item in channel.findall("item")[:limit]]
119
+ title = _clean(_text(channel, "title"), 300)
120
+ site_url = _text(channel, "link")
121
+ description = _clean(_text(channel, "description"))
122
+ elif tag.endswith("}rdf"):
123
+ feed_format = "rss1"
124
+ channel = root.find(f"{_RSS1}channel")
125
+ items = [_rss_item(item, rss1=True) for item in root.findall(f"{_RSS1}item")[:limit]]
126
+ title = _clean(_text(channel, f"{_RSS1}title") if channel is not None else "", 300)
127
+ site_url = _text(channel, f"{_RSS1}link") if channel is not None else ""
128
+ description = _clean(_text(channel, f"{_RSS1}description") if channel is not None else "")
129
+ else:
130
+ raise engine.ExtractionError(f"not a feed: unexpected root element {root.tag}")
131
+
132
+ lines = [f"# {title or 'Feed'}", ""]
133
+ if description:
134
+ lines.extend([description, ""])
135
+ if site_url:
136
+ lines.extend([f"> {site_url}", ""])
137
+ for index, item in enumerate(items, start=1):
138
+ lines.append(f"## {index}. {item['title'] or item['url'] or 'Untitled'}")
139
+ if item["published"]:
140
+ lines.append(f"**Published:** {item['published']}")
141
+ if item["authors"]:
142
+ lines.append(f"**Authors:** {', '.join(item['authors'])}")
143
+ if item["url"]:
144
+ lines.append(f"**URL:** {item['url']}")
145
+ if item["summary"]:
146
+ lines.extend(["", item["summary"]])
147
+ lines.append("")
148
+
149
+ return {
150
+ "kind": "feed",
151
+ "format": feed_format,
152
+ "title": title,
153
+ "site_url": site_url,
154
+ "description": description,
155
+ "count": len(items),
156
+ "items": items,
157
+ "markdown": "\n".join(lines).strip(),
158
+ }
159
+
160
+
161
+ def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
162
+ return {
163
+ "title": document.get("title", ""),
164
+ "authors": [],
165
+ "published": "",
166
+ "modified": "",
167
+ "journal": "",
168
+ "volume": "",
169
+ "issue": "",
170
+ "first_page": "",
171
+ "last_page": "",
172
+ "doi": "",
173
+ "arxiv_id": "",
174
+ "abstract": document.get("description", ""),
175
+ "keywords": [],
176
+ "language": "",
177
+ "site_name": document.get("format", "feed"),
178
+ "canonical_url": url or document.get("site_url", ""),
179
+ "og_type": "feed",
180
+ }
package/scripts/pdf.py ADDED
@@ -0,0 +1,156 @@
1
+ """PDF text extraction for web-forager.
2
+
3
+ The only third-party dependency here is ``pypdf`` (BSD-3-Clause). It is an
4
+ optional dependency: when it is absent, callers receive a clear
5
+ ``PdfUnavailable`` error instead of a traceback.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import io
11
+ import time
12
+ from typing import Any
13
+
14
+ PDF_MAGIC = b"%PDF-"
15
+ # pypdf has had denial-of-service advisories around page-tree and xref parsing.
16
+ # Keep the installed package current (>=6.21.4) and put a hard bound on the
17
+ # work this module will do on an untrusted document.
18
+ MAX_PAGES = 500
19
+ MAX_SECONDS = 20.0
20
+
21
+
22
+ class PdfError(Exception):
23
+ """Base class for PDF failures."""
24
+
25
+
26
+ class PdfUnavailable(PdfError):
27
+ """pypdf is not installed or could not be imported."""
28
+
29
+
30
+ class PdfExtractionError(PdfError):
31
+ """The file is not a readable PDF."""
32
+
33
+
34
+ def is_pdf(data: bytes) -> bool:
35
+ return bool(data) and data[:5] == PDF_MAGIC
36
+
37
+
38
+ def _meta_value(metadata: Any, key: str) -> str:
39
+ try:
40
+ value = metadata.get(key) if metadata else None
41
+ except Exception:
42
+ return ""
43
+ return str(value).strip() if value else ""
44
+
45
+
46
+ def extract(data: bytes) -> dict[str, Any]:
47
+ """Extract text and document metadata from PDF bytes."""
48
+
49
+ if not is_pdf(data):
50
+ raise PdfExtractionError("the supplied data is not a PDF (missing %PDF- header)")
51
+ try:
52
+ from pypdf import PdfReader # type: ignore
53
+ except ImportError as exc:
54
+ raise PdfUnavailable(
55
+ "pypdf is not installed; enable PDF support with: pip install pypdf"
56
+ ) from exc
57
+ except Exception as exc: # pragma: no cover - broken installation
58
+ raise PdfUnavailable(f"pypdf could not be imported ({exc}); reinstall it with: pip install pypdf") from exc
59
+
60
+ try:
61
+ reader = PdfReader(io.BytesIO(data))
62
+ except Exception as exc:
63
+ raise PdfExtractionError(f"cannot open PDF: {exc}") from exc
64
+
65
+ pages: list[str] = []
66
+ empty_pages = 0
67
+ truncated = False
68
+ deadline = time.monotonic() + MAX_SECONDS
69
+ for index, page in enumerate(reader.pages, start=1):
70
+ if index > MAX_PAGES or time.monotonic() > deadline:
71
+ truncated = True
72
+ break
73
+ try:
74
+ text = page.extract_text() or ""
75
+ except Exception as exc: # a single damaged page should not kill the run
76
+ text = f"[page {index}: extraction failed: {exc}]"
77
+ text = text.strip()
78
+ if not text:
79
+ empty_pages += 1
80
+ pages.append(text)
81
+
82
+ metadata = reader.metadata
83
+ warnings: list[str] = []
84
+ if truncated:
85
+ warnings.append(
86
+ f"PDF extraction stopped at {len(pages)} page(s) "
87
+ f"(limit: {MAX_PAGES} pages / {MAX_SECONDS:g} s)"
88
+ )
89
+ return {
90
+ "pages": pages,
91
+ "page_count": len(pages),
92
+ "empty_pages": empty_pages,
93
+ "truncated": truncated,
94
+ "warnings": warnings,
95
+ "text": "\n\n".join(pages).strip(),
96
+ "metadata": {
97
+ "title": _meta_value(metadata, "/Title"),
98
+ "author": _meta_value(metadata, "/Author"),
99
+ "subject": _meta_value(metadata, "/Subject"),
100
+ "creator": _meta_value(metadata, "/Creator"),
101
+ "producer": _meta_value(metadata, "/Producer"),
102
+ "creation_date": _meta_value(metadata, "/CreationDate"),
103
+ },
104
+ }
105
+
106
+
107
+ def to_markdown(document: dict[str, Any], url: str = "") -> str:
108
+ metadata = document.get("metadata") or {}
109
+ title = metadata.get("title") or (url.rsplit("/", 1)[-1] if url else "PDF document")
110
+ lines = [f"# {title}", ""]
111
+ info = " | ".join(
112
+ part
113
+ for part in (
114
+ f"作者:{metadata.get('author')}" if metadata.get("author") else "",
115
+ f"页数:{document.get('page_count')}" if document.get("page_count") else "",
116
+ f"来源:{url}" if url else "",
117
+ )
118
+ if part
119
+ )
120
+ if info:
121
+ lines.extend([f"> {info}", ""])
122
+ for warning in document.get("warnings") or []:
123
+ lines.extend([f"> 警告:{warning}", ""])
124
+ pages = document.get("pages") or []
125
+ for index, text in enumerate(pages, start=1):
126
+ if not text:
127
+ continue
128
+ if len(pages) > 1:
129
+ lines.extend([f"## 第 {index} 页", ""])
130
+ lines.extend([text, ""])
131
+ return "\n".join(lines).strip()
132
+
133
+
134
+ def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
135
+ """Return PDF metadata in the same shape as article metadata."""
136
+
137
+ source = document.get("metadata") or {}
138
+ return {
139
+ "title": source.get("title", ""),
140
+ "authors": [source["author"]] if source.get("author") else [],
141
+ "published": source.get("creation_date", ""),
142
+ "modified": "",
143
+ "journal": "",
144
+ "volume": "",
145
+ "issue": "",
146
+ "first_page": "",
147
+ "last_page": "",
148
+ "doi": "",
149
+ "arxiv_id": "",
150
+ "abstract": source.get("subject", ""),
151
+ "keywords": [],
152
+ "language": "",
153
+ "site_name": "PDF",
154
+ "canonical_url": url,
155
+ "og_type": "pdf",
156
+ }