web-forager 0.0.0-stage → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +119 -0
- package/CHANGELOG.md +508 -0
- package/LICENSE +21 -0
- package/README.md +498 -2
- package/README_zh.md +498 -0
- package/SKILL.md +451 -0
- package/THIRD_PARTY_LICENSES.txt +40 -0
- package/browser/bridges/playwright-bridge.js +382 -0
- package/browser/launch_chrome.py +193 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +90 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +141 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +130 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +215 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +252 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +107 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +441 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/research.md +35 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +349 -0
- package/docs/REFERENCE_zh.md +342 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +98 -0
- package/mcp_server.py +850 -0
- package/package.json +82 -6
- package/requirements.txt +15 -0
- package/run.py +908 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +882 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +5141 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +156 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +381 -0
- package/scripts/sitemap.py +195 -0
- package/scripts/untrusted.py +312 -0
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
"""Scholarly metadata through public APIs (arXiv, Crossref, PubMed).
|
|
2
|
+
|
|
3
|
+
These are documented query APIs, not scraped result pages, so they are both
|
|
4
|
+
more stable and more polite than reading a search engine's HTML. Only the
|
|
5
|
+
standard library and ``requests`` (through :mod:`scripts.engine`) are used.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import html as html_lib
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import xml.etree.ElementTree as ET
|
|
15
|
+
from typing import Any, Optional
|
|
16
|
+
|
|
17
|
+
from . import engine
|
|
18
|
+
|
|
19
|
+
SCHOLAR_SOURCES = ("arxiv", "crossref", "pubmed", "doi")
|
|
20
|
+
|
|
21
|
+
_ARXIV_API = (
|
|
22
|
+
"https://export.arxiv.org/api/query?search_query={field}:{query}"
|
|
23
|
+
"&start={start}&max_results={limit}&sortBy={sort}&sortOrder=descending"
|
|
24
|
+
)
|
|
25
|
+
_CROSSREF_API = (
|
|
26
|
+
"https://api.crossref.org/works?query.bibliographic={query}&rows={limit}"
|
|
27
|
+
"&select=DOI,title,author,container-title,issued,published-print,published-online,"
|
|
28
|
+
"abstract,URL,type,subject,ISSN,volume,issue,page"
|
|
29
|
+
)
|
|
30
|
+
_CROSSREF_DOI = "https://api.crossref.org/works/{doi}"
|
|
31
|
+
_PUBMED_SEARCH = (
|
|
32
|
+
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
|
|
33
|
+
"?db=pubmed&term={query}&retmode=json&retmax={limit}&sort=relevance"
|
|
34
|
+
)
|
|
35
|
+
_PUBMED_SUMMARY = (
|
|
36
|
+
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
|
|
37
|
+
"?db=pubmed&id={ids}&retmode=json"
|
|
38
|
+
)
|
|
39
|
+
_PUBMED_FETCH = (
|
|
40
|
+
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
|
|
41
|
+
"?db=pubmed&id={ids}&retmode=xml"
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
_ATOM = "{http://www.w3.org/2005/Atom}"
|
|
45
|
+
_ARXIV = "{http://arxiv.org/schemas/atom}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _text(node: Optional[ET.Element], path: str = "") -> str:
|
|
49
|
+
target = node.find(path) if node is not None and path else node
|
|
50
|
+
if target is None or target.text is None:
|
|
51
|
+
return ""
|
|
52
|
+
return re.sub(r"\s+", " ", target.text).strip()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _clean(value: Any) -> str:
|
|
56
|
+
if value is None:
|
|
57
|
+
return ""
|
|
58
|
+
text = html_lib.unescape(str(value))
|
|
59
|
+
if "<" in text:
|
|
60
|
+
text = re.sub(r"<[^>]+>", " ", text)
|
|
61
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _date_parts(value: Any) -> str:
|
|
65
|
+
if not isinstance(value, dict):
|
|
66
|
+
return ""
|
|
67
|
+
parts = (value.get("date-parts") or [[]])[0]
|
|
68
|
+
if not parts:
|
|
69
|
+
return ""
|
|
70
|
+
return "-".join(f"{int(part):02d}" if index else str(part) for index, part in enumerate(parts))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def parse_arxiv(xml_text: str, limit: int = 10) -> list[dict[str, Any]]:
|
|
74
|
+
"""Parse an arXiv Atom feed into normalised records."""
|
|
75
|
+
|
|
76
|
+
root = engine.parse_xml(xml_text)
|
|
77
|
+
|
|
78
|
+
records: list[dict[str, Any]] = []
|
|
79
|
+
for entry in root.findall(f"{_ATOM}entry")[:limit]:
|
|
80
|
+
identifier = _text(entry, f"{_ATOM}id")
|
|
81
|
+
abs_url = identifier
|
|
82
|
+
if abs_url.startswith("http://arxiv.org/"):
|
|
83
|
+
abs_url = "https://" + abs_url[len("http://") :]
|
|
84
|
+
pdf_url = ""
|
|
85
|
+
for link in entry.findall(f"{_ATOM}link"):
|
|
86
|
+
if link.get("title") == "pdf" or link.get("type") == "application/pdf":
|
|
87
|
+
pdf_url = link.get("href", "")
|
|
88
|
+
authors = [
|
|
89
|
+
_text(author, f"{_ATOM}name")
|
|
90
|
+
for author in entry.findall(f"{_ATOM}author")
|
|
91
|
+
if _text(author, f"{_ATOM}name")
|
|
92
|
+
]
|
|
93
|
+
categories = [
|
|
94
|
+
category.get("term", "")
|
|
95
|
+
for category in entry.findall(f"{_ATOM}category")
|
|
96
|
+
if category.get("term")
|
|
97
|
+
]
|
|
98
|
+
journal = _text(entry, f"{_ARXIV}journal_ref")
|
|
99
|
+
doi = _text(entry, f"{_ARXIV}doi")
|
|
100
|
+
records.append(
|
|
101
|
+
{
|
|
102
|
+
"source": "arxiv",
|
|
103
|
+
"id": identifier.rsplit("/", 1)[-1],
|
|
104
|
+
"title": _text(entry, f"{_ATOM}title"),
|
|
105
|
+
"authors": authors,
|
|
106
|
+
"published": _text(entry, f"{_ATOM}published")[:10],
|
|
107
|
+
"updated": _text(entry, f"{_ATOM}updated")[:10],
|
|
108
|
+
"journal": journal,
|
|
109
|
+
"volume": "",
|
|
110
|
+
"issue": "",
|
|
111
|
+
"pages": "",
|
|
112
|
+
"doi": doi,
|
|
113
|
+
"url": abs_url,
|
|
114
|
+
"pdf_url": pdf_url,
|
|
115
|
+
"abstract": _text(entry, f"{_ATOM}summary"),
|
|
116
|
+
"type": "preprint",
|
|
117
|
+
"keywords": categories,
|
|
118
|
+
}
|
|
119
|
+
)
|
|
120
|
+
return records
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def parse_crossref(payload: Any, limit: int = 10) -> list[dict[str, Any]]:
|
|
124
|
+
"""Parse a Crossref ``works`` response (already decoded JSON)."""
|
|
125
|
+
|
|
126
|
+
if isinstance(payload, str):
|
|
127
|
+
payload = json.loads(payload)
|
|
128
|
+
message = payload.get("message", payload) if isinstance(payload, dict) else {}
|
|
129
|
+
items = message.get("items")
|
|
130
|
+
if items is None and message.get("DOI"):
|
|
131
|
+
items = [message]
|
|
132
|
+
records: list[dict[str, Any]] = []
|
|
133
|
+
for item in (items or [])[:limit]:
|
|
134
|
+
authors: list[str] = []
|
|
135
|
+
for author in item.get("author") or []:
|
|
136
|
+
name = author.get("name") or " ".join(
|
|
137
|
+
part for part in (author.get("given"), author.get("family")) if part
|
|
138
|
+
)
|
|
139
|
+
if name:
|
|
140
|
+
authors.append(name)
|
|
141
|
+
titles = item.get("title") or []
|
|
142
|
+
title = titles[0] if isinstance(titles, list) and titles else str(titles or "")
|
|
143
|
+
containers = item.get("container-title") or []
|
|
144
|
+
journal = containers[0] if isinstance(containers, list) and containers else str(containers or "")
|
|
145
|
+
published = (
|
|
146
|
+
_date_parts(item.get("published-print"))
|
|
147
|
+
or _date_parts(item.get("published-online"))
|
|
148
|
+
or _date_parts(item.get("issued"))
|
|
149
|
+
)
|
|
150
|
+
records.append(
|
|
151
|
+
{
|
|
152
|
+
"source": "crossref",
|
|
153
|
+
"id": item.get("DOI", ""),
|
|
154
|
+
"title": _clean(title),
|
|
155
|
+
"authors": authors,
|
|
156
|
+
"published": published,
|
|
157
|
+
"updated": "",
|
|
158
|
+
"journal": _clean(journal),
|
|
159
|
+
"volume": str(item.get("volume", "") or ""),
|
|
160
|
+
"issue": str(item.get("issue", "") or ""),
|
|
161
|
+
"pages": str(item.get("page", "") or ""),
|
|
162
|
+
"doi": item.get("DOI", ""),
|
|
163
|
+
"url": item.get("URL", "") or (
|
|
164
|
+
f"https://doi.org/{item.get('DOI')}" if item.get("DOI") else ""
|
|
165
|
+
),
|
|
166
|
+
"pdf_url": "",
|
|
167
|
+
"abstract": _clean(item.get("abstract", "")),
|
|
168
|
+
"type": item.get("type", ""),
|
|
169
|
+
"keywords": [str(value) for value in item.get("subject") or []],
|
|
170
|
+
}
|
|
171
|
+
)
|
|
172
|
+
return records
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def parse_pubmed(
|
|
176
|
+
search_payload: Any,
|
|
177
|
+
summary_payload: Any,
|
|
178
|
+
abstracts: Optional[dict[str, str]] = None,
|
|
179
|
+
limit: int = 10,
|
|
180
|
+
) -> list[dict[str, Any]]:
|
|
181
|
+
"""Merge PubMed esearch + esummary (+ optional efetch abstracts)."""
|
|
182
|
+
|
|
183
|
+
if isinstance(search_payload, str):
|
|
184
|
+
search_payload = json.loads(search_payload)
|
|
185
|
+
if isinstance(summary_payload, str):
|
|
186
|
+
summary_payload = json.loads(summary_payload)
|
|
187
|
+
abstracts = abstracts or {}
|
|
188
|
+
|
|
189
|
+
ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
|
|
190
|
+
result = summary_payload.get("result", {}) or {}
|
|
191
|
+
records: list[dict[str, Any]] = []
|
|
192
|
+
for uid in ids:
|
|
193
|
+
item = result.get(uid) or {}
|
|
194
|
+
doi = ""
|
|
195
|
+
for article_id in item.get("articleids") or []:
|
|
196
|
+
if article_id.get("idtype") == "doi":
|
|
197
|
+
doi = article_id.get("value", "")
|
|
198
|
+
break
|
|
199
|
+
records.append(
|
|
200
|
+
{
|
|
201
|
+
"source": "pubmed",
|
|
202
|
+
"id": uid,
|
|
203
|
+
"title": _clean(item.get("title", "")),
|
|
204
|
+
"authors": [
|
|
205
|
+
author.get("name", "")
|
|
206
|
+
for author in item.get("authors") or []
|
|
207
|
+
if author.get("name")
|
|
208
|
+
],
|
|
209
|
+
"published": str(item.get("pubdate", "") or ""),
|
|
210
|
+
"updated": "",
|
|
211
|
+
"journal": str(item.get("fulljournalname") or item.get("source") or ""),
|
|
212
|
+
"volume": str(item.get("volume", "") or ""),
|
|
213
|
+
"issue": str(item.get("issue", "") or ""),
|
|
214
|
+
"pages": str(item.get("pages", "") or ""),
|
|
215
|
+
"doi": doi,
|
|
216
|
+
"url": f"https://pubmed.ncbi.nlm.nih.gov/{uid}/",
|
|
217
|
+
"pdf_url": "",
|
|
218
|
+
"abstract": abstracts.get(uid, ""),
|
|
219
|
+
"type": "journal-article",
|
|
220
|
+
"keywords": [],
|
|
221
|
+
}
|
|
222
|
+
)
|
|
223
|
+
return records
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _pubmed_abstracts(xml_text: str) -> dict[str, str]:
|
|
227
|
+
"""Map PMID → abstract from an efetch XML response."""
|
|
228
|
+
|
|
229
|
+
try:
|
|
230
|
+
root = engine.parse_xml(xml_text)
|
|
231
|
+
except engine.FetchError:
|
|
232
|
+
return {}
|
|
233
|
+
abstracts: dict[str, str] = {}
|
|
234
|
+
for article in root.findall(".//PubmedArticle"):
|
|
235
|
+
pmid = _text(article, ".//PMID")
|
|
236
|
+
parts = [
|
|
237
|
+
_clean(node.text)
|
|
238
|
+
for node in article.findall(".//Abstract/AbstractText")
|
|
239
|
+
if node.text
|
|
240
|
+
]
|
|
241
|
+
if pmid and parts:
|
|
242
|
+
abstracts[pmid] = " ".join(part for part in parts if part)
|
|
243
|
+
return abstracts
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _query_fetch(fetcher: engine.HttpFetcher, url: str) -> engine.HttpResponse:
|
|
247
|
+
"""Fetch a scholarly query endpoint under the user-level query policy."""
|
|
248
|
+
|
|
249
|
+
config = fetcher.config
|
|
250
|
+
return fetcher.fetch(
|
|
251
|
+
url,
|
|
252
|
+
check_robots=config.query_robots != "ignore",
|
|
253
|
+
robots_policy=config.query_robots,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _fetch_json(fetcher: engine.HttpFetcher, url: str) -> Any:
|
|
258
|
+
response = _query_fetch(fetcher, url)
|
|
259
|
+
try:
|
|
260
|
+
return json.loads(response.text)
|
|
261
|
+
except json.JSONDecodeError as exc:
|
|
262
|
+
raise engine.FetchError(f"invalid JSON from {url}: {exc}") from exc
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def search(
|
|
266
|
+
source: str,
|
|
267
|
+
query: str,
|
|
268
|
+
config: Optional[engine.FetchConfig] = None,
|
|
269
|
+
limit: Optional[int] = None,
|
|
270
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
271
|
+
sort: str = "relevance",
|
|
272
|
+
field: str = "all",
|
|
273
|
+
) -> dict[str, Any]:
|
|
274
|
+
"""Query one scholarly source and return normalised records.
|
|
275
|
+
|
|
276
|
+
``field`` is the arXiv query field (``all``, ``ti``, ``au`` or ``abs``) and
|
|
277
|
+
is ignored by the other sources. ``ti`` is useful when the caller already
|
|
278
|
+
has a paper title: it avoids the broad ``all`` search returning a different
|
|
279
|
+
paper that merely shares a phrase.
|
|
280
|
+
"""
|
|
281
|
+
|
|
282
|
+
source = (source or "arxiv").lower()
|
|
283
|
+
if source not in SCHOLAR_SOURCES:
|
|
284
|
+
raise ValueError(f"unsupported scholarly source: {source}")
|
|
285
|
+
query = (query or "").strip()
|
|
286
|
+
if not query:
|
|
287
|
+
raise ValueError("query must not be empty")
|
|
288
|
+
|
|
289
|
+
config = config or engine.FetchConfig()
|
|
290
|
+
limit = max(1, min(limit or config.num_results or 10, 50))
|
|
291
|
+
own_fetcher = fetcher is None
|
|
292
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
293
|
+
|
|
294
|
+
try:
|
|
295
|
+
if source == "arxiv":
|
|
296
|
+
if sort not in ("relevance", "submittedDate", "lastUpdatedDate"):
|
|
297
|
+
raise ValueError("arXiv sort must be relevance, submittedDate or lastUpdatedDate")
|
|
298
|
+
if field not in ("all", "ti", "au", "abs"):
|
|
299
|
+
raise ValueError("arXiv field must be all, ti, au or abs")
|
|
300
|
+
url = _ARXIV_API.format(
|
|
301
|
+
field=field,
|
|
302
|
+
query=urllib.parse.quote_plus(query),
|
|
303
|
+
start=0,
|
|
304
|
+
limit=limit,
|
|
305
|
+
sort=sort,
|
|
306
|
+
)
|
|
307
|
+
response = _query_fetch(fetcher, url)
|
|
308
|
+
records = parse_arxiv(response.text, limit=limit)
|
|
309
|
+
elif source == "crossref":
|
|
310
|
+
url = _CROSSREF_API.format(query=urllib.parse.quote_plus(query), limit=limit)
|
|
311
|
+
records = parse_crossref(_fetch_json(fetcher, url), limit=limit)
|
|
312
|
+
elif source == "doi":
|
|
313
|
+
doi = query
|
|
314
|
+
if doi.lower().startswith("https://doi.org/"):
|
|
315
|
+
doi = doi[len("https://doi.org/"):]
|
|
316
|
+
elif doi.lower().startswith("doi:"):
|
|
317
|
+
doi = doi[4:]
|
|
318
|
+
url = _CROSSREF_DOI.format(doi=urllib.parse.quote(doi, safe="/"))
|
|
319
|
+
records = parse_crossref(_fetch_json(fetcher, url), limit=1)
|
|
320
|
+
else: # pubmed
|
|
321
|
+
search_url = _PUBMED_SEARCH.format(
|
|
322
|
+
query=urllib.parse.quote_plus(query), limit=limit
|
|
323
|
+
)
|
|
324
|
+
search_payload = _fetch_json(fetcher, search_url)
|
|
325
|
+
ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
|
|
326
|
+
if not ids:
|
|
327
|
+
records = []
|
|
328
|
+
else:
|
|
329
|
+
summary_url = _PUBMED_SUMMARY.format(ids=",".join(ids))
|
|
330
|
+
summary_payload = _fetch_json(fetcher, summary_url)
|
|
331
|
+
abstracts: dict[str, str] = {}
|
|
332
|
+
try:
|
|
333
|
+
fetch_url = _PUBMED_FETCH.format(ids=",".join(ids))
|
|
334
|
+
abstracts = _pubmed_abstracts(_query_fetch(fetcher, fetch_url).text)
|
|
335
|
+
except engine.FetchError:
|
|
336
|
+
abstracts = {}
|
|
337
|
+
records = parse_pubmed(search_payload, summary_payload, abstracts, limit=limit)
|
|
338
|
+
|
|
339
|
+
return {
|
|
340
|
+
"source": source,
|
|
341
|
+
"query": query,
|
|
342
|
+
"query_robots": config.query_robots,
|
|
343
|
+
"count": len(records),
|
|
344
|
+
"results": records,
|
|
345
|
+
}
|
|
346
|
+
finally:
|
|
347
|
+
if own_fetcher:
|
|
348
|
+
fetcher.close()
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def to_markdown(payload: dict[str, Any]) -> str:
|
|
352
|
+
"""Render scholarly records as a compact Markdown bibliography."""
|
|
353
|
+
|
|
354
|
+
lines = [
|
|
355
|
+
f"# {payload.get('source', 'scholar')}: {payload.get('query', '')}",
|
|
356
|
+
"",
|
|
357
|
+
f"> {payload.get('count', 0)} record(s)",
|
|
358
|
+
"",
|
|
359
|
+
]
|
|
360
|
+
for index, record in enumerate(payload.get("results") or [], start=1):
|
|
361
|
+
title = record.get("title") or record.get("id") or "Untitled"
|
|
362
|
+
lines.append(f"## {index}. {title}")
|
|
363
|
+
if record.get("authors"):
|
|
364
|
+
lines.append(f"**Authors:** {', '.join(record['authors'])}")
|
|
365
|
+
if record.get("published"):
|
|
366
|
+
lines.append(f"**Published:** {record['published']}")
|
|
367
|
+
if record.get("journal"):
|
|
368
|
+
volume_issue = " ".join(
|
|
369
|
+
part for part in (record.get("volume"), record.get("issue")) if part
|
|
370
|
+
)
|
|
371
|
+
lines.append(f"**Journal:** {record['journal']}" + (f" {volume_issue}" if volume_issue else ""))
|
|
372
|
+
if record.get("doi"):
|
|
373
|
+
lines.append(f"**DOI:** {record['doi']}")
|
|
374
|
+
if record.get("url"):
|
|
375
|
+
lines.append(f"**URL:** {record['url']}")
|
|
376
|
+
if record.get("pdf_url"):
|
|
377
|
+
lines.append(f"**PDF:** {record['pdf_url']}")
|
|
378
|
+
if record.get("abstract"):
|
|
379
|
+
lines.extend(["", record["abstract"]])
|
|
380
|
+
lines.append("")
|
|
381
|
+
return "\n".join(lines).strip()
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Sitemap discovery: robots.txt ``Sitemap:`` lines and sitemap indexes.
|
|
2
|
+
|
|
3
|
+
Sitemaps are published specifically for crawlers, so they are fetched without
|
|
4
|
+
a per-URL robots check. Every discovered page URL is still checked against
|
|
5
|
+
robots.txt by :func:`engine.fetch_page` before its content is read.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import zlib
|
|
11
|
+
import urllib.parse
|
|
12
|
+
from typing import Any, Optional
|
|
13
|
+
|
|
14
|
+
from . import engine
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _local(tag: str) -> str:
|
|
18
|
+
return tag.rsplit("}", 1)[-1].lower()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
MAX_DECOMPRESSED_BYTES = 32 * 1024 * 1024
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _decode_sitemap(response: engine.HttpResponse) -> str:
|
|
25
|
+
raw = response.content
|
|
26
|
+
# Some servers answer .xml.gz URLs with plain XML, so trust the magic bytes
|
|
27
|
+
# rather than the URL suffix. A gzip payload is decompressed with a hard
|
|
28
|
+
# output cap: a few kilobytes can otherwise expand to gigabytes and take
|
|
29
|
+
# the process down before XML parsing even starts.
|
|
30
|
+
if raw[:2] == bytes([0x1f, 0x8b]):
|
|
31
|
+
try:
|
|
32
|
+
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
|
33
|
+
raw = decompressor.decompress(raw, MAX_DECOMPRESSED_BYTES)
|
|
34
|
+
if decompressor.unconsumed_tail or len(raw) >= MAX_DECOMPRESSED_BYTES:
|
|
35
|
+
raise engine.ExtractionError(
|
|
36
|
+
"sitemap expands beyond the 32 MiB safety limit"
|
|
37
|
+
)
|
|
38
|
+
raw += decompressor.flush()
|
|
39
|
+
except zlib.error as exc:
|
|
40
|
+
raise engine.FetchError(f"cannot decompress sitemap: {exc}") from exc
|
|
41
|
+
if len(raw) > MAX_DECOMPRESSED_BYTES:
|
|
42
|
+
raise engine.ExtractionError(
|
|
43
|
+
"sitemap expands beyond the 32 MiB safety limit"
|
|
44
|
+
)
|
|
45
|
+
return raw.decode(response.encoding or "utf-8", errors="replace")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def parse_sitemap(xml_text: str, limit: int = 5_000) -> dict[str, list[dict[str, str]]]:
|
|
49
|
+
"""Parse a urlset or sitemapindex into URL/sitemap entries."""
|
|
50
|
+
|
|
51
|
+
root = engine.parse_xml(xml_text)
|
|
52
|
+
kind = _local(root.tag)
|
|
53
|
+
urls: list[dict[str, str]] = []
|
|
54
|
+
sitemaps: list[dict[str, str]] = []
|
|
55
|
+
|
|
56
|
+
if kind == "sitemapindex":
|
|
57
|
+
for node in root:
|
|
58
|
+
if _local(node.tag) != "sitemap":
|
|
59
|
+
continue
|
|
60
|
+
loc = ""
|
|
61
|
+
lastmod = ""
|
|
62
|
+
for child in node:
|
|
63
|
+
name = _local(child.tag)
|
|
64
|
+
if name == "loc":
|
|
65
|
+
loc = (child.text or "").strip()
|
|
66
|
+
elif name == "lastmod":
|
|
67
|
+
lastmod = (child.text or "").strip()
|
|
68
|
+
if loc:
|
|
69
|
+
sitemaps.append({"url": loc, "lastmod": lastmod})
|
|
70
|
+
elif kind == "urlset":
|
|
71
|
+
for node in root:
|
|
72
|
+
if _local(node.tag) != "url":
|
|
73
|
+
continue
|
|
74
|
+
entry = {"url": "", "lastmod": "", "changefreq": "", "priority": ""}
|
|
75
|
+
for child in node:
|
|
76
|
+
name = _local(child.tag)
|
|
77
|
+
if name == "loc":
|
|
78
|
+
entry["url"] = (child.text or "").strip()
|
|
79
|
+
elif name in entry:
|
|
80
|
+
entry[name] = (child.text or "").strip()
|
|
81
|
+
if entry["url"]:
|
|
82
|
+
urls.append(entry)
|
|
83
|
+
if len(urls) >= limit:
|
|
84
|
+
break
|
|
85
|
+
else:
|
|
86
|
+
raise engine.FetchError(f"not a sitemap: unexpected root element {root.tag}")
|
|
87
|
+
|
|
88
|
+
return {"urls": urls[:limit], "sitemaps": sitemaps}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def discover(
|
|
92
|
+
url: str,
|
|
93
|
+
config: Optional[engine.FetchConfig] = None,
|
|
94
|
+
limit: int = 500,
|
|
95
|
+
max_sitemaps: int = 5,
|
|
96
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
97
|
+
) -> dict[str, Any]:
|
|
98
|
+
"""Find sitemap URLs for a site and return their page URLs."""
|
|
99
|
+
|
|
100
|
+
config = config or engine.FetchConfig()
|
|
101
|
+
seed = engine.normalize_url(url)
|
|
102
|
+
parsed = urllib.parse.urlsplit(seed)
|
|
103
|
+
origin = f"{parsed.scheme}://{parsed.netloc}"
|
|
104
|
+
lowered = seed.lower().split("?", 1)[0]
|
|
105
|
+
|
|
106
|
+
own_fetcher = fetcher is None
|
|
107
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
108
|
+
errors: list[str] = []
|
|
109
|
+
candidates: list[str] = []
|
|
110
|
+
sources: list[str] = []
|
|
111
|
+
|
|
112
|
+
def add_candidate(value: str, source: str) -> None:
|
|
113
|
+
value = value.strip()
|
|
114
|
+
if not value:
|
|
115
|
+
return
|
|
116
|
+
if not value.lower().startswith(("http://", "https://")):
|
|
117
|
+
value = urllib.parse.urljoin(origin + "/", value)
|
|
118
|
+
if value not in candidates:
|
|
119
|
+
candidates.append(value)
|
|
120
|
+
sources.append(source)
|
|
121
|
+
|
|
122
|
+
try:
|
|
123
|
+
if lowered.endswith((".xml", ".xml.gz", ".txt")) or "sitemap" in lowered:
|
|
124
|
+
add_candidate(seed, "seed")
|
|
125
|
+
else:
|
|
126
|
+
robots_url = origin + "/robots.txt"
|
|
127
|
+
try:
|
|
128
|
+
robots = fetcher.fetch(robots_url, check_robots=False).text
|
|
129
|
+
for line in robots.splitlines():
|
|
130
|
+
if line.lower().startswith("sitemap:"):
|
|
131
|
+
add_candidate(line.split(":", 1)[1], "robots.txt")
|
|
132
|
+
except engine.FetchError as exc:
|
|
133
|
+
errors.append(f"robots.txt: {exc}")
|
|
134
|
+
add_candidate(origin + "/sitemap.xml", "fallback")
|
|
135
|
+
add_candidate(origin + "/sitemap_index.xml", "fallback")
|
|
136
|
+
add_candidate(origin + "/sitemap.xml.gz", "fallback")
|
|
137
|
+
|
|
138
|
+
pages: list[dict[str, str]] = []
|
|
139
|
+
seen_pages: set[str] = set()
|
|
140
|
+
visited_maps: set[str] = set()
|
|
141
|
+
queue = list(candidates[:max_sitemaps])
|
|
142
|
+
while queue and len(pages) < limit and len(visited_maps) < max_sitemaps:
|
|
143
|
+
sitemap_url = queue.pop(0)
|
|
144
|
+
if sitemap_url in visited_maps:
|
|
145
|
+
continue
|
|
146
|
+
visited_maps.add(sitemap_url)
|
|
147
|
+
try:
|
|
148
|
+
response = fetcher.fetch(sitemap_url, check_robots=False)
|
|
149
|
+
parsed_map = parse_sitemap(_decode_sitemap(response), limit=limit)
|
|
150
|
+
except (engine.FetchError, engine.ExtractionError) as exc:
|
|
151
|
+
errors.append(f"{sitemap_url}: {exc}")
|
|
152
|
+
continue
|
|
153
|
+
for entry in parsed_map["urls"]:
|
|
154
|
+
page_url = entry["url"].strip()
|
|
155
|
+
if not page_url or page_url in seen_pages:
|
|
156
|
+
continue
|
|
157
|
+
seen_pages.add(page_url)
|
|
158
|
+
pages.append(entry)
|
|
159
|
+
if len(pages) >= limit:
|
|
160
|
+
break
|
|
161
|
+
for entry in parsed_map["sitemaps"]:
|
|
162
|
+
if entry["url"] not in visited_maps and len(queue) < max_sitemaps * 2:
|
|
163
|
+
queue.append(entry["url"])
|
|
164
|
+
|
|
165
|
+
note = "" if pages else "no sitemap was found for this site"
|
|
166
|
+
return {
|
|
167
|
+
"seed": seed,
|
|
168
|
+
"origin": origin,
|
|
169
|
+
"candidates": candidates,
|
|
170
|
+
"sources": sources,
|
|
171
|
+
"sitemaps_read": sorted(visited_maps),
|
|
172
|
+
"count": len(pages),
|
|
173
|
+
"urls": pages,
|
|
174
|
+
"errors": errors,
|
|
175
|
+
"note": note,
|
|
176
|
+
}
|
|
177
|
+
finally:
|
|
178
|
+
if own_fetcher:
|
|
179
|
+
fetcher.close()
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def to_markdown(payload: dict[str, Any]) -> str:
|
|
183
|
+
lines = [
|
|
184
|
+
f"# Sitemap: {payload.get('origin', '')}",
|
|
185
|
+
"",
|
|
186
|
+
f"> {payload.get('count', 0)} URL(s) from {len(payload.get('sitemaps_read') or [])} sitemap(s)",
|
|
187
|
+
"",
|
|
188
|
+
]
|
|
189
|
+
for entry in payload.get("urls") or []:
|
|
190
|
+
suffix = f" — {entry['lastmod']}" if entry.get("lastmod") else ""
|
|
191
|
+
lines.append(f"- {entry['url']}{suffix}")
|
|
192
|
+
if payload.get("errors"):
|
|
193
|
+
lines.extend(["", "## Errors", ""])
|
|
194
|
+
lines.extend(f"- {error}" for error in payload["errors"])
|
|
195
|
+
return "\n".join(lines).strip()
|