web-forager 0.0.0-stage → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +119 -0
  18. package/CHANGELOG.md +508 -0
  19. package/LICENSE +21 -0
  20. package/README.md +498 -2
  21. package/README_zh.md +498 -0
  22. package/SKILL.md +451 -0
  23. package/THIRD_PARTY_LICENSES.txt +40 -0
  24. package/browser/bridges/playwright-bridge.js +382 -0
  25. package/browser/launch_chrome.py +193 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +90 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +141 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +130 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +215 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +252 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +107 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +441 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/research.md +35 -0
  58. package/commands/scholar.md +28 -0
  59. package/commands/surf.md +28 -0
  60. package/cordis.patch.yml +19 -0
  61. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  62. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  63. package/docs/DEPENDENCIES.md +40 -0
  64. package/docs/INSTALL_COMMANDS.md +113 -0
  65. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  66. package/docs/REFERENCE.md +349 -0
  67. package/docs/REFERENCE_zh.md +342 -0
  68. package/index.js +7 -0
  69. package/install.py +254 -0
  70. package/manifest.json +98 -0
  71. package/mcp_server.py +850 -0
  72. package/package.json +82 -6
  73. package/requirements.txt +15 -0
  74. package/run.py +908 -0
  75. package/scripts/__init__.py +1 -0
  76. package/scripts/bilibili.py +882 -0
  77. package/scripts/documents.py +437 -0
  78. package/scripts/engine.py +5141 -0
  79. package/scripts/feeds.py +180 -0
  80. package/scripts/pdf.py +156 -0
  81. package/scripts/research.py +2087 -0
  82. package/scripts/scholar.py +381 -0
  83. package/scripts/sitemap.py +195 -0
  84. package/scripts/untrusted.py +312 -0
@@ -0,0 +1,381 @@
1
+ """Scholarly metadata through public APIs (arXiv, Crossref, PubMed).
2
+
3
+ These are documented query APIs, not scraped result pages, so they are both
4
+ more stable and more polite than reading a search engine's HTML. Only the
5
+ standard library and ``requests`` (through :mod:`scripts.engine`) are used.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import html as html_lib
11
+ import json
12
+ import re
13
+ import urllib.parse
14
+ import xml.etree.ElementTree as ET
15
+ from typing import Any, Optional
16
+
17
+ from . import engine
18
+
19
+ SCHOLAR_SOURCES = ("arxiv", "crossref", "pubmed", "doi")
20
+
21
+ _ARXIV_API = (
22
+ "https://export.arxiv.org/api/query?search_query={field}:{query}"
23
+ "&start={start}&max_results={limit}&sortBy={sort}&sortOrder=descending"
24
+ )
25
+ _CROSSREF_API = (
26
+ "https://api.crossref.org/works?query.bibliographic={query}&rows={limit}"
27
+ "&select=DOI,title,author,container-title,issued,published-print,published-online,"
28
+ "abstract,URL,type,subject,ISSN,volume,issue,page"
29
+ )
30
+ _CROSSREF_DOI = "https://api.crossref.org/works/{doi}"
31
+ _PUBMED_SEARCH = (
32
+ "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
33
+ "?db=pubmed&term={query}&retmode=json&retmax={limit}&sort=relevance"
34
+ )
35
+ _PUBMED_SUMMARY = (
36
+ "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
37
+ "?db=pubmed&id={ids}&retmode=json"
38
+ )
39
+ _PUBMED_FETCH = (
40
+ "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
41
+ "?db=pubmed&id={ids}&retmode=xml"
42
+ )
43
+
44
+ _ATOM = "{http://www.w3.org/2005/Atom}"
45
+ _ARXIV = "{http://arxiv.org/schemas/atom}"
46
+
47
+
48
+ def _text(node: Optional[ET.Element], path: str = "") -> str:
49
+ target = node.find(path) if node is not None and path else node
50
+ if target is None or target.text is None:
51
+ return ""
52
+ return re.sub(r"\s+", " ", target.text).strip()
53
+
54
+
55
+ def _clean(value: Any) -> str:
56
+ if value is None:
57
+ return ""
58
+ text = html_lib.unescape(str(value))
59
+ if "<" in text:
60
+ text = re.sub(r"<[^>]+>", " ", text)
61
+ return re.sub(r"\s+", " ", text).strip()
62
+
63
+
64
+ def _date_parts(value: Any) -> str:
65
+ if not isinstance(value, dict):
66
+ return ""
67
+ parts = (value.get("date-parts") or [[]])[0]
68
+ if not parts:
69
+ return ""
70
+ return "-".join(f"{int(part):02d}" if index else str(part) for index, part in enumerate(parts))
71
+
72
+
73
+ def parse_arxiv(xml_text: str, limit: int = 10) -> list[dict[str, Any]]:
74
+ """Parse an arXiv Atom feed into normalised records."""
75
+
76
+ root = engine.parse_xml(xml_text)
77
+
78
+ records: list[dict[str, Any]] = []
79
+ for entry in root.findall(f"{_ATOM}entry")[:limit]:
80
+ identifier = _text(entry, f"{_ATOM}id")
81
+ abs_url = identifier
82
+ if abs_url.startswith("http://arxiv.org/"):
83
+ abs_url = "https://" + abs_url[len("http://") :]
84
+ pdf_url = ""
85
+ for link in entry.findall(f"{_ATOM}link"):
86
+ if link.get("title") == "pdf" or link.get("type") == "application/pdf":
87
+ pdf_url = link.get("href", "")
88
+ authors = [
89
+ _text(author, f"{_ATOM}name")
90
+ for author in entry.findall(f"{_ATOM}author")
91
+ if _text(author, f"{_ATOM}name")
92
+ ]
93
+ categories = [
94
+ category.get("term", "")
95
+ for category in entry.findall(f"{_ATOM}category")
96
+ if category.get("term")
97
+ ]
98
+ journal = _text(entry, f"{_ARXIV}journal_ref")
99
+ doi = _text(entry, f"{_ARXIV}doi")
100
+ records.append(
101
+ {
102
+ "source": "arxiv",
103
+ "id": identifier.rsplit("/", 1)[-1],
104
+ "title": _text(entry, f"{_ATOM}title"),
105
+ "authors": authors,
106
+ "published": _text(entry, f"{_ATOM}published")[:10],
107
+ "updated": _text(entry, f"{_ATOM}updated")[:10],
108
+ "journal": journal,
109
+ "volume": "",
110
+ "issue": "",
111
+ "pages": "",
112
+ "doi": doi,
113
+ "url": abs_url,
114
+ "pdf_url": pdf_url,
115
+ "abstract": _text(entry, f"{_ATOM}summary"),
116
+ "type": "preprint",
117
+ "keywords": categories,
118
+ }
119
+ )
120
+ return records
121
+
122
+
123
+ def parse_crossref(payload: Any, limit: int = 10) -> list[dict[str, Any]]:
124
+ """Parse a Crossref ``works`` response (already decoded JSON)."""
125
+
126
+ if isinstance(payload, str):
127
+ payload = json.loads(payload)
128
+ message = payload.get("message", payload) if isinstance(payload, dict) else {}
129
+ items = message.get("items")
130
+ if items is None and message.get("DOI"):
131
+ items = [message]
132
+ records: list[dict[str, Any]] = []
133
+ for item in (items or [])[:limit]:
134
+ authors: list[str] = []
135
+ for author in item.get("author") or []:
136
+ name = author.get("name") or " ".join(
137
+ part for part in (author.get("given"), author.get("family")) if part
138
+ )
139
+ if name:
140
+ authors.append(name)
141
+ titles = item.get("title") or []
142
+ title = titles[0] if isinstance(titles, list) and titles else str(titles or "")
143
+ containers = item.get("container-title") or []
144
+ journal = containers[0] if isinstance(containers, list) and containers else str(containers or "")
145
+ published = (
146
+ _date_parts(item.get("published-print"))
147
+ or _date_parts(item.get("published-online"))
148
+ or _date_parts(item.get("issued"))
149
+ )
150
+ records.append(
151
+ {
152
+ "source": "crossref",
153
+ "id": item.get("DOI", ""),
154
+ "title": _clean(title),
155
+ "authors": authors,
156
+ "published": published,
157
+ "updated": "",
158
+ "journal": _clean(journal),
159
+ "volume": str(item.get("volume", "") or ""),
160
+ "issue": str(item.get("issue", "") or ""),
161
+ "pages": str(item.get("page", "") or ""),
162
+ "doi": item.get("DOI", ""),
163
+ "url": item.get("URL", "") or (
164
+ f"https://doi.org/{item.get('DOI')}" if item.get("DOI") else ""
165
+ ),
166
+ "pdf_url": "",
167
+ "abstract": _clean(item.get("abstract", "")),
168
+ "type": item.get("type", ""),
169
+ "keywords": [str(value) for value in item.get("subject") or []],
170
+ }
171
+ )
172
+ return records
173
+
174
+
175
+ def parse_pubmed(
176
+ search_payload: Any,
177
+ summary_payload: Any,
178
+ abstracts: Optional[dict[str, str]] = None,
179
+ limit: int = 10,
180
+ ) -> list[dict[str, Any]]:
181
+ """Merge PubMed esearch + esummary (+ optional efetch abstracts)."""
182
+
183
+ if isinstance(search_payload, str):
184
+ search_payload = json.loads(search_payload)
185
+ if isinstance(summary_payload, str):
186
+ summary_payload = json.loads(summary_payload)
187
+ abstracts = abstracts or {}
188
+
189
+ ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
190
+ result = summary_payload.get("result", {}) or {}
191
+ records: list[dict[str, Any]] = []
192
+ for uid in ids:
193
+ item = result.get(uid) or {}
194
+ doi = ""
195
+ for article_id in item.get("articleids") or []:
196
+ if article_id.get("idtype") == "doi":
197
+ doi = article_id.get("value", "")
198
+ break
199
+ records.append(
200
+ {
201
+ "source": "pubmed",
202
+ "id": uid,
203
+ "title": _clean(item.get("title", "")),
204
+ "authors": [
205
+ author.get("name", "")
206
+ for author in item.get("authors") or []
207
+ if author.get("name")
208
+ ],
209
+ "published": str(item.get("pubdate", "") or ""),
210
+ "updated": "",
211
+ "journal": str(item.get("fulljournalname") or item.get("source") or ""),
212
+ "volume": str(item.get("volume", "") or ""),
213
+ "issue": str(item.get("issue", "") or ""),
214
+ "pages": str(item.get("pages", "") or ""),
215
+ "doi": doi,
216
+ "url": f"https://pubmed.ncbi.nlm.nih.gov/{uid}/",
217
+ "pdf_url": "",
218
+ "abstract": abstracts.get(uid, ""),
219
+ "type": "journal-article",
220
+ "keywords": [],
221
+ }
222
+ )
223
+ return records
224
+
225
+
226
+ def _pubmed_abstracts(xml_text: str) -> dict[str, str]:
227
+ """Map PMID → abstract from an efetch XML response."""
228
+
229
+ try:
230
+ root = engine.parse_xml(xml_text)
231
+ except engine.FetchError:
232
+ return {}
233
+ abstracts: dict[str, str] = {}
234
+ for article in root.findall(".//PubmedArticle"):
235
+ pmid = _text(article, ".//PMID")
236
+ parts = [
237
+ _clean(node.text)
238
+ for node in article.findall(".//Abstract/AbstractText")
239
+ if node.text
240
+ ]
241
+ if pmid and parts:
242
+ abstracts[pmid] = " ".join(part for part in parts if part)
243
+ return abstracts
244
+
245
+
246
+ def _query_fetch(fetcher: engine.HttpFetcher, url: str) -> engine.HttpResponse:
247
+ """Fetch a scholarly query endpoint under the user-level query policy."""
248
+
249
+ config = fetcher.config
250
+ return fetcher.fetch(
251
+ url,
252
+ check_robots=config.query_robots != "ignore",
253
+ robots_policy=config.query_robots,
254
+ )
255
+
256
+
257
+ def _fetch_json(fetcher: engine.HttpFetcher, url: str) -> Any:
258
+ response = _query_fetch(fetcher, url)
259
+ try:
260
+ return json.loads(response.text)
261
+ except json.JSONDecodeError as exc:
262
+ raise engine.FetchError(f"invalid JSON from {url}: {exc}") from exc
263
+
264
+
265
+ def search(
266
+ source: str,
267
+ query: str,
268
+ config: Optional[engine.FetchConfig] = None,
269
+ limit: Optional[int] = None,
270
+ fetcher: Optional[engine.HttpFetcher] = None,
271
+ sort: str = "relevance",
272
+ field: str = "all",
273
+ ) -> dict[str, Any]:
274
+ """Query one scholarly source and return normalised records.
275
+
276
+ ``field`` is the arXiv query field (``all``, ``ti``, ``au`` or ``abs``) and
277
+ is ignored by the other sources. ``ti`` is useful when the caller already
278
+ has a paper title: it avoids the broad ``all`` search returning a different
279
+ paper that merely shares a phrase.
280
+ """
281
+
282
+ source = (source or "arxiv").lower()
283
+ if source not in SCHOLAR_SOURCES:
284
+ raise ValueError(f"unsupported scholarly source: {source}")
285
+ query = (query or "").strip()
286
+ if not query:
287
+ raise ValueError("query must not be empty")
288
+
289
+ config = config or engine.FetchConfig()
290
+ limit = max(1, min(limit or config.num_results or 10, 50))
291
+ own_fetcher = fetcher is None
292
+ fetcher = fetcher or engine.HttpFetcher(config)
293
+
294
+ try:
295
+ if source == "arxiv":
296
+ if sort not in ("relevance", "submittedDate", "lastUpdatedDate"):
297
+ raise ValueError("arXiv sort must be relevance, submittedDate or lastUpdatedDate")
298
+ if field not in ("all", "ti", "au", "abs"):
299
+ raise ValueError("arXiv field must be all, ti, au or abs")
300
+ url = _ARXIV_API.format(
301
+ field=field,
302
+ query=urllib.parse.quote_plus(query),
303
+ start=0,
304
+ limit=limit,
305
+ sort=sort,
306
+ )
307
+ response = _query_fetch(fetcher, url)
308
+ records = parse_arxiv(response.text, limit=limit)
309
+ elif source == "crossref":
310
+ url = _CROSSREF_API.format(query=urllib.parse.quote_plus(query), limit=limit)
311
+ records = parse_crossref(_fetch_json(fetcher, url), limit=limit)
312
+ elif source == "doi":
313
+ doi = query
314
+ if doi.lower().startswith("https://doi.org/"):
315
+ doi = doi[len("https://doi.org/"):]
316
+ elif doi.lower().startswith("doi:"):
317
+ doi = doi[4:]
318
+ url = _CROSSREF_DOI.format(doi=urllib.parse.quote(doi, safe="/"))
319
+ records = parse_crossref(_fetch_json(fetcher, url), limit=1)
320
+ else: # pubmed
321
+ search_url = _PUBMED_SEARCH.format(
322
+ query=urllib.parse.quote_plus(query), limit=limit
323
+ )
324
+ search_payload = _fetch_json(fetcher, search_url)
325
+ ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
326
+ if not ids:
327
+ records = []
328
+ else:
329
+ summary_url = _PUBMED_SUMMARY.format(ids=",".join(ids))
330
+ summary_payload = _fetch_json(fetcher, summary_url)
331
+ abstracts: dict[str, str] = {}
332
+ try:
333
+ fetch_url = _PUBMED_FETCH.format(ids=",".join(ids))
334
+ abstracts = _pubmed_abstracts(_query_fetch(fetcher, fetch_url).text)
335
+ except engine.FetchError:
336
+ abstracts = {}
337
+ records = parse_pubmed(search_payload, summary_payload, abstracts, limit=limit)
338
+
339
+ return {
340
+ "source": source,
341
+ "query": query,
342
+ "query_robots": config.query_robots,
343
+ "count": len(records),
344
+ "results": records,
345
+ }
346
+ finally:
347
+ if own_fetcher:
348
+ fetcher.close()
349
+
350
+
351
+ def to_markdown(payload: dict[str, Any]) -> str:
352
+ """Render scholarly records as a compact Markdown bibliography."""
353
+
354
+ lines = [
355
+ f"# {payload.get('source', 'scholar')}: {payload.get('query', '')}",
356
+ "",
357
+ f"> {payload.get('count', 0)} record(s)",
358
+ "",
359
+ ]
360
+ for index, record in enumerate(payload.get("results") or [], start=1):
361
+ title = record.get("title") or record.get("id") or "Untitled"
362
+ lines.append(f"## {index}. {title}")
363
+ if record.get("authors"):
364
+ lines.append(f"**Authors:** {', '.join(record['authors'])}")
365
+ if record.get("published"):
366
+ lines.append(f"**Published:** {record['published']}")
367
+ if record.get("journal"):
368
+ volume_issue = " ".join(
369
+ part for part in (record.get("volume"), record.get("issue")) if part
370
+ )
371
+ lines.append(f"**Journal:** {record['journal']}" + (f" {volume_issue}" if volume_issue else ""))
372
+ if record.get("doi"):
373
+ lines.append(f"**DOI:** {record['doi']}")
374
+ if record.get("url"):
375
+ lines.append(f"**URL:** {record['url']}")
376
+ if record.get("pdf_url"):
377
+ lines.append(f"**PDF:** {record['pdf_url']}")
378
+ if record.get("abstract"):
379
+ lines.extend(["", record["abstract"]])
380
+ lines.append("")
381
+ return "\n".join(lines).strip()
@@ -0,0 +1,195 @@
1
+ """Sitemap discovery: robots.txt ``Sitemap:`` lines and sitemap indexes.
2
+
3
+ Sitemaps are published specifically for crawlers, so they are fetched without
4
+ a per-URL robots check. Every discovered page URL is still checked against
5
+ robots.txt by :func:`engine.fetch_page` before its content is read.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import zlib
11
+ import urllib.parse
12
+ from typing import Any, Optional
13
+
14
+ from . import engine
15
+
16
+
17
+ def _local(tag: str) -> str:
18
+ return tag.rsplit("}", 1)[-1].lower()
19
+
20
+
21
+ MAX_DECOMPRESSED_BYTES = 32 * 1024 * 1024
22
+
23
+
24
+ def _decode_sitemap(response: engine.HttpResponse) -> str:
25
+ raw = response.content
26
+ # Some servers answer .xml.gz URLs with plain XML, so trust the magic bytes
27
+ # rather than the URL suffix. A gzip payload is decompressed with a hard
28
+ # output cap: a few kilobytes can otherwise expand to gigabytes and take
29
+ # the process down before XML parsing even starts.
30
+ if raw[:2] == bytes([0x1f, 0x8b]):
31
+ try:
32
+ decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
33
+ raw = decompressor.decompress(raw, MAX_DECOMPRESSED_BYTES)
34
+ if decompressor.unconsumed_tail or len(raw) >= MAX_DECOMPRESSED_BYTES:
35
+ raise engine.ExtractionError(
36
+ "sitemap expands beyond the 32 MiB safety limit"
37
+ )
38
+ raw += decompressor.flush()
39
+ except zlib.error as exc:
40
+ raise engine.FetchError(f"cannot decompress sitemap: {exc}") from exc
41
+ if len(raw) > MAX_DECOMPRESSED_BYTES:
42
+ raise engine.ExtractionError(
43
+ "sitemap expands beyond the 32 MiB safety limit"
44
+ )
45
+ return raw.decode(response.encoding or "utf-8", errors="replace")
46
+
47
+
48
+ def parse_sitemap(xml_text: str, limit: int = 5_000) -> dict[str, list[dict[str, str]]]:
49
+ """Parse a urlset or sitemapindex into URL/sitemap entries."""
50
+
51
+ root = engine.parse_xml(xml_text)
52
+ kind = _local(root.tag)
53
+ urls: list[dict[str, str]] = []
54
+ sitemaps: list[dict[str, str]] = []
55
+
56
+ if kind == "sitemapindex":
57
+ for node in root:
58
+ if _local(node.tag) != "sitemap":
59
+ continue
60
+ loc = ""
61
+ lastmod = ""
62
+ for child in node:
63
+ name = _local(child.tag)
64
+ if name == "loc":
65
+ loc = (child.text or "").strip()
66
+ elif name == "lastmod":
67
+ lastmod = (child.text or "").strip()
68
+ if loc:
69
+ sitemaps.append({"url": loc, "lastmod": lastmod})
70
+ elif kind == "urlset":
71
+ for node in root:
72
+ if _local(node.tag) != "url":
73
+ continue
74
+ entry = {"url": "", "lastmod": "", "changefreq": "", "priority": ""}
75
+ for child in node:
76
+ name = _local(child.tag)
77
+ if name == "loc":
78
+ entry["url"] = (child.text or "").strip()
79
+ elif name in entry:
80
+ entry[name] = (child.text or "").strip()
81
+ if entry["url"]:
82
+ urls.append(entry)
83
+ if len(urls) >= limit:
84
+ break
85
+ else:
86
+ raise engine.FetchError(f"not a sitemap: unexpected root element {root.tag}")
87
+
88
+ return {"urls": urls[:limit], "sitemaps": sitemaps}
89
+
90
+
91
+ def discover(
92
+ url: str,
93
+ config: Optional[engine.FetchConfig] = None,
94
+ limit: int = 500,
95
+ max_sitemaps: int = 5,
96
+ fetcher: Optional[engine.HttpFetcher] = None,
97
+ ) -> dict[str, Any]:
98
+ """Find sitemap URLs for a site and return their page URLs."""
99
+
100
+ config = config or engine.FetchConfig()
101
+ seed = engine.normalize_url(url)
102
+ parsed = urllib.parse.urlsplit(seed)
103
+ origin = f"{parsed.scheme}://{parsed.netloc}"
104
+ lowered = seed.lower().split("?", 1)[0]
105
+
106
+ own_fetcher = fetcher is None
107
+ fetcher = fetcher or engine.HttpFetcher(config)
108
+ errors: list[str] = []
109
+ candidates: list[str] = []
110
+ sources: list[str] = []
111
+
112
+ def add_candidate(value: str, source: str) -> None:
113
+ value = value.strip()
114
+ if not value:
115
+ return
116
+ if not value.lower().startswith(("http://", "https://")):
117
+ value = urllib.parse.urljoin(origin + "/", value)
118
+ if value not in candidates:
119
+ candidates.append(value)
120
+ sources.append(source)
121
+
122
+ try:
123
+ if lowered.endswith((".xml", ".xml.gz", ".txt")) or "sitemap" in lowered:
124
+ add_candidate(seed, "seed")
125
+ else:
126
+ robots_url = origin + "/robots.txt"
127
+ try:
128
+ robots = fetcher.fetch(robots_url, check_robots=False).text
129
+ for line in robots.splitlines():
130
+ if line.lower().startswith("sitemap:"):
131
+ add_candidate(line.split(":", 1)[1], "robots.txt")
132
+ except engine.FetchError as exc:
133
+ errors.append(f"robots.txt: {exc}")
134
+ add_candidate(origin + "/sitemap.xml", "fallback")
135
+ add_candidate(origin + "/sitemap_index.xml", "fallback")
136
+ add_candidate(origin + "/sitemap.xml.gz", "fallback")
137
+
138
+ pages: list[dict[str, str]] = []
139
+ seen_pages: set[str] = set()
140
+ visited_maps: set[str] = set()
141
+ queue = list(candidates[:max_sitemaps])
142
+ while queue and len(pages) < limit and len(visited_maps) < max_sitemaps:
143
+ sitemap_url = queue.pop(0)
144
+ if sitemap_url in visited_maps:
145
+ continue
146
+ visited_maps.add(sitemap_url)
147
+ try:
148
+ response = fetcher.fetch(sitemap_url, check_robots=False)
149
+ parsed_map = parse_sitemap(_decode_sitemap(response), limit=limit)
150
+ except (engine.FetchError, engine.ExtractionError) as exc:
151
+ errors.append(f"{sitemap_url}: {exc}")
152
+ continue
153
+ for entry in parsed_map["urls"]:
154
+ page_url = entry["url"].strip()
155
+ if not page_url or page_url in seen_pages:
156
+ continue
157
+ seen_pages.add(page_url)
158
+ pages.append(entry)
159
+ if len(pages) >= limit:
160
+ break
161
+ for entry in parsed_map["sitemaps"]:
162
+ if entry["url"] not in visited_maps and len(queue) < max_sitemaps * 2:
163
+ queue.append(entry["url"])
164
+
165
+ note = "" if pages else "no sitemap was found for this site"
166
+ return {
167
+ "seed": seed,
168
+ "origin": origin,
169
+ "candidates": candidates,
170
+ "sources": sources,
171
+ "sitemaps_read": sorted(visited_maps),
172
+ "count": len(pages),
173
+ "urls": pages,
174
+ "errors": errors,
175
+ "note": note,
176
+ }
177
+ finally:
178
+ if own_fetcher:
179
+ fetcher.close()
180
+
181
+
182
+ def to_markdown(payload: dict[str, Any]) -> str:
183
+ lines = [
184
+ f"# Sitemap: {payload.get('origin', '')}",
185
+ "",
186
+ f"> {payload.get('count', 0)} URL(s) from {len(payload.get('sitemaps_read') or [])} sitemap(s)",
187
+ "",
188
+ ]
189
+ for entry in payload.get("urls") or []:
190
+ suffix = f" — {entry['lastmod']}" if entry.get("lastmod") else ""
191
+ lines.append(f"- {entry['url']}{suffix}")
192
+ if payload.get("errors"):
193
+ lines.extend(["", "## Errors", ""])
194
+ lines.extend(f"- {error}" for error in payload["errors"])
195
+ return "\n".join(lines).strip()