web-forager 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +112 -0
  18. package/CHANGELOG.md +454 -0
  19. package/LICENSE +21 -0
  20. package/README.md +477 -3
  21. package/README_zh.md +477 -0
  22. package/SKILL.md +351 -0
  23. package/THIRD_PARTY_LICENSES.txt +35 -0
  24. package/browser/bridges/playwright-bridge.js +375 -0
  25. package/browser/launch_chrome.py +144 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +79 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +115 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +116 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +56 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +272 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +105 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +432 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/scholar.md +28 -0
  58. package/commands/surf.md +28 -0
  59. package/cordis.patch.yml +19 -0
  60. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  61. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  62. package/docs/DEPENDENCIES.md +40 -0
  63. package/docs/INSTALL_COMMANDS.md +113 -0
  64. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  65. package/docs/REFERENCE.md +308 -0
  66. package/docs/REFERENCE_zh.md +309 -0
  67. package/index.js +7 -0
  68. package/install.py +254 -0
  69. package/manifest.json +97 -0
  70. package/mcp_server.py +597 -0
  71. package/package.json +80 -6
  72. package/requirements.txt +12 -0
  73. package/run.py +852 -0
  74. package/scripts/__init__.py +1 -0
  75. package/scripts/bilibili.py +874 -0
  76. package/scripts/documents.py +437 -0
  77. package/scripts/engine.py +4220 -0
  78. package/scripts/feeds.py +180 -0
  79. package/scripts/pdf.py +135 -0
  80. package/scripts/scholar.py +354 -0
  81. package/scripts/sitemap.py +180 -0
  82. package/scripts/untrusted.py +95 -0
@@ -0,0 +1,180 @@
1
+ """RSS 2.0 / Atom / RSS 1.0 feed parsing.
2
+
3
+ Feeds are structured XML, so they are far cheaper to read than the rendered
4
+ HTML of a blog index. Parsing goes through :func:`engine.parse_xml`, which
5
+ rejects DOCTYPE/ENTITY and uses defusedxml when available.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import html as html_lib
11
+ import re
12
+ from typing import Any
13
+
14
+ from . import engine
15
+
16
+ _ATOM = "{http://www.w3.org/2005/Atom}"
17
+ _RSS1 = "{http://purl.org/rss/1.0/}"
18
+ _DC = "{http://purl.org/dc/elements/1.1/}"
19
+ _CONTENT = "{http://purl.org/rss/1.0/modules/content/}"
20
+
21
+
22
+ def _clean(value: Any, limit: int = 1200) -> str:
23
+ text = html_lib.unescape(str(value or ""))
24
+ if "<" in text:
25
+ text = re.sub(r"<[^>]+>", " ", text)
26
+ text = re.sub(r"\s+", " ", text).strip()
27
+ return text[:limit]
28
+
29
+
30
+ def _text(node: Any, path: str = "") -> str:
31
+ target = node.find(path) if path else node
32
+ return (target.text or "").strip() if target is not None else ""
33
+
34
+
35
+ def _atom_link(entry: Any) -> str:
36
+ fallback = ""
37
+ for link in entry.findall(f"{_ATOM}link"):
38
+ href = link.get("href", "")
39
+ if link.get("rel", "alternate") == "alternate":
40
+ return href
41
+ fallback = fallback or href
42
+ return fallback
43
+
44
+
45
+ def _atom_entry(entry: Any) -> dict[str, Any]:
46
+ return {
47
+ "title": _clean(_text(entry, f"{_ATOM}title"), 300),
48
+ "url": _atom_link(entry),
49
+ "published": _text(entry, f"{_ATOM}published") or _text(entry, f"{_ATOM}updated"),
50
+ "summary": _clean(
51
+ _text(entry, f"{_ATOM}summary") or _text(entry, f"{_ATOM}content")
52
+ ),
53
+ "authors": [
54
+ _clean(_text(author, f"{_ATOM}name"), 120)
55
+ for author in entry.findall(f"{_ATOM}author")
56
+ if _text(author, f"{_ATOM}name")
57
+ ],
58
+ "categories": [
59
+ category.get("term", "")
60
+ for category in entry.findall(f"{_ATOM}category")
61
+ if category.get("term")
62
+ ],
63
+ }
64
+
65
+
66
+ def _rss_item(item: Any, rss1: bool = False) -> dict[str, Any]:
67
+ ns = _RSS1 if rss1 else ""
68
+ summary = _text(item, f"{ns}description")
69
+ if not summary:
70
+ summary = _text(item, f"{_CONTENT}encoded")
71
+ author = _text(item, f"{_DC}creator") or _text(item, f"{ns}author")
72
+ return {
73
+ "title": _clean(_text(item, f"{ns}title"), 300),
74
+ "url": _text(item, f"{ns}link") or _text(item, f"{ns}guid"),
75
+ "published": _text(item, f"{ns}pubDate") or _text(item, f"{_DC}date"),
76
+ "summary": _clean(summary),
77
+ "authors": [_clean(author, 120)] if author else [],
78
+ "categories": [
79
+ _clean(category.text, 80)
80
+ for category in item.findall(f"{ns}category")
81
+ if _clean(category.text)
82
+ ],
83
+ }
84
+
85
+
86
+ def is_feed(xml_text: str) -> bool:
87
+ """Cheaply decide whether an XML payload is an RSS/Atom feed."""
88
+
89
+ head = xml_text.lstrip()[:2000].lower()
90
+ if not head.startswith("<?xml") and not head.startswith("<rss") and not head.startswith("<feed"):
91
+ return False
92
+ try:
93
+ root = engine.parse_xml(xml_text)
94
+ except engine.FetchError:
95
+ return False
96
+ tag = root.tag.lower()
97
+ return tag == "rss" or tag.endswith("}feed") or tag.endswith("}rdf")
98
+
99
+
100
+ def extract(xml_text: str, limit: int = 50) -> dict[str, Any]:
101
+ """Parse a feed and return a Markdown digest plus structured items."""
102
+
103
+ root = engine.parse_xml(xml_text)
104
+ tag = root.tag.lower()
105
+ if tag.endswith("}feed") or tag == "feed":
106
+ feed_format = "atom"
107
+ channel = root
108
+ entries = root.findall(f"{_ATOM}entry")[:limit]
109
+ items = [_atom_entry(entry) for entry in entries]
110
+ title = _clean(_text(channel, f"{_ATOM}title"), 300)
111
+ site_url = _atom_link(channel)
112
+ description = _clean(_text(channel, f"{_ATOM}subtitle"))
113
+ elif tag == "rss":
114
+ feed_format = "rss"
115
+ channel = root.find("channel")
116
+ if channel is None:
117
+ raise engine.ExtractionError("RSS feed has no channel element")
118
+ items = [_rss_item(item) for item in channel.findall("item")[:limit]]
119
+ title = _clean(_text(channel, "title"), 300)
120
+ site_url = _text(channel, "link")
121
+ description = _clean(_text(channel, "description"))
122
+ elif tag.endswith("}rdf"):
123
+ feed_format = "rss1"
124
+ channel = root.find(f"{_RSS1}channel")
125
+ items = [_rss_item(item, rss1=True) for item in root.findall(f"{_RSS1}item")[:limit]]
126
+ title = _clean(_text(channel, f"{_RSS1}title") if channel is not None else "", 300)
127
+ site_url = _text(channel, f"{_RSS1}link") if channel is not None else ""
128
+ description = _clean(_text(channel, f"{_RSS1}description") if channel is not None else "")
129
+ else:
130
+ raise engine.ExtractionError(f"not a feed: unexpected root element {root.tag}")
131
+
132
+ lines = [f"# {title or 'Feed'}", ""]
133
+ if description:
134
+ lines.extend([description, ""])
135
+ if site_url:
136
+ lines.extend([f"> {site_url}", ""])
137
+ for index, item in enumerate(items, start=1):
138
+ lines.append(f"## {index}. {item['title'] or item['url'] or 'Untitled'}")
139
+ if item["published"]:
140
+ lines.append(f"**Published:** {item['published']}")
141
+ if item["authors"]:
142
+ lines.append(f"**Authors:** {', '.join(item['authors'])}")
143
+ if item["url"]:
144
+ lines.append(f"**URL:** {item['url']}")
145
+ if item["summary"]:
146
+ lines.extend(["", item["summary"]])
147
+ lines.append("")
148
+
149
+ return {
150
+ "kind": "feed",
151
+ "format": feed_format,
152
+ "title": title,
153
+ "site_url": site_url,
154
+ "description": description,
155
+ "count": len(items),
156
+ "items": items,
157
+ "markdown": "\n".join(lines).strip(),
158
+ }
159
+
160
+
161
+ def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
162
+ return {
163
+ "title": document.get("title", ""),
164
+ "authors": [],
165
+ "published": "",
166
+ "modified": "",
167
+ "journal": "",
168
+ "volume": "",
169
+ "issue": "",
170
+ "first_page": "",
171
+ "last_page": "",
172
+ "doi": "",
173
+ "arxiv_id": "",
174
+ "abstract": document.get("description", ""),
175
+ "keywords": [],
176
+ "language": "",
177
+ "site_name": document.get("format", "feed"),
178
+ "canonical_url": url or document.get("site_url", ""),
179
+ "og_type": "feed",
180
+ }
package/scripts/pdf.py ADDED
@@ -0,0 +1,135 @@
1
+ """PDF text extraction for web-forager.
2
+
3
+ The only third-party dependency here is ``pypdf`` (BSD-3-Clause). It is an
4
+ optional dependency: when it is absent, callers receive a clear
5
+ ``PdfUnavailable`` error instead of a traceback.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import io
11
+ from typing import Any
12
+
13
+ PDF_MAGIC = b"%PDF-"
14
+
15
+
16
+ class PdfError(Exception):
17
+ """Base class for PDF failures."""
18
+
19
+
20
+ class PdfUnavailable(PdfError):
21
+ """pypdf is not installed or could not be imported."""
22
+
23
+
24
+ class PdfExtractionError(PdfError):
25
+ """The file is not a readable PDF."""
26
+
27
+
28
+ def is_pdf(data: bytes) -> bool:
29
+ return bool(data) and data[:5] == PDF_MAGIC
30
+
31
+
32
+ def _meta_value(metadata: Any, key: str) -> str:
33
+ try:
34
+ value = metadata.get(key) if metadata else None
35
+ except Exception:
36
+ return ""
37
+ return str(value).strip() if value else ""
38
+
39
+
40
+ def extract(data: bytes) -> dict[str, Any]:
41
+ """Extract text and document metadata from PDF bytes."""
42
+
43
+ if not is_pdf(data):
44
+ raise PdfExtractionError("the supplied data is not a PDF (missing %PDF- header)")
45
+ try:
46
+ from pypdf import PdfReader # type: ignore
47
+ except ImportError as exc:
48
+ raise PdfUnavailable(
49
+ "pypdf is not installed; enable PDF support with: pip install pypdf"
50
+ ) from exc
51
+ except Exception as exc: # pragma: no cover - broken installation
52
+ raise PdfUnavailable(f"pypdf could not be imported ({exc}); reinstall it with: pip install pypdf") from exc
53
+
54
+ try:
55
+ reader = PdfReader(io.BytesIO(data))
56
+ except Exception as exc:
57
+ raise PdfExtractionError(f"cannot open PDF: {exc}") from exc
58
+
59
+ pages: list[str] = []
60
+ empty_pages = 0
61
+ for index, page in enumerate(reader.pages, start=1):
62
+ try:
63
+ text = page.extract_text() or ""
64
+ except Exception as exc: # a single damaged page should not kill the run
65
+ text = f"[page {index}: extraction failed: {exc}]"
66
+ text = text.strip()
67
+ if not text:
68
+ empty_pages += 1
69
+ pages.append(text)
70
+
71
+ metadata = reader.metadata
72
+ return {
73
+ "pages": pages,
74
+ "page_count": len(pages),
75
+ "empty_pages": empty_pages,
76
+ "text": "\n\n".join(pages).strip(),
77
+ "metadata": {
78
+ "title": _meta_value(metadata, "/Title"),
79
+ "author": _meta_value(metadata, "/Author"),
80
+ "subject": _meta_value(metadata, "/Subject"),
81
+ "creator": _meta_value(metadata, "/Creator"),
82
+ "producer": _meta_value(metadata, "/Producer"),
83
+ "creation_date": _meta_value(metadata, "/CreationDate"),
84
+ },
85
+ }
86
+
87
+
88
+ def to_markdown(document: dict[str, Any], url: str = "") -> str:
89
+ metadata = document.get("metadata") or {}
90
+ title = metadata.get("title") or (url.rsplit("/", 1)[-1] if url else "PDF document")
91
+ lines = [f"# {title}", ""]
92
+ info = " | ".join(
93
+ part
94
+ for part in (
95
+ f"作者:{metadata.get('author')}" if metadata.get("author") else "",
96
+ f"页数:{document.get('page_count')}" if document.get("page_count") else "",
97
+ f"来源:{url}" if url else "",
98
+ )
99
+ if part
100
+ )
101
+ if info:
102
+ lines.extend([f"> {info}", ""])
103
+ pages = document.get("pages") or []
104
+ for index, text in enumerate(pages, start=1):
105
+ if not text:
106
+ continue
107
+ if len(pages) > 1:
108
+ lines.extend([f"## 第 {index} 页", ""])
109
+ lines.extend([text, ""])
110
+ return "\n".join(lines).strip()
111
+
112
+
113
+ def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
114
+ """Return PDF metadata in the same shape as article metadata."""
115
+
116
+ source = document.get("metadata") or {}
117
+ return {
118
+ "title": source.get("title", ""),
119
+ "authors": [source["author"]] if source.get("author") else [],
120
+ "published": source.get("creation_date", ""),
121
+ "modified": "",
122
+ "journal": "",
123
+ "volume": "",
124
+ "issue": "",
125
+ "first_page": "",
126
+ "last_page": "",
127
+ "doi": "",
128
+ "arxiv_id": "",
129
+ "abstract": source.get("subject", ""),
130
+ "keywords": [],
131
+ "language": "",
132
+ "site_name": "PDF",
133
+ "canonical_url": url,
134
+ "og_type": "pdf",
135
+ }
@@ -0,0 +1,354 @@
1
+ """Scholarly metadata through public APIs (arXiv, Crossref, PubMed).
2
+
3
+ These are documented query APIs, not scraped result pages, so they are both
4
+ more stable and more polite than reading a search engine's HTML. Only the
5
+ standard library and ``requests`` (through :mod:`scripts.engine`) are used.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import html as html_lib
11
+ import json
12
+ import re
13
+ import urllib.parse
14
+ import xml.etree.ElementTree as ET
15
+ from typing import Any, Optional
16
+
17
+ from . import engine
18
+
19
+ SCHOLAR_SOURCES = ("arxiv", "crossref", "pubmed", "doi")
20
+
21
+ _ARXIV_API = (
22
+ "https://export.arxiv.org/api/query?search_query=all:{query}"
23
+ "&start={start}&max_results={limit}&sortBy={sort}&sortOrder=descending"
24
+ )
25
+ _CROSSREF_API = (
26
+ "https://api.crossref.org/works?query.bibliographic={query}&rows={limit}"
27
+ "&select=DOI,title,author,container-title,issued,published-print,published-online,"
28
+ "abstract,URL,type,subject,ISSN,volume,issue,page"
29
+ )
30
+ _CROSSREF_DOI = "https://api.crossref.org/works/{doi}"
31
+ _PUBMED_SEARCH = (
32
+ "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
33
+ "?db=pubmed&term={query}&retmode=json&retmax={limit}&sort=relevance"
34
+ )
35
+ _PUBMED_SUMMARY = (
36
+ "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
37
+ "?db=pubmed&id={ids}&retmode=json"
38
+ )
39
+ _PUBMED_FETCH = (
40
+ "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
41
+ "?db=pubmed&id={ids}&retmode=xml"
42
+ )
43
+
44
+ _ATOM = "{http://www.w3.org/2005/Atom}"
45
+ _ARXIV = "{http://arxiv.org/schemas/atom}"
46
+
47
+
48
+ def _text(node: Optional[ET.Element], path: str = "") -> str:
49
+ target = node.find(path) if node is not None and path else node
50
+ if target is None or target.text is None:
51
+ return ""
52
+ return re.sub(r"\s+", " ", target.text).strip()
53
+
54
+
55
+ def _clean(value: Any) -> str:
56
+ if value is None:
57
+ return ""
58
+ text = html_lib.unescape(str(value))
59
+ if "<" in text:
60
+ text = re.sub(r"<[^>]+>", " ", text)
61
+ return re.sub(r"\s+", " ", text).strip()
62
+
63
+
64
+ def _date_parts(value: Any) -> str:
65
+ if not isinstance(value, dict):
66
+ return ""
67
+ parts = (value.get("date-parts") or [[]])[0]
68
+ if not parts:
69
+ return ""
70
+ return "-".join(f"{int(part):02d}" if index else str(part) for index, part in enumerate(parts))
71
+
72
+
73
+ def parse_arxiv(xml_text: str, limit: int = 10) -> list[dict[str, Any]]:
74
+ """Parse an arXiv Atom feed into normalised records."""
75
+
76
+ root = engine.parse_xml(xml_text)
77
+
78
+ records: list[dict[str, Any]] = []
79
+ for entry in root.findall(f"{_ATOM}entry")[:limit]:
80
+ identifier = _text(entry, f"{_ATOM}id")
81
+ abs_url = identifier
82
+ pdf_url = ""
83
+ for link in entry.findall(f"{_ATOM}link"):
84
+ if link.get("title") == "pdf" or link.get("type") == "application/pdf":
85
+ pdf_url = link.get("href", "")
86
+ authors = [
87
+ _text(author, f"{_ATOM}name")
88
+ for author in entry.findall(f"{_ATOM}author")
89
+ if _text(author, f"{_ATOM}name")
90
+ ]
91
+ categories = [
92
+ category.get("term", "")
93
+ for category in entry.findall(f"{_ATOM}category")
94
+ if category.get("term")
95
+ ]
96
+ journal = _text(entry, f"{_ARXIV}journal_ref")
97
+ doi = _text(entry, f"{_ARXIV}doi")
98
+ records.append(
99
+ {
100
+ "source": "arxiv",
101
+ "id": identifier.rsplit("/", 1)[-1],
102
+ "title": _text(entry, f"{_ATOM}title"),
103
+ "authors": authors,
104
+ "published": _text(entry, f"{_ATOM}published")[:10],
105
+ "updated": _text(entry, f"{_ATOM}updated")[:10],
106
+ "journal": journal,
107
+ "volume": "",
108
+ "issue": "",
109
+ "pages": "",
110
+ "doi": doi,
111
+ "url": abs_url,
112
+ "pdf_url": pdf_url,
113
+ "abstract": _text(entry, f"{_ATOM}summary"),
114
+ "type": "preprint",
115
+ "keywords": categories,
116
+ }
117
+ )
118
+ return records
119
+
120
+
121
+ def parse_crossref(payload: Any, limit: int = 10) -> list[dict[str, Any]]:
122
+ """Parse a Crossref ``works`` response (already decoded JSON)."""
123
+
124
+ if isinstance(payload, str):
125
+ payload = json.loads(payload)
126
+ message = payload.get("message", payload) if isinstance(payload, dict) else {}
127
+ items = message.get("items")
128
+ if items is None and message.get("DOI"):
129
+ items = [message]
130
+ records: list[dict[str, Any]] = []
131
+ for item in (items or [])[:limit]:
132
+ authors: list[str] = []
133
+ for author in item.get("author") or []:
134
+ name = author.get("name") or " ".join(
135
+ part for part in (author.get("given"), author.get("family")) if part
136
+ )
137
+ if name:
138
+ authors.append(name)
139
+ titles = item.get("title") or []
140
+ title = titles[0] if isinstance(titles, list) and titles else str(titles or "")
141
+ containers = item.get("container-title") or []
142
+ journal = containers[0] if isinstance(containers, list) and containers else str(containers or "")
143
+ published = (
144
+ _date_parts(item.get("published-print"))
145
+ or _date_parts(item.get("published-online"))
146
+ or _date_parts(item.get("issued"))
147
+ )
148
+ records.append(
149
+ {
150
+ "source": "crossref",
151
+ "id": item.get("DOI", ""),
152
+ "title": _clean(title),
153
+ "authors": authors,
154
+ "published": published,
155
+ "updated": "",
156
+ "journal": _clean(journal),
157
+ "volume": str(item.get("volume", "") or ""),
158
+ "issue": str(item.get("issue", "") or ""),
159
+ "pages": str(item.get("page", "") or ""),
160
+ "doi": item.get("DOI", ""),
161
+ "url": item.get("URL", "") or (
162
+ f"https://doi.org/{item.get('DOI')}" if item.get("DOI") else ""
163
+ ),
164
+ "pdf_url": "",
165
+ "abstract": _clean(item.get("abstract", "")),
166
+ "type": item.get("type", ""),
167
+ "keywords": [str(value) for value in item.get("subject") or []],
168
+ }
169
+ )
170
+ return records
171
+
172
+
173
+ def parse_pubmed(
174
+ search_payload: Any,
175
+ summary_payload: Any,
176
+ abstracts: Optional[dict[str, str]] = None,
177
+ limit: int = 10,
178
+ ) -> list[dict[str, Any]]:
179
+ """Merge PubMed esearch + esummary (+ optional efetch abstracts)."""
180
+
181
+ if isinstance(search_payload, str):
182
+ search_payload = json.loads(search_payload)
183
+ if isinstance(summary_payload, str):
184
+ summary_payload = json.loads(summary_payload)
185
+ abstracts = abstracts or {}
186
+
187
+ ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
188
+ result = summary_payload.get("result", {}) or {}
189
+ records: list[dict[str, Any]] = []
190
+ for uid in ids:
191
+ item = result.get(uid) or {}
192
+ doi = ""
193
+ for article_id in item.get("articleids") or []:
194
+ if article_id.get("idtype") == "doi":
195
+ doi = article_id.get("value", "")
196
+ break
197
+ records.append(
198
+ {
199
+ "source": "pubmed",
200
+ "id": uid,
201
+ "title": _clean(item.get("title", "")),
202
+ "authors": [
203
+ author.get("name", "")
204
+ for author in item.get("authors") or []
205
+ if author.get("name")
206
+ ],
207
+ "published": str(item.get("pubdate", "") or ""),
208
+ "updated": "",
209
+ "journal": str(item.get("fulljournalname") or item.get("source") or ""),
210
+ "volume": str(item.get("volume", "") or ""),
211
+ "issue": str(item.get("issue", "") or ""),
212
+ "pages": str(item.get("pages", "") or ""),
213
+ "doi": doi,
214
+ "url": f"https://pubmed.ncbi.nlm.nih.gov/{uid}/",
215
+ "pdf_url": "",
216
+ "abstract": abstracts.get(uid, ""),
217
+ "type": "journal-article",
218
+ "keywords": [],
219
+ }
220
+ )
221
+ return records
222
+
223
+
224
+ def _pubmed_abstracts(xml_text: str) -> dict[str, str]:
225
+ """Map PMID → abstract from an efetch XML response."""
226
+
227
+ try:
228
+ root = engine.parse_xml(xml_text)
229
+ except engine.FetchError:
230
+ return {}
231
+ abstracts: dict[str, str] = {}
232
+ for article in root.findall(".//PubmedArticle"):
233
+ pmid = _text(article, ".//PMID")
234
+ parts = [
235
+ _clean(node.text)
236
+ for node in article.findall(".//Abstract/AbstractText")
237
+ if node.text
238
+ ]
239
+ if pmid and parts:
240
+ abstracts[pmid] = " ".join(part for part in parts if part)
241
+ return abstracts
242
+
243
+
244
+ def _fetch_json(fetcher: engine.HttpFetcher, url: str) -> Any:
245
+ response = fetcher.fetch(url, check_robots=False)
246
+ try:
247
+ return json.loads(response.text)
248
+ except json.JSONDecodeError as exc:
249
+ raise engine.FetchError(f"invalid JSON from {url}: {exc}") from exc
250
+
251
+
252
+ def search(
253
+ source: str,
254
+ query: str,
255
+ config: Optional[engine.FetchConfig] = None,
256
+ limit: Optional[int] = None,
257
+ fetcher: Optional[engine.HttpFetcher] = None,
258
+ sort: str = "relevance",
259
+ ) -> dict[str, Any]:
260
+ """Query one scholarly source and return normalised records."""
261
+
262
+ source = (source or "arxiv").lower()
263
+ if source not in SCHOLAR_SOURCES:
264
+ raise ValueError(f"unsupported scholarly source: {source}")
265
+ query = (query or "").strip()
266
+ if not query:
267
+ raise ValueError("query must not be empty")
268
+
269
+ config = config or engine.FetchConfig()
270
+ limit = max(1, min(limit or config.num_results or 10, 50))
271
+ own_fetcher = fetcher is None
272
+ fetcher = fetcher or engine.HttpFetcher(config)
273
+
274
+ try:
275
+ if source == "arxiv":
276
+ if sort not in ("relevance", "submittedDate", "lastUpdatedDate"):
277
+ raise ValueError("arXiv sort must be relevance, submittedDate or lastUpdatedDate")
278
+ url = _ARXIV_API.format(
279
+ query=urllib.parse.quote_plus(query), start=0, limit=limit, sort=sort
280
+ )
281
+ response = fetcher.fetch(url, check_robots=False)
282
+ records = parse_arxiv(response.text, limit=limit)
283
+ elif source == "crossref":
284
+ url = _CROSSREF_API.format(query=urllib.parse.quote_plus(query), limit=limit)
285
+ records = parse_crossref(_fetch_json(fetcher, url), limit=limit)
286
+ elif source == "doi":
287
+ doi = query
288
+ if doi.lower().startswith("https://doi.org/"):
289
+ doi = doi[len("https://doi.org/"):]
290
+ elif doi.lower().startswith("doi:"):
291
+ doi = doi[4:]
292
+ url = _CROSSREF_DOI.format(doi=urllib.parse.quote(doi, safe="/"))
293
+ records = parse_crossref(_fetch_json(fetcher, url), limit=1)
294
+ else: # pubmed
295
+ search_url = _PUBMED_SEARCH.format(
296
+ query=urllib.parse.quote_plus(query), limit=limit
297
+ )
298
+ search_payload = _fetch_json(fetcher, search_url)
299
+ ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
300
+ if not ids:
301
+ records = []
302
+ else:
303
+ summary_url = _PUBMED_SUMMARY.format(ids=",".join(ids))
304
+ summary_payload = _fetch_json(fetcher, summary_url)
305
+ abstracts: dict[str, str] = {}
306
+ try:
307
+ fetch_url = _PUBMED_FETCH.format(ids=",".join(ids))
308
+ abstracts = _pubmed_abstracts(fetcher.fetch(fetch_url, check_robots=False).text)
309
+ except engine.FetchError:
310
+ abstracts = {}
311
+ records = parse_pubmed(search_payload, summary_payload, abstracts, limit=limit)
312
+
313
+ return {
314
+ "source": source,
315
+ "query": query,
316
+ "count": len(records),
317
+ "results": records,
318
+ }
319
+ finally:
320
+ if own_fetcher:
321
+ fetcher.close()
322
+
323
+
324
+ def to_markdown(payload: dict[str, Any]) -> str:
325
+ """Render scholarly records as a compact Markdown bibliography."""
326
+
327
+ lines = [
328
+ f"# {payload.get('source', 'scholar')}: {payload.get('query', '')}",
329
+ "",
330
+ f"> {payload.get('count', 0)} record(s)",
331
+ "",
332
+ ]
333
+ for index, record in enumerate(payload.get("results") or [], start=1):
334
+ title = record.get("title") or record.get("id") or "Untitled"
335
+ lines.append(f"## {index}. {title}")
336
+ if record.get("authors"):
337
+ lines.append(f"**Authors:** {', '.join(record['authors'])}")
338
+ if record.get("published"):
339
+ lines.append(f"**Published:** {record['published']}")
340
+ if record.get("journal"):
341
+ volume_issue = " ".join(
342
+ part for part in (record.get("volume"), record.get("issue")) if part
343
+ )
344
+ lines.append(f"**Journal:** {record['journal']}" + (f" {volume_issue}" if volume_issue else ""))
345
+ if record.get("doi"):
346
+ lines.append(f"**DOI:** {record['doi']}")
347
+ if record.get("url"):
348
+ lines.append(f"**URL:** {record['url']}")
349
+ if record.get("pdf_url"):
350
+ lines.append(f"**PDF:** {record['pdf_url']}")
351
+ if record.get("abstract"):
352
+ lines.extend(["", record["abstract"]])
353
+ lines.append("")
354
+ return "\n".join(lines).strip()