web-forager 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +112 -0
- package/CHANGELOG.md +454 -0
- package/LICENSE +21 -0
- package/README.md +477 -3
- package/README_zh.md +477 -0
- package/SKILL.md +351 -0
- package/THIRD_PARTY_LICENSES.txt +35 -0
- package/browser/bridges/playwright-bridge.js +375 -0
- package/browser/launch_chrome.py +144 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +79 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +115 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +116 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +56 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +272 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +105 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +432 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +308 -0
- package/docs/REFERENCE_zh.md +309 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +97 -0
- package/mcp_server.py +597 -0
- package/package.json +80 -6
- package/requirements.txt +12 -0
- package/run.py +852 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +874 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +4220 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +135 -0
- package/scripts/scholar.py +354 -0
- package/scripts/sitemap.py +180 -0
- package/scripts/untrusted.py +95 -0
package/scripts/feeds.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""RSS 2.0 / Atom / RSS 1.0 feed parsing.
|
|
2
|
+
|
|
3
|
+
Feeds are structured XML, so they are far cheaper to read than the rendered
|
|
4
|
+
HTML of a blog index. Parsing goes through :func:`engine.parse_xml`, which
|
|
5
|
+
rejects DOCTYPE/ENTITY and uses defusedxml when available.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import html as html_lib
|
|
11
|
+
import re
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from . import engine
|
|
15
|
+
|
|
16
|
+
_ATOM = "{http://www.w3.org/2005/Atom}"
|
|
17
|
+
_RSS1 = "{http://purl.org/rss/1.0/}"
|
|
18
|
+
_DC = "{http://purl.org/dc/elements/1.1/}"
|
|
19
|
+
_CONTENT = "{http://purl.org/rss/1.0/modules/content/}"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _clean(value: Any, limit: int = 1200) -> str:
|
|
23
|
+
text = html_lib.unescape(str(value or ""))
|
|
24
|
+
if "<" in text:
|
|
25
|
+
text = re.sub(r"<[^>]+>", " ", text)
|
|
26
|
+
text = re.sub(r"\s+", " ", text).strip()
|
|
27
|
+
return text[:limit]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _text(node: Any, path: str = "") -> str:
|
|
31
|
+
target = node.find(path) if path else node
|
|
32
|
+
return (target.text or "").strip() if target is not None else ""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _atom_link(entry: Any) -> str:
|
|
36
|
+
fallback = ""
|
|
37
|
+
for link in entry.findall(f"{_ATOM}link"):
|
|
38
|
+
href = link.get("href", "")
|
|
39
|
+
if link.get("rel", "alternate") == "alternate":
|
|
40
|
+
return href
|
|
41
|
+
fallback = fallback or href
|
|
42
|
+
return fallback
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _atom_entry(entry: Any) -> dict[str, Any]:
|
|
46
|
+
return {
|
|
47
|
+
"title": _clean(_text(entry, f"{_ATOM}title"), 300),
|
|
48
|
+
"url": _atom_link(entry),
|
|
49
|
+
"published": _text(entry, f"{_ATOM}published") or _text(entry, f"{_ATOM}updated"),
|
|
50
|
+
"summary": _clean(
|
|
51
|
+
_text(entry, f"{_ATOM}summary") or _text(entry, f"{_ATOM}content")
|
|
52
|
+
),
|
|
53
|
+
"authors": [
|
|
54
|
+
_clean(_text(author, f"{_ATOM}name"), 120)
|
|
55
|
+
for author in entry.findall(f"{_ATOM}author")
|
|
56
|
+
if _text(author, f"{_ATOM}name")
|
|
57
|
+
],
|
|
58
|
+
"categories": [
|
|
59
|
+
category.get("term", "")
|
|
60
|
+
for category in entry.findall(f"{_ATOM}category")
|
|
61
|
+
if category.get("term")
|
|
62
|
+
],
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _rss_item(item: Any, rss1: bool = False) -> dict[str, Any]:
|
|
67
|
+
ns = _RSS1 if rss1 else ""
|
|
68
|
+
summary = _text(item, f"{ns}description")
|
|
69
|
+
if not summary:
|
|
70
|
+
summary = _text(item, f"{_CONTENT}encoded")
|
|
71
|
+
author = _text(item, f"{_DC}creator") or _text(item, f"{ns}author")
|
|
72
|
+
return {
|
|
73
|
+
"title": _clean(_text(item, f"{ns}title"), 300),
|
|
74
|
+
"url": _text(item, f"{ns}link") or _text(item, f"{ns}guid"),
|
|
75
|
+
"published": _text(item, f"{ns}pubDate") or _text(item, f"{_DC}date"),
|
|
76
|
+
"summary": _clean(summary),
|
|
77
|
+
"authors": [_clean(author, 120)] if author else [],
|
|
78
|
+
"categories": [
|
|
79
|
+
_clean(category.text, 80)
|
|
80
|
+
for category in item.findall(f"{ns}category")
|
|
81
|
+
if _clean(category.text)
|
|
82
|
+
],
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def is_feed(xml_text: str) -> bool:
|
|
87
|
+
"""Cheaply decide whether an XML payload is an RSS/Atom feed."""
|
|
88
|
+
|
|
89
|
+
head = xml_text.lstrip()[:2000].lower()
|
|
90
|
+
if not head.startswith("<?xml") and not head.startswith("<rss") and not head.startswith("<feed"):
|
|
91
|
+
return False
|
|
92
|
+
try:
|
|
93
|
+
root = engine.parse_xml(xml_text)
|
|
94
|
+
except engine.FetchError:
|
|
95
|
+
return False
|
|
96
|
+
tag = root.tag.lower()
|
|
97
|
+
return tag == "rss" or tag.endswith("}feed") or tag.endswith("}rdf")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def extract(xml_text: str, limit: int = 50) -> dict[str, Any]:
|
|
101
|
+
"""Parse a feed and return a Markdown digest plus structured items."""
|
|
102
|
+
|
|
103
|
+
root = engine.parse_xml(xml_text)
|
|
104
|
+
tag = root.tag.lower()
|
|
105
|
+
if tag.endswith("}feed") or tag == "feed":
|
|
106
|
+
feed_format = "atom"
|
|
107
|
+
channel = root
|
|
108
|
+
entries = root.findall(f"{_ATOM}entry")[:limit]
|
|
109
|
+
items = [_atom_entry(entry) for entry in entries]
|
|
110
|
+
title = _clean(_text(channel, f"{_ATOM}title"), 300)
|
|
111
|
+
site_url = _atom_link(channel)
|
|
112
|
+
description = _clean(_text(channel, f"{_ATOM}subtitle"))
|
|
113
|
+
elif tag == "rss":
|
|
114
|
+
feed_format = "rss"
|
|
115
|
+
channel = root.find("channel")
|
|
116
|
+
if channel is None:
|
|
117
|
+
raise engine.ExtractionError("RSS feed has no channel element")
|
|
118
|
+
items = [_rss_item(item) for item in channel.findall("item")[:limit]]
|
|
119
|
+
title = _clean(_text(channel, "title"), 300)
|
|
120
|
+
site_url = _text(channel, "link")
|
|
121
|
+
description = _clean(_text(channel, "description"))
|
|
122
|
+
elif tag.endswith("}rdf"):
|
|
123
|
+
feed_format = "rss1"
|
|
124
|
+
channel = root.find(f"{_RSS1}channel")
|
|
125
|
+
items = [_rss_item(item, rss1=True) for item in root.findall(f"{_RSS1}item")[:limit]]
|
|
126
|
+
title = _clean(_text(channel, f"{_RSS1}title") if channel is not None else "", 300)
|
|
127
|
+
site_url = _text(channel, f"{_RSS1}link") if channel is not None else ""
|
|
128
|
+
description = _clean(_text(channel, f"{_RSS1}description") if channel is not None else "")
|
|
129
|
+
else:
|
|
130
|
+
raise engine.ExtractionError(f"not a feed: unexpected root element {root.tag}")
|
|
131
|
+
|
|
132
|
+
lines = [f"# {title or 'Feed'}", ""]
|
|
133
|
+
if description:
|
|
134
|
+
lines.extend([description, ""])
|
|
135
|
+
if site_url:
|
|
136
|
+
lines.extend([f"> {site_url}", ""])
|
|
137
|
+
for index, item in enumerate(items, start=1):
|
|
138
|
+
lines.append(f"## {index}. {item['title'] or item['url'] or 'Untitled'}")
|
|
139
|
+
if item["published"]:
|
|
140
|
+
lines.append(f"**Published:** {item['published']}")
|
|
141
|
+
if item["authors"]:
|
|
142
|
+
lines.append(f"**Authors:** {', '.join(item['authors'])}")
|
|
143
|
+
if item["url"]:
|
|
144
|
+
lines.append(f"**URL:** {item['url']}")
|
|
145
|
+
if item["summary"]:
|
|
146
|
+
lines.extend(["", item["summary"]])
|
|
147
|
+
lines.append("")
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
"kind": "feed",
|
|
151
|
+
"format": feed_format,
|
|
152
|
+
"title": title,
|
|
153
|
+
"site_url": site_url,
|
|
154
|
+
"description": description,
|
|
155
|
+
"count": len(items),
|
|
156
|
+
"items": items,
|
|
157
|
+
"markdown": "\n".join(lines).strip(),
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
|
|
162
|
+
return {
|
|
163
|
+
"title": document.get("title", ""),
|
|
164
|
+
"authors": [],
|
|
165
|
+
"published": "",
|
|
166
|
+
"modified": "",
|
|
167
|
+
"journal": "",
|
|
168
|
+
"volume": "",
|
|
169
|
+
"issue": "",
|
|
170
|
+
"first_page": "",
|
|
171
|
+
"last_page": "",
|
|
172
|
+
"doi": "",
|
|
173
|
+
"arxiv_id": "",
|
|
174
|
+
"abstract": document.get("description", ""),
|
|
175
|
+
"keywords": [],
|
|
176
|
+
"language": "",
|
|
177
|
+
"site_name": document.get("format", "feed"),
|
|
178
|
+
"canonical_url": url or document.get("site_url", ""),
|
|
179
|
+
"og_type": "feed",
|
|
180
|
+
}
|
package/scripts/pdf.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""PDF text extraction for web-forager.
|
|
2
|
+
|
|
3
|
+
The only third-party dependency here is ``pypdf`` (BSD-3-Clause). It is an
|
|
4
|
+
optional dependency: when it is absent, callers receive a clear
|
|
5
|
+
``PdfUnavailable`` error instead of a traceback.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import io
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
PDF_MAGIC = b"%PDF-"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class PdfError(Exception):
|
|
17
|
+
"""Base class for PDF failures."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class PdfUnavailable(PdfError):
|
|
21
|
+
"""pypdf is not installed or could not be imported."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class PdfExtractionError(PdfError):
|
|
25
|
+
"""The file is not a readable PDF."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def is_pdf(data: bytes) -> bool:
|
|
29
|
+
return bool(data) and data[:5] == PDF_MAGIC
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _meta_value(metadata: Any, key: str) -> str:
|
|
33
|
+
try:
|
|
34
|
+
value = metadata.get(key) if metadata else None
|
|
35
|
+
except Exception:
|
|
36
|
+
return ""
|
|
37
|
+
return str(value).strip() if value else ""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def extract(data: bytes) -> dict[str, Any]:
|
|
41
|
+
"""Extract text and document metadata from PDF bytes."""
|
|
42
|
+
|
|
43
|
+
if not is_pdf(data):
|
|
44
|
+
raise PdfExtractionError("the supplied data is not a PDF (missing %PDF- header)")
|
|
45
|
+
try:
|
|
46
|
+
from pypdf import PdfReader # type: ignore
|
|
47
|
+
except ImportError as exc:
|
|
48
|
+
raise PdfUnavailable(
|
|
49
|
+
"pypdf is not installed; enable PDF support with: pip install pypdf"
|
|
50
|
+
) from exc
|
|
51
|
+
except Exception as exc: # pragma: no cover - broken installation
|
|
52
|
+
raise PdfUnavailable(f"pypdf could not be imported ({exc}); reinstall it with: pip install pypdf") from exc
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
reader = PdfReader(io.BytesIO(data))
|
|
56
|
+
except Exception as exc:
|
|
57
|
+
raise PdfExtractionError(f"cannot open PDF: {exc}") from exc
|
|
58
|
+
|
|
59
|
+
pages: list[str] = []
|
|
60
|
+
empty_pages = 0
|
|
61
|
+
for index, page in enumerate(reader.pages, start=1):
|
|
62
|
+
try:
|
|
63
|
+
text = page.extract_text() or ""
|
|
64
|
+
except Exception as exc: # a single damaged page should not kill the run
|
|
65
|
+
text = f"[page {index}: extraction failed: {exc}]"
|
|
66
|
+
text = text.strip()
|
|
67
|
+
if not text:
|
|
68
|
+
empty_pages += 1
|
|
69
|
+
pages.append(text)
|
|
70
|
+
|
|
71
|
+
metadata = reader.metadata
|
|
72
|
+
return {
|
|
73
|
+
"pages": pages,
|
|
74
|
+
"page_count": len(pages),
|
|
75
|
+
"empty_pages": empty_pages,
|
|
76
|
+
"text": "\n\n".join(pages).strip(),
|
|
77
|
+
"metadata": {
|
|
78
|
+
"title": _meta_value(metadata, "/Title"),
|
|
79
|
+
"author": _meta_value(metadata, "/Author"),
|
|
80
|
+
"subject": _meta_value(metadata, "/Subject"),
|
|
81
|
+
"creator": _meta_value(metadata, "/Creator"),
|
|
82
|
+
"producer": _meta_value(metadata, "/Producer"),
|
|
83
|
+
"creation_date": _meta_value(metadata, "/CreationDate"),
|
|
84
|
+
},
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def to_markdown(document: dict[str, Any], url: str = "") -> str:
|
|
89
|
+
metadata = document.get("metadata") or {}
|
|
90
|
+
title = metadata.get("title") or (url.rsplit("/", 1)[-1] if url else "PDF document")
|
|
91
|
+
lines = [f"# {title}", ""]
|
|
92
|
+
info = " | ".join(
|
|
93
|
+
part
|
|
94
|
+
for part in (
|
|
95
|
+
f"作者:{metadata.get('author')}" if metadata.get("author") else "",
|
|
96
|
+
f"页数:{document.get('page_count')}" if document.get("page_count") else "",
|
|
97
|
+
f"来源:{url}" if url else "",
|
|
98
|
+
)
|
|
99
|
+
if part
|
|
100
|
+
)
|
|
101
|
+
if info:
|
|
102
|
+
lines.extend([f"> {info}", ""])
|
|
103
|
+
pages = document.get("pages") or []
|
|
104
|
+
for index, text in enumerate(pages, start=1):
|
|
105
|
+
if not text:
|
|
106
|
+
continue
|
|
107
|
+
if len(pages) > 1:
|
|
108
|
+
lines.extend([f"## 第 {index} 页", ""])
|
|
109
|
+
lines.extend([text, ""])
|
|
110
|
+
return "\n".join(lines).strip()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def to_metadata(document: dict[str, Any], url: str = "") -> dict[str, Any]:
|
|
114
|
+
"""Return PDF metadata in the same shape as article metadata."""
|
|
115
|
+
|
|
116
|
+
source = document.get("metadata") or {}
|
|
117
|
+
return {
|
|
118
|
+
"title": source.get("title", ""),
|
|
119
|
+
"authors": [source["author"]] if source.get("author") else [],
|
|
120
|
+
"published": source.get("creation_date", ""),
|
|
121
|
+
"modified": "",
|
|
122
|
+
"journal": "",
|
|
123
|
+
"volume": "",
|
|
124
|
+
"issue": "",
|
|
125
|
+
"first_page": "",
|
|
126
|
+
"last_page": "",
|
|
127
|
+
"doi": "",
|
|
128
|
+
"arxiv_id": "",
|
|
129
|
+
"abstract": source.get("subject", ""),
|
|
130
|
+
"keywords": [],
|
|
131
|
+
"language": "",
|
|
132
|
+
"site_name": "PDF",
|
|
133
|
+
"canonical_url": url,
|
|
134
|
+
"og_type": "pdf",
|
|
135
|
+
}
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
"""Scholarly metadata through public APIs (arXiv, Crossref, PubMed).
|
|
2
|
+
|
|
3
|
+
These are documented query APIs, not scraped result pages, so they are both
|
|
4
|
+
more stable and more polite than reading a search engine's HTML. Only the
|
|
5
|
+
standard library and ``requests`` (through :mod:`scripts.engine`) are used.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import html as html_lib
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import xml.etree.ElementTree as ET
|
|
15
|
+
from typing import Any, Optional
|
|
16
|
+
|
|
17
|
+
from . import engine
|
|
18
|
+
|
|
19
|
+
SCHOLAR_SOURCES = ("arxiv", "crossref", "pubmed", "doi")
|
|
20
|
+
|
|
21
|
+
_ARXIV_API = (
|
|
22
|
+
"https://export.arxiv.org/api/query?search_query=all:{query}"
|
|
23
|
+
"&start={start}&max_results={limit}&sortBy={sort}&sortOrder=descending"
|
|
24
|
+
)
|
|
25
|
+
_CROSSREF_API = (
|
|
26
|
+
"https://api.crossref.org/works?query.bibliographic={query}&rows={limit}"
|
|
27
|
+
"&select=DOI,title,author,container-title,issued,published-print,published-online,"
|
|
28
|
+
"abstract,URL,type,subject,ISSN,volume,issue,page"
|
|
29
|
+
)
|
|
30
|
+
_CROSSREF_DOI = "https://api.crossref.org/works/{doi}"
|
|
31
|
+
_PUBMED_SEARCH = (
|
|
32
|
+
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
|
|
33
|
+
"?db=pubmed&term={query}&retmode=json&retmax={limit}&sort=relevance"
|
|
34
|
+
)
|
|
35
|
+
_PUBMED_SUMMARY = (
|
|
36
|
+
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
|
|
37
|
+
"?db=pubmed&id={ids}&retmode=json"
|
|
38
|
+
)
|
|
39
|
+
_PUBMED_FETCH = (
|
|
40
|
+
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
|
|
41
|
+
"?db=pubmed&id={ids}&retmode=xml"
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
_ATOM = "{http://www.w3.org/2005/Atom}"
|
|
45
|
+
_ARXIV = "{http://arxiv.org/schemas/atom}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _text(node: Optional[ET.Element], path: str = "") -> str:
|
|
49
|
+
target = node.find(path) if node is not None and path else node
|
|
50
|
+
if target is None or target.text is None:
|
|
51
|
+
return ""
|
|
52
|
+
return re.sub(r"\s+", " ", target.text).strip()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _clean(value: Any) -> str:
|
|
56
|
+
if value is None:
|
|
57
|
+
return ""
|
|
58
|
+
text = html_lib.unescape(str(value))
|
|
59
|
+
if "<" in text:
|
|
60
|
+
text = re.sub(r"<[^>]+>", " ", text)
|
|
61
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _date_parts(value: Any) -> str:
|
|
65
|
+
if not isinstance(value, dict):
|
|
66
|
+
return ""
|
|
67
|
+
parts = (value.get("date-parts") or [[]])[0]
|
|
68
|
+
if not parts:
|
|
69
|
+
return ""
|
|
70
|
+
return "-".join(f"{int(part):02d}" if index else str(part) for index, part in enumerate(parts))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def parse_arxiv(xml_text: str, limit: int = 10) -> list[dict[str, Any]]:
|
|
74
|
+
"""Parse an arXiv Atom feed into normalised records."""
|
|
75
|
+
|
|
76
|
+
root = engine.parse_xml(xml_text)
|
|
77
|
+
|
|
78
|
+
records: list[dict[str, Any]] = []
|
|
79
|
+
for entry in root.findall(f"{_ATOM}entry")[:limit]:
|
|
80
|
+
identifier = _text(entry, f"{_ATOM}id")
|
|
81
|
+
abs_url = identifier
|
|
82
|
+
pdf_url = ""
|
|
83
|
+
for link in entry.findall(f"{_ATOM}link"):
|
|
84
|
+
if link.get("title") == "pdf" or link.get("type") == "application/pdf":
|
|
85
|
+
pdf_url = link.get("href", "")
|
|
86
|
+
authors = [
|
|
87
|
+
_text(author, f"{_ATOM}name")
|
|
88
|
+
for author in entry.findall(f"{_ATOM}author")
|
|
89
|
+
if _text(author, f"{_ATOM}name")
|
|
90
|
+
]
|
|
91
|
+
categories = [
|
|
92
|
+
category.get("term", "")
|
|
93
|
+
for category in entry.findall(f"{_ATOM}category")
|
|
94
|
+
if category.get("term")
|
|
95
|
+
]
|
|
96
|
+
journal = _text(entry, f"{_ARXIV}journal_ref")
|
|
97
|
+
doi = _text(entry, f"{_ARXIV}doi")
|
|
98
|
+
records.append(
|
|
99
|
+
{
|
|
100
|
+
"source": "arxiv",
|
|
101
|
+
"id": identifier.rsplit("/", 1)[-1],
|
|
102
|
+
"title": _text(entry, f"{_ATOM}title"),
|
|
103
|
+
"authors": authors,
|
|
104
|
+
"published": _text(entry, f"{_ATOM}published")[:10],
|
|
105
|
+
"updated": _text(entry, f"{_ATOM}updated")[:10],
|
|
106
|
+
"journal": journal,
|
|
107
|
+
"volume": "",
|
|
108
|
+
"issue": "",
|
|
109
|
+
"pages": "",
|
|
110
|
+
"doi": doi,
|
|
111
|
+
"url": abs_url,
|
|
112
|
+
"pdf_url": pdf_url,
|
|
113
|
+
"abstract": _text(entry, f"{_ATOM}summary"),
|
|
114
|
+
"type": "preprint",
|
|
115
|
+
"keywords": categories,
|
|
116
|
+
}
|
|
117
|
+
)
|
|
118
|
+
return records
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def parse_crossref(payload: Any, limit: int = 10) -> list[dict[str, Any]]:
|
|
122
|
+
"""Parse a Crossref ``works`` response (already decoded JSON)."""
|
|
123
|
+
|
|
124
|
+
if isinstance(payload, str):
|
|
125
|
+
payload = json.loads(payload)
|
|
126
|
+
message = payload.get("message", payload) if isinstance(payload, dict) else {}
|
|
127
|
+
items = message.get("items")
|
|
128
|
+
if items is None and message.get("DOI"):
|
|
129
|
+
items = [message]
|
|
130
|
+
records: list[dict[str, Any]] = []
|
|
131
|
+
for item in (items or [])[:limit]:
|
|
132
|
+
authors: list[str] = []
|
|
133
|
+
for author in item.get("author") or []:
|
|
134
|
+
name = author.get("name") or " ".join(
|
|
135
|
+
part for part in (author.get("given"), author.get("family")) if part
|
|
136
|
+
)
|
|
137
|
+
if name:
|
|
138
|
+
authors.append(name)
|
|
139
|
+
titles = item.get("title") or []
|
|
140
|
+
title = titles[0] if isinstance(titles, list) and titles else str(titles or "")
|
|
141
|
+
containers = item.get("container-title") or []
|
|
142
|
+
journal = containers[0] if isinstance(containers, list) and containers else str(containers or "")
|
|
143
|
+
published = (
|
|
144
|
+
_date_parts(item.get("published-print"))
|
|
145
|
+
or _date_parts(item.get("published-online"))
|
|
146
|
+
or _date_parts(item.get("issued"))
|
|
147
|
+
)
|
|
148
|
+
records.append(
|
|
149
|
+
{
|
|
150
|
+
"source": "crossref",
|
|
151
|
+
"id": item.get("DOI", ""),
|
|
152
|
+
"title": _clean(title),
|
|
153
|
+
"authors": authors,
|
|
154
|
+
"published": published,
|
|
155
|
+
"updated": "",
|
|
156
|
+
"journal": _clean(journal),
|
|
157
|
+
"volume": str(item.get("volume", "") or ""),
|
|
158
|
+
"issue": str(item.get("issue", "") or ""),
|
|
159
|
+
"pages": str(item.get("page", "") or ""),
|
|
160
|
+
"doi": item.get("DOI", ""),
|
|
161
|
+
"url": item.get("URL", "") or (
|
|
162
|
+
f"https://doi.org/{item.get('DOI')}" if item.get("DOI") else ""
|
|
163
|
+
),
|
|
164
|
+
"pdf_url": "",
|
|
165
|
+
"abstract": _clean(item.get("abstract", "")),
|
|
166
|
+
"type": item.get("type", ""),
|
|
167
|
+
"keywords": [str(value) for value in item.get("subject") or []],
|
|
168
|
+
}
|
|
169
|
+
)
|
|
170
|
+
return records
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def parse_pubmed(
|
|
174
|
+
search_payload: Any,
|
|
175
|
+
summary_payload: Any,
|
|
176
|
+
abstracts: Optional[dict[str, str]] = None,
|
|
177
|
+
limit: int = 10,
|
|
178
|
+
) -> list[dict[str, Any]]:
|
|
179
|
+
"""Merge PubMed esearch + esummary (+ optional efetch abstracts)."""
|
|
180
|
+
|
|
181
|
+
if isinstance(search_payload, str):
|
|
182
|
+
search_payload = json.loads(search_payload)
|
|
183
|
+
if isinstance(summary_payload, str):
|
|
184
|
+
summary_payload = json.loads(summary_payload)
|
|
185
|
+
abstracts = abstracts or {}
|
|
186
|
+
|
|
187
|
+
ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
|
|
188
|
+
result = summary_payload.get("result", {}) or {}
|
|
189
|
+
records: list[dict[str, Any]] = []
|
|
190
|
+
for uid in ids:
|
|
191
|
+
item = result.get(uid) or {}
|
|
192
|
+
doi = ""
|
|
193
|
+
for article_id in item.get("articleids") or []:
|
|
194
|
+
if article_id.get("idtype") == "doi":
|
|
195
|
+
doi = article_id.get("value", "")
|
|
196
|
+
break
|
|
197
|
+
records.append(
|
|
198
|
+
{
|
|
199
|
+
"source": "pubmed",
|
|
200
|
+
"id": uid,
|
|
201
|
+
"title": _clean(item.get("title", "")),
|
|
202
|
+
"authors": [
|
|
203
|
+
author.get("name", "")
|
|
204
|
+
for author in item.get("authors") or []
|
|
205
|
+
if author.get("name")
|
|
206
|
+
],
|
|
207
|
+
"published": str(item.get("pubdate", "") or ""),
|
|
208
|
+
"updated": "",
|
|
209
|
+
"journal": str(item.get("fulljournalname") or item.get("source") or ""),
|
|
210
|
+
"volume": str(item.get("volume", "") or ""),
|
|
211
|
+
"issue": str(item.get("issue", "") or ""),
|
|
212
|
+
"pages": str(item.get("pages", "") or ""),
|
|
213
|
+
"doi": doi,
|
|
214
|
+
"url": f"https://pubmed.ncbi.nlm.nih.gov/{uid}/",
|
|
215
|
+
"pdf_url": "",
|
|
216
|
+
"abstract": abstracts.get(uid, ""),
|
|
217
|
+
"type": "journal-article",
|
|
218
|
+
"keywords": [],
|
|
219
|
+
}
|
|
220
|
+
)
|
|
221
|
+
return records
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _pubmed_abstracts(xml_text: str) -> dict[str, str]:
|
|
225
|
+
"""Map PMID → abstract from an efetch XML response."""
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
root = engine.parse_xml(xml_text)
|
|
229
|
+
except engine.FetchError:
|
|
230
|
+
return {}
|
|
231
|
+
abstracts: dict[str, str] = {}
|
|
232
|
+
for article in root.findall(".//PubmedArticle"):
|
|
233
|
+
pmid = _text(article, ".//PMID")
|
|
234
|
+
parts = [
|
|
235
|
+
_clean(node.text)
|
|
236
|
+
for node in article.findall(".//Abstract/AbstractText")
|
|
237
|
+
if node.text
|
|
238
|
+
]
|
|
239
|
+
if pmid and parts:
|
|
240
|
+
abstracts[pmid] = " ".join(part for part in parts if part)
|
|
241
|
+
return abstracts
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _fetch_json(fetcher: engine.HttpFetcher, url: str) -> Any:
|
|
245
|
+
response = fetcher.fetch(url, check_robots=False)
|
|
246
|
+
try:
|
|
247
|
+
return json.loads(response.text)
|
|
248
|
+
except json.JSONDecodeError as exc:
|
|
249
|
+
raise engine.FetchError(f"invalid JSON from {url}: {exc}") from exc
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def search(
|
|
253
|
+
source: str,
|
|
254
|
+
query: str,
|
|
255
|
+
config: Optional[engine.FetchConfig] = None,
|
|
256
|
+
limit: Optional[int] = None,
|
|
257
|
+
fetcher: Optional[engine.HttpFetcher] = None,
|
|
258
|
+
sort: str = "relevance",
|
|
259
|
+
) -> dict[str, Any]:
|
|
260
|
+
"""Query one scholarly source and return normalised records."""
|
|
261
|
+
|
|
262
|
+
source = (source or "arxiv").lower()
|
|
263
|
+
if source not in SCHOLAR_SOURCES:
|
|
264
|
+
raise ValueError(f"unsupported scholarly source: {source}")
|
|
265
|
+
query = (query or "").strip()
|
|
266
|
+
if not query:
|
|
267
|
+
raise ValueError("query must not be empty")
|
|
268
|
+
|
|
269
|
+
config = config or engine.FetchConfig()
|
|
270
|
+
limit = max(1, min(limit or config.num_results or 10, 50))
|
|
271
|
+
own_fetcher = fetcher is None
|
|
272
|
+
fetcher = fetcher or engine.HttpFetcher(config)
|
|
273
|
+
|
|
274
|
+
try:
|
|
275
|
+
if source == "arxiv":
|
|
276
|
+
if sort not in ("relevance", "submittedDate", "lastUpdatedDate"):
|
|
277
|
+
raise ValueError("arXiv sort must be relevance, submittedDate or lastUpdatedDate")
|
|
278
|
+
url = _ARXIV_API.format(
|
|
279
|
+
query=urllib.parse.quote_plus(query), start=0, limit=limit, sort=sort
|
|
280
|
+
)
|
|
281
|
+
response = fetcher.fetch(url, check_robots=False)
|
|
282
|
+
records = parse_arxiv(response.text, limit=limit)
|
|
283
|
+
elif source == "crossref":
|
|
284
|
+
url = _CROSSREF_API.format(query=urllib.parse.quote_plus(query), limit=limit)
|
|
285
|
+
records = parse_crossref(_fetch_json(fetcher, url), limit=limit)
|
|
286
|
+
elif source == "doi":
|
|
287
|
+
doi = query
|
|
288
|
+
if doi.lower().startswith("https://doi.org/"):
|
|
289
|
+
doi = doi[len("https://doi.org/"):]
|
|
290
|
+
elif doi.lower().startswith("doi:"):
|
|
291
|
+
doi = doi[4:]
|
|
292
|
+
url = _CROSSREF_DOI.format(doi=urllib.parse.quote(doi, safe="/"))
|
|
293
|
+
records = parse_crossref(_fetch_json(fetcher, url), limit=1)
|
|
294
|
+
else: # pubmed
|
|
295
|
+
search_url = _PUBMED_SEARCH.format(
|
|
296
|
+
query=urllib.parse.quote_plus(query), limit=limit
|
|
297
|
+
)
|
|
298
|
+
search_payload = _fetch_json(fetcher, search_url)
|
|
299
|
+
ids = (search_payload.get("esearchresult", {}) or {}).get("idlist", [])[:limit]
|
|
300
|
+
if not ids:
|
|
301
|
+
records = []
|
|
302
|
+
else:
|
|
303
|
+
summary_url = _PUBMED_SUMMARY.format(ids=",".join(ids))
|
|
304
|
+
summary_payload = _fetch_json(fetcher, summary_url)
|
|
305
|
+
abstracts: dict[str, str] = {}
|
|
306
|
+
try:
|
|
307
|
+
fetch_url = _PUBMED_FETCH.format(ids=",".join(ids))
|
|
308
|
+
abstracts = _pubmed_abstracts(fetcher.fetch(fetch_url, check_robots=False).text)
|
|
309
|
+
except engine.FetchError:
|
|
310
|
+
abstracts = {}
|
|
311
|
+
records = parse_pubmed(search_payload, summary_payload, abstracts, limit=limit)
|
|
312
|
+
|
|
313
|
+
return {
|
|
314
|
+
"source": source,
|
|
315
|
+
"query": query,
|
|
316
|
+
"count": len(records),
|
|
317
|
+
"results": records,
|
|
318
|
+
}
|
|
319
|
+
finally:
|
|
320
|
+
if own_fetcher:
|
|
321
|
+
fetcher.close()
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def to_markdown(payload: dict[str, Any]) -> str:
|
|
325
|
+
"""Render scholarly records as a compact Markdown bibliography."""
|
|
326
|
+
|
|
327
|
+
lines = [
|
|
328
|
+
f"# {payload.get('source', 'scholar')}: {payload.get('query', '')}",
|
|
329
|
+
"",
|
|
330
|
+
f"> {payload.get('count', 0)} record(s)",
|
|
331
|
+
"",
|
|
332
|
+
]
|
|
333
|
+
for index, record in enumerate(payload.get("results") or [], start=1):
|
|
334
|
+
title = record.get("title") or record.get("id") or "Untitled"
|
|
335
|
+
lines.append(f"## {index}. {title}")
|
|
336
|
+
if record.get("authors"):
|
|
337
|
+
lines.append(f"**Authors:** {', '.join(record['authors'])}")
|
|
338
|
+
if record.get("published"):
|
|
339
|
+
lines.append(f"**Published:** {record['published']}")
|
|
340
|
+
if record.get("journal"):
|
|
341
|
+
volume_issue = " ".join(
|
|
342
|
+
part for part in (record.get("volume"), record.get("issue")) if part
|
|
343
|
+
)
|
|
344
|
+
lines.append(f"**Journal:** {record['journal']}" + (f" {volume_issue}" if volume_issue else ""))
|
|
345
|
+
if record.get("doi"):
|
|
346
|
+
lines.append(f"**DOI:** {record['doi']}")
|
|
347
|
+
if record.get("url"):
|
|
348
|
+
lines.append(f"**URL:** {record['url']}")
|
|
349
|
+
if record.get("pdf_url"):
|
|
350
|
+
lines.append(f"**PDF:** {record['pdf_url']}")
|
|
351
|
+
if record.get("abstract"):
|
|
352
|
+
lines.extend(["", record["abstract"]])
|
|
353
|
+
lines.append("")
|
|
354
|
+
return "\n".join(lines).strip()
|