litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,620 @@
|
|
|
1
|
+
"""Elsevier RetrievalAPI integration for fetching full-text articles."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import xml.etree.ElementTree as ET
|
|
6
|
+
from urllib.parse import quote
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
ELSEVIER_API = "https://api.elsevier.com/content"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Full text through the API: off in settings (elsevier_api_fulltext), or learned
|
|
16
|
+
# off for this key after the API answered 403 "Requestor configuration settings
|
|
17
|
+
# insufficient" (the key has no full-text entitlement; abstracts and Scopus are
|
|
18
|
+
# unaffected). Remembered per key (a hash, never the key) for 30 days.
|
|
19
|
+
_NO_FULLTEXT_DAYS = 30
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _no_fulltext_file():
|
|
23
|
+
from ..config import DATA_DIR
|
|
24
|
+
return DATA_DIR / "elsevier_no_fulltext.json"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _key_id(api_key: str) -> str:
|
|
28
|
+
import hashlib
|
|
29
|
+
return hashlib.sha256(api_key.encode("utf-8")).hexdigest()[:16]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def mark_no_fulltext(api_key: str) -> None:
|
|
33
|
+
import json
|
|
34
|
+
import time
|
|
35
|
+
if not api_key:
|
|
36
|
+
return
|
|
37
|
+
path = _no_fulltext_file()
|
|
38
|
+
try:
|
|
39
|
+
data = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
|
40
|
+
except (OSError, ValueError):
|
|
41
|
+
data = {}
|
|
42
|
+
data[_key_id(api_key)] = time.time()
|
|
43
|
+
try:
|
|
44
|
+
from .. import store
|
|
45
|
+
store.write_json(path, data)
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def fulltext_enabled(config: dict | None) -> bool:
|
|
51
|
+
"""Whether to ask the Elsevier API for full text at all."""
|
|
52
|
+
import json
|
|
53
|
+
import time
|
|
54
|
+
config = config or {}
|
|
55
|
+
if not config.get("elsevier_api_fulltext", True):
|
|
56
|
+
return False
|
|
57
|
+
key = get_api_key(str(config.get("elsevier_api_key") or ""))
|
|
58
|
+
if not key:
|
|
59
|
+
return False
|
|
60
|
+
try:
|
|
61
|
+
data = json.loads(_no_fulltext_file().read_text(encoding="utf-8"))
|
|
62
|
+
return time.time() - float(data.get(_key_id(key), 0)) > _NO_FULLTEXT_DAYS * 86400
|
|
63
|
+
except (OSError, ValueError, TypeError):
|
|
64
|
+
return True
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_api_key(config_key: str = "") -> str:
|
|
68
|
+
"""Get Elsevier API key from config or environment."""
|
|
69
|
+
return config_key or os.environ.get("ELSEVIER_API_KEY", "")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _resolve_proxies(config: dict | None) -> dict[str, str] | None:
|
|
73
|
+
"""Explicit LitLoop proxy → requests proxies dict (None = direct).
|
|
74
|
+
|
|
75
|
+
Entitlement for subscription full text follows the request's egress IP:
|
|
76
|
+
off-campus direct requests 403 (NOT_ENTITLED) even with a valid key.
|
|
77
|
+
HTTP_PROXY/HTTPS_PROXY are ignored project-wide, so a campus-bound proxy
|
|
78
|
+
must be set via LITLOOP_PROXY / network_proxy — the same setting the
|
|
79
|
+
entitlement probe (elsevier_check) routes through.
|
|
80
|
+
"""
|
|
81
|
+
from ..network import configured_proxy
|
|
82
|
+
|
|
83
|
+
proxy = configured_proxy(config)
|
|
84
|
+
return {"http": proxy, "https": proxy} if proxy else None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def fetch_pdf(doi: str, api_key: str, inst_token: str = "", config: dict | None = None) -> bytes | None:
|
|
88
|
+
"""Download full PDF via Elsevier API using the attachment EID approach.
|
|
89
|
+
|
|
90
|
+
Strategy (from successful 32-paper batch experience):
|
|
91
|
+
1. GET /article/doi/{doi}?view=FULL → XML with attachment metadata
|
|
92
|
+
2. Parse XML to find MAIN PDF attachment-eid (main.pdf or mainext.pdf)
|
|
93
|
+
3. GET /object/eid/{attachment-eid} → official publisher PDF
|
|
94
|
+
|
|
95
|
+
This bypasses ScienceDirect web pages, Cloudflare, and CAPTCHAs entirely.
|
|
96
|
+
Direct PDF endpoint (/article/doi/{doi} + Accept: application/pdf) only
|
|
97
|
+
returns a 1-page preview for paywalled articles.
|
|
98
|
+
"""
|
|
99
|
+
if not api_key:
|
|
100
|
+
return None
|
|
101
|
+
if config is not None and not fulltext_enabled({**config, "elsevier_api_key": api_key}):
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
proxies = _resolve_proxies(config)
|
|
105
|
+
|
|
106
|
+
# Step 1: Get FULL XML with attachment metadata
|
|
107
|
+
eids = _fetch_attachment_eids(doi, api_key, inst_token, proxies)
|
|
108
|
+
if not eids:
|
|
109
|
+
# Fallback: try direct PDF endpoint (works for OA articles)
|
|
110
|
+
return _fetch_pdf_direct(doi, api_key, inst_token, proxies)
|
|
111
|
+
|
|
112
|
+
# Step 2: Try each attachment EID until we get a valid PDF
|
|
113
|
+
for eid in eids:
|
|
114
|
+
pdf_bytes = _fetch_pdf_by_eid(eid, api_key, inst_token, proxies)
|
|
115
|
+
if pdf_bytes:
|
|
116
|
+
return pdf_bytes
|
|
117
|
+
|
|
118
|
+
logger.info("Elsevier API: all attachment EIDs failed for %s, trying direct", doi)
|
|
119
|
+
return _fetch_pdf_direct(doi, api_key, inst_token, proxies)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _local_name(tag: str) -> str:
|
|
123
|
+
return tag.rsplit("}", 1)[-1].split(":", 1)[-1].lower()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _element_text(el: ET.Element) -> str:
|
|
127
|
+
return " ".join(" ".join(el.itertext()).split())
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _header(headers: dict, name: str) -> str:
|
|
131
|
+
value = headers.get(name)
|
|
132
|
+
if value is not None:
|
|
133
|
+
return str(value)
|
|
134
|
+
needle = name.lower()
|
|
135
|
+
for key, value in headers.items():
|
|
136
|
+
if str(key).lower() == needle:
|
|
137
|
+
return str(value)
|
|
138
|
+
return ""
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _response_is_pdf(resp: requests.Response) -> bool:
|
|
142
|
+
content_type = _header(resp.headers, "content-type").lower()
|
|
143
|
+
return "pdf" in content_type or resp.content[:5] == b"%PDF-"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _looks_like_pdf_eid(value: str) -> bool:
|
|
147
|
+
lowered = value.strip().lower()
|
|
148
|
+
return bool(lowered) and (lowered.endswith(".pdf") or ".pdf" in lowered)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _article_eid_to_main_pdf(value: str) -> str:
|
|
152
|
+
candidate = value.strip()
|
|
153
|
+
if candidate.lower().startswith("eid:"):
|
|
154
|
+
candidate = candidate.split(":", 1)[1].strip()
|
|
155
|
+
if not candidate.startswith("1-s2.0-"):
|
|
156
|
+
return ""
|
|
157
|
+
if candidate.lower().endswith(".pdf"):
|
|
158
|
+
return candidate
|
|
159
|
+
return f"{candidate}-main.pdf"
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _attachment_container(el: ET.Element, parent_map: dict[ET.Element, ET.Element]) -> ET.Element:
|
|
163
|
+
node = parent_map.get(el, el)
|
|
164
|
+
while node is not None:
|
|
165
|
+
local = _local_name(str(node.tag))
|
|
166
|
+
if "attachment" in local or "object" in local or local == "web-pdf":
|
|
167
|
+
return node
|
|
168
|
+
node = parent_map.get(node)
|
|
169
|
+
return parent_map.get(el, el)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _attachment_metadata(el: ET.Element) -> str:
|
|
173
|
+
parts: list[str] = []
|
|
174
|
+
for node in el.iter():
|
|
175
|
+
local = _local_name(str(node.tag))
|
|
176
|
+
text = _element_text(node)
|
|
177
|
+
if text:
|
|
178
|
+
parts.append(f"{local}:{text}")
|
|
179
|
+
for attr_name, attr_value in node.attrib.items():
|
|
180
|
+
attr_local = _local_name(str(attr_name))
|
|
181
|
+
if attr_value:
|
|
182
|
+
parts.append(f"{attr_local}:{attr_value}")
|
|
183
|
+
return " ".join(parts).lower()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _attachment_score(eid: str, metadata: str) -> int:
|
|
187
|
+
haystack = f"{eid} {metadata}".lower()
|
|
188
|
+
score = 0
|
|
189
|
+
if eid.lower().endswith(".pdf"):
|
|
190
|
+
score += 20
|
|
191
|
+
if "pdf" in haystack:
|
|
192
|
+
score += 10
|
|
193
|
+
if "main" in haystack or "full-text" in haystack or "fulltext" in haystack:
|
|
194
|
+
score += 100
|
|
195
|
+
if "page-count" in haystack or "pages" in haystack:
|
|
196
|
+
score += 5
|
|
197
|
+
if "attachment-size" in haystack or "filesize" in haystack or "file-size" in haystack:
|
|
198
|
+
score += 5
|
|
199
|
+
if any(
|
|
200
|
+
marker in haystack
|
|
201
|
+
for marker in ("supplement", "supplementary", "mmc", "appendix", "graphical")
|
|
202
|
+
):
|
|
203
|
+
score -= 100
|
|
204
|
+
return score
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _extract_pdf_attachment_eids(xml_text: str) -> list[str]:
|
|
208
|
+
try:
|
|
209
|
+
root = ET.fromstring(xml_text)
|
|
210
|
+
except ET.ParseError as e:
|
|
211
|
+
logger.warning("Failed to parse Elsevier XML attachments: %s", e)
|
|
212
|
+
return []
|
|
213
|
+
|
|
214
|
+
parent_map = {child: parent for parent in root.iter() for child in parent}
|
|
215
|
+
candidates: list[tuple[int, int, str]] = []
|
|
216
|
+
seen: set[str] = set()
|
|
217
|
+
|
|
218
|
+
for el in root.iter():
|
|
219
|
+
local = _local_name(str(el.tag))
|
|
220
|
+
found: list[str] = []
|
|
221
|
+
|
|
222
|
+
if local in {"attachment-eid", "object-eid"}:
|
|
223
|
+
text = _element_text(el)
|
|
224
|
+
if _looks_like_pdf_eid(text):
|
|
225
|
+
found.append(text)
|
|
226
|
+
elif local in {"eid", "identifier"}:
|
|
227
|
+
main_pdf = _article_eid_to_main_pdf(_element_text(el))
|
|
228
|
+
if main_pdf:
|
|
229
|
+
found.append(main_pdf)
|
|
230
|
+
|
|
231
|
+
for attr_name, attr_value in el.attrib.items():
|
|
232
|
+
attr_local = _local_name(str(attr_name))
|
|
233
|
+
if attr_local in {"attachment-eid", "object-eid", "eid"}:
|
|
234
|
+
value = str(attr_value).strip()
|
|
235
|
+
if _looks_like_pdf_eid(value):
|
|
236
|
+
found.append(value)
|
|
237
|
+
else:
|
|
238
|
+
main_pdf = _article_eid_to_main_pdf(value)
|
|
239
|
+
if main_pdf:
|
|
240
|
+
found.append(main_pdf)
|
|
241
|
+
|
|
242
|
+
for eid in found:
|
|
243
|
+
if eid in seen:
|
|
244
|
+
continue
|
|
245
|
+
seen.add(eid)
|
|
246
|
+
metadata = _attachment_metadata(_attachment_container(el, parent_map))
|
|
247
|
+
candidates.append((_attachment_score(eid, metadata), len(candidates), eid))
|
|
248
|
+
|
|
249
|
+
candidates.sort(key=lambda item: (-item[0], item[1]))
|
|
250
|
+
return [eid for _, _, eid in candidates]
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _pdf_page_count(content: bytes) -> int | None:
|
|
254
|
+
try:
|
|
255
|
+
import fitz # type: ignore[import-not-found]
|
|
256
|
+
except Exception:
|
|
257
|
+
return None
|
|
258
|
+
|
|
259
|
+
try:
|
|
260
|
+
with fitz.open(stream=content, filetype="pdf") as doc:
|
|
261
|
+
return int(doc.page_count)
|
|
262
|
+
except Exception:
|
|
263
|
+
return None
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _valid_pdf_bytes(content: bytes, label: str, *, reject_single_page: bool) -> bool:
|
|
267
|
+
if content[:5] != b"%PDF-":
|
|
268
|
+
logger.info("Elsevier API: %s returned non-PDF", label)
|
|
269
|
+
return False
|
|
270
|
+
if len(content) < 10000:
|
|
271
|
+
logger.warning("Elsevier API: %s too small (%d bytes)", label, len(content))
|
|
272
|
+
return False
|
|
273
|
+
if reject_single_page and _pdf_page_count(content) == 1:
|
|
274
|
+
logger.info("Elsevier API: %s is a 1-page preview", label)
|
|
275
|
+
return False
|
|
276
|
+
return True
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _fetch_attachment_eids(
|
|
280
|
+
doi: str, api_key: str, inst_token: str = "", proxies: dict[str, str] | None = None,
|
|
281
|
+
) -> list[str]:
|
|
282
|
+
"""Fetch FULL XML and extract MAIN PDF attachment EIDs."""
|
|
283
|
+
url = f"{ELSEVIER_API}/article/doi/{doi}"
|
|
284
|
+
headers = {
|
|
285
|
+
"X-ELS-APIKey": api_key,
|
|
286
|
+
"Accept": "application/xml",
|
|
287
|
+
}
|
|
288
|
+
if inst_token:
|
|
289
|
+
headers["X-ELS-InstToken"] = inst_token
|
|
290
|
+
|
|
291
|
+
resp = _api_request(url, headers, params={"view": "FULL"}, proxies=proxies)
|
|
292
|
+
if not resp or resp.status_code != 200:
|
|
293
|
+
# The XML → attachment-EID chain starts here; a 403 on this request
|
|
294
|
+
# means the requestor (key + egress IP) is not configured for full
|
|
295
|
+
# text at all — no EID can exist. Name the two real causes so the
|
|
296
|
+
# next step is actionable instead of a re-diagnosis.
|
|
297
|
+
if resp is not None and resp.status_code == 403:
|
|
298
|
+
text = _error_status_text(resp)
|
|
299
|
+
if "requestor configuration settings insufficient" in text.lower():
|
|
300
|
+
# Not a verdict on the exit: on 2026-10-17 a campus exit that
|
|
301
|
+
# ScienceDirect recognizes got this 403 while the browser route
|
|
302
|
+
# downloaded the same paper. Publisher sites stay in play; the
|
|
303
|
+
# API is not asked for full text with this key again for a while.
|
|
304
|
+
mark_no_fulltext(api_key)
|
|
305
|
+
logger.warning(
|
|
306
|
+
"Elsevier API: 403 '%s' on view=FULL for %s — requestor (key + egress IP) "
|
|
307
|
+
"has no full-text configuration. Either the egress IP is outside the "
|
|
308
|
+
"institution's registered range (off-campus: route via LITLOOP_PROXY / "
|
|
309
|
+
"network_proxy to campus egress), or the institution has not enabled API "
|
|
310
|
+
"full-text for this key (contact the institution's Elsevier admin; "
|
|
311
|
+
"insttoken is the off-campus alternative).",
|
|
312
|
+
text, doi,
|
|
313
|
+
)
|
|
314
|
+
return []
|
|
315
|
+
|
|
316
|
+
eids = _extract_pdf_attachment_eids(resp.text)
|
|
317
|
+
for eid in eids:
|
|
318
|
+
logger.info("Elsevier API: found PDF attachment %s", eid)
|
|
319
|
+
return eids
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _fetch_pdf_by_eid(
|
|
323
|
+
eid: str, api_key: str, inst_token: str = "", proxies: dict[str, str] | None = None,
|
|
324
|
+
) -> bytes | None:
|
|
325
|
+
"""Download PDF via Content Object API using attachment EID."""
|
|
326
|
+
url = f"{ELSEVIER_API}/object/eid/{quote(eid, safe='')}"
|
|
327
|
+
headers = {
|
|
328
|
+
"X-ELS-APIKey": api_key,
|
|
329
|
+
"Accept": "application/pdf",
|
|
330
|
+
}
|
|
331
|
+
if inst_token:
|
|
332
|
+
headers["X-ELS-InstToken"] = inst_token
|
|
333
|
+
|
|
334
|
+
resp = _api_request(url, headers, proxies=proxies)
|
|
335
|
+
if not resp:
|
|
336
|
+
return None
|
|
337
|
+
|
|
338
|
+
if resp.status_code == 404:
|
|
339
|
+
logger.info("Elsevier API: attachment %s not found (404)", eid)
|
|
340
|
+
return None
|
|
341
|
+
|
|
342
|
+
if resp.status_code != 200:
|
|
343
|
+
logger.info("Elsevier API: HTTP %d for attachment %s", resp.status_code, eid)
|
|
344
|
+
return None
|
|
345
|
+
|
|
346
|
+
if not _response_is_pdf(resp):
|
|
347
|
+
logger.info("Elsevier API: attachment %s returned non-PDF", eid)
|
|
348
|
+
return None
|
|
349
|
+
|
|
350
|
+
if not _valid_pdf_bytes(resp.content, f"attachment {eid}", reject_single_page=True):
|
|
351
|
+
return None
|
|
352
|
+
|
|
353
|
+
logger.info("Elsevier API: downloaded %d bytes via attachment %s", len(resp.content), eid)
|
|
354
|
+
return resp.content
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _fetch_pdf_direct(
|
|
358
|
+
doi: str, api_key: str, inst_token: str = "", proxies: dict[str, str] | None = None,
|
|
359
|
+
) -> bytes | None:
|
|
360
|
+
"""Fallback: download PDF directly from article endpoint (works for OA)."""
|
|
361
|
+
url = f"{ELSEVIER_API}/article/doi/{doi}"
|
|
362
|
+
headers = {
|
|
363
|
+
"X-ELS-APIKey": api_key,
|
|
364
|
+
"Accept": "application/pdf",
|
|
365
|
+
}
|
|
366
|
+
if inst_token:
|
|
367
|
+
headers["X-ELS-InstToken"] = inst_token
|
|
368
|
+
|
|
369
|
+
resp = _api_request(url, headers, proxies=proxies)
|
|
370
|
+
if not resp or resp.status_code != 200:
|
|
371
|
+
return None
|
|
372
|
+
|
|
373
|
+
content_type = _header(resp.headers, "content-type")
|
|
374
|
+
if not _response_is_pdf(resp):
|
|
375
|
+
logger.info("Elsevier API: direct endpoint returned non-PDF (%s)", content_type[:50])
|
|
376
|
+
return None
|
|
377
|
+
|
|
378
|
+
if not _valid_pdf_bytes(resp.content, "direct PDF", reject_single_page=True):
|
|
379
|
+
return None
|
|
380
|
+
|
|
381
|
+
logger.info("Elsevier API: downloaded %d bytes directly for %s", len(resp.content), doi)
|
|
382
|
+
return resp.content
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _error_status_text(resp: requests.Response) -> str:
|
|
386
|
+
"""Extract <statusText> from an Elsevier error XML body ("" if absent)."""
|
|
387
|
+
try:
|
|
388
|
+
root = ET.fromstring(resp.text)
|
|
389
|
+
except (ET.ParseError, ValueError):
|
|
390
|
+
return ""
|
|
391
|
+
for el in root.iter():
|
|
392
|
+
if el.tag.rsplit("}", 1)[-1] == "statusText":
|
|
393
|
+
return (el.text or "").strip()
|
|
394
|
+
return ""
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _api_request(
|
|
398
|
+
url: str,
|
|
399
|
+
headers: dict,
|
|
400
|
+
*,
|
|
401
|
+
params: dict[str, str] | None = None,
|
|
402
|
+
proxies: dict[str, str] | None = None,
|
|
403
|
+
) -> requests.Response | None:
|
|
404
|
+
"""Make an Elsevier API request with error handling."""
|
|
405
|
+
try:
|
|
406
|
+
session = requests.Session()
|
|
407
|
+
session.trust_env = False
|
|
408
|
+
if proxies:
|
|
409
|
+
session.proxies = proxies
|
|
410
|
+
resp = session.get(
|
|
411
|
+
url,
|
|
412
|
+
headers=headers,
|
|
413
|
+
params=params,
|
|
414
|
+
timeout=30,
|
|
415
|
+
allow_redirects=True,
|
|
416
|
+
)
|
|
417
|
+
except requests.exceptions.SSLError:
|
|
418
|
+
try:
|
|
419
|
+
session = requests.Session()
|
|
420
|
+
session.trust_env = False
|
|
421
|
+
if proxies:
|
|
422
|
+
session.proxies = proxies
|
|
423
|
+
resp = session.get(
|
|
424
|
+
url,
|
|
425
|
+
headers=headers,
|
|
426
|
+
params=params,
|
|
427
|
+
timeout=30,
|
|
428
|
+
allow_redirects=True,
|
|
429
|
+
verify=False,
|
|
430
|
+
)
|
|
431
|
+
except requests.RequestException as e:
|
|
432
|
+
logger.warning("Elsevier API request failed: %s", e)
|
|
433
|
+
return None
|
|
434
|
+
except requests.RequestException as e:
|
|
435
|
+
logger.warning("Elsevier API request failed: %s", e)
|
|
436
|
+
return None
|
|
437
|
+
|
|
438
|
+
if resp.status_code in (401, 403):
|
|
439
|
+
# 403 ≠ key invalid: valid keys 403 on unentitled resources (#56) —
|
|
440
|
+
# surface Elsevier's own statusText instead of guessing.
|
|
441
|
+
detail = _error_status_text(resp) or "key invalid or not entitled"
|
|
442
|
+
logger.warning("Elsevier API: HTTP %d (%s)", resp.status_code, detail)
|
|
443
|
+
elif resp.status_code == 429:
|
|
444
|
+
logger.warning("Elsevier API: rate limited")
|
|
445
|
+
|
|
446
|
+
return resp
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def fetch_fulltext(doi: str, api_key: str, inst_token: str = "", config: dict | None = None) -> dict | None:
|
|
450
|
+
"""Fetch article full text via Elsevier RetrievalAPI."""
|
|
451
|
+
if not api_key:
|
|
452
|
+
return None
|
|
453
|
+
|
|
454
|
+
url = f"{ELSEVIER_API}/article/doi/{doi}"
|
|
455
|
+
headers = {
|
|
456
|
+
"X-ELS-APIKey": api_key,
|
|
457
|
+
"Accept": "application/xml",
|
|
458
|
+
}
|
|
459
|
+
if inst_token:
|
|
460
|
+
headers["X-ELS-Insttoken"] = inst_token
|
|
461
|
+
|
|
462
|
+
resp = _api_request(url, headers, params={"view": "FULL"}, proxies=_resolve_proxies(config))
|
|
463
|
+
if not resp:
|
|
464
|
+
return None
|
|
465
|
+
|
|
466
|
+
if resp.status_code == 401:
|
|
467
|
+
logger.warning("Elsevier API: invalid API key")
|
|
468
|
+
return None
|
|
469
|
+
if resp.status_code == 404:
|
|
470
|
+
logger.info("Elsevier API: DOI %s not found", doi)
|
|
471
|
+
return None
|
|
472
|
+
if resp.status_code != 200:
|
|
473
|
+
logger.info("Elsevier API: HTTP %d for %s", resp.status_code, doi)
|
|
474
|
+
return None
|
|
475
|
+
|
|
476
|
+
return _parse_xml(resp.text)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _parse_xml(xml_text: str) -> dict | None:
|
|
480
|
+
"""Parse Elsevier XML response into structured data."""
|
|
481
|
+
try:
|
|
482
|
+
root = ET.fromstring(xml_text)
|
|
483
|
+
except ET.ParseError as e:
|
|
484
|
+
logger.warning("Failed to parse Elsevier XML: %s", e)
|
|
485
|
+
return None
|
|
486
|
+
|
|
487
|
+
result = {
|
|
488
|
+
"title": "",
|
|
489
|
+
"authors": [],
|
|
490
|
+
"abstract": "",
|
|
491
|
+
"full_text": "",
|
|
492
|
+
"figures": [],
|
|
493
|
+
"references": [],
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
for el in root.iter():
|
|
497
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
498
|
+
if local == "title" and el.text and el.text.strip():
|
|
499
|
+
result["title"] = el.text.strip()
|
|
500
|
+
break
|
|
501
|
+
|
|
502
|
+
for el in root.iter():
|
|
503
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
504
|
+
if local in ("creator", "author"):
|
|
505
|
+
if el.text and el.text.strip():
|
|
506
|
+
result["authors"].append(el.text.strip())
|
|
507
|
+
if not result["authors"]:
|
|
508
|
+
for el in root.iter():
|
|
509
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
510
|
+
if local == "author":
|
|
511
|
+
given = ""
|
|
512
|
+
surname = ""
|
|
513
|
+
for child in el:
|
|
514
|
+
child_local = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
|
515
|
+
if "given" in child_local:
|
|
516
|
+
given = (child.text or "").strip()
|
|
517
|
+
elif "surname" in child_local or "last" in child_local:
|
|
518
|
+
surname = (child.text or "").strip()
|
|
519
|
+
if given or surname:
|
|
520
|
+
result["authors"].append(f"{given} {surname}".strip())
|
|
521
|
+
|
|
522
|
+
result["abstract"] = _extract_abstract(root)
|
|
523
|
+
result["full_text"] = _extract_body(root)
|
|
524
|
+
result["references"] = _extract_references(root)
|
|
525
|
+
|
|
526
|
+
return result
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _extract_abstract(root: ET.Element) -> str:
|
|
530
|
+
for el in root.iter():
|
|
531
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
532
|
+
if local in ("abstract", "description") and _collect_text(el).strip():
|
|
533
|
+
return _collect_text(el).strip()
|
|
534
|
+
return ""
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _extract_body(root: ET.Element) -> str:
|
|
538
|
+
parts = []
|
|
539
|
+
|
|
540
|
+
body = None
|
|
541
|
+
for el in root.iter():
|
|
542
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
543
|
+
if local == "body":
|
|
544
|
+
body = el
|
|
545
|
+
break
|
|
546
|
+
|
|
547
|
+
if body is None:
|
|
548
|
+
return ""
|
|
549
|
+
|
|
550
|
+
def _find_sections(el):
|
|
551
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
552
|
+
if local == "section":
|
|
553
|
+
yield el
|
|
554
|
+
for child in el:
|
|
555
|
+
yield from _find_sections(child)
|
|
556
|
+
|
|
557
|
+
for section in _find_sections(body):
|
|
558
|
+
heading = ""
|
|
559
|
+
content_parts = []
|
|
560
|
+
|
|
561
|
+
for child in section:
|
|
562
|
+
tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
|
563
|
+
if tag in ("section-title", "sectiontitle", "heading"):
|
|
564
|
+
heading = _collect_text(child).strip()
|
|
565
|
+
elif tag == "para":
|
|
566
|
+
text = _collect_text(child).strip()
|
|
567
|
+
if text:
|
|
568
|
+
content_parts.append(text)
|
|
569
|
+
|
|
570
|
+
if heading and content_parts:
|
|
571
|
+
parts.append(f"## {heading}\n\n{' '.join(content_parts)}")
|
|
572
|
+
elif content_parts:
|
|
573
|
+
parts.append(" ".join(content_parts))
|
|
574
|
+
|
|
575
|
+
if not parts:
|
|
576
|
+
for child in body.iter():
|
|
577
|
+
tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
|
578
|
+
if tag == "para":
|
|
579
|
+
text = _collect_text(child).strip()
|
|
580
|
+
if text:
|
|
581
|
+
parts.append(text)
|
|
582
|
+
|
|
583
|
+
return "\n\n".join(parts)
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def _extract_references(root: ET.Element) -> list[str]:
|
|
587
|
+
refs = []
|
|
588
|
+
|
|
589
|
+
for el in root.iter():
|
|
590
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
591
|
+
if local in ("bib-reference", "reference"):
|
|
592
|
+
text = " ".join(_collect_text(el).split())
|
|
593
|
+
if text and len(text) > 10:
|
|
594
|
+
refs.append(text)
|
|
595
|
+
|
|
596
|
+
if refs:
|
|
597
|
+
return refs
|
|
598
|
+
|
|
599
|
+
for el in root.iter():
|
|
600
|
+
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
|
601
|
+
if local == "bibliography":
|
|
602
|
+
for ref in el:
|
|
603
|
+
text = " ".join(_collect_text(ref).split())
|
|
604
|
+
if text and len(text) > 10:
|
|
605
|
+
refs.append(text)
|
|
606
|
+
if refs:
|
|
607
|
+
return refs
|
|
608
|
+
|
|
609
|
+
return refs
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def _collect_text(el: ET.Element) -> str:
|
|
613
|
+
parts = []
|
|
614
|
+
if el.text:
|
|
615
|
+
parts.append(el.text)
|
|
616
|
+
for child in el:
|
|
617
|
+
parts.append(_collect_text(child))
|
|
618
|
+
if child.tail:
|
|
619
|
+
parts.append(child.tail)
|
|
620
|
+
return " ".join(parts)
|