litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,620 @@
1
+ """Elsevier RetrievalAPI integration for fetching full-text articles."""
2
+
3
+ import logging
4
+ import os
5
+ import xml.etree.ElementTree as ET
6
+ from urllib.parse import quote
7
+
8
+ import requests
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+ ELSEVIER_API = "https://api.elsevier.com/content"
13
+
14
+
15
+ # Full text through the API: off in settings (elsevier_api_fulltext), or learned
16
+ # off for this key after the API answered 403 "Requestor configuration settings
17
+ # insufficient" (the key has no full-text entitlement; abstracts and Scopus are
18
+ # unaffected). Remembered per key (a hash, never the key) for 30 days.
19
+ _NO_FULLTEXT_DAYS = 30
20
+
21
+
22
+ def _no_fulltext_file():
23
+ from ..config import DATA_DIR
24
+ return DATA_DIR / "elsevier_no_fulltext.json"
25
+
26
+
27
+ def _key_id(api_key: str) -> str:
28
+ import hashlib
29
+ return hashlib.sha256(api_key.encode("utf-8")).hexdigest()[:16]
30
+
31
+
32
+ def mark_no_fulltext(api_key: str) -> None:
33
+ import json
34
+ import time
35
+ if not api_key:
36
+ return
37
+ path = _no_fulltext_file()
38
+ try:
39
+ data = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
40
+ except (OSError, ValueError):
41
+ data = {}
42
+ data[_key_id(api_key)] = time.time()
43
+ try:
44
+ from .. import store
45
+ store.write_json(path, data)
46
+ except Exception:
47
+ pass
48
+
49
+
50
+ def fulltext_enabled(config: dict | None) -> bool:
51
+ """Whether to ask the Elsevier API for full text at all."""
52
+ import json
53
+ import time
54
+ config = config or {}
55
+ if not config.get("elsevier_api_fulltext", True):
56
+ return False
57
+ key = get_api_key(str(config.get("elsevier_api_key") or ""))
58
+ if not key:
59
+ return False
60
+ try:
61
+ data = json.loads(_no_fulltext_file().read_text(encoding="utf-8"))
62
+ return time.time() - float(data.get(_key_id(key), 0)) > _NO_FULLTEXT_DAYS * 86400
63
+ except (OSError, ValueError, TypeError):
64
+ return True
65
+
66
+
67
+ def get_api_key(config_key: str = "") -> str:
68
+ """Get Elsevier API key from config or environment."""
69
+ return config_key or os.environ.get("ELSEVIER_API_KEY", "")
70
+
71
+
72
+ def _resolve_proxies(config: dict | None) -> dict[str, str] | None:
73
+ """Explicit LitLoop proxy → requests proxies dict (None = direct).
74
+
75
+ Entitlement for subscription full text follows the request's egress IP:
76
+ off-campus direct requests 403 (NOT_ENTITLED) even with a valid key.
77
+ HTTP_PROXY/HTTPS_PROXY are ignored project-wide, so a campus-bound proxy
78
+ must be set via LITLOOP_PROXY / network_proxy — the same setting the
79
+ entitlement probe (elsevier_check) routes through.
80
+ """
81
+ from ..network import configured_proxy
82
+
83
+ proxy = configured_proxy(config)
84
+ return {"http": proxy, "https": proxy} if proxy else None
85
+
86
+
87
+ def fetch_pdf(doi: str, api_key: str, inst_token: str = "", config: dict | None = None) -> bytes | None:
88
+ """Download full PDF via Elsevier API using the attachment EID approach.
89
+
90
+ Strategy (from successful 32-paper batch experience):
91
+ 1. GET /article/doi/{doi}?view=FULL → XML with attachment metadata
92
+ 2. Parse XML to find MAIN PDF attachment-eid (main.pdf or mainext.pdf)
93
+ 3. GET /object/eid/{attachment-eid} → official publisher PDF
94
+
95
+ This bypasses ScienceDirect web pages, Cloudflare, and CAPTCHAs entirely.
96
+ Direct PDF endpoint (/article/doi/{doi} + Accept: application/pdf) only
97
+ returns a 1-page preview for paywalled articles.
98
+ """
99
+ if not api_key:
100
+ return None
101
+ if config is not None and not fulltext_enabled({**config, "elsevier_api_key": api_key}):
102
+ return None
103
+
104
+ proxies = _resolve_proxies(config)
105
+
106
+ # Step 1: Get FULL XML with attachment metadata
107
+ eids = _fetch_attachment_eids(doi, api_key, inst_token, proxies)
108
+ if not eids:
109
+ # Fallback: try direct PDF endpoint (works for OA articles)
110
+ return _fetch_pdf_direct(doi, api_key, inst_token, proxies)
111
+
112
+ # Step 2: Try each attachment EID until we get a valid PDF
113
+ for eid in eids:
114
+ pdf_bytes = _fetch_pdf_by_eid(eid, api_key, inst_token, proxies)
115
+ if pdf_bytes:
116
+ return pdf_bytes
117
+
118
+ logger.info("Elsevier API: all attachment EIDs failed for %s, trying direct", doi)
119
+ return _fetch_pdf_direct(doi, api_key, inst_token, proxies)
120
+
121
+
122
+ def _local_name(tag: str) -> str:
123
+ return tag.rsplit("}", 1)[-1].split(":", 1)[-1].lower()
124
+
125
+
126
+ def _element_text(el: ET.Element) -> str:
127
+ return " ".join(" ".join(el.itertext()).split())
128
+
129
+
130
+ def _header(headers: dict, name: str) -> str:
131
+ value = headers.get(name)
132
+ if value is not None:
133
+ return str(value)
134
+ needle = name.lower()
135
+ for key, value in headers.items():
136
+ if str(key).lower() == needle:
137
+ return str(value)
138
+ return ""
139
+
140
+
141
+ def _response_is_pdf(resp: requests.Response) -> bool:
142
+ content_type = _header(resp.headers, "content-type").lower()
143
+ return "pdf" in content_type or resp.content[:5] == b"%PDF-"
144
+
145
+
146
+ def _looks_like_pdf_eid(value: str) -> bool:
147
+ lowered = value.strip().lower()
148
+ return bool(lowered) and (lowered.endswith(".pdf") or ".pdf" in lowered)
149
+
150
+
151
+ def _article_eid_to_main_pdf(value: str) -> str:
152
+ candidate = value.strip()
153
+ if candidate.lower().startswith("eid:"):
154
+ candidate = candidate.split(":", 1)[1].strip()
155
+ if not candidate.startswith("1-s2.0-"):
156
+ return ""
157
+ if candidate.lower().endswith(".pdf"):
158
+ return candidate
159
+ return f"{candidate}-main.pdf"
160
+
161
+
162
+ def _attachment_container(el: ET.Element, parent_map: dict[ET.Element, ET.Element]) -> ET.Element:
163
+ node = parent_map.get(el, el)
164
+ while node is not None:
165
+ local = _local_name(str(node.tag))
166
+ if "attachment" in local or "object" in local or local == "web-pdf":
167
+ return node
168
+ node = parent_map.get(node)
169
+ return parent_map.get(el, el)
170
+
171
+
172
+ def _attachment_metadata(el: ET.Element) -> str:
173
+ parts: list[str] = []
174
+ for node in el.iter():
175
+ local = _local_name(str(node.tag))
176
+ text = _element_text(node)
177
+ if text:
178
+ parts.append(f"{local}:{text}")
179
+ for attr_name, attr_value in node.attrib.items():
180
+ attr_local = _local_name(str(attr_name))
181
+ if attr_value:
182
+ parts.append(f"{attr_local}:{attr_value}")
183
+ return " ".join(parts).lower()
184
+
185
+
186
+ def _attachment_score(eid: str, metadata: str) -> int:
187
+ haystack = f"{eid} {metadata}".lower()
188
+ score = 0
189
+ if eid.lower().endswith(".pdf"):
190
+ score += 20
191
+ if "pdf" in haystack:
192
+ score += 10
193
+ if "main" in haystack or "full-text" in haystack or "fulltext" in haystack:
194
+ score += 100
195
+ if "page-count" in haystack or "pages" in haystack:
196
+ score += 5
197
+ if "attachment-size" in haystack or "filesize" in haystack or "file-size" in haystack:
198
+ score += 5
199
+ if any(
200
+ marker in haystack
201
+ for marker in ("supplement", "supplementary", "mmc", "appendix", "graphical")
202
+ ):
203
+ score -= 100
204
+ return score
205
+
206
+
207
+ def _extract_pdf_attachment_eids(xml_text: str) -> list[str]:
208
+ try:
209
+ root = ET.fromstring(xml_text)
210
+ except ET.ParseError as e:
211
+ logger.warning("Failed to parse Elsevier XML attachments: %s", e)
212
+ return []
213
+
214
+ parent_map = {child: parent for parent in root.iter() for child in parent}
215
+ candidates: list[tuple[int, int, str]] = []
216
+ seen: set[str] = set()
217
+
218
+ for el in root.iter():
219
+ local = _local_name(str(el.tag))
220
+ found: list[str] = []
221
+
222
+ if local in {"attachment-eid", "object-eid"}:
223
+ text = _element_text(el)
224
+ if _looks_like_pdf_eid(text):
225
+ found.append(text)
226
+ elif local in {"eid", "identifier"}:
227
+ main_pdf = _article_eid_to_main_pdf(_element_text(el))
228
+ if main_pdf:
229
+ found.append(main_pdf)
230
+
231
+ for attr_name, attr_value in el.attrib.items():
232
+ attr_local = _local_name(str(attr_name))
233
+ if attr_local in {"attachment-eid", "object-eid", "eid"}:
234
+ value = str(attr_value).strip()
235
+ if _looks_like_pdf_eid(value):
236
+ found.append(value)
237
+ else:
238
+ main_pdf = _article_eid_to_main_pdf(value)
239
+ if main_pdf:
240
+ found.append(main_pdf)
241
+
242
+ for eid in found:
243
+ if eid in seen:
244
+ continue
245
+ seen.add(eid)
246
+ metadata = _attachment_metadata(_attachment_container(el, parent_map))
247
+ candidates.append((_attachment_score(eid, metadata), len(candidates), eid))
248
+
249
+ candidates.sort(key=lambda item: (-item[0], item[1]))
250
+ return [eid for _, _, eid in candidates]
251
+
252
+
253
+ def _pdf_page_count(content: bytes) -> int | None:
254
+ try:
255
+ import fitz # type: ignore[import-not-found]
256
+ except Exception:
257
+ return None
258
+
259
+ try:
260
+ with fitz.open(stream=content, filetype="pdf") as doc:
261
+ return int(doc.page_count)
262
+ except Exception:
263
+ return None
264
+
265
+
266
+ def _valid_pdf_bytes(content: bytes, label: str, *, reject_single_page: bool) -> bool:
267
+ if content[:5] != b"%PDF-":
268
+ logger.info("Elsevier API: %s returned non-PDF", label)
269
+ return False
270
+ if len(content) < 10000:
271
+ logger.warning("Elsevier API: %s too small (%d bytes)", label, len(content))
272
+ return False
273
+ if reject_single_page and _pdf_page_count(content) == 1:
274
+ logger.info("Elsevier API: %s is a 1-page preview", label)
275
+ return False
276
+ return True
277
+
278
+
279
+ def _fetch_attachment_eids(
280
+ doi: str, api_key: str, inst_token: str = "", proxies: dict[str, str] | None = None,
281
+ ) -> list[str]:
282
+ """Fetch FULL XML and extract MAIN PDF attachment EIDs."""
283
+ url = f"{ELSEVIER_API}/article/doi/{doi}"
284
+ headers = {
285
+ "X-ELS-APIKey": api_key,
286
+ "Accept": "application/xml",
287
+ }
288
+ if inst_token:
289
+ headers["X-ELS-InstToken"] = inst_token
290
+
291
+ resp = _api_request(url, headers, params={"view": "FULL"}, proxies=proxies)
292
+ if not resp or resp.status_code != 200:
293
+ # The XML → attachment-EID chain starts here; a 403 on this request
294
+ # means the requestor (key + egress IP) is not configured for full
295
+ # text at all — no EID can exist. Name the two real causes so the
296
+ # next step is actionable instead of a re-diagnosis.
297
+ if resp is not None and resp.status_code == 403:
298
+ text = _error_status_text(resp)
299
+ if "requestor configuration settings insufficient" in text.lower():
300
+ # Not a verdict on the exit: on 2026-10-17 a campus exit that
301
+ # ScienceDirect recognizes got this 403 while the browser route
302
+ # downloaded the same paper. Publisher sites stay in play; the
303
+ # API is not asked for full text with this key again for a while.
304
+ mark_no_fulltext(api_key)
305
+ logger.warning(
306
+ "Elsevier API: 403 '%s' on view=FULL for %s — requestor (key + egress IP) "
307
+ "has no full-text configuration. Either the egress IP is outside the "
308
+ "institution's registered range (off-campus: route via LITLOOP_PROXY / "
309
+ "network_proxy to campus egress), or the institution has not enabled API "
310
+ "full-text for this key (contact the institution's Elsevier admin; "
311
+ "insttoken is the off-campus alternative).",
312
+ text, doi,
313
+ )
314
+ return []
315
+
316
+ eids = _extract_pdf_attachment_eids(resp.text)
317
+ for eid in eids:
318
+ logger.info("Elsevier API: found PDF attachment %s", eid)
319
+ return eids
320
+
321
+
322
+ def _fetch_pdf_by_eid(
323
+ eid: str, api_key: str, inst_token: str = "", proxies: dict[str, str] | None = None,
324
+ ) -> bytes | None:
325
+ """Download PDF via Content Object API using attachment EID."""
326
+ url = f"{ELSEVIER_API}/object/eid/{quote(eid, safe='')}"
327
+ headers = {
328
+ "X-ELS-APIKey": api_key,
329
+ "Accept": "application/pdf",
330
+ }
331
+ if inst_token:
332
+ headers["X-ELS-InstToken"] = inst_token
333
+
334
+ resp = _api_request(url, headers, proxies=proxies)
335
+ if not resp:
336
+ return None
337
+
338
+ if resp.status_code == 404:
339
+ logger.info("Elsevier API: attachment %s not found (404)", eid)
340
+ return None
341
+
342
+ if resp.status_code != 200:
343
+ logger.info("Elsevier API: HTTP %d for attachment %s", resp.status_code, eid)
344
+ return None
345
+
346
+ if not _response_is_pdf(resp):
347
+ logger.info("Elsevier API: attachment %s returned non-PDF", eid)
348
+ return None
349
+
350
+ if not _valid_pdf_bytes(resp.content, f"attachment {eid}", reject_single_page=True):
351
+ return None
352
+
353
+ logger.info("Elsevier API: downloaded %d bytes via attachment %s", len(resp.content), eid)
354
+ return resp.content
355
+
356
+
357
+ def _fetch_pdf_direct(
358
+ doi: str, api_key: str, inst_token: str = "", proxies: dict[str, str] | None = None,
359
+ ) -> bytes | None:
360
+ """Fallback: download PDF directly from article endpoint (works for OA)."""
361
+ url = f"{ELSEVIER_API}/article/doi/{doi}"
362
+ headers = {
363
+ "X-ELS-APIKey": api_key,
364
+ "Accept": "application/pdf",
365
+ }
366
+ if inst_token:
367
+ headers["X-ELS-InstToken"] = inst_token
368
+
369
+ resp = _api_request(url, headers, proxies=proxies)
370
+ if not resp or resp.status_code != 200:
371
+ return None
372
+
373
+ content_type = _header(resp.headers, "content-type")
374
+ if not _response_is_pdf(resp):
375
+ logger.info("Elsevier API: direct endpoint returned non-PDF (%s)", content_type[:50])
376
+ return None
377
+
378
+ if not _valid_pdf_bytes(resp.content, "direct PDF", reject_single_page=True):
379
+ return None
380
+
381
+ logger.info("Elsevier API: downloaded %d bytes directly for %s", len(resp.content), doi)
382
+ return resp.content
383
+
384
+
385
+ def _error_status_text(resp: requests.Response) -> str:
386
+ """Extract <statusText> from an Elsevier error XML body ("" if absent)."""
387
+ try:
388
+ root = ET.fromstring(resp.text)
389
+ except (ET.ParseError, ValueError):
390
+ return ""
391
+ for el in root.iter():
392
+ if el.tag.rsplit("}", 1)[-1] == "statusText":
393
+ return (el.text or "").strip()
394
+ return ""
395
+
396
+
397
+ def _api_request(
398
+ url: str,
399
+ headers: dict,
400
+ *,
401
+ params: dict[str, str] | None = None,
402
+ proxies: dict[str, str] | None = None,
403
+ ) -> requests.Response | None:
404
+ """Make an Elsevier API request with error handling."""
405
+ try:
406
+ session = requests.Session()
407
+ session.trust_env = False
408
+ if proxies:
409
+ session.proxies = proxies
410
+ resp = session.get(
411
+ url,
412
+ headers=headers,
413
+ params=params,
414
+ timeout=30,
415
+ allow_redirects=True,
416
+ )
417
+ except requests.exceptions.SSLError:
418
+ try:
419
+ session = requests.Session()
420
+ session.trust_env = False
421
+ if proxies:
422
+ session.proxies = proxies
423
+ resp = session.get(
424
+ url,
425
+ headers=headers,
426
+ params=params,
427
+ timeout=30,
428
+ allow_redirects=True,
429
+ verify=False,
430
+ )
431
+ except requests.RequestException as e:
432
+ logger.warning("Elsevier API request failed: %s", e)
433
+ return None
434
+ except requests.RequestException as e:
435
+ logger.warning("Elsevier API request failed: %s", e)
436
+ return None
437
+
438
+ if resp.status_code in (401, 403):
439
+ # 403 ≠ key invalid: valid keys 403 on unentitled resources (#56) —
440
+ # surface Elsevier's own statusText instead of guessing.
441
+ detail = _error_status_text(resp) or "key invalid or not entitled"
442
+ logger.warning("Elsevier API: HTTP %d (%s)", resp.status_code, detail)
443
+ elif resp.status_code == 429:
444
+ logger.warning("Elsevier API: rate limited")
445
+
446
+ return resp
447
+
448
+
449
+ def fetch_fulltext(doi: str, api_key: str, inst_token: str = "", config: dict | None = None) -> dict | None:
450
+ """Fetch article full text via Elsevier RetrievalAPI."""
451
+ if not api_key:
452
+ return None
453
+
454
+ url = f"{ELSEVIER_API}/article/doi/{doi}"
455
+ headers = {
456
+ "X-ELS-APIKey": api_key,
457
+ "Accept": "application/xml",
458
+ }
459
+ if inst_token:
460
+ headers["X-ELS-Insttoken"] = inst_token
461
+
462
+ resp = _api_request(url, headers, params={"view": "FULL"}, proxies=_resolve_proxies(config))
463
+ if not resp:
464
+ return None
465
+
466
+ if resp.status_code == 401:
467
+ logger.warning("Elsevier API: invalid API key")
468
+ return None
469
+ if resp.status_code == 404:
470
+ logger.info("Elsevier API: DOI %s not found", doi)
471
+ return None
472
+ if resp.status_code != 200:
473
+ logger.info("Elsevier API: HTTP %d for %s", resp.status_code, doi)
474
+ return None
475
+
476
+ return _parse_xml(resp.text)
477
+
478
+
479
+ def _parse_xml(xml_text: str) -> dict | None:
480
+ """Parse Elsevier XML response into structured data."""
481
+ try:
482
+ root = ET.fromstring(xml_text)
483
+ except ET.ParseError as e:
484
+ logger.warning("Failed to parse Elsevier XML: %s", e)
485
+ return None
486
+
487
+ result = {
488
+ "title": "",
489
+ "authors": [],
490
+ "abstract": "",
491
+ "full_text": "",
492
+ "figures": [],
493
+ "references": [],
494
+ }
495
+
496
+ for el in root.iter():
497
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
498
+ if local == "title" and el.text and el.text.strip():
499
+ result["title"] = el.text.strip()
500
+ break
501
+
502
+ for el in root.iter():
503
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
504
+ if local in ("creator", "author"):
505
+ if el.text and el.text.strip():
506
+ result["authors"].append(el.text.strip())
507
+ if not result["authors"]:
508
+ for el in root.iter():
509
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
510
+ if local == "author":
511
+ given = ""
512
+ surname = ""
513
+ for child in el:
514
+ child_local = child.tag.split("}")[-1] if "}" in child.tag else child.tag
515
+ if "given" in child_local:
516
+ given = (child.text or "").strip()
517
+ elif "surname" in child_local or "last" in child_local:
518
+ surname = (child.text or "").strip()
519
+ if given or surname:
520
+ result["authors"].append(f"{given} {surname}".strip())
521
+
522
+ result["abstract"] = _extract_abstract(root)
523
+ result["full_text"] = _extract_body(root)
524
+ result["references"] = _extract_references(root)
525
+
526
+ return result
527
+
528
+
529
+ def _extract_abstract(root: ET.Element) -> str:
530
+ for el in root.iter():
531
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
532
+ if local in ("abstract", "description") and _collect_text(el).strip():
533
+ return _collect_text(el).strip()
534
+ return ""
535
+
536
+
537
+ def _extract_body(root: ET.Element) -> str:
538
+ parts = []
539
+
540
+ body = None
541
+ for el in root.iter():
542
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
543
+ if local == "body":
544
+ body = el
545
+ break
546
+
547
+ if body is None:
548
+ return ""
549
+
550
+ def _find_sections(el):
551
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
552
+ if local == "section":
553
+ yield el
554
+ for child in el:
555
+ yield from _find_sections(child)
556
+
557
+ for section in _find_sections(body):
558
+ heading = ""
559
+ content_parts = []
560
+
561
+ for child in section:
562
+ tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
563
+ if tag in ("section-title", "sectiontitle", "heading"):
564
+ heading = _collect_text(child).strip()
565
+ elif tag == "para":
566
+ text = _collect_text(child).strip()
567
+ if text:
568
+ content_parts.append(text)
569
+
570
+ if heading and content_parts:
571
+ parts.append(f"## {heading}\n\n{' '.join(content_parts)}")
572
+ elif content_parts:
573
+ parts.append(" ".join(content_parts))
574
+
575
+ if not parts:
576
+ for child in body.iter():
577
+ tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
578
+ if tag == "para":
579
+ text = _collect_text(child).strip()
580
+ if text:
581
+ parts.append(text)
582
+
583
+ return "\n\n".join(parts)
584
+
585
+
586
+ def _extract_references(root: ET.Element) -> list[str]:
587
+ refs = []
588
+
589
+ for el in root.iter():
590
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
591
+ if local in ("bib-reference", "reference"):
592
+ text = " ".join(_collect_text(el).split())
593
+ if text and len(text) > 10:
594
+ refs.append(text)
595
+
596
+ if refs:
597
+ return refs
598
+
599
+ for el in root.iter():
600
+ local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
601
+ if local == "bibliography":
602
+ for ref in el:
603
+ text = " ".join(_collect_text(ref).split())
604
+ if text and len(text) > 10:
605
+ refs.append(text)
606
+ if refs:
607
+ return refs
608
+
609
+ return refs
610
+
611
+
612
+ def _collect_text(el: ET.Element) -> str:
613
+ parts = []
614
+ if el.text:
615
+ parts.append(el.text)
616
+ for child in el:
617
+ parts.append(_collect_text(child))
618
+ if child.tail:
619
+ parts.append(child.tail)
620
+ return " ".join(parts)