litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,578 @@
1
+ """ScienceDirect's article API: ask it for the PDF attachment.
2
+
3
+ Split out of `_core.py` (2026-11-05): it is the largest self-contained chunk
4
+ there — no browser, no DOM, just requests and XML. `_core.py` re-exports the
5
+ names below, so callers keep importing from there.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ from ...log import get_logger
15
+
16
+ log = get_logger()
17
+
18
+
19
+ def _persist_api_cookies(session: Any, config: dict[str, Any]) -> int:
20
+ """Delegates to `_core`: the cookie store lives with the browser code."""
21
+ from ._core import _persist_api_cookies as impl
22
+
23
+ return impl(session, config)
24
+
25
+
26
+ def _elsevier_header(headers: Any, name: str) -> str:
27
+ """Return a response header value using case-insensitive matching."""
28
+ if not headers:
29
+ return ""
30
+ value = headers.get(name)
31
+ if value is not None:
32
+ return str(value)
33
+ needle = name.lower()
34
+ for key, value in headers.items():
35
+ if str(key).lower() == needle:
36
+ return str(value)
37
+ return ""
38
+
39
+
40
+ def _elsevier_response_is_pdf(resp: Any) -> bool:
41
+ content_type = _elsevier_header(getattr(resp, "headers", {}), "content-type").lower()
42
+ content = getattr(resp, "content", b"") or b""
43
+ return "pdf" in content_type or content[:5] == b"%PDF-"
44
+
45
+
46
+ def _elsevier_response_is_xml(resp: Any) -> bool:
47
+ content_type = _elsevier_header(getattr(resp, "headers", {}), "content-type").lower()
48
+ content = (getattr(resp, "content", b"") or b"").lstrip()
49
+ return (
50
+ "xml" in content_type
51
+ or content.startswith(b"<?xml")
52
+ or content.startswith(b"<full-text-retrieval-response")
53
+ )
54
+
55
+
56
+ def _elsevier_response_text(resp: Any) -> str:
57
+ text = getattr(resp, "text", None)
58
+ if isinstance(text, str):
59
+ return text
60
+ content = getattr(resp, "content", b"") or b""
61
+ return content.decode("utf-8", errors="replace")
62
+
63
+
64
+ def _elsevier_xml_local_name(name: str) -> str:
65
+ return name.rsplit("}", 1)[-1].split(":", 1)[-1].lower()
66
+
67
+
68
+ def _elsevier_element_text(el: Any) -> str:
69
+ return " ".join(" ".join(el.itertext()).split())
70
+
71
+
72
+ def _elsevier_looks_like_pdf_eid(value: str) -> bool:
73
+ lowered = value.strip().lower()
74
+ return bool(lowered) and (lowered.endswith(".pdf") or ".pdf" in lowered)
75
+
76
+
77
+ def _elsevier_article_eid_to_main_pdf(value: str) -> str:
78
+ candidate = value.strip()
79
+ if candidate.lower().startswith("eid:"):
80
+ candidate = candidate.split(":", 1)[1].strip()
81
+ if not candidate.startswith("1-s2.0-"):
82
+ return ""
83
+ if candidate.lower().endswith(".pdf"):
84
+ return candidate
85
+ return f"{candidate}-main.pdf"
86
+
87
+
88
+ def _elsevier_attachment_container(el: Any, parent_map: dict[Any, Any]) -> Any:
89
+ node = parent_map.get(el, el)
90
+ while node is not None:
91
+ local = _elsevier_xml_local_name(str(node.tag))
92
+ if "attachment" in local or "object" in local:
93
+ return node
94
+ node = parent_map.get(node)
95
+ return parent_map.get(el, el)
96
+
97
+
98
+ def _elsevier_attachment_metadata(el: Any) -> str:
99
+ parts: list[str] = []
100
+ for node in el.iter():
101
+ local = _elsevier_xml_local_name(str(node.tag))
102
+ text = _elsevier_element_text(node)
103
+ if text:
104
+ parts.append(f"{local}:{text}")
105
+ for attr_name, attr_value in node.attrib.items():
106
+ attr_local = _elsevier_xml_local_name(str(attr_name))
107
+ if attr_value:
108
+ parts.append(f"{attr_local}:{attr_value}")
109
+ return " ".join(parts).lower()
110
+
111
+
112
+ def _elsevier_attachment_score(eid: str, metadata: str) -> int:
113
+ haystack = f"{eid} {metadata}".lower()
114
+ score = 0
115
+ if eid.lower().endswith(".pdf"):
116
+ score += 20
117
+ if "pdf" in haystack:
118
+ score += 10
119
+ if "main" in haystack or "full-text" in haystack or "fulltext" in haystack:
120
+ score += 100
121
+ if "page-count" in haystack or "pages" in haystack:
122
+ score += 5
123
+ if "attachment-size" in haystack or "filesize" in haystack or "file-size" in haystack:
124
+ score += 5
125
+ if any(marker in haystack for marker in (
126
+ "supplement", "supplementary", "mmc", "appendix", "graphical", "thumbnail",
127
+ )):
128
+ score -= 100
129
+ return score
130
+
131
+
132
+ def _extract_elsevier_pdf_attachment_eids(xml_text: str) -> list[str]:
133
+ """Extract candidate publisher PDF object EIDs from Elsevier full-text XML."""
134
+ import xml.etree.ElementTree as ET
135
+
136
+ try:
137
+ root = ET.fromstring(xml_text)
138
+ except ET.ParseError as exc:
139
+ log.info(f" [ElsevierAPI] XML parse failed: {exc}")
140
+ return []
141
+
142
+ parent_map = {child: parent for parent in root.iter() for child in parent}
143
+ candidates: list[tuple[int, int, str]] = []
144
+ seen: set[str] = set()
145
+
146
+ for el in root.iter():
147
+ found: list[str] = []
148
+ local = _elsevier_xml_local_name(str(el.tag))
149
+ if local in {"attachment-eid", "object-eid"}:
150
+ text = _elsevier_element_text(el)
151
+ if _elsevier_looks_like_pdf_eid(text):
152
+ found.append(text)
153
+ elif local in {"eid", "identifier"}:
154
+ text = _elsevier_element_text(el)
155
+ main_pdf = _elsevier_article_eid_to_main_pdf(text)
156
+ if main_pdf:
157
+ found.append(main_pdf)
158
+
159
+ for attr_name, attr_value in el.attrib.items():
160
+ attr_local = _elsevier_xml_local_name(str(attr_name))
161
+ if attr_local in {"attachment-eid", "object-eid", "eid"}:
162
+ value = str(attr_value).strip()
163
+ if _elsevier_looks_like_pdf_eid(value):
164
+ found.append(value)
165
+ else:
166
+ main_pdf = _elsevier_article_eid_to_main_pdf(value)
167
+ if main_pdf:
168
+ found.append(main_pdf)
169
+
170
+ for eid in found:
171
+ if eid in seen:
172
+ continue
173
+ seen.add(eid)
174
+ container = _elsevier_attachment_container(el, parent_map)
175
+ metadata = _elsevier_attachment_metadata(container)
176
+ candidates.append((
177
+ _elsevier_attachment_score(eid, metadata),
178
+ len(candidates),
179
+ eid,
180
+ ))
181
+
182
+ candidates.sort(key=lambda item: (-item[0], item[1]))
183
+ return [eid for _, _, eid in candidates]
184
+
185
+
186
+ def _elsevier_pdf_page_count(content: bytes) -> int | None:
187
+ """Best-effort page count for detecting Elsevier one-page preview PDFs."""
188
+ try:
189
+ import fitz # type: ignore[import-not-found]
190
+ except Exception:
191
+ return None
192
+
193
+ try:
194
+ with fitz.open(stream=content, filetype="pdf") as doc:
195
+ return int(doc.page_count)
196
+ except Exception:
197
+ return None
198
+
199
+
200
+ def _save_elsevier_pdf_content(
201
+ doi: str,
202
+ output_path: Path,
203
+ content: bytes,
204
+ config: dict[str, Any],
205
+ source: str,
206
+ *,
207
+ reject_single_page: bool = False,
208
+ ) -> dict[str, Any] | None:
209
+ from ...download.pdf import is_pdf_file, success
210
+
211
+ if len(content) < config.get("min_pdf_size_bytes", 10000):
212
+ log.info(f" [ElsevierAPI] response too small ({len(content)} bytes)")
213
+ return None
214
+ if reject_single_page:
215
+ page_count = _elsevier_pdf_page_count(content)
216
+ if page_count == 1:
217
+ output_path.unlink(missing_ok=True)
218
+ log.info(f" [ElsevierAPI] direct PDF is a 1-page preview for {doi}")
219
+ return None
220
+
221
+ output_path.parent.mkdir(parents=True, exist_ok=True)
222
+ output_path.write_bytes(content)
223
+ if is_pdf_file(output_path):
224
+ log.info(f" [ElsevierAPI] downloaded {len(content)} bytes for {doi}")
225
+ return success(doi, output_path, source)
226
+
227
+ output_path.unlink(missing_ok=True)
228
+ return None
229
+
230
+
231
+ def _try_elsevier_object_pdf_from_xml(
232
+ doi: str,
233
+ output_path: Path,
234
+ config: dict[str, Any],
235
+ session: Any,
236
+ article_url: str,
237
+ headers: dict[str, str],
238
+ first_resp: Any,
239
+ *,
240
+ full_xml_already_tried: bool = False,
241
+ ) -> dict[str, Any] | None:
242
+ import urllib.parse
243
+
244
+ xml_resp = None
245
+ if (
246
+ first_resp is not None
247
+ and getattr(first_resp, "status_code", 0) == 200
248
+ and _elsevier_response_is_xml(first_resp)
249
+ ):
250
+ xml_resp = first_resp
251
+
252
+ if xml_resp is None:
253
+ xml_headers = dict(headers)
254
+ xml_headers["Accept"] = "application/xml"
255
+ param_options = [None] if full_xml_already_tried else [{"view": "FULL"}, None]
256
+ for params in param_options:
257
+ try:
258
+ request_kwargs: dict[str, Any] = {
259
+ "headers": xml_headers,
260
+ "timeout": 30,
261
+ "allow_redirects": True,
262
+ }
263
+ if params:
264
+ request_kwargs["params"] = params
265
+ candidate = session.get(article_url, **request_kwargs)
266
+ except Exception as exc:
267
+ log.info(f" [ElsevierAPI] XML request failed: {exc}")
268
+ continue
269
+
270
+ if getattr(candidate, "status_code", 0) != 200:
271
+ view_name = "FULL XML" if params else "XML"
272
+ log.info(
273
+ f" [ElsevierAPI] {view_name} HTTP "
274
+ f"{getattr(candidate, 'status_code', 0)} for {doi}"
275
+ )
276
+ continue
277
+ if not _elsevier_response_is_xml(candidate):
278
+ content_type = _elsevier_header(
279
+ getattr(candidate, "headers", {}),
280
+ "content-type",
281
+ )
282
+ view_name = "FULL XML" if params else "XML"
283
+ log.info(
284
+ f" [ElsevierAPI] {view_name} request returned non-XML "
285
+ f"({content_type[:50]})"
286
+ )
287
+ continue
288
+
289
+ xml_resp = candidate
290
+ break
291
+
292
+ if xml_resp is None:
293
+ return None
294
+
295
+ eids = _extract_elsevier_pdf_attachment_eids(_elsevier_response_text(xml_resp))
296
+ if not eids:
297
+ log.info(f" [ElsevierAPI] XML has no PDF attachment EID for {doi}")
298
+ return None
299
+
300
+ object_headers = dict(headers)
301
+ object_headers["Accept"] = "application/pdf"
302
+ for eid in eids:
303
+ object_url = (
304
+ "https://api.elsevier.com/content/object/eid/"
305
+ f"{urllib.parse.quote(eid, safe='')}"
306
+ )
307
+ try:
308
+ object_resp = session.get(
309
+ object_url,
310
+ headers=object_headers,
311
+ timeout=30,
312
+ allow_redirects=True,
313
+ )
314
+ except Exception as exc:
315
+ log.info(f" [ElsevierAPI] object {eid} request failed: {exc}")
316
+ continue
317
+
318
+ if getattr(object_resp, "status_code", 0) != 200:
319
+ log.info(
320
+ f" [ElsevierAPI] object {eid} HTTP "
321
+ f"{getattr(object_resp, 'status_code', 0)}"
322
+ )
323
+ continue
324
+ if not _elsevier_response_is_pdf(object_resp):
325
+ content_type = _elsevier_header(
326
+ getattr(object_resp, "headers", {}),
327
+ "content-type",
328
+ )
329
+ log.info(f" [ElsevierAPI] object {eid} returned non-PDF ({content_type[:50]})")
330
+ continue
331
+
332
+ result = _save_elsevier_pdf_content(
333
+ doi,
334
+ output_path,
335
+ getattr(object_resp, "content", b"") or b"",
336
+ config,
337
+ "ElsevierAPI",
338
+ reject_single_page=True,
339
+ )
340
+ if result:
341
+ log.info(f" [ElsevierAPI] downloaded object PDF via attachment EID {eid}")
342
+ return result
343
+
344
+ return None
345
+
346
+
347
+ def _try_elsevier_text_fallback(
348
+ doi: str,
349
+ output_path: Path,
350
+ session: Any,
351
+ article_url: str,
352
+ headers: dict[str, str],
353
+ config: dict[str, Any],
354
+ ) -> dict[str, Any] | None:
355
+ """Last API resort: save the httpAccept=text/plain full text as .txt.
356
+
357
+ Only kept when the body is a real plain-text payload above 100KB — the
358
+ same threshold pdf_utils.is_suspicious_pdf treats as certainly full text —
359
+ so the racing pipeline never flags or drops the artifact. API error
360
+ payloads (text/xml) are rejected by the content-type check.
361
+ """
362
+ txt_headers = dict(headers)
363
+ txt_headers["Accept"] = "text/plain"
364
+ try:
365
+ resp = session.get(
366
+ article_url,
367
+ headers=txt_headers,
368
+ params={"httpAccept": "text/plain"},
369
+ timeout=30,
370
+ allow_redirects=True,
371
+ )
372
+ except Exception as e:
373
+ log.info(f" [ElsevierAPI] text/plain request failed: {e}")
374
+ return None
375
+
376
+ if getattr(resp, "status_code", 0) != 200:
377
+ return None
378
+ content_type = _elsevier_header(getattr(resp, "headers", {}), "content-type").lower()
379
+ content = getattr(resp, "content", b"") or b""
380
+ if "text/plain" not in content_type or len(content) < 100_000:
381
+ log.info(
382
+ f" [ElsevierAPI] no usable full text ({content_type[:40]}, "
383
+ f"{len(content)} bytes)"
384
+ )
385
+ return None
386
+
387
+ txt_path = output_path.with_suffix(".txt")
388
+ try:
389
+ txt_path.parent.mkdir(parents=True, exist_ok=True)
390
+ txt_path.write_bytes(content)
391
+ except OSError as e:
392
+ log.info(f" [ElsevierAPI] full text save failed: {e}")
393
+ return None
394
+ log.info(f" [ElsevierAPI] saved {len(content)} bytes of full text as {txt_path.name}")
395
+ from ...download.pdf import success
396
+ return success(doi, txt_path, "ElsevierAPI(Text)")
397
+
398
+
399
+ def try_elsevier_api(
400
+ doi: str, output_path: Path, config: dict[str, Any],
401
+ ) -> dict[str, Any] | None:
402
+ from ...sources.elsevier_api import fulltext_enabled
403
+ if not fulltext_enabled(config):
404
+ return None # off in settings, or this key has no full-text entitlement
405
+ return _try_elsevier_api(doi, output_path, config)
406
+
407
+
408
+ def _try_elsevier_api(
409
+ doi: str, output_path: Path, config: dict[str, Any],
410
+ ) -> dict[str, Any] | None:
411
+ """Download Elsevier/ScienceDirect PDF via Article Retrieval API.
412
+
413
+ Uses the Elsevier Institutional API (api.elsevier.com) with an API key
414
+ and optional institutional token. This is far faster and more reliable
415
+ than browser-based login flows.
416
+
417
+ API-first request order per route (browser navigation is only a last
418
+ resort and lives in try_elsevier_browser):
419
+ 1. article endpoint with Accept: application/pdf — the full PDF when
420
+ the key's entitlement covers the article. Entitlement follows the
421
+ API key, not the client IP, so this works off-campus. The body is
422
+ validated as a real multi-page PDF before saving (rejects the
423
+ 1-page preview served to unentitled keys).
424
+ 2. FULL XML → main-PDF attachment EIDs → Content Object API
425
+ (existing chain; covers articles whose direct endpoint misbehaves).
426
+ 3. httpAccept=text/plain full text saved as .txt when no PDF is
427
+ returned at all.
428
+
429
+ When the API does not return a PDF directly (no institutional access),
430
+ cookies from the request chain (api.elsevier.com → linkinghub.elsevier.com
431
+ → sciencedirect.com) are persisted for the browser strategy to reuse.
432
+
433
+ Config keys:
434
+ elsevier_api_key — personal or institutional API key (required);
435
+ falls back to the ELSEVIER_API_KEY env var
436
+ elsevier_insttoken — institutional token for campus-level access
437
+ """
438
+ api_key = config.get("elsevier_api_key", "") or os.environ.get("ELSEVIER_API_KEY", "")
439
+ if not api_key:
440
+ log.info(f" [ElsevierAPI] no API key configured, skipping. "
441
+ f"Run litloop_elsevier_setup to configure (free).")
442
+ return None
443
+
444
+ from ...network import USER_AGENT, proxy_dict, select_proxy_for_url
445
+
446
+ url = f"https://api.elsevier.com/content/article/doi/{doi}"
447
+
448
+ headers: dict[str, str] = {
449
+ "User-Agent": USER_AGENT,
450
+ "X-ELS-APIKey": api_key,
451
+ }
452
+ insttoken = config.get("elsevier_insttoken", "")
453
+ if insttoken:
454
+ headers["X-ELS-InstToken"] = insttoken
455
+
456
+ proxy = select_proxy_for_url(url, config)
457
+ configured_proxies = proxy_dict(proxy)
458
+ route_options: list[tuple[str, dict[str, str]]] = [("direct", {})]
459
+ if configured_proxies:
460
+ route_options.append(("configured proxy", configured_proxies))
461
+
462
+ import requests
463
+
464
+ for route_name, proxies in route_options:
465
+ session = requests.Session()
466
+ session.trust_env = False
467
+ if proxies:
468
+ session.proxies = proxies
469
+
470
+ # Step 1: direct article PDF — entitlement follows the API key, so no
471
+ # campus IP is needed. Reject non-200 and 1-page preview responses.
472
+ pdf_headers = dict(headers)
473
+ pdf_headers["Accept"] = "application/pdf"
474
+ pdf_resp = None
475
+ try:
476
+ log.info(f" [ElsevierAPI] trying {route_name} route")
477
+ pdf_resp = session.get(
478
+ url,
479
+ headers=pdf_headers,
480
+ timeout=30,
481
+ allow_redirects=True,
482
+ )
483
+ except Exception as e:
484
+ log.info(f" [ElsevierAPI] {route_name} direct PDF request failed: {e}")
485
+
486
+ if pdf_resp is not None and pdf_resp.status_code == 200 and _elsevier_response_is_pdf(pdf_resp):
487
+ result = _save_elsevier_pdf_content(
488
+ doi,
489
+ output_path,
490
+ pdf_resp.content,
491
+ config,
492
+ "ElsevierAPI",
493
+ reject_single_page=True,
494
+ )
495
+ if result:
496
+ return result
497
+ elif pdf_resp is not None and pdf_resp.status_code != 200:
498
+ log.info(
499
+ f" [ElsevierAPI] {route_name} direct PDF HTTP "
500
+ f"{pdf_resp.status_code} for {doi}, trying FULL XML route"
501
+ )
502
+
503
+ try:
504
+ xml_headers = dict(headers)
505
+ xml_headers["Accept"] = "application/xml"
506
+ resp = session.get(
507
+ url,
508
+ headers=xml_headers,
509
+ params={"view": "FULL"},
510
+ timeout=30,
511
+ allow_redirects=True,
512
+ )
513
+ except Exception as e:
514
+ log.info(f" [ElsevierAPI] {route_name} FULL XML request failed: {e}")
515
+ continue
516
+
517
+ if resp is not None and resp.status_code != 200:
518
+ if resp.status_code in (403, 429):
519
+ log.info(
520
+ f" [ElsevierAPI] {route_name} HTTP {resp.status_code}; "
521
+ "API access may be unavailable or rate limited"
522
+ )
523
+ else:
524
+ log.info(f" [ElsevierAPI] {route_name} HTTP {resp.status_code} for {doi}")
525
+
526
+ if resp is not None and resp.status_code == 200 and _elsevier_response_is_pdf(resp):
527
+ result = _save_elsevier_pdf_content(
528
+ doi,
529
+ output_path,
530
+ resp.content,
531
+ config,
532
+ "ElsevierAPI",
533
+ reject_single_page=True,
534
+ )
535
+ if result:
536
+ try:
537
+ _persist_api_cookies(session, config)
538
+ except Exception:
539
+ pass
540
+ return result
541
+
542
+ result = _try_elsevier_object_pdf_from_xml(
543
+ doi,
544
+ output_path,
545
+ config,
546
+ session,
547
+ url,
548
+ headers,
549
+ resp,
550
+ full_xml_already_tried=True,
551
+ )
552
+ if result:
553
+ try:
554
+ _persist_api_cookies(session, config)
555
+ except Exception:
556
+ pass
557
+ return result
558
+
559
+ content_type = _elsevier_header(
560
+ getattr(resp, "headers", {}) if resp is not None else {},
561
+ "content-type",
562
+ )
563
+ log.info(
564
+ f" [ElsevierAPI] {route_name} non-PDF response "
565
+ f"({content_type[:50]}), trying next route if available"
566
+ )
567
+ try:
568
+ _persist_api_cookies(session, config)
569
+ except Exception as e:
570
+ log.info(f" [ElsevierAPI] cookie persist failed: {e}")
571
+
572
+ # Step 3: full text as .txt — the API returned no PDF deliverable for
573
+ # this DOI; grab the plain-text full text before falling back further.
574
+ txt_result = _try_elsevier_text_fallback(doi, output_path, session, url, headers, config)
575
+ if txt_result:
576
+ return txt_result
577
+
578
+ return None
@@ -0,0 +1,27 @@
1
+ """ACM (Association for Computing Machinery) publisher strategy."""
2
+
3
+ from .base import BasePublisherStrategy
4
+ from .registry import StrategyRegistry
5
+
6
+
7
+ @StrategyRegistry.register
8
+ class ACMStrategy(BasePublisherStrategy):
9
+ name = "ACM"
10
+ aliases = ("acm",)
11
+ doi_prefixes = ("10.1145/",)
12
+ base_domains = ("dl.acm.org", "acm.org")
13
+ sample_dois = ("10.1145/3292500.3330893",)
14
+ article_url_template = "https://dl.acm.org/doi/{doi}"
15
+ pdf_url_templates = (
16
+ "https://dl.acm.org/doi/pdf/{doi}",
17
+ )
18
+ success_url_markers = ("dl.acm.org/doi/",)
19
+ auth_url_markers = (
20
+ "id.tsinghua.edu.cn",
21
+ "idp.tsinghua.edu.cn",
22
+ "login.openathens.net",
23
+ )
24
+ sso_text_markers = ("Access through your institution",)
25
+ sso_url_patterns = ("/shibboleth", "/institutional")
26
+ institution_input_selectors = ("input[name='search']", "#searchInstitution")
27
+ institution_result_selectors = ("input[name='search']",)
@@ -0,0 +1,42 @@
1
+ """ACS (American Chemical Society) publisher strategy."""
2
+
3
+ from .base import BasePublisherStrategy
4
+ from .registry import StrategyRegistry
5
+
6
+
7
+ @StrategyRegistry.register
8
+ class ACSStrategy(BasePublisherStrategy):
9
+ name = "ACS"
10
+ aliases = ("acs",)
11
+ doi_prefixes = ("10.1021/",)
12
+ base_domains = ("pubs.acs.org",)
13
+ sample_dois = ("10.1021/jacs.5b00936",)
14
+ article_url_template = "https://pubs.acs.org/doi/{doi}"
15
+ pdf_url_templates = (
16
+ "https://pubs.acs.org/doi/pdf/{doi}?ref=article_openPDF",
17
+ "https://pubs.acs.org/doi/pdf/{doi}",
18
+ "https://pubs.acs.org/doi/epdf/{doi}",
19
+ )
20
+ success_url_markers = ("pubs.acs.org/doi/",)
21
+ auth_url_markers = (
22
+ "id.tsinghua.edu.cn",
23
+ "idp.tsinghua.edu.cn",
24
+ "login.openathens.net",
25
+ "acs.org/sso",
26
+ )
27
+ sso_text_markers = (
28
+ "Access through your institution",
29
+ "Institutional login",
30
+ "Find my institution",
31
+ "Access via your institution",
32
+ )
33
+ sso_url_patterns = ("/action/showLogin", "/action/ssostart")
34
+ institution_input_selectors = (
35
+ "input#search-input",
36
+ "input.institution-search__input",
37
+ 'input[name="search"]',
38
+ )
39
+ institution_result_selectors = (
40
+ ".institution-search__item",
41
+ ".institution-result",
42
+ )
@@ -0,0 +1,27 @@
1
+ """AIP (American Institute of Physics) publisher strategy."""
2
+
3
+ from .base import BasePublisherStrategy
4
+ from .registry import StrategyRegistry
5
+
6
+
7
+ @StrategyRegistry.register
8
+ class AIPStrategy(BasePublisherStrategy):
9
+ name = "AIP"
10
+ aliases = ("aip",)
11
+ doi_prefixes = ("10.1063/",)
12
+ base_domains = ("pubs.aip.org", "aip.org")
13
+ sample_dois = ("10.1063/1.1625376",)
14
+ article_url_template = "https://doi.org/{doi}"
15
+ pdf_url_templates = (
16
+ "https://pubs.aip.org/aip/apl/article-pdf/{doi}",
17
+ )
18
+ success_url_markers = ("pubs.aip.org/",)
19
+ auth_url_markers = (
20
+ "id.tsinghua.edu.cn",
21
+ "idp.tsinghua.edu.cn",
22
+ "login.openathens.net",
23
+ )
24
+ sso_text_markers = ("Access through your institution",)
25
+ sso_url_patterns = ("/shibboleth", "/institutional")
26
+ institution_input_selectors = ("input[name='search']", "#searchInstitution")
27
+ institution_result_selectors = ("input[name='search']",)