litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
"""ScienceDirect's article API: ask it for the PDF attachment.
|
|
2
|
+
|
|
3
|
+
Split out of `_core.py` (2026-11-05): it is the largest self-contained chunk
|
|
4
|
+
there — no browser, no DOM, just requests and XML. `_core.py` re-exports the
|
|
5
|
+
names below, so callers keep importing from there.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from ...log import get_logger
|
|
15
|
+
|
|
16
|
+
log = get_logger()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _persist_api_cookies(session: Any, config: dict[str, Any]) -> int:
|
|
20
|
+
"""Delegates to `_core`: the cookie store lives with the browser code."""
|
|
21
|
+
from ._core import _persist_api_cookies as impl
|
|
22
|
+
|
|
23
|
+
return impl(session, config)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _elsevier_header(headers: Any, name: str) -> str:
|
|
27
|
+
"""Return a response header value using case-insensitive matching."""
|
|
28
|
+
if not headers:
|
|
29
|
+
return ""
|
|
30
|
+
value = headers.get(name)
|
|
31
|
+
if value is not None:
|
|
32
|
+
return str(value)
|
|
33
|
+
needle = name.lower()
|
|
34
|
+
for key, value in headers.items():
|
|
35
|
+
if str(key).lower() == needle:
|
|
36
|
+
return str(value)
|
|
37
|
+
return ""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _elsevier_response_is_pdf(resp: Any) -> bool:
|
|
41
|
+
content_type = _elsevier_header(getattr(resp, "headers", {}), "content-type").lower()
|
|
42
|
+
content = getattr(resp, "content", b"") or b""
|
|
43
|
+
return "pdf" in content_type or content[:5] == b"%PDF-"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _elsevier_response_is_xml(resp: Any) -> bool:
|
|
47
|
+
content_type = _elsevier_header(getattr(resp, "headers", {}), "content-type").lower()
|
|
48
|
+
content = (getattr(resp, "content", b"") or b"").lstrip()
|
|
49
|
+
return (
|
|
50
|
+
"xml" in content_type
|
|
51
|
+
or content.startswith(b"<?xml")
|
|
52
|
+
or content.startswith(b"<full-text-retrieval-response")
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _elsevier_response_text(resp: Any) -> str:
|
|
57
|
+
text = getattr(resp, "text", None)
|
|
58
|
+
if isinstance(text, str):
|
|
59
|
+
return text
|
|
60
|
+
content = getattr(resp, "content", b"") or b""
|
|
61
|
+
return content.decode("utf-8", errors="replace")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _elsevier_xml_local_name(name: str) -> str:
|
|
65
|
+
return name.rsplit("}", 1)[-1].split(":", 1)[-1].lower()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _elsevier_element_text(el: Any) -> str:
|
|
69
|
+
return " ".join(" ".join(el.itertext()).split())
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _elsevier_looks_like_pdf_eid(value: str) -> bool:
|
|
73
|
+
lowered = value.strip().lower()
|
|
74
|
+
return bool(lowered) and (lowered.endswith(".pdf") or ".pdf" in lowered)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _elsevier_article_eid_to_main_pdf(value: str) -> str:
|
|
78
|
+
candidate = value.strip()
|
|
79
|
+
if candidate.lower().startswith("eid:"):
|
|
80
|
+
candidate = candidate.split(":", 1)[1].strip()
|
|
81
|
+
if not candidate.startswith("1-s2.0-"):
|
|
82
|
+
return ""
|
|
83
|
+
if candidate.lower().endswith(".pdf"):
|
|
84
|
+
return candidate
|
|
85
|
+
return f"{candidate}-main.pdf"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _elsevier_attachment_container(el: Any, parent_map: dict[Any, Any]) -> Any:
|
|
89
|
+
node = parent_map.get(el, el)
|
|
90
|
+
while node is not None:
|
|
91
|
+
local = _elsevier_xml_local_name(str(node.tag))
|
|
92
|
+
if "attachment" in local or "object" in local:
|
|
93
|
+
return node
|
|
94
|
+
node = parent_map.get(node)
|
|
95
|
+
return parent_map.get(el, el)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _elsevier_attachment_metadata(el: Any) -> str:
|
|
99
|
+
parts: list[str] = []
|
|
100
|
+
for node in el.iter():
|
|
101
|
+
local = _elsevier_xml_local_name(str(node.tag))
|
|
102
|
+
text = _elsevier_element_text(node)
|
|
103
|
+
if text:
|
|
104
|
+
parts.append(f"{local}:{text}")
|
|
105
|
+
for attr_name, attr_value in node.attrib.items():
|
|
106
|
+
attr_local = _elsevier_xml_local_name(str(attr_name))
|
|
107
|
+
if attr_value:
|
|
108
|
+
parts.append(f"{attr_local}:{attr_value}")
|
|
109
|
+
return " ".join(parts).lower()
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _elsevier_attachment_score(eid: str, metadata: str) -> int:
|
|
113
|
+
haystack = f"{eid} {metadata}".lower()
|
|
114
|
+
score = 0
|
|
115
|
+
if eid.lower().endswith(".pdf"):
|
|
116
|
+
score += 20
|
|
117
|
+
if "pdf" in haystack:
|
|
118
|
+
score += 10
|
|
119
|
+
if "main" in haystack or "full-text" in haystack or "fulltext" in haystack:
|
|
120
|
+
score += 100
|
|
121
|
+
if "page-count" in haystack or "pages" in haystack:
|
|
122
|
+
score += 5
|
|
123
|
+
if "attachment-size" in haystack or "filesize" in haystack or "file-size" in haystack:
|
|
124
|
+
score += 5
|
|
125
|
+
if any(marker in haystack for marker in (
|
|
126
|
+
"supplement", "supplementary", "mmc", "appendix", "graphical", "thumbnail",
|
|
127
|
+
)):
|
|
128
|
+
score -= 100
|
|
129
|
+
return score
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _extract_elsevier_pdf_attachment_eids(xml_text: str) -> list[str]:
|
|
133
|
+
"""Extract candidate publisher PDF object EIDs from Elsevier full-text XML."""
|
|
134
|
+
import xml.etree.ElementTree as ET
|
|
135
|
+
|
|
136
|
+
try:
|
|
137
|
+
root = ET.fromstring(xml_text)
|
|
138
|
+
except ET.ParseError as exc:
|
|
139
|
+
log.info(f" [ElsevierAPI] XML parse failed: {exc}")
|
|
140
|
+
return []
|
|
141
|
+
|
|
142
|
+
parent_map = {child: parent for parent in root.iter() for child in parent}
|
|
143
|
+
candidates: list[tuple[int, int, str]] = []
|
|
144
|
+
seen: set[str] = set()
|
|
145
|
+
|
|
146
|
+
for el in root.iter():
|
|
147
|
+
found: list[str] = []
|
|
148
|
+
local = _elsevier_xml_local_name(str(el.tag))
|
|
149
|
+
if local in {"attachment-eid", "object-eid"}:
|
|
150
|
+
text = _elsevier_element_text(el)
|
|
151
|
+
if _elsevier_looks_like_pdf_eid(text):
|
|
152
|
+
found.append(text)
|
|
153
|
+
elif local in {"eid", "identifier"}:
|
|
154
|
+
text = _elsevier_element_text(el)
|
|
155
|
+
main_pdf = _elsevier_article_eid_to_main_pdf(text)
|
|
156
|
+
if main_pdf:
|
|
157
|
+
found.append(main_pdf)
|
|
158
|
+
|
|
159
|
+
for attr_name, attr_value in el.attrib.items():
|
|
160
|
+
attr_local = _elsevier_xml_local_name(str(attr_name))
|
|
161
|
+
if attr_local in {"attachment-eid", "object-eid", "eid"}:
|
|
162
|
+
value = str(attr_value).strip()
|
|
163
|
+
if _elsevier_looks_like_pdf_eid(value):
|
|
164
|
+
found.append(value)
|
|
165
|
+
else:
|
|
166
|
+
main_pdf = _elsevier_article_eid_to_main_pdf(value)
|
|
167
|
+
if main_pdf:
|
|
168
|
+
found.append(main_pdf)
|
|
169
|
+
|
|
170
|
+
for eid in found:
|
|
171
|
+
if eid in seen:
|
|
172
|
+
continue
|
|
173
|
+
seen.add(eid)
|
|
174
|
+
container = _elsevier_attachment_container(el, parent_map)
|
|
175
|
+
metadata = _elsevier_attachment_metadata(container)
|
|
176
|
+
candidates.append((
|
|
177
|
+
_elsevier_attachment_score(eid, metadata),
|
|
178
|
+
len(candidates),
|
|
179
|
+
eid,
|
|
180
|
+
))
|
|
181
|
+
|
|
182
|
+
candidates.sort(key=lambda item: (-item[0], item[1]))
|
|
183
|
+
return [eid for _, _, eid in candidates]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _elsevier_pdf_page_count(content: bytes) -> int | None:
|
|
187
|
+
"""Best-effort page count for detecting Elsevier one-page preview PDFs."""
|
|
188
|
+
try:
|
|
189
|
+
import fitz # type: ignore[import-not-found]
|
|
190
|
+
except Exception:
|
|
191
|
+
return None
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
with fitz.open(stream=content, filetype="pdf") as doc:
|
|
195
|
+
return int(doc.page_count)
|
|
196
|
+
except Exception:
|
|
197
|
+
return None
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _save_elsevier_pdf_content(
|
|
201
|
+
doi: str,
|
|
202
|
+
output_path: Path,
|
|
203
|
+
content: bytes,
|
|
204
|
+
config: dict[str, Any],
|
|
205
|
+
source: str,
|
|
206
|
+
*,
|
|
207
|
+
reject_single_page: bool = False,
|
|
208
|
+
) -> dict[str, Any] | None:
|
|
209
|
+
from ...download.pdf import is_pdf_file, success
|
|
210
|
+
|
|
211
|
+
if len(content) < config.get("min_pdf_size_bytes", 10000):
|
|
212
|
+
log.info(f" [ElsevierAPI] response too small ({len(content)} bytes)")
|
|
213
|
+
return None
|
|
214
|
+
if reject_single_page:
|
|
215
|
+
page_count = _elsevier_pdf_page_count(content)
|
|
216
|
+
if page_count == 1:
|
|
217
|
+
output_path.unlink(missing_ok=True)
|
|
218
|
+
log.info(f" [ElsevierAPI] direct PDF is a 1-page preview for {doi}")
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
222
|
+
output_path.write_bytes(content)
|
|
223
|
+
if is_pdf_file(output_path):
|
|
224
|
+
log.info(f" [ElsevierAPI] downloaded {len(content)} bytes for {doi}")
|
|
225
|
+
return success(doi, output_path, source)
|
|
226
|
+
|
|
227
|
+
output_path.unlink(missing_ok=True)
|
|
228
|
+
return None
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _try_elsevier_object_pdf_from_xml(
|
|
232
|
+
doi: str,
|
|
233
|
+
output_path: Path,
|
|
234
|
+
config: dict[str, Any],
|
|
235
|
+
session: Any,
|
|
236
|
+
article_url: str,
|
|
237
|
+
headers: dict[str, str],
|
|
238
|
+
first_resp: Any,
|
|
239
|
+
*,
|
|
240
|
+
full_xml_already_tried: bool = False,
|
|
241
|
+
) -> dict[str, Any] | None:
|
|
242
|
+
import urllib.parse
|
|
243
|
+
|
|
244
|
+
xml_resp = None
|
|
245
|
+
if (
|
|
246
|
+
first_resp is not None
|
|
247
|
+
and getattr(first_resp, "status_code", 0) == 200
|
|
248
|
+
and _elsevier_response_is_xml(first_resp)
|
|
249
|
+
):
|
|
250
|
+
xml_resp = first_resp
|
|
251
|
+
|
|
252
|
+
if xml_resp is None:
|
|
253
|
+
xml_headers = dict(headers)
|
|
254
|
+
xml_headers["Accept"] = "application/xml"
|
|
255
|
+
param_options = [None] if full_xml_already_tried else [{"view": "FULL"}, None]
|
|
256
|
+
for params in param_options:
|
|
257
|
+
try:
|
|
258
|
+
request_kwargs: dict[str, Any] = {
|
|
259
|
+
"headers": xml_headers,
|
|
260
|
+
"timeout": 30,
|
|
261
|
+
"allow_redirects": True,
|
|
262
|
+
}
|
|
263
|
+
if params:
|
|
264
|
+
request_kwargs["params"] = params
|
|
265
|
+
candidate = session.get(article_url, **request_kwargs)
|
|
266
|
+
except Exception as exc:
|
|
267
|
+
log.info(f" [ElsevierAPI] XML request failed: {exc}")
|
|
268
|
+
continue
|
|
269
|
+
|
|
270
|
+
if getattr(candidate, "status_code", 0) != 200:
|
|
271
|
+
view_name = "FULL XML" if params else "XML"
|
|
272
|
+
log.info(
|
|
273
|
+
f" [ElsevierAPI] {view_name} HTTP "
|
|
274
|
+
f"{getattr(candidate, 'status_code', 0)} for {doi}"
|
|
275
|
+
)
|
|
276
|
+
continue
|
|
277
|
+
if not _elsevier_response_is_xml(candidate):
|
|
278
|
+
content_type = _elsevier_header(
|
|
279
|
+
getattr(candidate, "headers", {}),
|
|
280
|
+
"content-type",
|
|
281
|
+
)
|
|
282
|
+
view_name = "FULL XML" if params else "XML"
|
|
283
|
+
log.info(
|
|
284
|
+
f" [ElsevierAPI] {view_name} request returned non-XML "
|
|
285
|
+
f"({content_type[:50]})"
|
|
286
|
+
)
|
|
287
|
+
continue
|
|
288
|
+
|
|
289
|
+
xml_resp = candidate
|
|
290
|
+
break
|
|
291
|
+
|
|
292
|
+
if xml_resp is None:
|
|
293
|
+
return None
|
|
294
|
+
|
|
295
|
+
eids = _extract_elsevier_pdf_attachment_eids(_elsevier_response_text(xml_resp))
|
|
296
|
+
if not eids:
|
|
297
|
+
log.info(f" [ElsevierAPI] XML has no PDF attachment EID for {doi}")
|
|
298
|
+
return None
|
|
299
|
+
|
|
300
|
+
object_headers = dict(headers)
|
|
301
|
+
object_headers["Accept"] = "application/pdf"
|
|
302
|
+
for eid in eids:
|
|
303
|
+
object_url = (
|
|
304
|
+
"https://api.elsevier.com/content/object/eid/"
|
|
305
|
+
f"{urllib.parse.quote(eid, safe='')}"
|
|
306
|
+
)
|
|
307
|
+
try:
|
|
308
|
+
object_resp = session.get(
|
|
309
|
+
object_url,
|
|
310
|
+
headers=object_headers,
|
|
311
|
+
timeout=30,
|
|
312
|
+
allow_redirects=True,
|
|
313
|
+
)
|
|
314
|
+
except Exception as exc:
|
|
315
|
+
log.info(f" [ElsevierAPI] object {eid} request failed: {exc}")
|
|
316
|
+
continue
|
|
317
|
+
|
|
318
|
+
if getattr(object_resp, "status_code", 0) != 200:
|
|
319
|
+
log.info(
|
|
320
|
+
f" [ElsevierAPI] object {eid} HTTP "
|
|
321
|
+
f"{getattr(object_resp, 'status_code', 0)}"
|
|
322
|
+
)
|
|
323
|
+
continue
|
|
324
|
+
if not _elsevier_response_is_pdf(object_resp):
|
|
325
|
+
content_type = _elsevier_header(
|
|
326
|
+
getattr(object_resp, "headers", {}),
|
|
327
|
+
"content-type",
|
|
328
|
+
)
|
|
329
|
+
log.info(f" [ElsevierAPI] object {eid} returned non-PDF ({content_type[:50]})")
|
|
330
|
+
continue
|
|
331
|
+
|
|
332
|
+
result = _save_elsevier_pdf_content(
|
|
333
|
+
doi,
|
|
334
|
+
output_path,
|
|
335
|
+
getattr(object_resp, "content", b"") or b"",
|
|
336
|
+
config,
|
|
337
|
+
"ElsevierAPI",
|
|
338
|
+
reject_single_page=True,
|
|
339
|
+
)
|
|
340
|
+
if result:
|
|
341
|
+
log.info(f" [ElsevierAPI] downloaded object PDF via attachment EID {eid}")
|
|
342
|
+
return result
|
|
343
|
+
|
|
344
|
+
return None
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _try_elsevier_text_fallback(
|
|
348
|
+
doi: str,
|
|
349
|
+
output_path: Path,
|
|
350
|
+
session: Any,
|
|
351
|
+
article_url: str,
|
|
352
|
+
headers: dict[str, str],
|
|
353
|
+
config: dict[str, Any],
|
|
354
|
+
) -> dict[str, Any] | None:
|
|
355
|
+
"""Last API resort: save the httpAccept=text/plain full text as .txt.
|
|
356
|
+
|
|
357
|
+
Only kept when the body is a real plain-text payload above 100KB — the
|
|
358
|
+
same threshold pdf_utils.is_suspicious_pdf treats as certainly full text —
|
|
359
|
+
so the racing pipeline never flags or drops the artifact. API error
|
|
360
|
+
payloads (text/xml) are rejected by the content-type check.
|
|
361
|
+
"""
|
|
362
|
+
txt_headers = dict(headers)
|
|
363
|
+
txt_headers["Accept"] = "text/plain"
|
|
364
|
+
try:
|
|
365
|
+
resp = session.get(
|
|
366
|
+
article_url,
|
|
367
|
+
headers=txt_headers,
|
|
368
|
+
params={"httpAccept": "text/plain"},
|
|
369
|
+
timeout=30,
|
|
370
|
+
allow_redirects=True,
|
|
371
|
+
)
|
|
372
|
+
except Exception as e:
|
|
373
|
+
log.info(f" [ElsevierAPI] text/plain request failed: {e}")
|
|
374
|
+
return None
|
|
375
|
+
|
|
376
|
+
if getattr(resp, "status_code", 0) != 200:
|
|
377
|
+
return None
|
|
378
|
+
content_type = _elsevier_header(getattr(resp, "headers", {}), "content-type").lower()
|
|
379
|
+
content = getattr(resp, "content", b"") or b""
|
|
380
|
+
if "text/plain" not in content_type or len(content) < 100_000:
|
|
381
|
+
log.info(
|
|
382
|
+
f" [ElsevierAPI] no usable full text ({content_type[:40]}, "
|
|
383
|
+
f"{len(content)} bytes)"
|
|
384
|
+
)
|
|
385
|
+
return None
|
|
386
|
+
|
|
387
|
+
txt_path = output_path.with_suffix(".txt")
|
|
388
|
+
try:
|
|
389
|
+
txt_path.parent.mkdir(parents=True, exist_ok=True)
|
|
390
|
+
txt_path.write_bytes(content)
|
|
391
|
+
except OSError as e:
|
|
392
|
+
log.info(f" [ElsevierAPI] full text save failed: {e}")
|
|
393
|
+
return None
|
|
394
|
+
log.info(f" [ElsevierAPI] saved {len(content)} bytes of full text as {txt_path.name}")
|
|
395
|
+
from ...download.pdf import success
|
|
396
|
+
return success(doi, txt_path, "ElsevierAPI(Text)")
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def try_elsevier_api(
|
|
400
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
401
|
+
) -> dict[str, Any] | None:
|
|
402
|
+
from ...sources.elsevier_api import fulltext_enabled
|
|
403
|
+
if not fulltext_enabled(config):
|
|
404
|
+
return None # off in settings, or this key has no full-text entitlement
|
|
405
|
+
return _try_elsevier_api(doi, output_path, config)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _try_elsevier_api(
|
|
409
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
410
|
+
) -> dict[str, Any] | None:
|
|
411
|
+
"""Download Elsevier/ScienceDirect PDF via Article Retrieval API.
|
|
412
|
+
|
|
413
|
+
Uses the Elsevier Institutional API (api.elsevier.com) with an API key
|
|
414
|
+
and optional institutional token. This is far faster and more reliable
|
|
415
|
+
than browser-based login flows.
|
|
416
|
+
|
|
417
|
+
API-first request order per route (browser navigation is only a last
|
|
418
|
+
resort and lives in try_elsevier_browser):
|
|
419
|
+
1. article endpoint with Accept: application/pdf — the full PDF when
|
|
420
|
+
the key's entitlement covers the article. Entitlement follows the
|
|
421
|
+
API key, not the client IP, so this works off-campus. The body is
|
|
422
|
+
validated as a real multi-page PDF before saving (rejects the
|
|
423
|
+
1-page preview served to unentitled keys).
|
|
424
|
+
2. FULL XML → main-PDF attachment EIDs → Content Object API
|
|
425
|
+
(existing chain; covers articles whose direct endpoint misbehaves).
|
|
426
|
+
3. httpAccept=text/plain full text saved as .txt when no PDF is
|
|
427
|
+
returned at all.
|
|
428
|
+
|
|
429
|
+
When the API does not return a PDF directly (no institutional access),
|
|
430
|
+
cookies from the request chain (api.elsevier.com → linkinghub.elsevier.com
|
|
431
|
+
→ sciencedirect.com) are persisted for the browser strategy to reuse.
|
|
432
|
+
|
|
433
|
+
Config keys:
|
|
434
|
+
elsevier_api_key — personal or institutional API key (required);
|
|
435
|
+
falls back to the ELSEVIER_API_KEY env var
|
|
436
|
+
elsevier_insttoken — institutional token for campus-level access
|
|
437
|
+
"""
|
|
438
|
+
api_key = config.get("elsevier_api_key", "") or os.environ.get("ELSEVIER_API_KEY", "")
|
|
439
|
+
if not api_key:
|
|
440
|
+
log.info(f" [ElsevierAPI] no API key configured, skipping. "
|
|
441
|
+
f"Run litloop_elsevier_setup to configure (free).")
|
|
442
|
+
return None
|
|
443
|
+
|
|
444
|
+
from ...network import USER_AGENT, proxy_dict, select_proxy_for_url
|
|
445
|
+
|
|
446
|
+
url = f"https://api.elsevier.com/content/article/doi/{doi}"
|
|
447
|
+
|
|
448
|
+
headers: dict[str, str] = {
|
|
449
|
+
"User-Agent": USER_AGENT,
|
|
450
|
+
"X-ELS-APIKey": api_key,
|
|
451
|
+
}
|
|
452
|
+
insttoken = config.get("elsevier_insttoken", "")
|
|
453
|
+
if insttoken:
|
|
454
|
+
headers["X-ELS-InstToken"] = insttoken
|
|
455
|
+
|
|
456
|
+
proxy = select_proxy_for_url(url, config)
|
|
457
|
+
configured_proxies = proxy_dict(proxy)
|
|
458
|
+
route_options: list[tuple[str, dict[str, str]]] = [("direct", {})]
|
|
459
|
+
if configured_proxies:
|
|
460
|
+
route_options.append(("configured proxy", configured_proxies))
|
|
461
|
+
|
|
462
|
+
import requests
|
|
463
|
+
|
|
464
|
+
for route_name, proxies in route_options:
|
|
465
|
+
session = requests.Session()
|
|
466
|
+
session.trust_env = False
|
|
467
|
+
if proxies:
|
|
468
|
+
session.proxies = proxies
|
|
469
|
+
|
|
470
|
+
# Step 1: direct article PDF — entitlement follows the API key, so no
|
|
471
|
+
# campus IP is needed. Reject non-200 and 1-page preview responses.
|
|
472
|
+
pdf_headers = dict(headers)
|
|
473
|
+
pdf_headers["Accept"] = "application/pdf"
|
|
474
|
+
pdf_resp = None
|
|
475
|
+
try:
|
|
476
|
+
log.info(f" [ElsevierAPI] trying {route_name} route")
|
|
477
|
+
pdf_resp = session.get(
|
|
478
|
+
url,
|
|
479
|
+
headers=pdf_headers,
|
|
480
|
+
timeout=30,
|
|
481
|
+
allow_redirects=True,
|
|
482
|
+
)
|
|
483
|
+
except Exception as e:
|
|
484
|
+
log.info(f" [ElsevierAPI] {route_name} direct PDF request failed: {e}")
|
|
485
|
+
|
|
486
|
+
if pdf_resp is not None and pdf_resp.status_code == 200 and _elsevier_response_is_pdf(pdf_resp):
|
|
487
|
+
result = _save_elsevier_pdf_content(
|
|
488
|
+
doi,
|
|
489
|
+
output_path,
|
|
490
|
+
pdf_resp.content,
|
|
491
|
+
config,
|
|
492
|
+
"ElsevierAPI",
|
|
493
|
+
reject_single_page=True,
|
|
494
|
+
)
|
|
495
|
+
if result:
|
|
496
|
+
return result
|
|
497
|
+
elif pdf_resp is not None and pdf_resp.status_code != 200:
|
|
498
|
+
log.info(
|
|
499
|
+
f" [ElsevierAPI] {route_name} direct PDF HTTP "
|
|
500
|
+
f"{pdf_resp.status_code} for {doi}, trying FULL XML route"
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
try:
|
|
504
|
+
xml_headers = dict(headers)
|
|
505
|
+
xml_headers["Accept"] = "application/xml"
|
|
506
|
+
resp = session.get(
|
|
507
|
+
url,
|
|
508
|
+
headers=xml_headers,
|
|
509
|
+
params={"view": "FULL"},
|
|
510
|
+
timeout=30,
|
|
511
|
+
allow_redirects=True,
|
|
512
|
+
)
|
|
513
|
+
except Exception as e:
|
|
514
|
+
log.info(f" [ElsevierAPI] {route_name} FULL XML request failed: {e}")
|
|
515
|
+
continue
|
|
516
|
+
|
|
517
|
+
if resp is not None and resp.status_code != 200:
|
|
518
|
+
if resp.status_code in (403, 429):
|
|
519
|
+
log.info(
|
|
520
|
+
f" [ElsevierAPI] {route_name} HTTP {resp.status_code}; "
|
|
521
|
+
"API access may be unavailable or rate limited"
|
|
522
|
+
)
|
|
523
|
+
else:
|
|
524
|
+
log.info(f" [ElsevierAPI] {route_name} HTTP {resp.status_code} for {doi}")
|
|
525
|
+
|
|
526
|
+
if resp is not None and resp.status_code == 200 and _elsevier_response_is_pdf(resp):
|
|
527
|
+
result = _save_elsevier_pdf_content(
|
|
528
|
+
doi,
|
|
529
|
+
output_path,
|
|
530
|
+
resp.content,
|
|
531
|
+
config,
|
|
532
|
+
"ElsevierAPI",
|
|
533
|
+
reject_single_page=True,
|
|
534
|
+
)
|
|
535
|
+
if result:
|
|
536
|
+
try:
|
|
537
|
+
_persist_api_cookies(session, config)
|
|
538
|
+
except Exception:
|
|
539
|
+
pass
|
|
540
|
+
return result
|
|
541
|
+
|
|
542
|
+
result = _try_elsevier_object_pdf_from_xml(
|
|
543
|
+
doi,
|
|
544
|
+
output_path,
|
|
545
|
+
config,
|
|
546
|
+
session,
|
|
547
|
+
url,
|
|
548
|
+
headers,
|
|
549
|
+
resp,
|
|
550
|
+
full_xml_already_tried=True,
|
|
551
|
+
)
|
|
552
|
+
if result:
|
|
553
|
+
try:
|
|
554
|
+
_persist_api_cookies(session, config)
|
|
555
|
+
except Exception:
|
|
556
|
+
pass
|
|
557
|
+
return result
|
|
558
|
+
|
|
559
|
+
content_type = _elsevier_header(
|
|
560
|
+
getattr(resp, "headers", {}) if resp is not None else {},
|
|
561
|
+
"content-type",
|
|
562
|
+
)
|
|
563
|
+
log.info(
|
|
564
|
+
f" [ElsevierAPI] {route_name} non-PDF response "
|
|
565
|
+
f"({content_type[:50]}), trying next route if available"
|
|
566
|
+
)
|
|
567
|
+
try:
|
|
568
|
+
_persist_api_cookies(session, config)
|
|
569
|
+
except Exception as e:
|
|
570
|
+
log.info(f" [ElsevierAPI] cookie persist failed: {e}")
|
|
571
|
+
|
|
572
|
+
# Step 3: full text as .txt — the API returned no PDF deliverable for
|
|
573
|
+
# this DOI; grab the plain-text full text before falling back further.
|
|
574
|
+
txt_result = _try_elsevier_text_fallback(doi, output_path, session, url, headers, config)
|
|
575
|
+
if txt_result:
|
|
576
|
+
return txt_result
|
|
577
|
+
|
|
578
|
+
return None
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""ACM (Association for Computing Machinery) publisher strategy."""
|
|
2
|
+
|
|
3
|
+
from .base import BasePublisherStrategy
|
|
4
|
+
from .registry import StrategyRegistry
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@StrategyRegistry.register
|
|
8
|
+
class ACMStrategy(BasePublisherStrategy):
|
|
9
|
+
name = "ACM"
|
|
10
|
+
aliases = ("acm",)
|
|
11
|
+
doi_prefixes = ("10.1145/",)
|
|
12
|
+
base_domains = ("dl.acm.org", "acm.org")
|
|
13
|
+
sample_dois = ("10.1145/3292500.3330893",)
|
|
14
|
+
article_url_template = "https://dl.acm.org/doi/{doi}"
|
|
15
|
+
pdf_url_templates = (
|
|
16
|
+
"https://dl.acm.org/doi/pdf/{doi}",
|
|
17
|
+
)
|
|
18
|
+
success_url_markers = ("dl.acm.org/doi/",)
|
|
19
|
+
auth_url_markers = (
|
|
20
|
+
"id.tsinghua.edu.cn",
|
|
21
|
+
"idp.tsinghua.edu.cn",
|
|
22
|
+
"login.openathens.net",
|
|
23
|
+
)
|
|
24
|
+
sso_text_markers = ("Access through your institution",)
|
|
25
|
+
sso_url_patterns = ("/shibboleth", "/institutional")
|
|
26
|
+
institution_input_selectors = ("input[name='search']", "#searchInstitution")
|
|
27
|
+
institution_result_selectors = ("input[name='search']",)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""ACS (American Chemical Society) publisher strategy."""
|
|
2
|
+
|
|
3
|
+
from .base import BasePublisherStrategy
|
|
4
|
+
from .registry import StrategyRegistry
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@StrategyRegistry.register
|
|
8
|
+
class ACSStrategy(BasePublisherStrategy):
|
|
9
|
+
name = "ACS"
|
|
10
|
+
aliases = ("acs",)
|
|
11
|
+
doi_prefixes = ("10.1021/",)
|
|
12
|
+
base_domains = ("pubs.acs.org",)
|
|
13
|
+
sample_dois = ("10.1021/jacs.5b00936",)
|
|
14
|
+
article_url_template = "https://pubs.acs.org/doi/{doi}"
|
|
15
|
+
pdf_url_templates = (
|
|
16
|
+
"https://pubs.acs.org/doi/pdf/{doi}?ref=article_openPDF",
|
|
17
|
+
"https://pubs.acs.org/doi/pdf/{doi}",
|
|
18
|
+
"https://pubs.acs.org/doi/epdf/{doi}",
|
|
19
|
+
)
|
|
20
|
+
success_url_markers = ("pubs.acs.org/doi/",)
|
|
21
|
+
auth_url_markers = (
|
|
22
|
+
"id.tsinghua.edu.cn",
|
|
23
|
+
"idp.tsinghua.edu.cn",
|
|
24
|
+
"login.openathens.net",
|
|
25
|
+
"acs.org/sso",
|
|
26
|
+
)
|
|
27
|
+
sso_text_markers = (
|
|
28
|
+
"Access through your institution",
|
|
29
|
+
"Institutional login",
|
|
30
|
+
"Find my institution",
|
|
31
|
+
"Access via your institution",
|
|
32
|
+
)
|
|
33
|
+
sso_url_patterns = ("/action/showLogin", "/action/ssostart")
|
|
34
|
+
institution_input_selectors = (
|
|
35
|
+
"input#search-input",
|
|
36
|
+
"input.institution-search__input",
|
|
37
|
+
'input[name="search"]',
|
|
38
|
+
)
|
|
39
|
+
institution_result_selectors = (
|
|
40
|
+
".institution-search__item",
|
|
41
|
+
".institution-result",
|
|
42
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""AIP (American Institute of Physics) publisher strategy."""
|
|
2
|
+
|
|
3
|
+
from .base import BasePublisherStrategy
|
|
4
|
+
from .registry import StrategyRegistry
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@StrategyRegistry.register
|
|
8
|
+
class AIPStrategy(BasePublisherStrategy):
|
|
9
|
+
name = "AIP"
|
|
10
|
+
aliases = ("aip",)
|
|
11
|
+
doi_prefixes = ("10.1063/",)
|
|
12
|
+
base_domains = ("pubs.aip.org", "aip.org")
|
|
13
|
+
sample_dois = ("10.1063/1.1625376",)
|
|
14
|
+
article_url_template = "https://doi.org/{doi}"
|
|
15
|
+
pdf_url_templates = (
|
|
16
|
+
"https://pubs.aip.org/aip/apl/article-pdf/{doi}",
|
|
17
|
+
)
|
|
18
|
+
success_url_markers = ("pubs.aip.org/",)
|
|
19
|
+
auth_url_markers = (
|
|
20
|
+
"id.tsinghua.edu.cn",
|
|
21
|
+
"idp.tsinghua.edu.cn",
|
|
22
|
+
"login.openathens.net",
|
|
23
|
+
)
|
|
24
|
+
sso_text_markers = ("Access through your institution",)
|
|
25
|
+
sso_url_patterns = ("/shibboleth", "/institutional")
|
|
26
|
+
institution_input_selectors = ("input[name='search']", "#searchInstitution")
|
|
27
|
+
institution_result_selectors = ("input[name='search']",)
|