litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,684 @@
1
+ """Publisher routing table and direct download strategies.
2
+
3
+ Supports Nature direct download and MDPI open access.
4
+ Other publishers fall through to Crossref/Unpaywall/OpenAlex.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import re
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ import requests
14
+
15
+ from .crossref import try_crossref
16
+
17
+
18
+ def _load_publisher_cookies(session: requests.Session, config: dict[str, Any]) -> None:
19
+ """Load saved publisher cookies into a session (and its per-host proxy routes)."""
20
+ from ..network import apply_host_proxy
21
+ apply_host_proxy(session, config)
22
+ try:
23
+ from ..browser.cookies import inject_cookies
24
+ inject_cookies(session, config)
25
+ except Exception:
26
+ pass
27
+ # Also load CARSI/institutional login cookies
28
+ try:
29
+ from ..config import DATA_DIR
30
+ import json as _json
31
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
32
+ carsi_dir = cache_dir / "carsi_cookies"
33
+ if carsi_dir.is_dir():
34
+ for cf in carsi_dir.glob("*.json"):
35
+ try:
36
+ for c in _json.loads(cf.read_text(encoding="utf-8")):
37
+ session.cookies.set(
38
+ c.get("name", ""), c.get("value", ""),
39
+ domain=c.get("domain", ""), path=c.get("path", "/"),
40
+ )
41
+ except Exception:
42
+ pass
43
+ except Exception:
44
+ pass
45
+ from .nature import try_nature_direct
46
+ from .openalex import try_openalex_oa
47
+ from .semantic_scholar import try_semanticscholar
48
+ from .springer_tdm import try_springer_tdm
49
+ from .unpaywall import try_unpaywall
50
+
51
+ # ============================================================
52
+ # Publisher URL templates: (name, pdf_template, match_patterns, gdpr, bibtex_template)
53
+ # {doi} = full DOI, {doi_suffix} = part after prefix, {base} = resolved URL base
54
+ # ============================================================
55
+ # Only publishers with working direct download (no subscription required)
56
+ PUBLISHERS: list[tuple[str, str | None, list[str], bool, str | None]] = [
57
+ ("Nature", "https://www.nature.com/articles/{doi_suffix}.pdf",
58
+ ["nature.com", "10.1038/"], False, None),
59
+ ("Frontiers", None,
60
+ ["frontiersin.org", "10.3389/"], False, None),
61
+ ("BMC", None,
62
+ ["biomedcentral.com", "springeropen.com", "10.1186/"], False, None),
63
+ ]
64
+
65
+ DOI_PREFIX_TO_PUBLISHER: dict[str, str] = {
66
+ "10.1038/": "Nature",
67
+ "10.1016/": "Elsevier",
68
+ "10.1002/": "Wiley",
69
+ "10.1126/": "Science",
70
+ "10.1073/": "PNAS",
71
+ "10.1093/": "Oxford",
72
+ "10.1021/": "ACS",
73
+ "10.1007/": "Springer",
74
+ "10.1103/": "APS",
75
+ "10.1088/": "IOP",
76
+ "10.1101/": "bioRxiv",
77
+ "10.3390/": "MDPI",
78
+ "10.1371/": "PLOS",
79
+ "10.3389/": "Frontiers",
80
+ "10.1186/": "BMC",
81
+ "10.15252/": "EMBO",
82
+ "10.1039/": "RSC",
83
+ "10.1063/": "AIP",
84
+ "10.1111/": "Wiley",
85
+ "10.1145/": "ACM",
86
+ "10.1109/": "IEEE",
87
+ "10.1364/": "OSA",
88
+ "10.1080/": "Tandfonline",
89
+ "10.1116/": "AIP", # AVS (pubs.aip.org)
90
+ "10.1177/": "SAGE",
91
+ "10.1061/": "ASCE",
92
+ "10.1146/": "AnnualReviews",
93
+ "10.31219/": "Research Square",
94
+ "10.21203/": "Research Square",
95
+ "10.20944/": "Preprints.org",
96
+ "10.26434/": "chemRxiv",
97
+ "10.48550/": "arXiv",
98
+ "10.1143/": "IOP", # JJAP (IOP-hosted)
99
+ "10.3938/": "KPS",
100
+ "10.3762/": "Beilstein",
101
+ }
102
+
103
+ # Fast sources per publisher (used by tiered racing)
104
+ # Browser strategies (ElsevierBrowser, etc.) use camofox for anti-bot bypass
105
+ PUBLISHER_TOOL_MAP: dict[str, list[str]] = {
106
+ "Nature": ["NatureDirect", "PublisherDirect", "NatureBrowser", "Crossref", "Unpaywall"],
107
+ "MDPI": ["MDPIDirect", "Crossref", "Unpaywall"],
108
+ "arXiv": ["arXiv"],
109
+ "bioRxiv": ["arXiv", "Unpaywall"],
110
+ "Elsevier": ["Crossref", "Unpaywall", "ElsevierAPI", "ElsevierBrowser"],
111
+ "Wiley": ["WileyBrowser", "Crossref", "Unpaywall"],
112
+ "Science": ["ScienceDirect", "ScienceBrowser", "Crossref", "Unpaywall"],
113
+ "PNAS": ["PNASDirect", "Crossref", "Unpaywall"],
114
+ "Oxford": ["OxfordBrowser", "Crossref", "Unpaywall"],
115
+ "ACS": ["ACSBrowser", "Crossref", "Unpaywall"],
116
+ "Springer": ["SpringerTDM", "SpringerBrowser", "Crossref", "Unpaywall"],
117
+ "APS": ["APSBrowser", "Crossref", "Unpaywall"],
118
+ "IOP": ["IOPBrowser", "Crossref", "Unpaywall"],
119
+ "PLOS": ["PLOSDirect", "Crossref", "Unpaywall"],
120
+ "Frontiers": ["FrontiersDirect", "Crossref", "Unpaywall"],
121
+ "BMC": ["BMCDirect", "Crossref", "Unpaywall"],
122
+ "RSC": ["RSCBrowser", "Crossref", "Unpaywall"],
123
+ "AIP": ["AIPBrowser", "Crossref", "Unpaywall"],
124
+ "ACM": ["ACMBrowser", "Crossref", "Unpaywall"],
125
+ "IEEE": ["IEEEBrowser", "Crossref", "Unpaywall"],
126
+ "OSA": ["Crossref", "Unpaywall"],
127
+ "Tandfonline": ["TandFBrowser", "Crossref", "Unpaywall"],
128
+ "SAGE": ["SAGEBrowser", "Crossref", "Unpaywall"],
129
+ "ASCE": ["ASCEBrowser", "Crossref", "Unpaywall"],
130
+ "AnnualReviews": ["Crossref", "Unpaywall"],
131
+ "KPS": ["Crossref", "Unpaywall"],
132
+ "Beilstein": ["Crossref", "Unpaywall"],
133
+ "EMBO": ["Crossref", "Unpaywall"],
134
+ "Research Square": ["Crossref", "Unpaywall"],
135
+ "chemRxiv": ["Crossref", "Unpaywall"],
136
+ "Preprints.org": ["Crossref", "Unpaywall"],
137
+ "WorldScientific": ["Crossref", "Unpaywall"],
138
+ }
139
+
140
+ _FN_MAP: dict[str, Any] = {}
141
+
142
+ from ..log import get_logger
143
+ log = get_logger()
144
+
145
+
146
+ def _write_pdf_atomic(output_path: Path, first: bytes, iterator: Any) -> bool:
147
+ """Write PDF to .part file then atomically rename. Cleans up on failure."""
148
+ output_path.parent.mkdir(parents=True, exist_ok=True)
149
+ tmp_path = output_path.with_suffix(output_path.suffix + ".part")
150
+ try:
151
+ with tmp_path.open("wb") as fh:
152
+ fh.write(first)
153
+ for chunk in iterator:
154
+ if chunk:
155
+ fh.write(chunk)
156
+ tmp_path.replace(output_path)
157
+ return True
158
+ except Exception:
159
+ tmp_path.unlink(missing_ok=True)
160
+ return False
161
+
162
+
163
+ # ============================================================
164
+ # Publisher matching
165
+ # ============================================================
166
+
167
+ def _match_publisher(doi_or_url: str, patterns: list[str]) -> bool:
168
+ lower = doi_or_url.lower()
169
+ return any(p.lower() in lower for p in patterns)
170
+
171
+
172
+ def _format_pdf_url(template: str, doi: str, url: str) -> str | None:
173
+ if not template:
174
+ return None
175
+ doi_suffix = doi.split("/", 1)[1] if "/" in doi else doi
176
+ base = ""
177
+ if url:
178
+ m = re.match(r"(https?://[^/]+)", url)
179
+ if m:
180
+ base = m.group(1)
181
+ return (template
182
+ .replace("{doi}", doi)
183
+ .replace("{doi_suffix}", doi_suffix)
184
+ .replace("{base}", base))
185
+
186
+
187
+ # ============================================================
188
+ # DOI resolution
189
+ # ============================================================
190
+
191
+ def resolve_doi(doi: str, config: dict[str, Any]) -> str | None:
192
+ """Resolve DOI to publisher URL via doi.org redirect."""
193
+ from ..network import USER_AGENT
194
+ try:
195
+ s = requests.Session()
196
+ s.trust_env = False
197
+ # doi.org's own redirect already names the publisher URL; following it
198
+ # to the publisher (6+ s on the slow route to Nature's CDN) adds nothing.
199
+ from ..network import apply_host_proxy
200
+ apply_host_proxy(s, config) # doi.org may be on the proxy exception list
201
+ resp = s.head(f"https://doi.org/{doi}", allow_redirects=False,
202
+ timeout=10, headers={"User-Agent": USER_AGENT})
203
+ url = resp.headers.get("location", "") if resp.is_redirect else resp.url
204
+ known_patterns = []
205
+ for pub in PUBLISHERS:
206
+ known_patterns.extend(pub[2])
207
+ if url and any(p.lower() in url.lower() for p in known_patterns):
208
+ return url.replace("http://", "https://", 1)
209
+ except Exception:
210
+ pass
211
+ return None
212
+
213
+
214
+ # ============================================================
215
+ # Publisher direct download (with GDPR cookie bypass)
216
+ # ============================================================
217
+
218
+ def try_publisher_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
219
+ """Try publisher direct download with HTTP first, then browser fallback."""
220
+ from ..network import USER_AGENT, polite_delay
221
+ from ..download.pdf import is_pdf_file, success, _response_looks_pdf
222
+
223
+ log.info(f" [PublisherDirect] Starting for {doi}")
224
+ resolved_url = resolve_doi(doi, config)
225
+ log.info(f" [PublisherDirect] Resolved URL: {resolved_url}")
226
+
227
+ # Phase 1: HTTP-based direct download (fast path)
228
+ for pub in PUBLISHERS:
229
+ name, template, matches, gdpr, _ = pub
230
+
231
+ if not _match_publisher(doi, matches) and not _match_publisher(resolved_url or "", matches):
232
+ continue
233
+
234
+ pdf_url = _format_pdf_url(template or "", doi, resolved_url or "")
235
+ if not pdf_url:
236
+ continue
237
+
238
+ log.info(f" [Publisher] {name}: {pdf_url[:80]}...")
239
+ polite_delay(config)
240
+
241
+ try:
242
+ s = requests.Session()
243
+ s.trust_env = False
244
+ s.headers.update({"User-Agent": USER_AGENT})
245
+ _load_publisher_cookies(s, config)
246
+ resp = s.get(pdf_url, timeout=15, stream=True,
247
+ headers={"Accept": "application/pdf,*/*"},
248
+ allow_redirects=True)
249
+
250
+ if resp.status_code >= 400:
251
+ log.info(f" [Publisher] {name}: HTTP {resp.status_code}")
252
+ continue
253
+
254
+ iterator = resp.iter_content(chunk_size=8192)
255
+ first = next(iterator, b"")
256
+ if not _response_looks_pdf(resp, first):
257
+ continue
258
+
259
+ if not _write_pdf_atomic(output_path, first, iterator):
260
+ continue
261
+ if is_pdf_file(output_path):
262
+ return success(doi, output_path, f"Publisher({name})")
263
+
264
+ except Exception as e:
265
+ log.info(f" [Publisher] {name}: {e}")
266
+
267
+ # Phase 2: Browser-based download via camofox (handles anti-bot)
268
+ publisher = get_publisher(doi)
269
+ if publisher and config.get("camofox_enabled", True):
270
+ from ..browser.camofox_api import is_available as _camofox_avail
271
+ if _camofox_avail(config):
272
+ log.info(f" [PublisherDirect] Trying browser strategy for {publisher}")
273
+ fn = _FN_MAP.get(f"{publisher}Browser") or _FN_MAP.get("GenericBrowser")
274
+ if fn:
275
+ try:
276
+ result = fn(doi, output_path, config)
277
+ if result:
278
+ return result
279
+ except Exception as e:
280
+ log.info(f" [PublisherDirect] Browser strategy failed: {e}")
281
+
282
+ return None
283
+
284
+
285
+ # ============================================================
286
+ # MDPI direct download
287
+ # ============================================================
288
+
289
+ def try_mdpi_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
290
+ """Download MDPI open access papers directly."""
291
+ if not doi.startswith("10.3390/"):
292
+ return None
293
+
294
+ from ..download.pdf import is_pdf_file, success, _response_looks_pdf
295
+ from ..network import USER_AGENT, polite_delay
296
+
297
+ article_id = doi.split("10.3390/")[-1]
298
+
299
+ urls = [
300
+ f"https://www.mdpi.com/{article_id}/pdf",
301
+ ]
302
+
303
+ try:
304
+ session = requests.Session()
305
+ session.trust_env = False
306
+ session.headers.update({"User-Agent": USER_AGENT})
307
+ polite_delay(config)
308
+
309
+ # First try landing page to get real PDF URL
310
+ try:
311
+ resp = session.get(f"https://www.mdpi.com/{article_id}", timeout=10, allow_redirects=True)
312
+ if resp.status_code == 200:
313
+ pdf_match = re.search(r'citation_pdf_url["\s]+content="([^"]+)"', resp.text[:5000], re.I)
314
+ if not pdf_match:
315
+ pdf_match = re.search(r'href="(/[^"]*?/pdf[^"]*)"', resp.text[:10000], re.I)
316
+ if pdf_match:
317
+ pdf_url = pdf_match.group(1)
318
+ if pdf_url.startswith("/"):
319
+ pdf_url = f"https://www.mdpi.com{pdf_url}"
320
+ urls.insert(0, pdf_url)
321
+ except Exception:
322
+ pass
323
+
324
+ for pdf_url in urls:
325
+ try:
326
+ resp2 = session.get(pdf_url, timeout=15, stream=True,
327
+ headers={"Accept": "application/pdf,*/*"})
328
+ if resp2.status_code >= 400:
329
+ continue
330
+
331
+ iterator = resp2.iter_content(chunk_size=8192)
332
+ first = next(iterator, b"")
333
+ if not _response_looks_pdf(resp2, first):
334
+ continue
335
+
336
+ if not _write_pdf_atomic(output_path, first, iterator):
337
+ continue
338
+ if is_pdf_file(output_path):
339
+ return success(doi, output_path, "MDPIDirect")
340
+ except Exception:
341
+ continue
342
+ except Exception:
343
+ pass
344
+ return None
345
+
346
+
347
+ # ============================================================
348
+ # Science/AAAS direct download (Science Advances, Science, etc.)
349
+ # ============================================================
350
+
351
+ def try_science_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
352
+ """Download Science/AAAS papers directly. Science Advances is OA."""
353
+ if not doi.startswith("10.1126/"):
354
+ return None
355
+
356
+ from ..network import polite_delay
357
+ from ..download.pdf import is_pdf_file, success, _response_looks_pdf
358
+
359
+ def _fetch_pdf_camofox(pdf_url: str) -> "requests.Response | None": # type: ignore[name-defined]
360
+ from ..browser.camofox_api import is_available as _camofox_avail, solve_url as _camofox_solve
361
+ if not _camofox_avail(config):
362
+ return None
363
+ result = _camofox_solve(pdf_url, config)
364
+ if not result:
365
+ return None
366
+ solution = result.get("solution", {})
367
+ if solution.get("status", 0) >= 400:
368
+ return None
369
+ import requests as _req
370
+ resp = _req.Response()
371
+ resp.status_code = solution.get("status", 200)
372
+ resp._content = solution.get("response", "").encode("utf-8")
373
+ resp.url = solution.get("url", pdf_url)
374
+ return resp
375
+
376
+ # Extract article ID from DOI (e.g., sciadv.1600983 -> 1600983)
377
+ doi_suffix = doi.split("10.1126/")[-1]
378
+
379
+ # Try Science Advances PDF URL patterns
380
+ pdf_urls = []
381
+ if "sciadv" in doi_suffix:
382
+ # Science Advances: https://advances.sciencemag.org/content/3/11/e1600983.full.pdf
383
+ article_id = doi_suffix.split(".")[-1] if "." in doi_suffix else doi_suffix
384
+ pdf_urls.append(f"https://advances.sciencemag.org/content/3/11/{article_id}.full.pdf")
385
+ pdf_urls.append(f"https://advances.sciencemag.org/content/early/{article_id}.full.pdf")
386
+ elif "science" in doi_suffix.lower():
387
+ # Science magazine
388
+ pdf_urls.append(f"https://www.science.org/doi/pdf/{doi}")
389
+
390
+ for pdf_url in pdf_urls:
391
+ try:
392
+ polite_delay(config)
393
+ resp = _fetch_pdf_camofox(pdf_url)
394
+ if resp is None:
395
+ continue
396
+
397
+ iterator = resp.iter_content(chunk_size=8192)
398
+ first = next(iterator, b"")
399
+ if not _response_looks_pdf(resp, first):
400
+ continue
401
+
402
+ if not _write_pdf_atomic(output_path, first, iterator):
403
+ continue
404
+ if is_pdf_file(output_path):
405
+ return success(doi, output_path, "ScienceDirect")
406
+ except Exception:
407
+ continue
408
+ return None
409
+
410
+
411
+ # ============================================================
412
+ # PNAS direct download
413
+ # ============================================================
414
+
415
+ def try_pnas_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
416
+ """Download PNAS papers directly. PNAS is OA after 6 months."""
417
+ if not doi.startswith("10.1073/"):
418
+ return None
419
+
420
+ from ..network import USER_AGENT, polite_delay
421
+ from ..download.pdf import is_pdf_file, success, _response_looks_pdf
422
+
423
+ doi_suffix = doi.split("10.1073/")[-1]
424
+ pdf_url = f"https://www.pnas.org/doi/epdf/{doi}"
425
+
426
+ try:
427
+ polite_delay(config)
428
+ s = requests.Session()
429
+ s.trust_env = False
430
+ s.headers.update({"User-Agent": USER_AGENT})
431
+ resp = s.get(pdf_url, timeout=15, stream=True,
432
+ headers={"Accept": "application/pdf,*/*"},
433
+ allow_redirects=True)
434
+
435
+ if resp.status_code >= 400:
436
+ return None
437
+
438
+ iterator = resp.iter_content(chunk_size=8192)
439
+ first = next(iterator, b"")
440
+ if not _response_looks_pdf(resp, first):
441
+ return None
442
+
443
+ if not _write_pdf_atomic(output_path, first, iterator):
444
+ return None
445
+ if is_pdf_file(output_path):
446
+ return success(doi, output_path, "PNASDirect")
447
+ except Exception:
448
+ pass
449
+ return None
450
+
451
+
452
+ # ============================================================
453
+ # PLOS direct download
454
+ # ============================================================
455
+
456
+ def try_plos_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
457
+ """Download PLOS papers directly. All PLOS journals are OA."""
458
+ if not doi.startswith("10.1371/"):
459
+ return None
460
+
461
+ from ..network import USER_AGENT, polite_delay
462
+ from ..download.pdf import is_pdf_file, success, _response_looks_pdf
463
+
464
+ # PLOS PDF URL: https://journals.plos.org/{journal}/article/file?id={doi}&type=printable
465
+ # Determine journal from DOI: 10.1371/journal.pone.0123456 -> plosone
466
+ doi_suffix = doi.split("10.1371/")[-1] if "10.1371/" in doi else doi
467
+ journal_map = {
468
+ "journal.pone": "plosone",
469
+ "journal.pbio": "plosbiology",
470
+ "journal.pmed": "plosmedicine",
471
+ "journal.pgen": "plosgenetics",
472
+ "journal.ppat": "plospathogens",
473
+ "journal.pcbi": "ploscompbiol",
474
+ "journal.pntd": "plosntds",
475
+ "journal.pone": "plosone",
476
+ "journal.pclm": "plosclimate",
477
+ "journal.pdig": "plosdigitalhealth",
478
+ }
479
+ journal = "plosone" # default
480
+ for key, val in journal_map.items():
481
+ if doi_suffix.startswith(key):
482
+ journal = val
483
+ break
484
+ pdf_url = f"https://journals.plos.org/{journal}/article/file?id={doi}&type=printable"
485
+
486
+ try:
487
+ polite_delay(config)
488
+ s = requests.Session()
489
+ s.trust_env = False
490
+ s.headers.update({"User-Agent": USER_AGENT})
491
+ resp = s.get(pdf_url, timeout=15, stream=True,
492
+ headers={"Accept": "application/pdf,*/*"},
493
+ allow_redirects=True)
494
+
495
+ if resp.status_code >= 400:
496
+ return None
497
+
498
+ iterator = resp.iter_content(chunk_size=8192)
499
+ first = next(iterator, b"")
500
+ if not _response_looks_pdf(resp, first):
501
+ return None
502
+
503
+ if not _write_pdf_atomic(output_path, first, iterator):
504
+ return None
505
+ if is_pdf_file(output_path):
506
+ return success(doi, output_path, "PLOSDirect")
507
+ except Exception:
508
+ pass
509
+ return None
510
+
511
+
512
+ # ============================================================
513
+ # Frontiers direct download
514
+ # ============================================================
515
+
516
+ def try_frontiers_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
517
+ """Download Frontiers papers directly. All Frontiers journals are OA."""
518
+ if not doi.startswith("10.3389/"):
519
+ return None
520
+
521
+ from ..network import USER_AGENT, polite_delay
522
+ from ..download.pdf import is_pdf_file, success, _response_looks_pdf
523
+
524
+ # Frontiers PDF URL: https://www.frontiersin.org/articles/{doi}/pdf
525
+ pdf_url = f"https://www.frontiersin.org/articles/{doi}/pdf"
526
+
527
+ try:
528
+ polite_delay(config)
529
+ s = requests.Session()
530
+ s.trust_env = False
531
+ s.headers.update({"User-Agent": USER_AGENT})
532
+ resp = s.get(pdf_url, timeout=15, stream=True,
533
+ headers={"Accept": "application/pdf,*/*"},
534
+ allow_redirects=True)
535
+
536
+ if resp.status_code >= 400:
537
+ return None
538
+
539
+ iterator = resp.iter_content(chunk_size=8192)
540
+ first = next(iterator, b"")
541
+ if not _response_looks_pdf(resp, first):
542
+ return None
543
+
544
+ if not _write_pdf_atomic(output_path, first, iterator):
545
+ return None
546
+ if is_pdf_file(output_path):
547
+ return success(doi, output_path, "FrontiersDirect")
548
+ except Exception:
549
+ pass
550
+ return None
551
+
552
+
553
+ # ============================================================
554
+ # BMC/SpringerOpen direct download
555
+ # ============================================================
556
+
557
+ def try_bmc_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
558
+ """Download BMC/SpringerOpen papers directly. All BMC journals are OA."""
559
+ if not doi.startswith("10.1186/"):
560
+ return None
561
+
562
+ from ..network import USER_AGENT, polite_delay
563
+ from ..download.pdf import is_pdf_file, success, _response_looks_pdf
564
+
565
+ # BMC/SpringerOpen PDF: try Springer link first, then direct biomedcentral
566
+ pdf_urls = [
567
+ f"https://link.springer.com/content/pdf/{doi}.pdf",
568
+ f"https://link.springer.com/content/pdf/{doi}.pdf?pdf=button%20sticky",
569
+ ]
570
+
571
+ for pdf_url in pdf_urls:
572
+ try:
573
+ polite_delay(config)
574
+ s = requests.Session()
575
+ s.trust_env = False
576
+ s.headers.update({"User-Agent": USER_AGENT})
577
+ resp = s.get(pdf_url, timeout=15, stream=True,
578
+ headers={"Accept": "application/pdf,*/*"},
579
+ allow_redirects=True)
580
+
581
+ if resp.status_code >= 400:
582
+ continue
583
+
584
+ iterator = resp.iter_content(chunk_size=8192)
585
+ first = next(iterator, b"")
586
+ if not _response_looks_pdf(resp, first):
587
+ continue
588
+
589
+ if not _write_pdf_atomic(output_path, first, iterator):
590
+ continue
591
+ if is_pdf_file(output_path):
592
+ return success(doi, output_path, "BMCDirect")
593
+ except Exception:
594
+ continue
595
+ return None
596
+
597
+
598
+ # ============================================================
599
+ # Helpers
600
+ # ============================================================
601
+
602
+ def get_publisher(doi: str) -> str:
603
+ for prefix, publisher in DOI_PREFIX_TO_PUBLISHER.items():
604
+ if doi.startswith(prefix):
605
+ return publisher
606
+ return ""
607
+
608
+
609
+ def _elsevier_api_fn() -> Any:
610
+ """Lazy import of the Elsevier API download function."""
611
+ from ..publishers.strategies import try_elsevier_api
612
+ return try_elsevier_api
613
+
614
+
615
+ def _browser_strategy(publisher: str) -> Any:
616
+ """Create a browser download function for a publisher.
617
+
618
+ Uses the new StrategyRegistry to validate the publisher, then returns
619
+ the legacy ``try_*_browser`` function via the compatibility bridge.
620
+ """
621
+ from ..publishers.strategies import StrategyRegistry
622
+ from ..publishers.strategies._core import try_generic_browser
623
+
624
+ strategy = StrategyRegistry.get_by_name(publisher)
625
+ if not strategy:
626
+ return try_generic_browser
627
+
628
+ # Resolve the legacy function name from the strategy's data
629
+ # Standard naming: try_{publisher.lower()}_browser
630
+ # Special cases handled by the strategy's aliases
631
+ fn_name = f"try_{strategy.name.lower().replace(' ', '_')}_browser"
632
+ from ..publishers.strategies import _core
633
+ return getattr(_core, fn_name, try_generic_browser)
634
+
635
+
636
+ def get_publisher_fast_sources(doi: str) -> list[tuple[Any, str]]:
637
+ publisher = get_publisher(doi)
638
+ if not publisher:
639
+ return []
640
+ tools = PUBLISHER_TOOL_MAP.get(publisher, ["Crossref", "Unpaywall"])
641
+ sources = []
642
+ for tool in tools:
643
+ fn = _FN_MAP.get(tool)
644
+ if fn:
645
+ sources.append((fn, tool))
646
+ return sources
647
+
648
+
649
+ # Populate function map
650
+ _FN_MAP.update({
651
+ "NatureDirect": try_nature_direct,
652
+ "PublisherDirect": try_publisher_direct,
653
+ "Crossref": try_crossref,
654
+ "Unpaywall": try_unpaywall,
655
+ "OpenAlexOA": try_openalex_oa,
656
+ "SemanticScholar": try_semanticscholar,
657
+ "MDPIDirect": try_mdpi_direct,
658
+ "SpringerTDM": try_springer_tdm,
659
+ "ScienceDirect": try_science_direct,
660
+ "PNASDirect": try_pnas_direct,
661
+ "PLOSDirect": try_plos_direct,
662
+ "FrontiersDirect": try_frontiers_direct,
663
+ "BMCDirect": try_bmc_direct,
664
+ # API strategies
665
+ "ElsevierAPI": _elsevier_api_fn(),
666
+ # Browser strategies via camofox
667
+ "ElsevierBrowser": _browser_strategy("Elsevier"),
668
+ "WileyBrowser": _browser_strategy("Wiley"),
669
+ "IEEEBrowser": _browser_strategy("IEEE"),
670
+ "ACSBrowser": _browser_strategy("ACS"),
671
+ "RSCBrowser": _browser_strategy("RSC"),
672
+ "AIPBrowser": _browser_strategy("AIP"),
673
+ "SpringerBrowser": _browser_strategy("Springer"),
674
+ "APSBrowser": _browser_strategy("APS"),
675
+ "TandFBrowser": _browser_strategy("Tandfonline"),
676
+ "IOPBrowser": _browser_strategy("IOP"),
677
+ "OxfordBrowser": _browser_strategy("Oxford"),
678
+ "ACMBrowser": _browser_strategy("ACM"),
679
+ "NatureBrowser": _browser_strategy("Nature"),
680
+ "ScienceBrowser": _browser_strategy("Science"),
681
+ "SAGEBrowser": _browser_strategy("SAGE"),
682
+ "ASCEBrowser": _browser_strategy("ASCE"),
683
+ "GenericBrowser": _browser_strategy(""),
684
+ })