litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2340 @@
1
+ """Publisher-specific browser download strategies.
2
+
3
+ Implements tailored download paths for major academic publishers,
4
+ handling anti-bot challenges, PDF viewers, and publisher-specific DOM structures.
5
+ Inspired by ref-downloader's publisher strategy registry.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import contextlib
11
+ import json
12
+ import os
13
+ import re
14
+ import threading
15
+ import time
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+ try:
20
+ from ...browser.backend import launch, launch_persistent_context
21
+ from ...browser.backend import is_available as _browser_backend_available
22
+ _HAS_CLOAKBROWSER = _browser_backend_available()
23
+ except ImportError:
24
+ launch = None # type: ignore[assignment]
25
+ launch_persistent_context = None # type: ignore[assignment]
26
+ _HAS_CLOAKBROWSER = False
27
+ from ...log import get_logger
28
+
29
+ log = get_logger()
30
+
31
+ # Per-thread error tracking for caller introspection: racing lanes run in
32
+ # parallel threads, so a module-global would let one lane's error leak into
33
+ # another lane's fallback decision.
34
+ _ERR_TLS = threading.local()
35
+
36
+ # Page titles of human checks (Cloudflare and the like), lower case.
37
+ _CHALLENGE_TITLES = ("just a moment", "attention required", "verify", "security check",
38
+ "请稍候", "正在验证", "checking", "cloudflare")
39
+
40
+ # DOIs whose publisher site was logged into with the school's identity and then
41
+ # still offered the paper for sale: the school does not subscribe, and the
42
+ # school routes (WebVPN, CARSI) end at the same publisher with the same identity
43
+ # (2026-10-18: a Springer chapter, 3 minutes and a WebVPN login for nothing).
44
+ # DOI -> when it was seen. Kept NO_ACCESS_HOURS (自定值) or until the paper is
45
+ # downloaded, so a retry does not walk the school routes again.
46
+ LOGIN_NO_ACCESS: dict[str, float] = {}
47
+ NO_ACCESS_HOURS = 6
48
+ # What an article page says about access, read from the article itself (side
49
+ # panels, recommendations, navigation and cookie banners removed: their "Buy
50
+ # eBook" / "Purchase PDF" belong to other papers). Narrower than _detect_paywall:
51
+ # "(Get) access through your institution" stays on entitled pages too, so it is
52
+ # no cue; a page offering "View PDF" / "Download PDF" is never "no access".
53
+ _SALE_CUES = ("buy article", "buy this article", "buy chapter", "buy this chapter", "purchase pdf",
54
+ "purchase this article", "purchase access", "rent this article", "subscribe to access",
55
+ "you do not have access", "you don't have access")
56
+ _BUY_WORDS = ("buy", "purchase", "rent")
57
+ _PRICE = re.compile(r"(?:[$€£¥]\s?\d|\b\d[\d.,]*\s?(?:usd|eur|gbp)\b)", re.I)
58
+ # A login prompt in place of the full text. Before the school login it only says
59
+ # "not logged in"; after it, "the school has no access".
60
+ _LOGIN_CUES = ("log in via an institution", "access this chapter", "access this article", "access this book")
61
+ _ARTICLE_TEXT_JS = r"""() => {
62
+ const root = document.querySelector('main') || document.querySelector('article') || document.body;
63
+ if (!root) return {text: '', pdf: false};
64
+ const pdf = /\b(view|download) pdf\b/i.test(root.innerText || '');
65
+ const clone = root.cloneNode(true);
66
+ clone.querySelectorAll('aside, nav, footer, script, style, noscript, [class*="recommend" i], [id*="recommend" i],'
67
+ + ' [class*="related" i], [id*="related" i], [class*="cookie" i], [id*="cookie" i], [class*="consent" i]')
68
+ .forEach((e) => e.remove());
69
+ return {text: (clone.textContent || '').replace(/\s+/g, ' '), pdf};
70
+ }"""
71
+
72
+
73
+ def note_no_access(doi: str | None) -> None:
74
+ LOGIN_NO_ACCESS[(doi or "").strip().lower()] = time.time()
75
+
76
+
77
+ def no_access(doi: str | None) -> bool:
78
+ key = (doi or "").strip().lower()
79
+ seen = LOGIN_NO_ACCESS.get(key)
80
+ if seen is not None and time.time() - seen >= NO_ACCESS_HOURS * 3600:
81
+ LOGIN_NO_ACCESS.pop(key, None) # expired: drop it
82
+ return False
83
+ return seen is not None
84
+
85
+
86
+ def clear_no_access(doi: str | None) -> None:
87
+ LOGIN_NO_ACCESS.pop((doi or "").strip().lower(), None)
88
+
89
+
90
+ def _page_access(page: Any) -> dict[str, bool]:
91
+ """{"pdf": offers View/Download PDF, "sells": sells the paper, "asks_login": asks to log in}."""
92
+ try:
93
+ got = page.evaluate(_ARTICLE_TEXT_JS) or {}
94
+ except Exception:
95
+ return {"pdf": False, "sells": False, "asks_login": False}
96
+ text = str(got.get("text") or "").lower()
97
+ if got.get("pdf"):
98
+ return {"pdf": True, "sells": False, "asks_login": False}
99
+ sells = any(c in text for c in _SALE_CUES) or (
100
+ bool(_PRICE.search(text)) and any(w in text for w in _BUY_WORDS))
101
+ return {"pdf": False, "sells": sells, "asks_login": any(c in text for c in _LOGIN_CUES)}
102
+
103
+
104
+ def _page_sells_paper(page: Any) -> bool:
105
+ """After the school login: the page still sells the paper or asks to log in."""
106
+ state = _page_access(page)
107
+ return state["sells"] or state["asks_login"]
108
+
109
+
110
+ def _wait_for_pdf_link(page: Any, doi: str, publisher: str) -> tuple[str | None, str]:
111
+ """The article page's own PDF link, by what the page says once it is quiet.
112
+
113
+ Returns (href, state), state being "pdf", "no_access" or "unclear":
114
+ - a PDF link or button: take it;
115
+ - the page sells the paper or asks to log in: no link will come, so say so
116
+ at once (before the school login: go and log in; after it: not
117
+ subscribed) instead of looking for 30 s and reloading (2026-10-19,
118
+ Springer: 80 s lost on a page that said "Log in via an institution");
119
+ - neither: the access check may still be running ("View PDF" came after
120
+ 15 s on ScienceDirect, 2026-10-18), so look longer and reload once.
121
+ """
122
+ from ...browser.pdf import find_page_pdf_link
123
+ for attempt in range(2):
124
+ try:
125
+ page.wait_for_load_state("networkidle", timeout=10000)
126
+ except Exception:
127
+ pass
128
+ href = find_page_pdf_link(page, max_wait_s=2.0, doi=doi)
129
+ if href:
130
+ return href, "pdf"
131
+ state = _page_access(page)
132
+ if state["sells"] or state["asks_login"]:
133
+ time.sleep(3) # the cue must still be there, not a flash before the access check
134
+ href = find_page_pdf_link(page, max_wait_s=1.0, doi=doi)
135
+ if href:
136
+ return href, "pdf"
137
+ again = _page_access(page)
138
+ if again["sells"] or again["asks_login"]:
139
+ log.info(f" [{publisher}] the page {'sells the paper' if again['sells'] else 'asks to log in'}; "
140
+ "not waiting for a PDF link")
141
+ return None, "no_access"
142
+ href = find_page_pdf_link(page, max_wait_s=18.0 if attempt == 0 else 8.0, doi=doi)
143
+ if href:
144
+ return href, "pdf"
145
+ if attempt:
146
+ break
147
+ log.info(f" [{publisher}] no PDF link yet; reloading the page once")
148
+ try:
149
+ page.reload(wait_until="domcontentloaded", timeout=30000)
150
+ except Exception:
151
+ pass
152
+ return None, "unclear"
153
+
154
+
155
+ def get_last_error() -> tuple[str, str]:
156
+ """(error_type, action) of the last browser strategy run in this thread.
157
+
158
+ Only this thread's: the download's own thread reads its lanes' errors from
159
+ its DownloadAttempt (``sources._try_source`` copies them there), never a
160
+ process-wide value another paper's lane may have written.
161
+ """
162
+ return getattr(_ERR_TLS, "type", ""), getattr(_ERR_TLS, "action", "")
163
+
164
+
165
+ def _set_error(error_type: str, action: str = "") -> None:
166
+ _ERR_TLS.type = error_type
167
+ _ERR_TLS.action = action
168
+
169
+
170
+ def _is_browser_error_page(page: Any) -> bool:
171
+ """Chrome's own "This site can't be reached" / timeout page, not the site's."""
172
+ try:
173
+ url = page.url or ""
174
+ if url.startswith("chrome-error://"):
175
+ return True
176
+ title = (page.title() or "").lower()
177
+ except Exception:
178
+ return False
179
+ return any(s in title for s in ("无法访问此网站", "this site can’t be reached",
180
+ "this site can't be reached", "err_timed_out", "err_connection"))
181
+
182
+
183
+ def _clear_error() -> None:
184
+ _set_error("", "")
185
+
186
+
187
+ # Click the institution in a search-result list. Containers (the whole result
188
+ # panel, the "Access content through your institution" heading block) also
189
+ # contain the name, so take the shortest visible match: the result row itself.
190
+ _PICK_INSTITUTION_JS = """
191
+ (name) => {
192
+ const items = document.querySelectorAll('[role="option"], [class*="result"], [class*="suggestion"], [class*="federation"], li, a, button');
193
+ let best = null;
194
+ for (const el of items) {
195
+ const text = (el.textContent || '').trim();
196
+ if (!text.toLowerCase().includes(name.toLowerCase()) || el.offsetParent === null) continue;
197
+ if (!best || text.length < best.text.length) best = {el, text};
198
+ }
199
+ if (!best) return null;
200
+ best.el.click();
201
+ return best.text.substring(0, 60);
202
+ }
203
+ """
204
+
205
+
206
+ # Publishers' own PDF links the generic finder (find_page_pdf_link) may miss;
207
+ # moved from the shared-browser first pass (2026-10-20): Cell Press showPdf,
208
+ # ScienceDirect's View PDF button and pdfft, Wiley pdfdirect, IEEE stamp, and
209
+ # SAGE's China platform, whose PDF button has no href (the file's path sits in
210
+ # the page data as /storage/.../<doi with _ for />.pdf, seen 2026-10-17).
211
+ _PUBLISHER_PDF_LINK_JS = r"""
212
+ (doi) => {
213
+ const first = (sel) => { for (const a of document.querySelectorAll(sel)) if (a.href) return a.href; return null; };
214
+ const hit = first('a[href*="showPdf"]') || first('a[aria-label*="View PDF" i], a.pdf-download-btn-link')
215
+ || first('a[href*="pdfft"]') || first('a[href*="pdfdirect"]')
216
+ || first('a[href*="/stamp/"], a[href*="getPDF"], a.xpl-pdf-btn');
217
+ if (hit) return hit;
218
+ for (const a of document.querySelectorAll('a[href*="/doi/pdf/"]')) {
219
+ const h = a.href.toLowerCase();
220
+ if (!h.includes('supplement') && !h.includes('supporting')) return a.href;
221
+ }
222
+ if (location.hostname.endsWith('cnpereading.com')) {
223
+ const m = document.documentElement.outerHTML.match(/\/storage\/[^"'\\\s<>]+?\.pdf/);
224
+ if (m && m[0].toLowerCase().includes(doi.replace(/\//g, '_').toLowerCase())) return location.origin + m[0];
225
+ }
226
+ return null;
227
+ }
228
+ """
229
+
230
+
231
+ def _known_landing_url(doi: str) -> str:
232
+ """Article URL built from the DOI, for publishers whose pattern is fixed.
233
+
234
+ Saves the doi.org hop (and its occasional timeouts) when opening the page
235
+ in a browser. Publishers whose URL needs data the DOI lacks (AIP, RSC)
236
+ keep going through doi.org.
237
+ """
238
+ d = doi.strip()
239
+ low = d.lower()
240
+ suffix = d.split("/", 1)[1] if "/" in d else ""
241
+ if low.startswith("10.1126/"):
242
+ return f"https://www.science.org/doi/{d}"
243
+ if low.startswith("10.1021/"):
244
+ return f"https://pubs.acs.org/doi/{d}"
245
+ if low.startswith("10.1038/") and suffix:
246
+ return f"https://www.nature.com/articles/{suffix}"
247
+ if low.startswith("10.1007/") and suffix:
248
+ kind = "chapter" if suffix.startswith("978-") else "article"
249
+ return f"https://link.springer.com/{kind}/{d}"
250
+ return ""
251
+
252
+
253
+ def _resolve_doi_landing(doi: str) -> str:
254
+ """Publisher landing URL for a DOI via the doi.org handle API ("" if unknown)."""
255
+ import requests
256
+ try:
257
+ from ...config import load_config
258
+ from ...network import apply_host_proxy
259
+ s = requests.Session()
260
+ s.trust_env = False
261
+ apply_host_proxy(s, load_config())
262
+ r = s.get(f"https://doi.org/api/handles/{doi}", params={"type": "URL"}, timeout=15)
263
+ for v in r.json().get("values", []):
264
+ if v.get("type") == "URL":
265
+ return str(v.get("data", {}).get("value", ""))
266
+ except Exception:
267
+ pass
268
+ return ""
269
+
270
+
271
+ def _on_login_page(where: Any) -> bool:
272
+ """Whether a page (or any page of a browser context) shows a login."""
273
+ pages = list(getattr(where, "pages", None) or [where])
274
+ for pg in pages:
275
+ try:
276
+ title, url = pg.title() or "", (pg.url or "").lower()
277
+ except Exception:
278
+ continue
279
+ if any(x in title for x in _AUTH_TITLES) or any(x in url for x in _AUTH_KEYWORDS + ("carsi",)):
280
+ return True
281
+ return False
282
+
283
+
284
+ def _user_waiting(what: str = "verify", where: Any = None) -> None:
285
+ """Heartbeat while the user is logging in or passing a check, so racing does
286
+ not give up on us; the app shows what the user should do.
287
+
288
+ With ``where`` (a page or browser context) the heartbeat is sent only while
289
+ a login page is actually open: the steps before it (finding the
290
+ institution's link, an SSO click that goes nowhere) are not the user's to
291
+ wait for, and saying so would show a notice with nothing to log into."""
292
+ try:
293
+ from ...sources import SourceWaived, _source_may_continue, note_user_waiting
294
+ except Exception:
295
+ return
296
+ if where is not None and not _on_login_page(where):
297
+ if not _source_may_continue():
298
+ raise SourceWaived("download already finished; stop waiting")
299
+ return
300
+ try:
301
+ note_user_waiting(what)
302
+ except SourceWaived:
303
+ raise
304
+ except Exception:
305
+ pass
306
+ _show_window_for_user(what)
307
+
308
+
309
+ # The publisher window of this thread (``_visible_browser``): it starts
310
+ # minimized and comes up when the user has to act (2026-10-20, user's choice).
311
+ _WINDOW_TLS = threading.local()
312
+ _VERIFY_GRACE_S = 8 # most human checks pass by themselves within seconds (own value)
313
+
314
+
315
+ def _set_window_state(page: Any, state: str) -> bool:
316
+ """Minimize the window of ``page`` or bring it back ("minimized" / "normal")."""
317
+ try:
318
+ cdp = page.context.new_cdp_session(page)
319
+ win = cdp.send("Browser.getWindowForTarget")["windowId"]
320
+ cdp.send("Browser.setWindowBounds", {"windowId": win, "bounds": {"windowState": state}})
321
+ if state == "normal":
322
+ page.bring_to_front()
323
+ return True
324
+ except Exception as exc:
325
+ log.info(f" publisher window: cannot set it {state} ({type(exc).__name__})")
326
+ return False
327
+
328
+
329
+ def _show_window_for_user(what: str) -> None:
330
+ """Bring this thread's publisher window up, once: at once for a login, after
331
+ a few seconds for a human check (one that passes by itself needs no one)."""
332
+ page = getattr(_WINDOW_TLS, "page", None)
333
+ if page is None or getattr(_WINDOW_TLS, "shown", False):
334
+ return
335
+ if what == "verify":
336
+ since = getattr(_WINDOW_TLS, "verify_since", None)
337
+ if since is None:
338
+ _WINDOW_TLS.verify_since = since = time.time()
339
+ if time.time() - since < _VERIFY_GRACE_S:
340
+ return
341
+ _WINDOW_TLS.shown = _set_window_state(page, "normal")
342
+
343
+
344
+ def _get_profile_dir(config: dict[str, Any], publisher: str = "shared") -> Path:
345
+ from ...config import DATA_DIR
346
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
347
+ profile_dir = cache_dir / f"publisher_profile_{publisher.lower()}"
348
+ profile_dir.mkdir(parents=True, exist_ok=True)
349
+ return profile_dir
350
+
351
+
352
+ def _restore_cookies_to_context(context: Any, config: dict[str, Any]) -> None:
353
+ try:
354
+ from ...browser.cookies import load_saved_cookies
355
+ saved = load_saved_cookies(config)
356
+ if saved:
357
+ pw_cookies = []
358
+ for c in saved:
359
+ pw_c = {"name": c.get("name", ""), "value": c.get("value", ""),
360
+ "domain": c.get("domain", ""), "path": c.get("path", "/")}
361
+ if pw_c["domain"]:
362
+ pw_cookies.append(pw_c)
363
+ if pw_cookies:
364
+ context.add_cookies(pw_cookies)
365
+ except Exception:
366
+ pass
367
+
368
+
369
+ @contextlib.contextmanager
370
+ def _visible_browser(config: dict[str, Any], publisher: str, *, viewport: dict | None = None):
371
+ """Open visible stealth browser with persistent profile. Falls back to ephemeral."""
372
+ from ...browser.engine import close_shared_browser, is_available
373
+
374
+ # Any resolved backend works (camoufox included); the old _HAS_CLOAKBROWSER
375
+ # gate wrongly required the cloakbrowser package.
376
+ if not is_available(config):
377
+ raise RuntimeError(
378
+ "no browser backend available. Run: pip install patchright / cloakbrowser / camoufox")
379
+ # The shared headless browser holds this thread's sync-API loop — a second
380
+ # (visible) launch on the same thread dies with "Sync API inside the
381
+ # asyncio loop". Tear it down first; it relaunches lazily when needed.
382
+ close_shared_browser(config)
383
+ profile_dir = _get_profile_dir(config, publisher)
384
+ browser = None
385
+
386
+ try:
387
+ ctx = launch_persistent_context(
388
+ str(profile_dir),
389
+ headless=False, humanize=True,
390
+ args=["--disable-features=CrossOriginOpenerPolicy"],
391
+ config=config,
392
+ )
393
+ page = ctx.new_page()
394
+ log.info(f" [{publisher}] persistent browser profile: {profile_dir}")
395
+ # Ensure cookies are loaded from saved file
396
+ _restore_cookies_to_context(ctx, config)
397
+ except Exception as _e:
398
+ log.info(f" [{publisher}] persistent context unavailable ({_e}), using ephemeral")
399
+ _vp = viewport or {"width": 1440, "height": 900}
400
+ browser = launch(headless=False, humanize=True,
401
+ args=["--disable-features=CrossOriginOpenerPolicy"],
402
+ config=config)
403
+ ctx = browser.new_context(viewport=_vp)
404
+ _restore_cookies_to_context(ctx, config)
405
+ page = ctx.new_page()
406
+
407
+ _WINDOW_TLS.page, _WINDOW_TLS.shown, _WINDOW_TLS.verify_since = page, False, None
408
+ if config.get("publisher_window_minimized", True):
409
+ _set_window_state(page, "minimized")
410
+ try:
411
+ yield ctx, page
412
+ finally:
413
+ _WINDOW_TLS.page = None
414
+ try:
415
+ if browser:
416
+ browser.close()
417
+ else:
418
+ ctx.close()
419
+ except Exception:
420
+ pass
421
+
422
+
423
+ def _save_all_cookie_formats(
424
+ cookies: list[dict[str, Any]],
425
+ publisher: str,
426
+ config: dict[str, Any],
427
+ ) -> None:
428
+ """Save cookies in all formats: JSON, Netscape, publisher subset, + bridge to browser-engine."""
429
+ from ...config import DATA_DIR
430
+ from ...browser.cookies import cookies_to_netscape, _save_cookies_json
431
+
432
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
433
+
434
+ # 1. Save full cookies to carsi_cookies/{publisher}.json
435
+ carsi_dir = cache_dir / "carsi_cookies"
436
+ carsi_dir.mkdir(parents=True, exist_ok=True)
437
+ cookie_data = [
438
+ {"name": c.get("name", ""), "value": c.get("value", ""),
439
+ "domain": c.get("domain", ""), "path": c.get("path", "/"),
440
+ "secure": c.get("secure", False), "expires": c.get("expires", 0),
441
+ "httpOnly": c.get("httpOnly", False)}
442
+ for c in cookies
443
+ ]
444
+ (carsi_dir / f"{publisher.lower()}.json").write_text(
445
+ json.dumps(cookie_data, indent=2, ensure_ascii=False), encoding="utf-8")
446
+
447
+ # 2. Save Netscape format (for browser-engine import)
448
+ netscape_text = cookies_to_netscape(cookies)
449
+ netscape_file = cache_dir / "publisher_cookies.txt"
450
+ netscape_file.write_text(netscape_text, encoding="utf-8")
451
+
452
+ # 3. Save publisher-domain subset
453
+ _pub_domains = [
454
+ "wiley.com", "onlinelibrary.wiley.com", "elsevier.com", "sciencedirect.com",
455
+ "cell.com", "springer.com", "link.springer.com", "nature.com",
456
+ "ieee.org", "ieeexplore.ieee.org", "acs.org", "pubs.acs.org",
457
+ "rsc.org", "pubs.rsc.org", "tandfonline.com", "oup.com",
458
+ "academic.oup.com", "iop.org", "iopscience.iop.org",
459
+ "aps.org", "journals.aps.org", "aip.org", "pubs.aip.org",
460
+ "dl.acm.org", "acm.org", "science.org", "sciencemag.org",
461
+ "ascelibrary.org", "sagepub.com", "journals.sagepub.com",
462
+ "sage.cnpereading.com",
463
+ "royalsocietypublishing.org", "copernicus.org",
464
+ ]
465
+ pub_cookies = [c for c in cookie_data
466
+ if any(c.get("domain", "").endswith(d) for d in _pub_domains)]
467
+ if pub_cookies:
468
+ _save_cookies_json(pub_cookies, cache_dir / "publisher_cookies.json")
469
+
470
+ # 4. Bridge to browser-engine headless service
471
+ try:
472
+ from ...browser.engine import import_cookies, is_available
473
+ if is_available(config):
474
+ imported = import_cookies(str(netscape_file), config)
475
+ log.info(f" [{publisher}] bridged {imported} cookies to browser-engine service")
476
+ except Exception as _e:
477
+ log.info(f" [{publisher}] browser-engine bridge note: {_e}")
478
+
479
+ log.info(f" [{publisher}] saved {len(cookies)} cookies in all formats")
480
+
481
+
482
+ def _detect_paywall(html: str, status_code: int = 0) -> bool:
483
+ """Detect if the page is a paywall/login wall rather than anti-bot challenge."""
484
+ lower = html.lower()
485
+ # Paywall indicators
486
+ paywall_signals = [
487
+ "sign in to access", "login to access", "institutional access",
488
+ "purchase access", "subscribe to access", "buy article",
489
+ "access through your institution", "get access",
490
+ "this article is behind a paywall", "subscription required",
491
+ "please log in", "register to continue",
492
+ "access denied", "you do not have access",
493
+ "institutional login", "shibboleth", "openathens",
494
+ "check access", # Science (AAAS): the only access cue on its article page
495
+ ]
496
+ if any(sig in lower for sig in paywall_signals):
497
+ return True
498
+ # 403 with article content (not Cloudflare challenge) = paywall
499
+ if status_code == 403 and not _is_challenge_page(html):
500
+ return True
501
+ return False
502
+
503
+
504
+ def _has_publisher_cookies(config: dict[str, Any]) -> bool:
505
+ """Check if we have saved publisher cookies for the current session."""
506
+ try:
507
+ from ...browser.cookies import load_saved_cookies
508
+ if len(load_saved_cookies(config)) > 0:
509
+ return True
510
+ except Exception:
511
+ pass
512
+ # Also check CARSI cookies
513
+ try:
514
+ from ...config import DATA_DIR
515
+ carsi_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "carsi_cookies"
516
+ if carsi_dir.is_dir() and any(carsi_dir.glob("*.json")):
517
+ return True
518
+ except Exception:
519
+ pass
520
+ return False
521
+
522
+
523
+ def _inject_cookies_to_tab(tab_id: str, config: dict[str, Any], publisher: str) -> None:
524
+ """Inject saved publisher + CARSI cookies into browser-engine session."""
525
+ from ...browser.engine import import_cookies
526
+ from ...config import DATA_DIR
527
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
528
+ total = 0
529
+
530
+ # Import publisher cookies (Netscape format if exists)
531
+ netscape_file = cache_dir / "publisher_cookies.txt"
532
+ if netscape_file.exists():
533
+ try:
534
+ total += import_cookies(str(netscape_file), config)
535
+ except Exception:
536
+ pass
537
+
538
+ # Import CARSI cookies
539
+ carsi_dir = cache_dir / "carsi_cookies"
540
+ if carsi_dir.is_dir():
541
+ for cf in carsi_dir.glob("*.json"):
542
+ try:
543
+ # Convert JSON → Netscape format
544
+ from ...browser.cookies import cookies_to_netscape
545
+ cookies = json.loads(cf.read_text(encoding="utf-8"))
546
+ tmp = cache_dir / f"_tmp_{cf.stem}.txt"
547
+ tmp.write_text(cookies_to_netscape(cookies), encoding="utf-8")
548
+ total += import_cookies(str(tmp), config)
549
+ tmp.unlink(missing_ok=True)
550
+ except Exception:
551
+ pass
552
+
553
+ if total > 0:
554
+ log.info(f" [{publisher}] imported {total} cookies into browser session")
555
+
556
+
557
+ # ============================================================
558
+ # Publisher direct PDF URL patterns
559
+ # ============================================================
560
+
561
+ def _visible_browser_download(
562
+ doi: str,
563
+ article_url: str,
564
+ output_path: Path,
565
+ config: dict[str, Any],
566
+ publisher: str,
567
+ *,
568
+ wait_for_loading: float = 0,
569
+ ) -> dict[str, Any] | None:
570
+ """Open the publisher's window: the page's own PDF, else SSO login, then the PDF.
571
+
572
+ Works for any publisher. Uses _PUBLISHER_SSO_CONFIG for publisher-specific
573
+ SSO link patterns, institution search selectors, and PDF fetch paths.
574
+ Without a school set (carsi_idp_name) the login step is skipped: the window
575
+ still passes human checks and uses the campus network's access.
576
+ """
577
+ from ...download.pdf import is_pdf_file, success
578
+ import base64
579
+
580
+ idp_name = config.get("carsi_idp_name", "")
581
+ idp_en = _IDP_MAP.get(idp_name, idp_name)
582
+
583
+ # Publisher-specific SSO config
584
+ sso_cfg = _PUBLISHER_SSO_CONFIG.get(publisher, _PUBLISHER_SSO_CONFIG["_default"])
585
+ sso_link_js = sso_cfg["sso_link_js"]
586
+ # Publisher-specific selectors first, then the shared generic ones.
587
+ search_selectors = list(sso_cfg["search_selectors"]) + [
588
+ s for s in _INSTITUTION_SEARCH_SELECTORS if s not in sso_cfg["search_selectors"]]
589
+ pdf_paths = sso_cfg["pdf_paths"](doi)
590
+
591
+ with _visible_browser(config, publisher) as (context, page):
592
+
593
+ # For Elsevier DOIs, avoid linkinghub.elsevier.com by using direct URL
594
+ if publisher == "Elsevier" and ("doi.org/" in article_url or "linkinghub" in article_url):
595
+ cell_url = _build_cell_press_url(doi)
596
+ if cell_url:
597
+ log.info(f" [{publisher}] bypassing linkinghub, using direct cell.com URL")
598
+ article_url = cell_url
599
+ else:
600
+ # Resolve DOI → sciencedirect.com PII URL via HTTP
601
+ sd_url = _resolve_elsevier_pii(doi, config)
602
+ if sd_url:
603
+ log.info(f" [{publisher}] bypassing linkinghub, using resolved URL")
604
+ article_url = sd_url
605
+
606
+ # Navigate to article page
607
+ log.info(f" [{publisher}] visible browser: opening {article_url[:60]}")
608
+ try:
609
+ page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
610
+ time.sleep(5)
611
+ except Exception as exc:
612
+ log.info(f" [{publisher}] page load warning: {exc}")
613
+ if "doi.org/" in article_url and _is_browser_error_page(page):
614
+ # doi.org itself sometimes times out from here: ask its API for the
615
+ # publisher URL and go there directly.
616
+ landing = _resolve_doi_landing(doi)
617
+ if landing:
618
+ log.info(f" [{publisher}] doi.org did not load, opening {landing[:80]}")
619
+ article_url = landing
620
+ try:
621
+ page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
622
+ time.sleep(5)
623
+ except Exception as exc:
624
+ log.info(f" [{publisher}] page load warning: {exc}")
625
+ if wait_for_loading > 0: # AIP/AVS "Loading" pages
626
+ time.sleep(wait_for_loading)
627
+
628
+ # If stuck on linkinghub redirect, extract target and navigate directly
629
+ url = page.url
630
+ if "linkinghub" in url or ("retrieve/pii" in url and "Loading" in (page.title() or "")):
631
+ log.info(f" [{publisher}] stuck on linkinghub, extracting direct URL...")
632
+ pii_match = re.search(r"pii/([A-Z0-9]+)", url)
633
+ if pii_match:
634
+ pii = pii_match.group(1)
635
+ # Try cell.com only if it's a known Cell Press journal, else sciencedirect.com
636
+ cell_url = _build_cell_press_url(doi) if publisher == "Elsevier" else None
637
+ direct_urls = []
638
+ if cell_url:
639
+ direct_urls.append(cell_url)
640
+ direct_urls.append(f"https://www.sciencedirect.com/science/article/pii/{pii}")
641
+ for direct_url in direct_urls:
642
+ try:
643
+ page.goto(direct_url, wait_until="domcontentloaded", timeout=30000)
644
+ time.sleep(5)
645
+ if "linkinghub" not in page.url:
646
+ log.info(f" [{publisher}] navigated to {page.url[:60]}")
647
+ break
648
+ except Exception:
649
+ pass
650
+
651
+ # Wait for Cloudflare challenge to resolve (visible browser can pass it).
652
+ # The user may have to tick it: say so (the app shows a notice, the race
653
+ # keeps waiting) and give them up to 5 minutes, not one.
654
+ for _cf_wait in range(60):
655
+ _cf_title = (page.title() or "").lower()
656
+ if any(_sig in _cf_title for _sig in _CHALLENGE_TITLES):
657
+ if _cf_wait % 6 == 0:
658
+ log.info(f" [{publisher}] Cloudflare challenge detected, waiting for it or the user... ({_cf_wait * 5}s)")
659
+ try:
660
+ if _is_challenge_dead_end(page.content()):
661
+ log.info(f" [{publisher}] the check cannot run in this browser; giving up on the site")
662
+ break
663
+ except Exception:
664
+ pass
665
+ _user_waiting("verify")
666
+ time.sleep(5)
667
+ else:
668
+ break
669
+ else:
670
+ log.info(f" [{publisher}] Cloudflare challenge did not resolve")
671
+
672
+ # Let interstitials ("Loading https://...") and auth hops finish before
673
+ # reading the page; evaluating mid-navigation raises.
674
+ from ...browser.pdf import capture_pdf_via_cdp, find_page_pdf_link, wait_page_settled, wait_url_stable
675
+ wait_url_stable(page)
676
+ title, url = wait_page_settled(page)
677
+ log.info(f" [{publisher}] page: '{title[:40]}' {url[:60]}")
678
+ if _is_browser_error_page(page):
679
+ # 2026-10-18: SSRN timed out; the lane waited on it for 16 minutes.
680
+ log.info(f" [{publisher}] the site did not load (browser error page); giving up on it")
681
+ _set_error("navigate_failed", "site_unreachable")
682
+ return None
683
+ try:
684
+ html = page.content()
685
+ if publisher == "Elsevier" and _is_elsevier_crasolve(page.url, html):
686
+ log.info(f" [{publisher}] crasolve shell detected, waiting...")
687
+ time.sleep(5)
688
+ elif publisher == "AIP" and _is_aip_loading_page(html):
689
+ log.info(f" [{publisher}] loading page detected, waiting...")
690
+ time.sleep(10)
691
+ except Exception:
692
+ pass
693
+
694
+ def _page_pdf_via_cdp() -> dict[str, Any] | None:
695
+ """Open the page's own PDF link in the browser and read the body via CDP."""
696
+ href, state = _wait_for_pdf_link(page, doi, publisher)
697
+ if not href:
698
+ try: # publishers' own link patterns the generic finder misses
699
+ href = page.evaluate(_PUBLISHER_PDF_LINK_JS, doi) or None
700
+ except Exception:
701
+ href = None
702
+ if not href:
703
+ return None
704
+ log.info(f" [{publisher}] page PDF link: {href[:80]}")
705
+ try: # as the headless path says it (2026-10-19: this one stayed on "opening the window")
706
+ from ...download import progress as _pr
707
+ _pr.item_stage(doi, f"在 {publisher} 找到 PDF 链接,正在打开" if publisher not in ("", "Unknown")
708
+ else "在出版商官网找到 PDF 链接,正在打开")
709
+ except Exception:
710
+ pass
711
+ body = capture_pdf_via_cdp(context, page, href)
712
+ if body:
713
+ output_path.parent.mkdir(parents=True, exist_ok=True)
714
+ output_path.write_bytes(body)
715
+ if is_pdf_file(output_path):
716
+ return success(doi, output_path, f"{publisher}(Visible)")
717
+ return None
718
+
719
+ # Check if already on auth page (previous cookies caused redirect)
720
+ already_on_auth = any(x in (title or "") for x in _AUTH_TITLES) or \
721
+ any(x in (url or "").lower() for x in _AUTH_KEYWORDS)
722
+
723
+ # Try fetching PDF first — if it works, no login needed
724
+ pdf_fetched = False
725
+ if not already_on_auth:
726
+ result = _page_pdf_via_cdp()
727
+ if result:
728
+ return result
729
+ log.info(f" [{publisher}] trying direct PDF fetch...")
730
+ fetch_result = _try_browser_fetch_pdf(page, pdf_paths)
731
+ if fetch_result:
732
+ pdf_bytes = fetch_result
733
+ output_path.parent.mkdir(parents=True, exist_ok=True)
734
+ output_path.write_bytes(pdf_bytes)
735
+ log.info(f" [{publisher}] downloaded {len(pdf_bytes)} bytes via direct fetch")
736
+ if is_pdf_file(output_path):
737
+ return success(doi, output_path, f"{publisher}(Visible)")
738
+ pdf_fetched = True
739
+ else:
740
+ log.info(f" [{publisher}] direct PDF fetch failed, need SSO login")
741
+
742
+ if not idp_name and not pdf_fetched:
743
+ # No school set: nobody to log in as. Say why there is no PDF.
744
+ log.info(f" [{publisher}] no PDF and no school set (carsi_idp_name): not logging in")
745
+ if _page_sells_paper(page) or already_on_auth:
746
+ _set_error("paywall", "login_required")
747
+ elif not get_last_error()[0]:
748
+ _set_error("no_pdf_found", "try_other_source")
749
+ return None
750
+
751
+ # SSO login needed
752
+ if not already_on_auth and not pdf_fetched:
753
+ log.info(f" [{publisher}] starting SSO login...")
754
+ page.evaluate(sso_link_js)
755
+ time.sleep(8)
756
+ if not any(any(x in (pg.url or "").lower() for x in _AUTH_KEYWORDS) for pg in context.pages):
757
+ # The click went nowhere (e.g. Springer's "Log in via an
758
+ # institution" is a JS widget): follow the page's SSO/WAYF link.
759
+ try:
760
+ sso_href = page.evaluate(_SSO_LINK_FINDER_JS)
761
+ if sso_href:
762
+ log.info(f" [{publisher}] opening SSO link: {sso_href[:80]}")
763
+ page.goto(sso_href, wait_until="domcontentloaded", timeout=30000)
764
+ time.sleep(5)
765
+ except Exception as e:
766
+ log.info(f" [{publisher}] SSO link fallback: {e}")
767
+
768
+ # SSO click may have navigated the page or opened a popup
769
+ # Check all pages in the context for the SSO/IDP page
770
+ sso_page = page
771
+ for ctx_page in context.pages:
772
+ try:
773
+ p_url = ctx_page.url
774
+ p_title = ctx_page.title()
775
+ if any(x in p_url.lower() for x in _AUTH_KEYWORDS) or \
776
+ any(x in p_title for x in _AUTH_TITLES):
777
+ sso_page = ctx_page
778
+ log.info(f" [{publisher}] SSO detected on tab: '{p_title[:30]}' {p_url[:60]}")
779
+ break
780
+ except Exception:
781
+ pass
782
+
783
+ log.info(f" [{publisher}] after SSO click: '{sso_page.title()[:30] if sso_page else '?'}' {sso_page.url[:60] if sso_page else '?'}")
784
+
785
+ # Search for institution on the SSO page. Pick as soon as the
786
+ # results show (2026-10-19, Springer's WAYF: a fixed 3 s per
787
+ # selector added up to 31 s before the school was picked).
788
+ t_pick = time.time()
789
+ for sel in search_selectors:
790
+ si = sso_page.query_selector(sel)
791
+ if not si:
792
+ continue
793
+ # fill() first waits for the box to take input, by default up
794
+ # to 30 s, outside the polling below (2026-10-19: Springer's
795
+ # WAYF still took 30.5 s after the polling fix).
796
+ # Not ready in 5 s (covered by a banner, still animating): type
797
+ # into it anyway, which skips those checks.
798
+ try:
799
+ si.fill(idp_en, timeout=5000)
800
+ except Exception:
801
+ try:
802
+ si.fill(idp_en, force=True, timeout=3000)
803
+ log.info(f" [{publisher}] institution search via {sel!r}: box not ready, typed in anyway")
804
+ except Exception as exc:
805
+ log.info(f" [{publisher}] institution search via {sel!r}: cannot type "
806
+ f"({type(exc).__name__}) after {time.time() - t_pick:.1f}s")
807
+ continue
808
+ clicked = False
809
+ for _ in range(16): # up to 8 s for the results to appear
810
+ time.sleep(0.5)
811
+ try:
812
+ clicked = sso_page.evaluate(_PICK_INSTITUTION_JS, idp_en)
813
+ except Exception:
814
+ clicked = False
815
+ if clicked:
816
+ break
817
+ log.info(f" [{publisher}] institution search via {sel!r}: "
818
+ f"{'picked' if clicked else 'no match'} after {time.time() - t_pick:.1f}s")
819
+ if clicked:
820
+ log.info(f" [{publisher}] selected institution '{idp_en}'")
821
+ time.sleep(5)
822
+ break
823
+
824
+ # Wait for user to complete CAS login
825
+ print(f"\n 请在浏览器中完成机构登录 ({idp_name})")
826
+ print(" 登录成功后会自动下载 PDF\n")
827
+
828
+ login_ok = False
829
+ # "No tab is on a login page" only means success once one was: if the
830
+ # SSO click never left the article, it is not a login.
831
+ seen_auth = already_on_auth
832
+ for i in range(100):
833
+ # A human check on the way to the login (2026-10-18: Wiley's
834
+ # Cloudflare page after the SSO click) is the user's to pass: say
835
+ # "verify", not "login", and keep waiting while it is shown.
836
+ challenge = False
837
+ for ctx_page in context.pages:
838
+ try:
839
+ if any(s in (ctx_page.title() or "").lower() for s in _CHALLENGE_TITLES):
840
+ challenge = True
841
+ break
842
+ except Exception:
843
+ pass
844
+ _user_waiting("verify" if challenge else "login", None if challenge else context)
845
+ time.sleep(3)
846
+ if not seen_auth and i >= 10 and not challenge:
847
+ break # ~30 s and the login page never appeared
848
+ # Check ALL pages in context — SSO may happen in any tab
849
+ any_auth = False
850
+ for ctx_page in context.pages:
851
+ try:
852
+ p_title = ctx_page.title()
853
+ p_url = ctx_page.url
854
+ is_auth = any(x in p_title for x in _AUTH_TITLES)
855
+ is_auth_url = any(x in p_url.lower() for x in _AUTH_KEYWORDS)
856
+ if is_auth or is_auth_url:
857
+ any_auth = True
858
+ break
859
+ except Exception:
860
+ pass
861
+ if any_auth:
862
+ seen_auth = True
863
+ elif seen_auth:
864
+ login_ok = True
865
+ break
866
+
867
+ if not login_ok and seen_auth:
868
+ log.info(f" [{publisher}] visible login timed out")
869
+ _set_error("login_abandoned", "retry_login")
870
+ return None
871
+ if not login_ok:
872
+ # The SSO entry led back to the article, not to a login: the school's
873
+ # exit may need none (2026-10-17, Wiley after a late Cloudflare
874
+ # check). Try the article page as it is rather than giving up.
875
+ log.info(f" [{publisher}] SSO entry did not reach a login page: trying the article page as it is")
876
+ else:
877
+ log.info(f" [{publisher}] login successful, downloading PDF...")
878
+ time.sleep(3)
879
+
880
+ # Save cookies for future use (all formats + bridge to browser-engine)
881
+ try:
882
+ cookies = context.cookies()
883
+ _save_all_cookie_formats(cookies, publisher, config)
884
+ except Exception as e:
885
+ log.info(f" [{publisher}] cookie save note: {e}")
886
+
887
+ # Navigate back to the article page and reload to pick up auth cookies
888
+ log.info(f" [{publisher}] reloading article page with new auth cookies...")
889
+ try:
890
+ page.goto(article_url, wait_until="domcontentloaded", timeout=30000)
891
+ time.sleep(5)
892
+ except Exception:
893
+ pass
894
+ wait_url_stable(page)
895
+ wait_page_settled(page)
896
+ result = _page_pdf_via_cdp()
897
+ if result:
898
+ return result
899
+
900
+ # Try downloading PDF via in-browser fetch (post-login)
901
+ fetch_result = _try_browser_fetch_pdf(page, pdf_paths)
902
+ if fetch_result:
903
+ pdf_bytes = fetch_result
904
+ output_path.parent.mkdir(parents=True, exist_ok=True)
905
+ output_path.write_bytes(pdf_bytes)
906
+ log.info(f" [{publisher}] downloaded {len(pdf_bytes)} bytes via post-login fetch")
907
+ if is_pdf_file(output_path):
908
+ return success(doi, output_path, f"{publisher}(Visible)")
909
+
910
+ # Fallback: look for PDF link on the page and fetch it
911
+ log.info(f" [{publisher}] fetch failed, trying page PDF links...")
912
+ pdf_link = page.evaluate("""
913
+ (() => {
914
+ for (const a of document.querySelectorAll('a')) {
915
+ const href = (a.href || '').toLowerCase();
916
+ if (href.includes('pdfdirect') || href.includes('/pdf/') || href.includes('.pdf')) {
917
+ if (!href.includes('supplement') && !href.includes('supporting')) return a.href;
918
+ }
919
+ }
920
+ for (const el of document.querySelectorAll('iframe, embed, object')) {
921
+ const src = el.src || el.data || '';
922
+ if (src.includes('.pdf') || src.includes('pdfdirect')) return src;
923
+ }
924
+ return null;
925
+ })()
926
+ """)
927
+
928
+ if pdf_link and isinstance(pdf_link, str):
929
+ log.info(f" [{publisher}] found PDF link: {pdf_link[:60]}")
930
+ try:
931
+ page.goto(pdf_link, wait_until="domcontentloaded", timeout=30000)
932
+ time.sleep(3)
933
+ except Exception:
934
+ pass
935
+
936
+ # Try fetch from current page context
937
+ current_url = page.url
938
+ from urllib.parse import urlparse
939
+ parsed = urlparse(current_url)
940
+ # Try absolute fetch
941
+ abs_paths = [parsed.path]
942
+ fetch_result2 = _try_browser_fetch_pdf(page, abs_paths)
943
+ if fetch_result2:
944
+ pdf_bytes = fetch_result2
945
+ output_path.parent.mkdir(parents=True, exist_ok=True)
946
+ output_path.write_bytes(pdf_bytes)
947
+ log.info(f" [{publisher}] downloaded {len(pdf_bytes)} bytes via PDF link")
948
+ if is_pdf_file(output_path):
949
+ return success(doi, output_path, f"{publisher}(Visible)")
950
+
951
+ log.info(f" [{publisher}] visible browser download failed")
952
+ if login_ok and _page_sells_paper(page):
953
+ # Logged in as the school and the page still sells it: not subscribed.
954
+ note_no_access(doi)
955
+ _set_error("not_entitled", "no_subscription")
956
+ elif login_ok:
957
+ # Logged in, no PDF, no sign of a missing subscription: the school
958
+ # routes may still get it, so this says nothing about entitlement.
959
+ _set_error("no_pdf_found", "try_other_source")
960
+ return None
961
+
962
+
963
+ def _try_browser_fetch_pdf(page: Any, paths: list[str]) -> bytes | None:
964
+ """Try fetching PDF via browser JS fetch for given paths. Returns PDF bytes or None."""
965
+ import base64
966
+ paths_js = json.dumps(paths)
967
+ pdf_b64 = page.evaluate(f"""
968
+ (async () => {{
969
+ const paths = {paths_js};
970
+ for (const p of paths) {{
971
+ try {{
972
+ const resp = await fetch(p, {{credentials: 'include', headers: {{'Accept': 'application/pdf,*/*'}}}});
973
+ if (!resp.ok) return 'status:' + resp.status;
974
+ const ct = resp.headers.get('content-type') || '';
975
+ if (!ct.includes('pdf') && !ct.includes('octet')) return 'ct:' + ct;
976
+ const blob = await resp.blob();
977
+ return await new Promise((resolve) => {{
978
+ const reader = new FileReader();
979
+ reader.onload = () => resolve(reader.result);
980
+ reader.readAsDataURL(blob);
981
+ }});
982
+ }} catch(e) {{ return 'error:' + e.message; }}
983
+ }}
984
+ return null;
985
+ }})()
986
+ """)
987
+
988
+ if isinstance(pdf_b64, str) and pdf_b64.startswith("data:"):
989
+ _, data = pdf_b64.split(",", 1)
990
+ pdf_bytes = base64.b64decode(data)
991
+ if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
992
+ return pdf_bytes
993
+ return None
994
+
995
+
996
+ # Sites whose bot wall answers plain HTTP clients with 403 whatever cookies
997
+ # they carry (the clearance is bound to the browser fingerprint). The direct
998
+ # HTTP try only delayed the browser: pubs.acs.org takes ~12.6 s to say 403.
999
+ # Publishers whose PDF URLs never answer a plain HTTP request (Cloudflare and
1000
+ # the like): skip straight to the browser. SAGE added 2026-10-19: two direct
1001
+ # tries at journals.sagepub.com hung 20.8 + 31.6 s before failing, every paper.
1002
+ _HTTP_DIRECT_BLOCKED = frozenset({"ACS", "Wiley", "RSC", "IOP", "SAGE"})
1003
+
1004
+ # Hosts whose direct PDF request timed out: skipped for _STALLED_HOST_SECONDS
1005
+ # (自定值), so a site that hangs costs one timeout per session, not one per paper.
1006
+ _STALLED_HOSTS: dict[str, float] = {}
1007
+ _STALLED_HOST_SECONDS = 1800
1008
+
1009
+
1010
+ def _direct_pdf_urls(doi: str, publisher: str) -> list[str]:
1011
+ """Build direct PDF URL candidates for a publisher (deduplicated)."""
1012
+ if publisher in _HTTP_DIRECT_BLOCKED:
1013
+ return []
1014
+ return list(dict.fromkeys(_direct_pdf_url_candidates(doi, publisher)))
1015
+
1016
+
1017
+ def _direct_pdf_url_candidates(doi: str, publisher: str) -> list[str]:
1018
+ urls: list[str] = []
1019
+
1020
+ if publisher == "Elsevier":
1021
+ # ScienceDirect doesn't have a simple direct PDF URL pattern
1022
+ # Need to go through the article page
1023
+ pass
1024
+
1025
+ elif publisher == "Wiley":
1026
+ # PDFDirect is the fastest path
1027
+ urls.append(f"https://onlinelibrary.wiley.com/doi/pdfdirect/{doi}")
1028
+ urls.append(f"https://onlinelibrary.wiley.com/doi/pdf/{doi}")
1029
+
1030
+ elif publisher == "IEEE":
1031
+ # IEEE stamp URL — classic redirect-based PDF
1032
+ # Need to resolve DOI first to get the article number
1033
+ doi_suffix = doi.split("/", 1)[1] if "/" in doi else doi
1034
+ urls.append(f"https://ieeexplore.ieee.org/stampPDF/getPDF.jsp?tp=&arnumber={doi_suffix}")
1035
+
1036
+ elif publisher == "ACS":
1037
+ urls.append(f"https://pubs.acs.org/doi/pdf/{doi}")
1038
+ urls.append(f"https://pubs.acs.org/doi/pdf/10.1021/{doi.split('10.1021/')[-1]}")
1039
+
1040
+ elif publisher == "RSC":
1041
+ urls.append(f"https://pubs.rsc.org/en/content/articlepdf/{doi}")
1042
+
1043
+ elif publisher == "AIP":
1044
+ # AIP has a loading page, direct URL may not work
1045
+ urls.append(f"https://pubs.aip.org/aip/apl/article-pdf/{doi}")
1046
+
1047
+ elif publisher == "Springer":
1048
+ urls.append(f"https://link.springer.com/content/pdf/{doi}.pdf")
1049
+
1050
+ elif publisher == "APS":
1051
+ # Physical Review journals
1052
+ doi_suffix = doi.split("10.1103/")[-1] if "10.1103/" in doi else doi
1053
+ urls.append(f"https://journals.aps.org/prl/pdf/{doi}")
1054
+ urls.append(f"https://journals.aps.org/prb/pdf/{doi}")
1055
+ urls.append(f"https://journals.aps.org/pdf/{doi}")
1056
+
1057
+ elif publisher == "Tandfonline":
1058
+ urls.append(f"https://www.tandfonline.com/doi/pdf/{doi}")
1059
+
1060
+ elif publisher == "OSA":
1061
+ urls.append(f"https://opg.optica.org/viewmedia.cfm?r=1&uri={doi}")
1062
+
1063
+ elif publisher == "IOP":
1064
+ urls.append(f"https://iopscience.iop.org/article/{doi}/pdf")
1065
+
1066
+ elif publisher == "Oxford":
1067
+ urls.append(f"https://academic.oup.com/downloadpdf/{doi}")
1068
+
1069
+ elif publisher == "ACM":
1070
+ urls.append(f"https://dl.acm.org/doi/pdf/{doi}")
1071
+
1072
+ elif publisher == "SAGE":
1073
+ # DOI-resolved host may be journals.sagepub.com or sage.cnpereading.com
1074
+ urls.append(f"https://journals.sagepub.com/doi/pdf/{doi}?download=true")
1075
+ urls.append(f"https://journals.sagepub.com/doi/pdf/{doi}")
1076
+
1077
+ elif publisher == "ASCE":
1078
+ doi_upper = doi.upper()
1079
+ urls.append(f"https://ascelibrary.org/doi/pdf/{doi_upper}?download=true")
1080
+ urls.append(f"https://ascelibrary.org/doi/pdf/{doi_upper}")
1081
+ urls.append(f"https://ascelibrary.org/doi/pdf/{doi}?download=true")
1082
+
1083
+ elif publisher == "Royal Society":
1084
+ urls.append(f"https://royalsocietypublishing.org/doi/pdf/{doi}")
1085
+ urls.append(f"https://royalsocietypublishing.org/doi/epdf/{doi}")
1086
+
1087
+ elif publisher == "Copernicus":
1088
+ # Copernicus uses journal-specific subdomains; resolve DOI first
1089
+ # Typical DOI: 10.5194/hess-22-3433-2018
1090
+ urls.append(f"https://www.copernicus.org/publications/{doi}")
1091
+ urls.append(f"https://www.copernicus.org/articles/{doi}")
1092
+
1093
+ return urls
1094
+
1095
+
1096
+ # ============================================================
1097
+ # Publisher-specific article page selectors
1098
+ # ============================================================
1099
+
1100
+ # CSS selectors to find PDF links on article landing pages
1101
+ _PDF_LINK_SELECTORS: dict[str, list[str]] = {
1102
+ "Elsevier": [
1103
+ 'a[aria-label*="View PDF" i]',
1104
+ 'a[aria-label*="pdf" i]',
1105
+ 'a.pdf-download-btn-link',
1106
+ 'a[data-aa-name="download-pdf"]',
1107
+ 'a[href*="/pdf/"]',
1108
+ 'a[href*="pdfft"]',
1109
+ 'a[href*=".pdf"]',
1110
+ 'a[href*="showPdf"]',
1111
+ 'a[href*="pdfExtended"]',
1112
+ ],
1113
+ "Wiley": [
1114
+ 'a[href*="pdfdirect"]',
1115
+ 'a[href*="/doi/pdf/"]',
1116
+ 'a[href*="/doi/epdf/"]',
1117
+ 'a.c-pdf-download__link',
1118
+ 'a[title*="PDF" i]',
1119
+ ],
1120
+ "IEEE": [
1121
+ 'a[href*="/stamp/"]',
1122
+ 'a[href*="getPDF"]',
1123
+ 'a[title*="PDF" i]',
1124
+ 'a.xpl-pdf-btn',
1125
+ ],
1126
+ "ACS": [
1127
+ 'a[href*="/doi/pdf/"]',
1128
+ 'a[title*="Download PDF" i]',
1129
+ 'a.nav-item-link-download',
1130
+ ],
1131
+ "RSC": [
1132
+ 'a[href*="/articlepdf/"]',
1133
+ 'a[title*="PDF" i]',
1134
+ ],
1135
+ "AIP": [
1136
+ 'a[href*="/article-pdf/"]',
1137
+ 'a[title*="PDF" i]',
1138
+ 'a[href*=".pdf"]',
1139
+ ],
1140
+ "Springer": [
1141
+ 'a[href*="/content/pdf/"]',
1142
+ 'a[data-track-action="download pdf"]',
1143
+ 'a.c-pdf-download__link',
1144
+ ],
1145
+ "Nature": [
1146
+ 'a.c-pdf-download__link',
1147
+ 'a[data-track-action="download pdf"]',
1148
+ 'a[href*=".pdf"]',
1149
+ ],
1150
+ "Science": [
1151
+ 'a[href*="/doi/pdf/"]',
1152
+ 'a[title*="PDF" i]',
1153
+ 'a.article-pdf',
1154
+ ],
1155
+ "APS": [
1156
+ 'a[href*="/pdf/"]',
1157
+ 'a[title*="PDF" i]',
1158
+ ],
1159
+ "Tandfonline": [
1160
+ 'a[href*="/doi/pdf/"]',
1161
+ 'a[title*="PDF" i]',
1162
+ ],
1163
+ "IOP": [
1164
+ 'a[href*="/article/"][href$="/pdf"]',
1165
+ 'a[title*="PDF" i]',
1166
+ ],
1167
+ "Oxford": [
1168
+ 'a[href*="/downloadpdf/"]',
1169
+ 'a[title*="PDF" i]',
1170
+ ],
1171
+ "ACM": [
1172
+ 'a[href*="/doi/pdf/"]',
1173
+ 'a[title*="PDF" i]',
1174
+ ],
1175
+ "SAGE": [
1176
+ 'a[href*="/doi/pdf/"]',
1177
+ 'a[title*="PDF" i]',
1178
+ 'a[href*=".pdf"]',
1179
+ ],
1180
+ "ASCE": [
1181
+ 'a[href*="/doi/pdf/"]',
1182
+ 'a[title*="PDF" i]',
1183
+ 'a[href*="download=true"]',
1184
+ ],
1185
+ "Royal Society": [
1186
+ 'a[href*="/doi/pdf/"]',
1187
+ 'a[href*="/doi/epdf/"]',
1188
+ 'a[title*="PDF" i]',
1189
+ ],
1190
+ "Copernicus": [
1191
+ 'a[href*="/pdf"]',
1192
+ 'a[href*=".pdf"]',
1193
+ 'a[title*="PDF" i]',
1194
+ 'a[class*="pdf"]',
1195
+ ],
1196
+ }
1197
+
1198
+ # ============================================================
1199
+ # Anti-bot challenge detection
1200
+ # ============================================================
1201
+
1202
+ # Auth detection keywords for institutional login flows
1203
+ _AUTH_KEYWORDS = ("cas", "idp", "saml", "wayf", "sso", "passport", "accounts", "oauth", "/login", "/signin")
1204
+ _AUTH_TITLES = ("登录", "身份", "二次认证", "Login", "Sign in", "Log in")
1205
+
1206
+
1207
+ def _school_auth_patterns(config: dict[str, Any]) -> tuple[str, ...]:
1208
+ """Return school-specific URL auth patterns derived from carsi_idp_name.
1209
+
1210
+ Uses _IDP_MAP to translate the configured Chinese institution name to
1211
+ English, then lowercases it so the caller can match it against IDP URLs
1212
+ (e.g. 'tsinghua' matches 'id.tsinghua.edu.cn').
1213
+
1214
+ For universities not in _IDP_MAP, falls back to pinyin conversion
1215
+ so that e.g. "兰州大学" → "lanzhou" can match "lzu.edu.cn" IDP URLs.
1216
+ """
1217
+ name = (config.get("carsi_idp_name", "") or "").strip()
1218
+ if not name:
1219
+ return ()
1220
+ en = _IDP_MAP.get(name, "").lower()
1221
+ if not en:
1222
+ # Dynamic fallback: try pinyin conversion for Chinese names
1223
+ en = _chinese_to_pinyin_token(name).lower()
1224
+ if not en:
1225
+ return ()
1226
+ # Split multi-word names into individual tokens for broad matching
1227
+ tokens = tuple(t for t in en.replace("-", " ").split() if len(t) > 2)
1228
+ return tokens
1229
+
1230
+
1231
+ def _chinese_to_pinyin_token(name: str) -> str:
1232
+ """Convert a Chinese university name to a pinyin token for URL matching.
1233
+
1234
+ E.g. "兰州大学" → "lanzhou", "哈尔滨工业大学" → "haerbingongye".
1235
+ Falls back to empty string if pypinyin is not installed.
1236
+ """
1237
+ try:
1238
+ from pypinyin import lazy_pinyin
1239
+ return "".join(lazy_pinyin(name))
1240
+ except ImportError:
1241
+ pass
1242
+ # Strip common suffixes and return what we have
1243
+ for suffix in ("大学", "学院", "研究院"):
1244
+ name = name.replace(suffix, "")
1245
+ return name
1246
+
1247
+ # Chinese → English university name mapping for institution search (CARSI/OpenAthens WAYF).
1248
+ # This is a lookup table used by _translate_institution_name(). Order is not significant —
1249
+ # the tool does NOT default to any school; the user must configure their own institution.
1250
+ _IDP_MAP: dict[str, str] = {
1251
+ "清华大学": "Tsinghua", "北京大学": "Peking", "浙江大学": "Zhejiang",
1252
+ "复旦大学": "Fudan", "上海交通大学": "Shanghai Jiao Tong",
1253
+ "南京大学": "Nanjing", "中国科学技术大学": "USTC",
1254
+ "武汉大学": "Wuhan", "中山大学": "Sun Yat-sen",
1255
+ "华中科技大学": "Huazhong", "哈尔滨工业大学": "Harbin Institute",
1256
+ "西安交通大学": "Xi'an Jiaotong", "同济大学": "Tongji",
1257
+ "东南大学": "Southeast", "厦门大学": "Xiamen",
1258
+ "北京师范大学": "Beijing Normal", "南开大学": "Nankai",
1259
+ "四川大学": "Sichuan", "天津大学": "Tianjin",
1260
+ "中国人民大学": "Renmin", "北京航空航天大学": "Beihang",
1261
+ "山东大学": "Shandong", "吉林大学": "Jilin",
1262
+ }
1263
+
1264
+ # Institution search input selectors, tried in order on WAYF pages
1265
+ _INSTITUTION_SEARCH_SELECTORS: list[str] = [
1266
+ "#searchInstitution", "#bdd-email", 'input[name="institution"]',
1267
+ "#institution-search", 'input[placeholder*="institution"]',
1268
+ 'input[placeholder*="University"]',
1269
+ # Generic fallbacks (e.g. wayf.springernature.com renders an unlabelled
1270
+ # text box under "Find your institution").
1271
+ 'input[id*="institution" i]', 'input[name*="institution" i]',
1272
+ 'input[aria-label*="institution" i]', 'input[placeholder*="institution" i]',
1273
+ 'input[type="search"]', 'main input[type="text"]', 'form input[type="text"]',
1274
+ ]
1275
+
1276
+ # Shared JS snippets for SSO/institutional login interaction.
1277
+ # These are Playwright-style (parametrized) — callers using string interpolation
1278
+ # (e.g. evaluate_js) must wrap them in an IIFE and inject params via json.dumps.
1279
+
1280
+ # Find and click SSO/institutional login link on article page
1281
+ _SSO_LINK_FINDER_JS: str = (
1282
+ "() => {"
1283
+ " const links = [...document.querySelectorAll('a')];"
1284
+ " const sso = links.find(a => a.href &&"
1285
+ " (a.href.includes('ssostart') || a.href.includes('shibboleth')"
1286
+ " || a.href.includes('saml') || a.href.includes('institutional-login')"
1287
+ " || a.href.includes('federation') || a.href.includes('/action/showLogin')"
1288
+ " || a.href.includes('/institutional-access') || a.href.includes('wayf')));"
1289
+ " if (sso) { return sso.href; }"
1290
+ " return false;"
1291
+ "}"
1292
+ )
1293
+
1294
+ # Click institution result that matches the given name (shortest match, see
1295
+ # _PICK_INSTITUTION_JS: the first match used to be a container or heading)
1296
+ _INSTITUTION_CLICK_JS: str = _PICK_INSTITUTION_JS
1297
+
1298
+ # Publisher-specific SSO configuration for visible browser download
1299
+ # Each entry: sso_link_js (JS to click institutional login), search_selectors, pdf_paths(doi)
1300
+
1301
+
1302
+ def _make_sso_click_js(*href_patterns: str, text_patterns: tuple[str, ...] = ()) -> str:
1303
+ """Generate a JS IIFE that clicks the first <a> matching the given patterns.
1304
+
1305
+ Args:
1306
+ href_patterns: Substrings to match against a.href (lowercased).
1307
+ text_patterns: Substrings to match against a.innerText (lowercased).
1308
+ """
1309
+ conditions = " || ".join(f"href.includes('{p}')" for p in href_patterns)
1310
+ if text_patterns:
1311
+ text_conds = " || ".join(f"text.includes('{p}')" for p in text_patterns)
1312
+ conditions = f"({conditions}) || ({text_conds})"
1313
+ return f"""
1314
+ (() => {{
1315
+ const a = [...document.querySelectorAll('a')].find(a => {{
1316
+ const href = (a.href || '').toLowerCase();
1317
+ const text = (a.innerText || '').toLowerCase();
1318
+ return {conditions};
1319
+ }});
1320
+ if (a) a.click();
1321
+ }})()
1322
+ """
1323
+
1324
+
1325
+ def _elsevier_pdf_paths(doi: str) -> list[str]:
1326
+ """Build Elsevier/Cell Press PDF fetch paths from DOI."""
1327
+ paths: list[str] = []
1328
+ doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
1329
+ # Cell Press showPdf uses PII — try common Cell journal DOI patterns
1330
+ import re
1331
+ cell_url = _build_cell_press_url(doi)
1332
+ if cell_url and "/abstract/" in cell_url:
1333
+ pii = cell_url.split("/abstract/")[-1]
1334
+ paths.append(f"/action/showPdf?pii={pii}")
1335
+ paths.append(f"/science/article/pii/{pii}/pdfft")
1336
+ # General ScienceDirect path with DOI
1337
+ paths.append(f"/doi/pdfdirect/{doi}")
1338
+ paths.append(f"/science/article/pii/{doi_suffix.replace('j.', 'j.')}/pdfft")
1339
+ return paths
1340
+
1341
+
1342
+ _PUBLISHER_SSO_CONFIG: dict[str, dict[str, Any]] = {
1343
+ "Wiley": {
1344
+ "sso_link_js": _make_sso_click_js("ssostart"),
1345
+ "search_selectors": ["#searchInstitution"],
1346
+ "pdf_paths": lambda doi: [f"/doi/pdfdirect/{doi}", f"/doi/pdf/{doi}"],
1347
+ },
1348
+ "Elsevier": {
1349
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional",
1350
+ text_patterns=("access through your institution", "institutional access")),
1351
+ "search_selectors": ["#institution-search", "input[name='query']", "#bdd-email"],
1352
+ "pdf_paths": lambda doi: _elsevier_pdf_paths(doi),
1353
+ },
1354
+ "Springer": {
1355
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional-login"),
1356
+ "search_selectors": ["#idp-search", "input[name='idpSearch']", "#searchInstitution"],
1357
+ "pdf_paths": lambda doi: [f"/content/pdf/{doi}.pdf"],
1358
+ },
1359
+ "ACS": {
1360
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1361
+ "search_selectors": ["input[name='search']", "#searchInstitution"],
1362
+ "pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
1363
+ },
1364
+ "IEEE": {
1365
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional",
1366
+ text_patterns=("institutional sign in",)),
1367
+ "search_selectors": ["input[name='idpSearch']", "#searchInstitution"],
1368
+ "pdf_paths": lambda doi: [], # IEEE uses stamp URL, needs page-level extraction
1369
+ },
1370
+ "Tandfonline": {
1371
+ "sso_link_js": _make_sso_click_js("ssostart", "shibboleth", "institutional",
1372
+ text_patterns=("access through your institution",)),
1373
+ "search_selectors": [
1374
+ 'input[placeholder*="institution"]',
1375
+ 'input[placeholder*="Type the name"]',
1376
+ "#searchInstitution",
1377
+ "input[name='query']",
1378
+ ],
1379
+ "pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
1380
+ },
1381
+ "Oxford": {
1382
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1383
+ "search_selectors": ["#searchInstitution", "input[name='query']"],
1384
+ "pdf_paths": lambda doi: [f"/downloadpdf/{doi}"],
1385
+ },
1386
+ "RSC": {
1387
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1388
+ "search_selectors": ["input[name='search']", "#searchInstitution"],
1389
+ "pdf_paths": lambda doi: [f"/content/articlepdf/{doi}"],
1390
+ },
1391
+ "IOP": {
1392
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1393
+ "search_selectors": ["input[name='search']", "#searchInstitution"],
1394
+ "pdf_paths": lambda doi: [f"/article/{doi}/pdf"],
1395
+ },
1396
+ "APS": {
1397
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1398
+ "search_selectors": ["input[name='search']", "#searchInstitution"],
1399
+ "pdf_paths": lambda doi: [f"/pdf/{doi}"],
1400
+ },
1401
+ "AIP": {
1402
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1403
+ "search_selectors": ["input[name='search']", "#searchInstitution"],
1404
+ "pdf_paths": lambda doi: [],
1405
+ },
1406
+ "Nature": {
1407
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1408
+ "search_selectors": ["input[name='idpSearch']", "#searchInstitution"],
1409
+ "pdf_paths": lambda doi: [f"/articles/{doi}.pdf"],
1410
+ },
1411
+ "Science": {
1412
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1413
+ "search_selectors": ["input[name='search']", "#searchInstitution"],
1414
+ "pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
1415
+ },
1416
+ "ACM": {
1417
+ "sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
1418
+ "search_selectors": ["input[name='search']", "#searchInstitution"],
1419
+ "pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
1420
+ },
1421
+ "_default": {
1422
+ "sso_link_js": """
1423
+ (() => {
1424
+ const links = [...document.querySelectorAll('a')];
1425
+ for (const a of links) {
1426
+ const href = (a.href || '').toLowerCase();
1427
+ const text = (a.innerText || '').toLowerCase();
1428
+ if (href.includes('ssostart') || href.includes('shibboleth') ||
1429
+ href.includes('saml') || href.includes('institutional-login') ||
1430
+ href.includes('federation') ||
1431
+ (text.includes('institutional') && text.includes('login')) ||
1432
+ text.includes('access through your institution')) {
1433
+ a.click();
1434
+ return;
1435
+ }
1436
+ }
1437
+ })()
1438
+ """,
1439
+ "search_selectors": ["#searchInstitution", "input[name='query']", "input[name='search']", "#institution-search"],
1440
+ "pdf_paths": lambda doi: [f"/doi/pdf/{doi}", f"/doi/pdfdirect/{doi}", f"/content/pdf/{doi}.pdf"],
1441
+ },
1442
+ }
1443
+
1444
+ _CHALLENGE_SIGNATURES = [
1445
+ "cf-browser-verification",
1446
+ "challenge-platform",
1447
+ "just a moment",
1448
+ "attention required",
1449
+ "security check",
1450
+ "verify you are human",
1451
+ "access to this page has been denied",
1452
+ "please complete the security check",
1453
+ "px-captcha",
1454
+ "hcaptcha",
1455
+ "recaptcha",
1456
+ "radware",
1457
+ "request rejected",
1458
+ "crasolve",
1459
+ ]
1460
+
1461
+
1462
+ # Challenge pages that cannot be passed in this browser at all: the check did
1463
+ # not even run ("extension or network configuration incompatible"). Waiting
1464
+ # for the user to click is pointless here: nothing to click.
1465
+ _CHALLENGE_DEAD_END = (
1466
+ "浏览器扩展或网络配置不兼容",
1467
+ "已阻止",
1468
+ "browser extensions or network settings have blocked",
1469
+ "incompatible browser extension or network configuration",
1470
+ )
1471
+
1472
+
1473
+ def _is_challenge_dead_end(html: str) -> bool:
1474
+ lower = html[:20000].lower()
1475
+ return any(sig.lower() in lower for sig in _CHALLENGE_DEAD_END)
1476
+
1477
+
1478
+ def _is_challenge_page(html: str) -> bool:
1479
+ """Detect if the page is an anti-bot challenge."""
1480
+ lower = html[:5000].lower()
1481
+ return any(sig in lower for sig in _CHALLENGE_SIGNATURES)
1482
+
1483
+
1484
+ # ============================================================
1485
+ # Elsevier-specific detection
1486
+ # ============================================================
1487
+
1488
+ def _is_elsevier_crasolve(url: str, html: str) -> bool:
1489
+ """Detect Elsevier's crasolve anti-bot shell."""
1490
+ if "crasolve=1" in url.lower():
1491
+ return True
1492
+ lower = html[:3000].lower()
1493
+ return "crasolve" in lower and "sciencedirect" in url.lower()
1494
+
1495
+
1496
+ # ============================================================
1497
+ # AIP/AVS loading page detection
1498
+ # ============================================================
1499
+
1500
+ def _is_aip_loading_page(html: str) -> bool:
1501
+ """Detect AIP's loading/waiting page."""
1502
+ lower = html[:3000].lower()
1503
+ return "loading" in lower and ("aip" in lower or "pubs.aip.org" in lower)
1504
+
1505
+
1506
+ # ============================================================
1507
+ # Core browser strategy
1508
+ # ============================================================
1509
+
1510
+ def _try_http_download(
1511
+ pdf_url: str,
1512
+ output_path: Path,
1513
+ config: dict[str, Any],
1514
+ cookies: dict[str, str] | None = None,
1515
+ ) -> bool:
1516
+ """Try direct HTTP download of a PDF URL. Returns True on success.
1517
+
1518
+ A host that timed out is skipped for a while (``_STALLED_HOSTS``); every
1519
+ failure says why and how long it took (2026-10-19: SAGE's two silent
1520
+ timeouts cost 52 s a paper and showed nowhere in the log)."""
1521
+ import requests
1522
+ from urllib.parse import urlparse
1523
+ from ...network import USER_AGENT
1524
+ from ...download.pdf import _response_looks_pdf, is_pdf_file
1525
+ from ...sources.publishers import _write_pdf_atomic, _load_publisher_cookies
1526
+
1527
+ host = (urlparse(pdf_url).hostname or "").lower()
1528
+ stalled_at = _STALLED_HOSTS.get(host)
1529
+ if stalled_at and time.time() - stalled_at < _STALLED_HOST_SECONDS:
1530
+ log.info(f" direct PDF: skipped {host} (it timed out {time.time() - stalled_at:.0f}s ago)")
1531
+ return False
1532
+ t_start = time.time()
1533
+ try:
1534
+ s = requests.Session()
1535
+ s.trust_env = False
1536
+ s.headers.update({"User-Agent": USER_AGENT})
1537
+ _load_publisher_cookies(s, config)
1538
+ if cookies:
1539
+ for name, value in cookies.items():
1540
+ s.cookies.set(name, value)
1541
+
1542
+ resp = s.get(pdf_url, timeout=20, stream=True,
1543
+ headers={"Accept": "application/pdf,*/*"},
1544
+ allow_redirects=True)
1545
+
1546
+ if resp.status_code >= 400:
1547
+ log.info(f" direct PDF: HTTP {resp.status_code} from {host} ({time.time() - t_start:.1f}s)")
1548
+ return False
1549
+
1550
+ iterator = resp.iter_content(chunk_size=8192)
1551
+ first = next(iterator, b"")
1552
+ if not _response_looks_pdf(resp, first):
1553
+ log.info(f" direct PDF: not a PDF from {host} ({time.time() - t_start:.1f}s)")
1554
+ return False
1555
+
1556
+ # timeout=20 bounds each read, not the transfer: a server that trickles
1557
+ # bytes (seen on link.springer.com) kept this lane — and the whole race —
1558
+ # waiting for minutes. Same wall-clock bound as pdf_utils' streams.
1559
+ deadline = time.monotonic() + float(config.get("download_deadline_seconds", 60))
1560
+
1561
+ def _bounded(chunks):
1562
+ for chunk in chunks:
1563
+ if time.monotonic() > deadline:
1564
+ log.info(f" HTTP PDF download too slow, giving up: {pdf_url[:80]}")
1565
+ raise TimeoutError("stream deadline exceeded")
1566
+ yield chunk
1567
+
1568
+ if not _write_pdf_atomic(output_path, first, _bounded(iterator)):
1569
+ return False
1570
+ return is_pdf_file(output_path)
1571
+ except (requests.Timeout, requests.ConnectionError) as exc:
1572
+ if isinstance(exc, requests.Timeout):
1573
+ _STALLED_HOSTS[host] = time.time()
1574
+ log.info(f" direct PDF: {type(exc).__name__} from {host} after {time.time() - t_start:.1f}s")
1575
+ return False
1576
+ except Exception as exc:
1577
+ log.info(f" direct PDF: {type(exc).__name__} from {host} after {time.time() - t_start:.1f}s")
1578
+ return False
1579
+
1580
+
1581
+ def _is_campus_network(config: dict[str, Any]) -> bool:
1582
+ """Check if user is on campus network (IP-authenticated access)."""
1583
+ return config.get("is_campus_network", False)
1584
+
1585
+
1586
+ def _find_pdf_link(page: Any, publisher: str) -> str | None:
1587
+ """Find PDF download link on article page using CSS selectors and JS."""
1588
+ from urllib.parse import urljoin
1589
+
1590
+ selectors = _PDF_LINK_SELECTORS.get(publisher, _PDF_LINK_SELECTORS.get("Elsevier", []))
1591
+ for sel in selectors:
1592
+ try:
1593
+ el = page.query_selector(sel)
1594
+ if el:
1595
+ href = el.get_attribute("href")
1596
+ if href:
1597
+ if href.startswith("/"):
1598
+ href = urljoin(page.url, href)
1599
+ return href
1600
+ except Exception:
1601
+ continue
1602
+
1603
+ # JS extraction fallback
1604
+ try:
1605
+ url = page.evaluate("""
1606
+ (() => {
1607
+ const meta = document.querySelector('meta[name="citation_pdf_url"]');
1608
+ if (meta) return meta.content;
1609
+ const links = [...document.querySelectorAll('a')];
1610
+ for (const a of links) {
1611
+ const href = (a.href || '').toLowerCase();
1612
+ if (href.includes('/pdf/') || href.includes('.pdf') || href.includes('pdfdirect')) {
1613
+ return a.href;
1614
+ }
1615
+ }
1616
+ return null;
1617
+ })()
1618
+ """)
1619
+ return url
1620
+ except Exception:
1621
+ return None
1622
+
1623
+
1624
+ def _browser_download_with_fallback(
1625
+ doi: str,
1626
+ article_url: str,
1627
+ output_path: Path,
1628
+ config: dict[str, Any],
1629
+ publisher: str,
1630
+ *,
1631
+ wait_for_loading: float = 0,
1632
+ ) -> bool:
1633
+ """The publisher's site, in its own window (persistent profile per publisher).
1634
+
1635
+ 2026-10-20: the first pass in the shared browser is gone. It judged pages
1636
+ too early, failed to open some sites the window opens, and could not keep a
1637
+ passed human check between papers (40-60 s per paper on T&F and ACS); the
1638
+ window's profile keeps it. The window starts minimized and comes up only
1639
+ when the user has to log in or pass a check (``_show_window_for_user``).
1640
+ """
1641
+ from ...browser.engine import _is_pdf_url, is_available
1642
+ from ...download.pdf import is_pdf_file, success
1643
+ _clear_error()
1644
+ if article_url.startswith("https://doi.org/"):
1645
+ article_url = _known_landing_url(doi) or article_url
1646
+ # A PDF URL and saved publisher cookies: try it without a browser first.
1647
+ if _has_publisher_cookies(config) and _is_pdf_url(article_url):
1648
+ log.info(f" [{publisher}] trying HTTP with cookies first (fast-path)")
1649
+ if _try_http_download(article_url, output_path, config) and is_pdf_file(output_path):
1650
+ return success(doi, output_path, f"{publisher}(HTTP)")
1651
+ if not is_available(config):
1652
+ log.info(f" [{publisher}] browser not available, skipping browser strategy")
1653
+ _set_error("browser_unavailable", "start_browser")
1654
+ return False
1655
+ try:
1656
+ from ...download import progress as _pr
1657
+ _pr.item_stage(doi, f"正在打开 {publisher} 的页面" if publisher not in ("", "Unknown") else "正在打开出版商官网")
1658
+ except Exception:
1659
+ pass
1660
+ try:
1661
+ from ...sources import SourceWaived
1662
+ except Exception: # pragma: no cover
1663
+ SourceWaived = () # type: ignore[assignment]
1664
+ try:
1665
+ return bool(_visible_browser_download(doi, article_url, output_path, config, publisher,
1666
+ wait_for_loading=wait_for_loading))
1667
+ except SourceWaived:
1668
+ raise
1669
+ except Exception as e:
1670
+ log.info(f" [{publisher}] publisher window error: {e}")
1671
+ if not get_last_error()[0]:
1672
+ _set_error("navigate_failed", "site_unreachable")
1673
+ return False
1674
+
1675
+
1676
+ # ============================================================
1677
+ # Elsevier API (no browser needed)
1678
+ # ============================================================
1679
+
1680
+ _CLOUDFLARE_COOKIE_NAMES = {"__cf_bm", "_cfuvid", "cf_clearance", "cf_chl_rc_ni"}
1681
+
1682
+
1683
+ def _persist_api_cookies(session: Any, config: dict[str, Any]) -> int:
1684
+ """Extract cookies from a requests.Session and merge into publisher_cookies.json.
1685
+
1686
+ Skips cookies that are only Cloudflare bot-management cookies (__cf_bm, _cfuvid, etc.)
1687
+ since those don't carry authentication and don't help the browser strategy.
1688
+
1689
+ Returns the number of cookies persisted.
1690
+ """
1691
+ from ...browser.cookies import merge_cookies
1692
+
1693
+ raw = []
1694
+ meaningful = 0
1695
+ for c in session.cookies:
1696
+ cookie = {
1697
+ "name": c.name,
1698
+ "value": c.value,
1699
+ "domain": c.domain,
1700
+ "path": c.path,
1701
+ "secure": c.secure,
1702
+ "expires": c.expires or 0,
1703
+ }
1704
+ raw.append(cookie)
1705
+ if c.name not in _CLOUDFLARE_COOKIE_NAMES:
1706
+ meaningful += 1
1707
+
1708
+ if not raw:
1709
+ return 0
1710
+
1711
+ if meaningful == 0:
1712
+ log.info(f" [ElsevierAPI] {len(raw)} cookies are Cloudflare-only, skipping persist")
1713
+ return 0
1714
+
1715
+ merged = merge_cookies(raw, config)
1716
+ log.info(f" [ElsevierAPI] persisted {meaningful} meaningful cookies "
1717
+ f"({len(raw)} total, merged: {len(merged)})")
1718
+ return meaningful
1719
+
1720
+
1721
+ from ._elsevier_api import ( # noqa: F401 — re-exported, callers import from here
1722
+ _elsevier_header,
1723
+ _elsevier_response_is_pdf,
1724
+ _elsevier_response_is_xml,
1725
+ _elsevier_response_text,
1726
+ _elsevier_xml_local_name,
1727
+ _elsevier_element_text,
1728
+ _elsevier_looks_like_pdf_eid,
1729
+ _elsevier_article_eid_to_main_pdf,
1730
+ _elsevier_attachment_container,
1731
+ _elsevier_attachment_metadata,
1732
+ _elsevier_attachment_score,
1733
+ _extract_elsevier_pdf_attachment_eids,
1734
+ _elsevier_pdf_page_count,
1735
+ _save_elsevier_pdf_content,
1736
+ _try_elsevier_object_pdf_from_xml,
1737
+ _try_elsevier_text_fallback,
1738
+ try_elsevier_api,
1739
+ _try_elsevier_api,
1740
+ )
1741
+ # ============================================================
1742
+ # Publisher entry points
1743
+ # ============================================================
1744
+
1745
+ def try_elsevier_browser(
1746
+ doi: str, output_path: Path, config: dict[str, Any],
1747
+ ) -> dict[str, Any] | None:
1748
+ """Elsevier/ScienceDirect/Cell Press browser strategy."""
1749
+ from ...download.pdf import is_pdf_file
1750
+ from ...download.pdf import success
1751
+ from ...browser.engine import is_available as browser_available, download_pdf_via_browser
1752
+
1753
+ # Campus network fast-path: skip HTTP, go directly to CloakBrowser
1754
+ if _is_campus_network(config) and browser_available(config):
1755
+ cell_url = _build_cell_press_url(doi)
1756
+ if cell_url:
1757
+ log.info(f" [Elsevier] campus network detected, trying CloakBrowser directly: {cell_url[:80]}")
1758
+ if download_pdf_via_browser(cell_url, output_path, config):
1759
+ if is_pdf_file(output_path):
1760
+ log.info(f" [Elsevier] campus network download succeeded")
1761
+ return success(doi, output_path, "CellPress(Campus)")
1762
+
1763
+ # Fast-path: if we have publisher cookies, try HTTP download first
1764
+ # This avoids 15-30s browser startup overhead
1765
+ if _has_publisher_cookies(config):
1766
+ # Try Cell Press showPdf with PII first (most reliable for Cell Press)
1767
+ cell_url = _build_cell_press_url(doi)
1768
+ if cell_url and "/abstract/" in cell_url:
1769
+ pii = cell_url.split("/abstract/")[-1]
1770
+ show_pdf_url = f"https://www.cell.com/action/showPdf?pii={pii}"
1771
+ log.info(f" [Elsevier] trying Cell Press showPdf HTTP with cookies: {show_pdf_url[:80]}")
1772
+ if _try_http_download(show_pdf_url, output_path, config):
1773
+ if is_pdf_file(output_path):
1774
+ log.info(f" [Elsevier] Cell Press HTTP download succeeded with cached cookies")
1775
+ return success(doi, output_path, "CellPress(HTTP)")
1776
+
1777
+ # For Cell Press DOIs, try direct showPdf download first (bypasses linkinghub)
1778
+ cell_url = _build_cell_press_url(doi)
1779
+ if cell_url and "/abstract/" in cell_url:
1780
+ pii = cell_url.split("/abstract/")[-1]
1781
+ show_pdf_url = f"https://www.cell.com/action/showPdf?pii={pii}"
1782
+ log.info(f" [CellPress] trying showPdf via network capture: {show_pdf_url[:80]}")
1783
+ result = _cell_press_showpdf_download(show_pdf_url, output_path, config)
1784
+ if result and is_pdf_file(output_path):
1785
+ return result
1786
+
1787
+ # If showPdf failed, navigate to the abstract page directly
1788
+ log.info(f" [CellPress] trying abstract page: {cell_url[:80]}")
1789
+ if _browser_download_with_fallback(doi, cell_url, output_path, config, "Elsevier"):
1790
+ if is_pdf_file(output_path):
1791
+ return success(doi, output_path, "CellPress(Browser)")
1792
+
1793
+ # Fallback to general browser download for ScienceDirect articles
1794
+ # Only try if we haven't already detected a paywall
1795
+ if not get_last_error()[0]:
1796
+ # Resolve DOI to direct sciencedirect.com URL (avoid linkinghub)
1797
+ article_url = _resolve_elsevier_pii(doi, config) or f"https://doi.org/{doi}"
1798
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Elsevier"):
1799
+ if is_pdf_file(output_path):
1800
+ return success(doi, output_path, "Elsevier(Browser)")
1801
+ return None
1802
+
1803
+
1804
+ def _cell_press_showpdf_download(
1805
+ show_pdf_url: str,
1806
+ output_path: Path,
1807
+ config: dict[str, Any],
1808
+ ) -> dict[str, Any] | None:
1809
+ """Download a Cell Press showPdf URL via browser network capture."""
1810
+ from ...browser.engine import create_tab, close_tab, fetch_url
1811
+ from ...download.pdf import success
1812
+
1813
+ tab_id = create_tab("https://example.com", config, timeout=15.0)
1814
+ if not tab_id:
1815
+ return None
1816
+
1817
+ try:
1818
+ _inject_cookies_to_tab(tab_id, config, "Elsevier")
1819
+ result = fetch_url(tab_id, show_pdf_url, config, timeout=30.0)
1820
+ if result and result.get("data"):
1821
+ pdf_bytes = result["data"]
1822
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1823
+ output_path.write_bytes(pdf_bytes)
1824
+ log.info(f" [CellPress] downloaded {len(pdf_bytes)} bytes via showPdf")
1825
+ return success(show_pdf_url, output_path, "CellPress(Network)")
1826
+ except Exception as e:
1827
+ log.info(f" [CellPress] showPdf failed: {e}")
1828
+ finally:
1829
+ close_tab(tab_id, config)
1830
+ return None
1831
+
1832
+
1833
+ # Cell Press journal mapping from DOI prefix to cell.com path
1834
+ _CELL_JOURNAL_MAP = {
1835
+ "j.oneear": "one-earth",
1836
+ "j.cell": "cell",
1837
+ "j.cels": "cancer-cell",
1838
+ "j.cub": "current-biology",
1839
+ "j.neuron": "neuron",
1840
+ "j.molcel": "molecular-cell",
1841
+ "j.devcel": "developmental-cell",
1842
+ "j.immuni": "immunity",
1843
+ "j.chom": "cell-host-microbe",
1844
+ "j.cmet": "cell-metabolism",
1845
+ "j.stem": "cell-stem-cell",
1846
+ "j.celrep": "cell-reports",
1847
+ "j.isci": "iscience",
1848
+ "j.xcr": "cell-reports-physical-science",
1849
+ "j.heliyon": "heliyon",
1850
+ "j.applanim": "ajhg",
1851
+ "j.ajhg": "ajhg",
1852
+ }
1853
+
1854
+
1855
+ def _build_cell_press_url(doi: str) -> str | None:
1856
+ """Build direct cell.com abstract URL from DOI if it's a Cell Press journal."""
1857
+ import re
1858
+ doi_suffix = doi.split("/", 1)[-1] if "/" in doi else ""
1859
+ if not doi_suffix.startswith("j."):
1860
+ return None
1861
+
1862
+ parts = doi_suffix.split(".")
1863
+ if len(parts) < 3:
1864
+ return None
1865
+
1866
+ # Try to match journal: j.oneear, j.cell, etc.
1867
+ journal_key = f"{parts[0]}.{parts[1]}"
1868
+ journal_path = _CELL_JOURNAL_MAP.get(journal_key)
1869
+ if not journal_path:
1870
+ return None
1871
+
1872
+ # Try to get PII from Crossref alternative-id
1873
+ pii = _get_elsevier_pii(doi)
1874
+ if pii:
1875
+ return f"https://www.cell.com/{journal_path}/abstract/{pii}"
1876
+
1877
+ # Fallback: use /doi/ path (may not have showPdf links)
1878
+ return f"https://www.cell.com/{journal_path}/doi/{doi}"
1879
+
1880
+
1881
+ def _get_elsevier_pii(doi: str) -> str | None:
1882
+ """Get PII (Publisher Item Identifier) from Crossref API."""
1883
+ try:
1884
+ import requests
1885
+ from ...network import USER_AGENT
1886
+ resp = requests.get(
1887
+ f"https://api.crossref.org/works/{doi}",
1888
+ timeout=10, verify=False,
1889
+ headers={"User-Agent": USER_AGENT},
1890
+ )
1891
+ if resp.status_code != 200:
1892
+ return None
1893
+ data = resp.json().get("message", {})
1894
+ alt_ids = data.get("alternative-id", [])
1895
+ for aid in alt_ids:
1896
+ if aid.startswith("S") and len(aid) >= 16:
1897
+ return aid
1898
+ except Exception:
1899
+ pass
1900
+ return None
1901
+
1902
+
1903
+ def _resolve_elsevier_pii(doi: str, config: dict[str, Any]) -> str | None:
1904
+ """Resolve Elsevier DOI to direct article URL, bypassing linkinghub.elsevier.com.
1905
+
1906
+ Uses HTTP to get the PII from linkinghub, then uses Crossref to determine the journal.
1907
+ Returns a direct cell.com/sciencedirect.com URL, or None.
1908
+ """
1909
+ import re
1910
+ try:
1911
+ import requests
1912
+ from ...network import USER_AGENT
1913
+ s = requests.Session()
1914
+ s.trust_env = False
1915
+ s.headers.update({"User-Agent": USER_AGENT, "Accept": "text/html"})
1916
+ from ...network import apply_host_proxy
1917
+ apply_host_proxy(s, config)
1918
+ resp = s.get(f"https://doi.org/{doi}", timeout=10, allow_redirects=True)
1919
+ final_url = str(resp.url)
1920
+ html = resp.text
1921
+
1922
+ # Extract PII from URL
1923
+ pii_match = re.search(r'pii/(S[0-9A-Za-z]+)', final_url) # PII may end in check char 'X'
1924
+ if not pii_match:
1925
+ return None
1926
+ pii = pii_match.group(1)
1927
+
1928
+ # Check if meta-refresh points to sciencedirect.com or cell.com
1929
+ if "sciencedirect.com" in html:
1930
+ return f"https://www.sciencedirect.com/science/article/pii/{pii}"
1931
+
1932
+ # Check for cell.com in meta-refresh
1933
+ if "cell.com" in html:
1934
+ cell_match = re.search(r'cell\.com/([^/]+)/', html)
1935
+ if cell_match:
1936
+ journal = cell_match.group(1)
1937
+ return f"https://www.cell.com/{journal}/abstract/{pii}"
1938
+
1939
+ return None
1940
+ except Exception:
1941
+ return None
1942
+
1943
+
1944
+ def try_wiley_browser(
1945
+ doi: str, output_path: Path, config: dict[str, Any],
1946
+ ) -> dict[str, Any] | None:
1947
+ """Wiley browser strategy with PDFDirect."""
1948
+ from ...download.pdf import is_pdf_file
1949
+ from ...download.pdf import success
1950
+ from ...browser.engine import is_available as browser_available, download_pdf_via_browser
1951
+
1952
+ # Campus network fast-path: skip HTTP, go directly to CloakBrowser
1953
+ if _is_campus_network(config) and browser_available(config):
1954
+ article_url = f"https://doi.org/{doi}"
1955
+ log.info(f" [Wiley] campus network detected, trying CloakBrowser directly")
1956
+ if download_pdf_via_browser(article_url, output_path, config):
1957
+ if is_pdf_file(output_path):
1958
+ log.info(f" [Wiley] campus network download succeeded")
1959
+ return success(doi, output_path, "Wiley(Campus)")
1960
+
1961
+ # Try direct PDF URLs first
1962
+ for url in _direct_pdf_urls(doi, "Wiley"):
1963
+ if _try_http_download(url, output_path, config):
1964
+ if is_pdf_file(output_path):
1965
+ return success(doi, output_path, "Wiley(PDFDirect)")
1966
+
1967
+ # Fall back to browser — use direct Wiley URL to avoid slow DOI redirect
1968
+ article_url = f"https://onlinelibrary.wiley.com/doi/{doi}"
1969
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Wiley"):
1970
+ if is_pdf_file(output_path):
1971
+ return success(doi, output_path, "Wiley(Browser)")
1972
+ return None
1973
+
1974
+
1975
+ def try_ieee_browser(
1976
+ doi: str, output_path: Path, config: dict[str, Any],
1977
+ ) -> dict[str, Any] | None:
1978
+ """IEEE browser strategy."""
1979
+ from ...download.pdf import is_pdf_file
1980
+ from ...download.pdf import success
1981
+
1982
+ article_url = f"https://doi.org/{doi}"
1983
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "IEEE"):
1984
+ if is_pdf_file(output_path):
1985
+ return success(doi, output_path, "IEEE(Browser)")
1986
+ return None
1987
+
1988
+
1989
+ def try_acs_browser(
1990
+ doi: str, output_path: Path, config: dict[str, Any],
1991
+ ) -> dict[str, Any] | None:
1992
+ """ACS browser strategy."""
1993
+ from ...download.pdf import is_pdf_file
1994
+ from ...download.pdf import success
1995
+
1996
+ # Try direct PDF URL first
1997
+ for url in _direct_pdf_urls(doi, "ACS"):
1998
+ if _try_http_download(url, output_path, config):
1999
+ if is_pdf_file(output_path):
2000
+ return success(doi, output_path, "ACS(Direct)")
2001
+
2002
+ article_url = f"https://doi.org/{doi}"
2003
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "ACS"):
2004
+ if is_pdf_file(output_path):
2005
+ return success(doi, output_path, "ACS(Browser)")
2006
+ return None
2007
+
2008
+
2009
+ def try_rsc_browser(
2010
+ doi: str, output_path: Path, config: dict[str, Any],
2011
+ ) -> dict[str, Any] | None:
2012
+ """RSC browser strategy."""
2013
+ from ...download.pdf import is_pdf_file
2014
+ from ...download.pdf import success
2015
+
2016
+ for url in _direct_pdf_urls(doi, "RSC"):
2017
+ if _try_http_download(url, output_path, config):
2018
+ if is_pdf_file(output_path):
2019
+ return success(doi, output_path, "RSC(Direct)")
2020
+
2021
+ article_url = f"https://doi.org/{doi}"
2022
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "RSC"):
2023
+ if is_pdf_file(output_path):
2024
+ return success(doi, output_path, "RSC(Browser)")
2025
+ return None
2026
+
2027
+
2028
+ def try_aip_browser(
2029
+ doi: str, output_path: Path, config: dict[str, Any],
2030
+ ) -> dict[str, Any] | None:
2031
+ """AIP browser strategy with loading page wait."""
2032
+ from ...download.pdf import is_pdf_file
2033
+ from ...download.pdf import success
2034
+
2035
+ article_url = f"https://doi.org/{doi}"
2036
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "AIP", wait_for_loading=10):
2037
+ if is_pdf_file(output_path):
2038
+ return success(doi, output_path, "AIP(Browser)")
2039
+ return None
2040
+
2041
+
2042
+ def try_springer_browser(
2043
+ doi: str, output_path: Path, config: dict[str, Any],
2044
+ ) -> dict[str, Any] | None:
2045
+ """Springer browser strategy."""
2046
+ from ...download.pdf import is_pdf_file
2047
+ from ...download.pdf import success
2048
+
2049
+ # Try direct PDF URL first
2050
+ for url in _direct_pdf_urls(doi, "Springer"):
2051
+ if _try_http_download(url, output_path, config):
2052
+ if is_pdf_file(output_path):
2053
+ return success(doi, output_path, "Springer(Direct)")
2054
+
2055
+ article_url = f"https://doi.org/{doi}"
2056
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Springer"):
2057
+ if is_pdf_file(output_path):
2058
+ return success(doi, output_path, "Springer(Browser)")
2059
+ return None
2060
+
2061
+
2062
+ def try_aps_browser(
2063
+ doi: str, output_path: Path, config: dict[str, Any],
2064
+ ) -> dict[str, Any] | None:
2065
+ """APS (Physical Review) browser strategy."""
2066
+ from ...download.pdf import is_pdf_file
2067
+ from ...download.pdf import success
2068
+
2069
+ for url in _direct_pdf_urls(doi, "APS"):
2070
+ if _try_http_download(url, output_path, config):
2071
+ if is_pdf_file(output_path):
2072
+ return success(doi, output_path, "APS(Direct)")
2073
+
2074
+ article_url = f"https://doi.org/{doi}"
2075
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "APS"):
2076
+ if is_pdf_file(output_path):
2077
+ return success(doi, output_path, "APS(Browser)")
2078
+ return None
2079
+
2080
+
2081
+ def try_tandfonline_browser(
2082
+ doi: str, output_path: Path, config: dict[str, Any],
2083
+ ) -> dict[str, Any] | None:
2084
+ """Taylor & Francis browser strategy."""
2085
+ from ...download.pdf import is_pdf_file
2086
+ from ...download.pdf import success
2087
+
2088
+ for url in _direct_pdf_urls(doi, "Tandfonline"):
2089
+ if _try_http_download(url, output_path, config):
2090
+ if is_pdf_file(output_path):
2091
+ return success(doi, output_path, "T&F(Direct)")
2092
+
2093
+ article_url = f"https://doi.org/{doi}"
2094
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Tandfonline"):
2095
+ if is_pdf_file(output_path):
2096
+ return success(doi, output_path, "T&F(Browser)")
2097
+ return None
2098
+
2099
+
2100
+ def try_iop_browser(
2101
+ doi: str, output_path: Path, config: dict[str, Any],
2102
+ ) -> dict[str, Any] | None:
2103
+ """IOP browser strategy."""
2104
+ from ...download.pdf import is_pdf_file
2105
+ from ...download.pdf import success
2106
+
2107
+ for url in _direct_pdf_urls(doi, "IOP"):
2108
+ if _try_http_download(url, output_path, config):
2109
+ if is_pdf_file(output_path):
2110
+ return success(doi, output_path, "IOP(Direct)")
2111
+
2112
+ article_url = f"https://doi.org/{doi}"
2113
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "IOP"):
2114
+ if is_pdf_file(output_path):
2115
+ return success(doi, output_path, "IOP(Browser)")
2116
+ return None
2117
+
2118
+
2119
+ def try_oxford_browser(
2120
+ doi: str, output_path: Path, config: dict[str, Any],
2121
+ ) -> dict[str, Any] | None:
2122
+ """Oxford Academic browser strategy."""
2123
+ from ...download.pdf import is_pdf_file
2124
+ from ...download.pdf import success
2125
+
2126
+ for url in _direct_pdf_urls(doi, "Oxford"):
2127
+ if _try_http_download(url, output_path, config):
2128
+ if is_pdf_file(output_path):
2129
+ return success(doi, output_path, "Oxford(Direct)")
2130
+
2131
+ article_url = f"https://doi.org/{doi}"
2132
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Oxford"):
2133
+ if is_pdf_file(output_path):
2134
+ return success(doi, output_path, "Oxford(Browser)")
2135
+ return None
2136
+
2137
+
2138
+ def try_acm_browser(
2139
+ doi: str, output_path: Path, config: dict[str, Any],
2140
+ ) -> dict[str, Any] | None:
2141
+ """ACM browser strategy."""
2142
+ from ...download.pdf import is_pdf_file
2143
+ from ...download.pdf import success
2144
+
2145
+ for url in _direct_pdf_urls(doi, "ACM"):
2146
+ if _try_http_download(url, output_path, config):
2147
+ if is_pdf_file(output_path):
2148
+ return success(doi, output_path, "ACM(Direct)")
2149
+
2150
+ article_url = f"https://doi.org/{doi}"
2151
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "ACM"):
2152
+ if is_pdf_file(output_path):
2153
+ return success(doi, output_path, "ACM(Browser)")
2154
+ return None
2155
+
2156
+
2157
+ def try_nature_browser(
2158
+ doi: str, output_path: Path, config: dict[str, Any],
2159
+ ) -> dict[str, Any] | None:
2160
+ """Nature browser strategy (fallback when direct fails)."""
2161
+ from ...download.pdf import is_pdf_file
2162
+ from ...download.pdf import success
2163
+
2164
+ article_url = f"https://doi.org/{doi}"
2165
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Nature"):
2166
+ if is_pdf_file(output_path):
2167
+ return success(doi, output_path, "Nature(Browser)")
2168
+ return None
2169
+
2170
+
2171
+ def try_science_browser(
2172
+ doi: str, output_path: Path, config: dict[str, Any],
2173
+ ) -> dict[str, Any] | None:
2174
+ """Science/AAAS browser strategy."""
2175
+ from ...download.pdf import is_pdf_file
2176
+ from ...download.pdf import success
2177
+
2178
+ article_url = f"https://doi.org/{doi}"
2179
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Science"):
2180
+ if is_pdf_file(output_path):
2181
+ return success(doi, output_path, "Science(Browser)")
2182
+ return None
2183
+
2184
+
2185
+ def try_sage_browser(
2186
+ doi: str, output_path: Path, config: dict[str, Any],
2187
+ ) -> dict[str, Any] | None:
2188
+ """SAGE Journals browser strategy.
2189
+
2190
+ Supports journals.sagepub.com and sage.cnpereading.com regional frontends
2191
+ by resolving DOI first and using host-relative PDF candidates.
2192
+ """
2193
+ from ...download.pdf import is_pdf_file, success
2194
+ from ...sources.sage_cn import try_sage_cn_authorized
2195
+
2196
+ if try_sage_cn_authorized(doi, output_path, config):
2197
+ return success(doi, output_path, "SAGE-CN(CARSI)")
2198
+
2199
+ for url in _direct_pdf_urls(doi, "SAGE"):
2200
+ if _try_http_download(url, output_path, config):
2201
+ if is_pdf_file(output_path):
2202
+ return success(doi, output_path, "SAGE(Direct)")
2203
+
2204
+ article_url = f"https://doi.org/{doi}"
2205
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "SAGE"):
2206
+ if is_pdf_file(output_path):
2207
+ return success(doi, output_path, "SAGE(Browser)")
2208
+ return None
2209
+
2210
+
2211
+ def try_asce_browser(
2212
+ doi: str, output_path: Path, config: dict[str, Any],
2213
+ ) -> dict[str, Any] | None:
2214
+ """ASCE Library browser strategy.
2215
+
2216
+ ASCE challenges plain HTTP requests with Cloudflare, so browser path
2217
+ is the primary strategy. DOI case must be uppercased for ASCE URLs.
2218
+ """
2219
+ from ...download.pdf import is_pdf_file, success
2220
+
2221
+ for url in _direct_pdf_urls(doi, "ASCE"):
2222
+ if _try_http_download(url, output_path, config):
2223
+ if is_pdf_file(output_path):
2224
+ return success(doi, output_path, "ASCE(Direct)")
2225
+
2226
+ article_url = f"https://ascelibrary.org/doi/{doi.upper()}"
2227
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "ASCE"):
2228
+ if is_pdf_file(output_path):
2229
+ return success(doi, output_path, "ASCE(Browser)")
2230
+ return None
2231
+
2232
+
2233
+ # ============================================================
2234
+ # Royal Society browser
2235
+ # ============================================================
2236
+
2237
+ def try_royalsociety_browser(
2238
+ doi: str, output_path: Path, config: dict[str, Any],
2239
+ ) -> dict[str, Any] | None:
2240
+ """Royal Society Publishing browser strategy.
2241
+
2242
+ Tries direct HTTP PDF URLs first (royalsocietypublishing.org serves PDFs
2243
+ without anti-bot for many articles), then falls back to browser for
2244
+ paywalled or Cloudflare-protected content.
2245
+ """
2246
+ from ...download.pdf import is_pdf_file, success
2247
+
2248
+ for url in _direct_pdf_urls(doi, "Royal Society"):
2249
+ if _try_http_download(url, output_path, config):
2250
+ if is_pdf_file(output_path):
2251
+ return success(doi, output_path, "Royal Society(Direct)")
2252
+
2253
+ article_url = f"https://doi.org/{doi}"
2254
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Royal Society"):
2255
+ if is_pdf_file(output_path):
2256
+ return success(doi, output_path, "Royal Society(Browser)")
2257
+ return None
2258
+
2259
+
2260
+ # ============================================================
2261
+ # Copernicus direct download
2262
+ # ============================================================
2263
+
2264
+ def try_copernicus_direct(
2265
+ doi: str, output_path: Path, config: dict[str, Any],
2266
+ ) -> dict[str, Any] | None:
2267
+ """Copernicus Publications direct download strategy.
2268
+
2269
+ Copernicus (EGU journals) provides open-access PDFs at predictable URLs.
2270
+ Resolves DOI to article page, then constructs PDF URL from article path.
2271
+ """
2272
+ from ...download.pdf import is_pdf_file, success
2273
+ import requests
2274
+
2275
+ article_url = f"https://doi.org/{doi}"
2276
+ try:
2277
+ resp = requests.head(article_url, allow_redirects=True, timeout=8,
2278
+ headers={"User-Agent": "Mozilla/5.0"})
2279
+ final_url = resp.url
2280
+
2281
+ # Strategy 1: Extract citation_pdf_url from article page HTML
2282
+ try:
2283
+ page_resp = requests.get(final_url, timeout=10,
2284
+ headers={"User-Agent": "Mozilla/5.0"})
2285
+ pdf_match = re.search(
2286
+ r'citation_pdf_url["\s]+content="([^"]+)"', page_resp.text[:20000], re.I)
2287
+ if pdf_match:
2288
+ pdf_url = pdf_match.group(1)
2289
+ if pdf_url.startswith("/"):
2290
+ from urllib.parse import urlparse as _up
2291
+ base = _up(final_url)
2292
+ pdf_url = f"{base.scheme}://{base.netloc}{pdf_url}"
2293
+ if _try_http_download(pdf_url, output_path, config):
2294
+ if is_pdf_file(output_path):
2295
+ return success(doi, output_path, "Copernicus(Direct)")
2296
+ except Exception:
2297
+ pass
2298
+
2299
+ # Strategy 2: Construct PDF URL from article path
2300
+ # Resolved URL: https://www.{journal}.net/articles/{vol}/{id}/{year}/
2301
+ # PDF URL: https://www.{journal}.net/articles/{vol}/{id}/{year}/{slug}.pdf
2302
+ # where slug = journal-abbrev-vol-id-year (from DOI: 10.5194/journal-vol-id-year)
2303
+ doi_parts = doi.split("/")[-1] if "/" in doi else doi # e.g. "hess-26-3433-2024"
2304
+ base_url = final_url.rstrip("/")
2305
+ for pdf_candidate in [
2306
+ f"{base_url}/{doi_parts}.pdf",
2307
+ f"{base_url}.pdf",
2308
+ ]:
2309
+ if _try_http_download(pdf_candidate, output_path, config):
2310
+ if is_pdf_file(output_path):
2311
+ return success(doi, output_path, "Copernicus(Direct)")
2312
+ except Exception:
2313
+ pass
2314
+
2315
+ # Fallback to browser
2316
+ if _browser_download_with_fallback(doi, article_url, output_path, config, "Copernicus"):
2317
+ if is_pdf_file(output_path):
2318
+ return success(doi, output_path, "Copernicus(Browser)")
2319
+ return None
2320
+
2321
+
2322
+ # ============================================================
2323
+ # Generic publisher browser fallback
2324
+ # ============================================================
2325
+
2326
+ def try_generic_browser(
2327
+ doi: str, output_path: Path, config: dict[str, Any],
2328
+ ) -> dict[str, Any] | None:
2329
+ """Generic browser fallback for unknown publishers."""
2330
+ from ...download.pdf import is_pdf_file
2331
+ from ...sources.publishers import get_publisher
2332
+ from ...download.pdf import success
2333
+
2334
+ publisher = get_publisher(doi) or "Unknown"
2335
+ article_url = f"https://doi.org/{doi}"
2336
+
2337
+ if _browser_download_with_fallback(doi, article_url, output_path, config, publisher):
2338
+ if is_pdf_file(output_path):
2339
+ return success(doi, output_path, f"{publisher}(Browser)")
2340
+ return None