litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1295 @@
1
+ """WebVPN institutional proxy source (multi-university support).
2
+
3
+ Uses AES-CFB encrypted URL conversion to access papers through
4
+ Chinese university WebVPN systems. Supports 100+ schools with
5
+ per-school encryption keys.
6
+
7
+ Password safety: Login happens in your browser via CAS.
8
+ The code only stores session cookies, never your password.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import binascii
14
+ import json
15
+ import re
16
+ import time
17
+ import urllib.parse
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ import requests
22
+ from bs4 import BeautifulSoup
23
+
24
+ from ..browser.pdf import capture_pdf_via_cdp, find_page_pdf_link, wait_page_settled
25
+ from ..log import get_logger
26
+ from ..download.pdf import (
27
+ _response_looks_pdf,
28
+ extract_pdf_url_from_html,
29
+ is_pdf_file,
30
+ is_plausible_pdf_url,
31
+ success,
32
+ )
33
+
34
+ log = get_logger()
35
+
36
+ # Config key migration: v1.7.0 renamed instsci_* → vpnsci_*. Prefer the new
37
+ # vpnsci_* keys, fall back to legacy instsci_* so existing user configs keep
38
+ # working. Both naming conventions resolve to the same value.
39
+ def _cfg(config: dict[str, Any], suffix: str, default: Any = None) -> Any:
40
+ """Read a config value trying vpnsci_<suffix> then instsci_<suffix>."""
41
+ if "vpnsci_" + suffix in config:
42
+ return config["vpnsci_" + suffix]
43
+ return config.get("instsci_" + suffix, default)
44
+
45
+
46
+ # Rate limiting between WebVPN requests
47
+ _last_instsci_time = 0.0
48
+ _INSTSCI_DELAY_MIN = 2.0
49
+ _INSTSCI_DELAY_MAX = 5.0
50
+
51
+
52
+ def _instsci_rate_limit() -> None:
53
+ global _last_instsci_time
54
+ now = time.time()
55
+ elapsed = now - _last_instsci_time
56
+ delay = __import__("random").uniform(_INSTSCI_DELAY_MIN, _INSTSCI_DELAY_MAX)
57
+ if elapsed < delay:
58
+ time.sleep(delay - elapsed)
59
+ _last_instsci_time = time.time()
60
+
61
+
62
+ def instsci_cookie_path(config: dict[str, Any]) -> Path:
63
+ configured = _cfg(config, "cookie_file")
64
+ if configured:
65
+ return Path(configured).expanduser()
66
+ from ..config import DEFAULT_CONFIG
67
+ return Path(config.get("cache_dir", DEFAULT_CONFIG["cache_dir"])).expanduser() / "instsci-cookies.json"
68
+
69
+
70
+ def instsci_is_configured(config: dict[str, Any]) -> bool:
71
+ return bool(_cfg(config, "enabled", False) and _get_webvpn_base(config))
72
+
73
+
74
+ def _get_webvpn_base(config: dict[str, Any]) -> str:
75
+ """Get WebVPN base URL, resolving from school if needed."""
76
+ base = _cfg(config, "base_url", "").strip()
77
+ if base:
78
+ return base.rstrip("/")
79
+ school = _cfg(config, "school", "")
80
+ if school:
81
+ try:
82
+ from ..institutional.schools import get_school
83
+ entry = get_school(school)
84
+ return entry.host.rstrip("/")
85
+ except ValueError:
86
+ pass
87
+ return ""
88
+
89
+
90
+ def _get_aes():
91
+ """Lazy import AES (pycryptodome may not be installed)."""
92
+ try:
93
+ from Crypto.Cipher import AES
94
+ return AES
95
+ except ImportError:
96
+ try:
97
+ from Cryptodome.Cipher import AES
98
+ return AES
99
+ except ImportError:
100
+ raise ImportError(
101
+ "pycryptodome required for WebVPN. Install: pip install pycryptodome"
102
+ )
103
+
104
+
105
+ def _get_school_keys(config: dict[str, Any]) -> tuple[bytes, bytes]:
106
+ """Get AES key and IV for the configured school."""
107
+ default_key = b"wrdvpnisthebest!"
108
+ school = _cfg(config, "school", "")
109
+ if school:
110
+ try:
111
+ from ..institutional.schools import get_school
112
+ entry = get_school(school)
113
+ return entry.key, entry.iv
114
+ except ValueError:
115
+ pass
116
+ return default_key, default_key
117
+
118
+
119
+ def convert_url(url: str, webvpn_base: str, config: dict[str, Any] | None = None) -> str:
120
+ """Convert a regular URL to a WebVPN URL using AES-CFB encryption.
121
+
122
+ Encrypts only the hostname; path and query are kept as-is.
123
+ Uses per-school encryption keys when config is provided.
124
+ """
125
+ key, iv = _get_school_keys(config) if config else (b"wrdvpnisthebest!", b"wrdvpnisthebest!")
126
+
127
+ parsed = urllib.parse.urlparse(url)
128
+ scheme = parsed.scheme.lower()
129
+ hostname = parsed.hostname
130
+ port = parsed.port
131
+ path = parsed.path
132
+ query = parsed.query
133
+
134
+ if not hostname:
135
+ return url
136
+
137
+ AES = _get_aes()
138
+ cipher = AES.new(key, AES.MODE_CFB, iv, segment_size=128)
139
+ encrypted = cipher.encrypt(hostname.encode("utf-8"))
140
+
141
+ encrypted_hex = binascii.hexlify(iv).decode() + binascii.hexlify(encrypted).decode()
142
+
143
+ scheme_part = scheme
144
+ if port:
145
+ scheme_part = f"{scheme}-{port}"
146
+
147
+ result = f"{webvpn_base.rstrip('/')}/{scheme_part}/{encrypted_hex}{path}"
148
+ if query:
149
+ result += f"?{query}"
150
+ return result
151
+
152
+
153
+ def _load_cookies(config: dict[str, Any]) -> requests.cookies.RequestsCookieJar:
154
+ path = instsci_cookie_path(config)
155
+ jar = requests.cookies.RequestsCookieJar()
156
+ if not path.exists():
157
+ # Legacy underscore variant written by older browser_login versions.
158
+ legacy = Path(str(path).replace("instsci-cookies.json", "instsci_cookies.json"))
159
+ if legacy.exists():
160
+ path = legacy
161
+ else:
162
+ return jar
163
+ try:
164
+ cookies = json.loads(path.read_text(encoding="utf-8"))
165
+ for c in cookies:
166
+ name = c.get("name")
167
+ value = c.get("value")
168
+ if name and value is not None:
169
+ kwargs: dict[str, Any] = {}
170
+ if c.get("domain"):
171
+ kwargs["domain"] = c["domain"]
172
+ if c.get("path"):
173
+ kwargs["path"] = c["path"]
174
+ jar.set(name, value, **kwargs)
175
+ except Exception:
176
+ pass
177
+ return jar
178
+
179
+
180
+ def session_status(config: dict[str, Any]) -> str:
181
+ """Classify the WebVPN session: ``none`` | ``valid`` | ``expired`` | ``unreachable``.
182
+
183
+ ``none``: no cookies or no base configured (fresh user — never auto-login).
184
+ ``expired``: cookies exist but the probe lands on a CAS/login page.
185
+ ``unreachable``: the probe failed for network reasons — do not treat as
186
+ expired, so auto re-login is not triggered during an outage.
187
+ """
188
+ jar = _load_cookies(config)
189
+ if not jar:
190
+ return "none"
191
+ base = _get_webvpn_base(config)
192
+ if not base:
193
+ return "none"
194
+ from ..network import USER_AGENT
195
+ # Probe the gateway root, not a proxied publisher: an expired session is
196
+ # redirected to the gateway's /login (then the school SSO) either way, but a
197
+ # proxied site adds its own latency/failures and WebVPN remembers it as the
198
+ # post-login target (users landed on nature.com after logging in).
199
+ test_url = base.rstrip("/") + "/"
200
+ try:
201
+ s = requests.Session()
202
+ s.trust_env = False
203
+ s.cookies.update(jar)
204
+ resp = s.get(test_url, timeout=15, allow_redirects=True,
205
+ headers={"User-Agent": USER_AGENT})
206
+ if "cas" in resp.url.lower() or "login" in resp.url.lower():
207
+ return "expired"
208
+ return "valid" if resp.status_code == 200 else "unreachable"
209
+ except Exception:
210
+ return "unreachable"
211
+
212
+
213
+ def _validate_session(config: dict[str, Any]) -> bool:
214
+ """Check if saved cookies still work."""
215
+ return session_status(config) == "valid"
216
+
217
+
218
+ def instsci_login(config: dict[str, Any]) -> bool:
219
+ """Open browser for CAS login. Called from MCP tool, not interactively."""
220
+ return _browser_login(config)
221
+
222
+
223
+ def _browser_login(config: dict[str, Any]) -> bool:
224
+ """Open browser for CAS login via CloakBrowser."""
225
+ try:
226
+ from ..browser.login import webvpn_login
227
+ if webvpn_login(config):
228
+ return True
229
+ except Exception as exc:
230
+ log.info(f" [WebVPN] stealth browser login failed: {exc}")
231
+ return False
232
+
233
+
234
+ def _get_socks5_proxy(config: dict[str, Any]) -> str:
235
+ """Get SOCKS5 proxy URL from config (for EasyConnect/aTrust campus connectors)."""
236
+ proxy = config.get("network_proxy", "").strip()
237
+ if proxy and proxy.lower().startswith("socks5://"):
238
+ return proxy
239
+ return ""
240
+
241
+
242
+ def _is_campus_connector_mode(config: dict[str, Any]) -> bool:
243
+ """Check if we should use direct access via SOCKS5 campus connector instead of WebVPN."""
244
+ proxy = _get_socks5_proxy(config)
245
+ if not proxy:
246
+ return False
247
+ # If school type is easyconnect or atrust, use direct access via SOCKS5
248
+ school = _cfg(config, "school", "")
249
+ if school:
250
+ try:
251
+ from ..institutional.schools import get_school
252
+ entry = get_school(school)
253
+ if entry.school_type in ("easyconnect", "atrust"):
254
+ return True
255
+ except (ValueError, AttributeError):
256
+ pass
257
+ return False
258
+
259
+
260
+ def _fetch_via_webvpn(url: str, config: dict[str, Any], *, stream: bool = False) -> requests.Response:
261
+ from ..network import USER_AGENT, request_timeout
262
+ base = _get_webvpn_base(config)
263
+ proxied = convert_url(url, base, config)
264
+
265
+ s = requests.Session()
266
+ s.trust_env = False
267
+ s.headers.update({"User-Agent": USER_AGENT})
268
+ s.cookies.update(_load_cookies(config))
269
+
270
+ # Use SOCKS5 proxy if configured (for campus connectors like EasyConnect/aTrust)
271
+ socks5 = _get_socks5_proxy(config)
272
+ if socks5:
273
+ s.proxies = {"http": socks5, "https": socks5}
274
+ s.verify = False # Campus connectors often use self-signed certs
275
+
276
+ return s.get(proxied, timeout=request_timeout(config), allow_redirects=True, stream=stream)
277
+
278
+
279
+ def _fetch_direct_via_socks5(url: str, config: dict[str, Any], *, stream: bool = False) -> requests.Response:
280
+ """Fetch a URL directly through SOCKS5 campus connector (no WebVPN URL conversion)."""
281
+ from ..network import USER_AGENT, request_timeout
282
+ socks5 = _get_socks5_proxy(config)
283
+
284
+ s = requests.Session()
285
+ s.trust_env = False
286
+ s.headers.update({"User-Agent": USER_AGENT})
287
+ s.proxies = {"http": socks5, "https": socks5}
288
+ s.verify = False
289
+
290
+ return s.get(url, timeout=request_timeout(config), allow_redirects=True, stream=stream)
291
+
292
+
293
+ def _resolve_doi_url(doi: str) -> str | None:
294
+ """Resolve DOI to get the publisher URL."""
295
+ try:
296
+ from ..config import load_config
297
+ from ..network import apply_host_proxy
298
+ session = requests.Session()
299
+ session.trust_env = False
300
+ apply_host_proxy(session, load_config())
301
+ resp = session.get(
302
+ f"https://doi.org/{doi}",
303
+ allow_redirects=True,
304
+ timeout=15,
305
+ headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"},
306
+ stream=True,
307
+ verify=False,
308
+ )
309
+ resp.close()
310
+ if resp.url and resp.url != f"https://doi.org/{doi}":
311
+ return resp.url
312
+ except Exception:
313
+ pass
314
+ return None
315
+
316
+
317
+ # taylorfrancis.com itself, or its WebVPN-rewritten form (/https/<hex>/books/...)
318
+ _TF_BOOK = re.compile(r"(?:taylorfrancis\.com|/https?/[0-9a-f]{16,})/books/(mono|edit)/10\.", re.I)
319
+
320
+
321
+ def _taylorfrancis_book_pdf_url(doi: str, resolved_url: str) -> str | None:
322
+ """Whole Taylor & Francis / CRC / Routledge books: the PDF comes from one
323
+ download endpoint. Its link is on authored books' pages but not on edited
324
+ ones (chapter PDFs only), though the endpoint serves both (checked through
325
+ WebVPN 2026-10-17). Single chapters are not covered: the same endpoint with
326
+ chapters/ returned a page, not the PDF."""
327
+ m = _TF_BOOK.search(resolved_url or "")
328
+ if not m:
329
+ return None
330
+ return (f"https://api.taylorfrancis.com/content/books/{m.group(1).lower()}/download?identifierName=doi"
331
+ f"&identifierValue={doi}&type=googlepdf")
332
+
333
+
334
+ _TF_CHAPTER = re.compile(r"(?:taylorfrancis\.com|/https?/[0-9a-f]{16,})/chapters/(mono|edit)/10\.", re.I)
335
+ _TF_CHAPTER_DOI = re.compile(r"^(10\.\d{4,9}/\d{10,13})-\d+$")
336
+
337
+
338
+ def _taylorfrancis_chapter_book(doi: str, landing_url: str) -> tuple[str, str] | None:
339
+ """A Taylor & Francis chapter (DOI "<book DOI>-<n>"): (book DOI, mono|edit).
340
+
341
+ T&F serves no PDF of a single chapter, only of the whole book (checked
342
+ through WebVPN 2026-10-17), so a chapter is downloaded as its book.
343
+ """
344
+ m = _TF_CHAPTER.search(landing_url or "")
345
+ b = _TF_CHAPTER_DOI.match((doi or "").strip())
346
+ return (b.group(1), m.group(1).lower()) if m and b else None
347
+
348
+
349
+ def _chapter_note(doi: str, config: dict[str, Any]) -> str:
350
+ """Where the chapter is in the book, for the user: from Crossref's page range."""
351
+ try:
352
+ from ..citations.formats import fetch_metadata
353
+ meta = fetch_metadata(doi, config) or {}
354
+ page = str(meta.get("page") or "").strip()
355
+ except Exception:
356
+ page = ""
357
+ where = f",本章在第 {page} 页" if page else ""
358
+ return f"T&F 只提供整本书的 PDF:下载的是本章所在的整本书{where}"
359
+
360
+
361
+ def _construct_publisher_pdf_url(doi: str, resolved_url: str) -> str | None:
362
+ """Try to construct a direct publisher PDF URL from the resolved URL."""
363
+ tf = _taylorfrancis_book_pdf_url(doi, resolved_url)
364
+ if tf:
365
+ return tf
366
+
367
+ parsed = urllib.parse.urlparse(resolved_url)
368
+ hostname = parsed.netloc.lower()
369
+ doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
370
+
371
+ if "pubs.acs.org" in hostname:
372
+ return f"https://pubs.acs.org/doi/pdf/{doi}"
373
+ elif "onlinelibrary.wiley.com" in hostname:
374
+ return f"https://onlinelibrary.wiley.com/doi/pdfdirect/{doi}"
375
+ elif "tandfonline.com" in hostname:
376
+ return f"https://www.tandfonline.com/doi/pdf/{doi}?needAccess=true"
377
+ elif "nature.com" in hostname:
378
+ return f"https://www.nature.com/articles/{doi_suffix}.pdf"
379
+ elif "link.springer.com" in hostname:
380
+ return f"https://link.springer.com/content/pdf/{doi}.pdf"
381
+ elif "pubs.rsc.org" in hostname:
382
+ pdf_url = resolved_url.replace("/articlelanding/", "/articlepdf/")
383
+ return pdf_url if pdf_url != resolved_url else None
384
+ elif "pnas.org" in hostname:
385
+ return f"https://www.pnas.org/doi/pdf/{doi}"
386
+ elif "science.org" in hostname or "sciencemag.org" in hostname:
387
+ return f"https://www.science.org/doi/pdf/{doi}"
388
+ elif "elsevier.com" in hostname or "sciencedirect.com" in hostname:
389
+ pii_match = re.search(r"pii/([A-Z0-9]+)", resolved_url)
390
+ if pii_match:
391
+ return f"https://www.sciencedirect.com/science/article/pii/{pii_match.group(1)}/pdfft"
392
+
393
+ return None
394
+
395
+
396
+ def _find_pdf_link(html: str, base_url: str) -> str | None:
397
+ """Find a PDF download link in an HTML page.
398
+
399
+ Tries: citation_pdf_url meta, <a> tags with PDF text/class,
400
+ and publisher-specific URL patterns.
401
+ """
402
+ try:
403
+ soup = BeautifulSoup(html, "lxml")
404
+ except Exception:
405
+ soup = BeautifulSoup(html, "html.parser")
406
+ parsed = urllib.parse.urlparse(base_url)
407
+ base = f"{parsed.scheme}://{parsed.netloc}"
408
+ hostname = parsed.netloc.lower()
409
+
410
+ # Strategy 1: <meta name="citation_pdf_url">
411
+ meta_pdf = soup.find("meta", attrs={"name": "citation_pdf_url"})
412
+ if meta_pdf and meta_pdf.get("content"):
413
+ pdf_url = meta_pdf["content"]
414
+ if pdf_url.startswith("http"):
415
+ return pdf_url
416
+ return base + pdf_url
417
+
418
+ # Strategy 2: <a> tags with PDF-related text/class/href
419
+ for a in soup.find_all("a", href=True):
420
+ href = a["href"]
421
+ text = a.get_text(strip=True).lower()
422
+ classes = " ".join(a.get("class", []))
423
+
424
+ if any(kw in text for kw in ["pdf", "download pdf", "full text pdf", "view pdf", "get pdf"]):
425
+ return _resolve_href(href, base)
426
+ if any(kw in classes for kw in ["pdf", "download-pdf", "pdf-download", "article-pdf"]):
427
+ return _resolve_href(href, base)
428
+ if href.endswith(".pdf"):
429
+ return _resolve_href(href, base)
430
+ if "/doi/pdf/" in href or "/doi/pdfdirect/" in href:
431
+ return _resolve_href(href, base)
432
+
433
+ # Strategy 3: Publisher-specific URL patterns
434
+ path = parsed.path
435
+ if "pubs.acs.org" in hostname and "/doi/" in path and "/pdf/" not in path:
436
+ doi_part = path.split("/doi/")[-1]
437
+ if doi_part:
438
+ return f"{base}/doi/pdf/{doi_part}"
439
+
440
+ if "onlinelibrary.wiley.com" in hostname and "/doi/" in path and "/pdfdirect/" not in path:
441
+ doi_part = path.split("/doi/")[-1]
442
+ if doi_part:
443
+ return f"{base}/doi/pdfdirect/{doi_part}"
444
+
445
+ if "pubs.rsc.org" in hostname and "/articlelanding/" in path:
446
+ return base_url.replace("/articlelanding/", "/articlepdf/")
447
+
448
+ if "tandfonline.com" in hostname and "/doi/" in path and "/pdf/" not in path:
449
+ doi_part = re.sub(r"/doi/(?:full|abs)/", "/doi/pdf/", path)
450
+ if doi_part != path:
451
+ return f"{base}{doi_part}"
452
+
453
+ if "pnas.org" in hostname and "/doi/" in path and "/pdf/" not in path:
454
+ doi_part = path.split("/doi/")[-1]
455
+ if doi_part:
456
+ return f"{base}/doi/pdf/{doi_part}"
457
+
458
+ if "science.org" in hostname and "/doi/" in path and "/pdf/" not in path:
459
+ doi_part = path.split("/doi/")[-1]
460
+ if doi_part:
461
+ return f"{base}/doi/pdf/{doi_part}"
462
+
463
+ if ("elsevier.com" in hostname or "sciencedirect.com" in hostname):
464
+ pii_match = re.search(r"pii/([A-Z0-9]+)", path)
465
+ if pii_match:
466
+ return f"https://www.sciencedirect.com/science/article/pii/{pii_match.group(1)}/pdfft"
467
+
468
+ return None
469
+
470
+
471
+ def _resolve_href(href: str, base: str) -> str:
472
+ if href.startswith("http"):
473
+ return href
474
+ if href.startswith("//"):
475
+ return "https:" + href
476
+ if href.startswith("/"):
477
+ return base + href
478
+ return base + "/" + href
479
+
480
+
481
+ def _is_inline_pdf_page(page: Any) -> bool:
482
+ """Check if the page is displaying an inline PDF."""
483
+ try:
484
+ url = page.url.lower()
485
+ if url.endswith(".pdf"):
486
+ return True
487
+ # Check for PDF embed/object
488
+ has_embed = page.evaluate("""
489
+ (() => {
490
+ const e = document.querySelector('embed[type="application/pdf"], object[type="application/pdf"], iframe[src*=".pdf"]');
491
+ return !!e;
492
+ })()
493
+ """)
494
+ if has_embed:
495
+ return True
496
+ # Check if page content starts with %PDF
497
+ content = page.evaluate("document.contentType || ''")
498
+ if "pdf" in content.lower():
499
+ return True
500
+ except Exception:
501
+ pass
502
+ return False
503
+
504
+
505
+ def _extract_inline_pdf(page: Any) -> bytes | None:
506
+ """Extract PDF bytes from an inline PDF page via JS fetch."""
507
+ try:
508
+ result = page.evaluate("""
509
+ (() => {
510
+ // Try embed/object src first
511
+ const embed = document.querySelector('embed[type="application/pdf"], object[type="application/pdf"]');
512
+ if (embed) {
513
+ const src = embed.src || embed.data;
514
+ if (src) return src;
515
+ }
516
+ // Try iframe
517
+ const iframe = document.querySelector('iframe[src*=".pdf"]');
518
+ if (iframe) return iframe.src;
519
+ // Use current URL
520
+ return window.location.href;
521
+ })()
522
+ """)
523
+ if not result:
524
+ return None
525
+
526
+ # Fetch the PDF bytes using JS fetch in the page context
527
+ pdf_bytes = page.evaluate(f"""
528
+ (async () => {{
529
+ try {{
530
+ const resp = await fetch({json.dumps(result)});
531
+ if (!resp.ok) return null;
532
+ const buf = await resp.arrayBuffer();
533
+ const bytes = new Uint8Array(buf);
534
+ // Check PDF magic
535
+ if (bytes[0] !== 0x25 || bytes[1] !== 0x50 || bytes[2] !== 0x44 || bytes[3] !== 0x46) return null;
536
+ if (bytes.length < 5000) return null;
537
+ // Convert to base64
538
+ let binary = '';
539
+ for (let i = 0; i < bytes.length; i++) {{
540
+ binary += String.fromCharCode(bytes[i]);
541
+ }}
542
+ return btoa(binary);
543
+ }} catch(e) {{
544
+ return null;
545
+ }}
546
+ }})()
547
+ """)
548
+ if pdf_bytes:
549
+ import base64
550
+ return base64.b64decode(pdf_bytes)
551
+ except Exception:
552
+ pass
553
+ return None
554
+
555
+
556
+ def _download_pdf_with_browser_cookies(
557
+ pdf_url: str, output_path: Path, config: dict[str, Any], doi: str, context: Any
558
+ ) -> dict[str, Any] | None:
559
+ """Download PDF via WebVPN using cookies from a live browser context.
560
+
561
+ Bypasses stale cookie files by pulling cookies directly from the
562
+ Playwright browser context that was just used for login/page-navigation.
563
+ """
564
+ if not is_plausible_pdf_url(pdf_url):
565
+ return None
566
+ try:
567
+ from ..network import USER_AGENT, request_timeout
568
+
569
+ base = _get_webvpn_base(config)
570
+ if not base:
571
+ return None
572
+ proxied = convert_url(pdf_url, base, config)
573
+
574
+ s = requests.Session()
575
+ s.trust_env = False
576
+ s.headers.update({"User-Agent": USER_AGENT})
577
+ for c in context.cookies():
578
+ name = c.get("name")
579
+ value = c.get("value")
580
+ if name and value is not None:
581
+ s.cookies.set(name, value, domain=c.get("domain", ""), path=c.get("path", "/"))
582
+
583
+ resp = s.get(proxied, timeout=request_timeout(config), allow_redirects=True, stream=True)
584
+ if resp.status_code >= 400:
585
+ log.info(f" [WebVPN-Browser] Browser-cookie HTTP status={resp.status_code} for PDF URL")
586
+ return None
587
+
588
+ iterator = resp.iter_content(chunk_size=8192)
589
+ first = next(iterator, b"")
590
+ if not _response_looks_pdf(resp, first):
591
+ return None
592
+
593
+ output_path.parent.mkdir(parents=True, exist_ok=True)
594
+ tmp = output_path.with_suffix(output_path.suffix + ".part")
595
+ try:
596
+ with tmp.open("wb") as fh:
597
+ fh.write(first)
598
+ for chunk in iterator:
599
+ if chunk:
600
+ fh.write(chunk)
601
+ tmp.replace(output_path)
602
+ except Exception:
603
+ tmp.unlink(missing_ok=True)
604
+ raise
605
+
606
+ if is_pdf_file(output_path):
607
+ log.info(f" [WebVPN-Browser] PDF downloaded via browser-cookie HTTP")
608
+ return success(doi, output_path, "WebVPN(Browser)")
609
+ except Exception as e:
610
+ log.info(f" [WebVPN-Browser] Browser-cookie HTTP error: {e}")
611
+ return None
612
+
613
+
614
+ def _try_instsci_socks5(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
615
+ """Try downloading via SOCKS5 campus connector (EasyConnect/aTrust).
616
+
617
+ The SOCKS5 connector handles authentication at the network level,
618
+ so no WebVPN URL conversion or CAS login is needed — just fetch
619
+ the publisher URL directly through the proxy.
620
+ """
621
+ _instsci_rate_limit()
622
+ socks5 = _get_socks5_proxy(config)
623
+ log.info(f" [CampusConnector] Trying {doi} via {socks5}")
624
+
625
+ # Step 1: Resolve DOI to publisher URL
626
+ resolved_url = _resolve_doi_url(doi)
627
+ if not resolved_url:
628
+ resolved_url = f"https://doi.org/{doi}"
629
+
630
+ # Step 2: Try direct publisher PDF URL
631
+ pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
632
+ if pdf_url:
633
+ log.info(f" [CampusConnector] Trying publisher PDF: {pdf_url[:80]}")
634
+ result = _download_pdf_socks5(pdf_url, output_path, config, doi)
635
+ if result:
636
+ return result
637
+
638
+ # Step 3: Fetch landing page via SOCKS5 and find PDF link
639
+ try:
640
+ resp = _fetch_direct_via_socks5(resolved_url, config, stream=True)
641
+ if resp.status_code >= 400:
642
+ log.info(f" [CampusConnector] HTTP {resp.status_code} for {resolved_url[:60]}")
643
+ return None
644
+
645
+ iterator = resp.iter_content(chunk_size=8192)
646
+ first = next(iterator, b"")
647
+
648
+ # Direct PDF response
649
+ if _response_looks_pdf(resp, first):
650
+ output_path.parent.mkdir(parents=True, exist_ok=True)
651
+ tmp_path = output_path.with_suffix(output_path.suffix + ".part")
652
+ try:
653
+ with tmp_path.open("wb") as fh:
654
+ fh.write(first)
655
+ for chunk in iterator:
656
+ if chunk:
657
+ fh.write(chunk)
658
+ tmp_path.replace(output_path)
659
+ except Exception:
660
+ tmp_path.unlink(missing_ok=True)
661
+ raise
662
+ if is_pdf_file(output_path):
663
+ log.info(" [CampusConnector] PDF downloaded directly")
664
+ return success(doi, output_path, "CampusConnector")
665
+
666
+ # HTML response - look for PDF link
667
+ html = first + resp.raw.read(512_000, decode_content=True)
668
+ html_str = html.decode("utf-8", errors="ignore")
669
+
670
+ found_pdf = _find_pdf_link(html_str, resp.url)
671
+ if found_pdf:
672
+ log.info(f" [CampusConnector] Found PDF link: {found_pdf[:80]}")
673
+ result = _download_pdf_socks5(found_pdf, output_path, config, doi)
674
+ if result:
675
+ return result
676
+
677
+ pdf_url = extract_pdf_url_from_html(html_str, resp.url)
678
+ if pdf_url:
679
+ result = _download_pdf_socks5(pdf_url, output_path, config, doi)
680
+ if result:
681
+ return result
682
+
683
+ except Exception as e:
684
+ log.info(f" [CampusConnector] {e}")
685
+
686
+ return None
687
+
688
+
689
+ def _download_pdf_socks5(
690
+ url: str,
691
+ output_path: Path,
692
+ config: dict[str, Any],
693
+ doi: str,
694
+ ) -> dict[str, Any] | None:
695
+ """Download a PDF URL directly through SOCKS5 campus connector."""
696
+ if not is_plausible_pdf_url(url):
697
+ return None
698
+ try:
699
+ _instsci_rate_limit()
700
+ resp = _fetch_direct_via_socks5(url, config, stream=True)
701
+ if resp.status_code >= 400:
702
+ return None
703
+
704
+ iterator = resp.iter_content(chunk_size=8192)
705
+ first_chunk = next(iterator, b"")
706
+ if not _response_looks_pdf(resp, first_chunk):
707
+ return None
708
+
709
+ output_path.parent.mkdir(parents=True, exist_ok=True)
710
+ tmp_path = output_path.with_suffix(output_path.suffix + ".part")
711
+ try:
712
+ with tmp_path.open("wb") as fh:
713
+ fh.write(first_chunk)
714
+ for chunk in iterator:
715
+ if chunk:
716
+ fh.write(chunk)
717
+ tmp_path.replace(output_path)
718
+ except Exception:
719
+ tmp_path.unlink(missing_ok=True)
720
+ raise
721
+
722
+ if is_pdf_file(output_path):
723
+ log.info(f" [CampusConnector] PDF downloaded: {doi}")
724
+ return success(doi, output_path, "CampusConnector")
725
+ except Exception as e:
726
+ log.info(f" [CampusConnector] Download error: {e}")
727
+ return None
728
+
729
+
730
+ def _try_instsci_browser(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
731
+ """Download via visible stealth browser browser. Login + download in same session."""
732
+ try:
733
+ from ..browser.backend import launch
734
+ from ..browser.backend import is_available as _browser_backend_available
735
+ if not _browser_backend_available():
736
+ log.info(" [WebVPN-Browser] no browser backend available")
737
+ return None
738
+ except ImportError:
739
+ log.info(" [WebVPN-Browser] no browser backend available")
740
+ return None
741
+
742
+ base = _get_webvpn_base(config)
743
+ if not base:
744
+ return None
745
+
746
+ resolved_url = _resolve_doi_url(doi)
747
+ if not resolved_url:
748
+ resolved_url = f"https://doi.org/{doi}"
749
+
750
+ webvpn_url = convert_url(resolved_url, base, config)
751
+ log.info(f" [WebVPN-Browser] Target: {webvpn_url[:80]}")
752
+ print(f"\n [WebVPN] 正在打开浏览器,请在浏览器中登录 WebVPN...")
753
+ print(f" 登录完成后等待 5 秒,程序会自动继续下载。\n")
754
+
755
+ try:
756
+ browser = launch(headless=False, humanize=True,
757
+ args=["--disable-features=CrossOriginOpenerPolicy"])
758
+ context = browser.new_context()
759
+ page = context.new_page()
760
+
761
+ # Restore saved cookies before navigating
762
+ cookie_path = instsci_cookie_path(config)
763
+ if cookie_path.exists():
764
+ try:
765
+ saved_cookies = json.loads(cookie_path.read_text(encoding="utf-8"))
766
+ if saved_cookies:
767
+ # Convert to Playwright cookie format
768
+ pw_cookies = []
769
+ for c in saved_cookies:
770
+ pw_c = {
771
+ "name": c["name"],
772
+ "value": c["value"],
773
+ "domain": c.get("domain", ""),
774
+ "path": c.get("path", "/"),
775
+ }
776
+ if c.get("secure"):
777
+ pw_c["secure"] = True
778
+ if c.get("httpOnly"):
779
+ pw_c["httpOnly"] = True
780
+ if pw_c["domain"]:
781
+ pw_cookies.append(pw_c)
782
+ if pw_cookies:
783
+ context.add_cookies(pw_cookies)
784
+ log.info(f" [WebVPN-Browser] Restored {len(pw_cookies)} cookies")
785
+ except Exception:
786
+ pass
787
+
788
+ # Capture PDF from network responses
789
+ captured_pdf = []
790
+
791
+ def on_response(response):
792
+ try:
793
+ ct = response.headers.get("content-type", "")
794
+ url = response.url
795
+ # Capture PDF responses (any content type that's actually a PDF)
796
+ is_pdf_ct = "pdf" in ct.lower() or "octet-stream" in ct.lower()
797
+ is_pdf_url = url.lower().endswith(".pdf") or "/pdfdirect/" in url or "/doi/pdf/" in url
798
+ if not (is_pdf_ct or is_pdf_url):
799
+ return
800
+ if response.status >= 400:
801
+ return
802
+ body = response.body()
803
+ if len(body) > 5000 and body[:4] == b"%PDF-":
804
+ captured_pdf.append(body)
805
+ log.info(f" [WebVPN-Browser] PDF captured: {len(body)} bytes from {url[:60]}")
806
+ except Exception:
807
+ pass
808
+
809
+ page.on("response", on_response)
810
+
811
+ # Navigate to paper URL directly via WebVPN
812
+ # If not logged in, will redirect to login page
813
+ try:
814
+ page.goto(webvpn_url, wait_until="domcontentloaded", timeout=60000)
815
+ except Exception:
816
+ pass
817
+ # WebVPN first shows a "Loading https://..." interstitial, then redirects
818
+ # (possibly to the school's SSO); judge login state only once it settles.
819
+ title, url_now = wait_page_settled(page)
820
+
821
+ # If on login page, wait for user to login then retry
822
+ from ..publishers.strategies import _school_auth_patterns
823
+ _stoks = _school_auth_patterns(config)
824
+ _auth_url_signals = list(_stoks) + ["/do/off/ui/auth"]
825
+
826
+ log.info(f" [WebVPN-Browser] Page title: '{title}' URL: {url_now[:80]}")
827
+ # The publisher sent the browser out of WebVPN to its own institution
828
+ # login (2026-10-17: ScienceDirect via WebVPN -> auth.elsevier.com ->
829
+ # the school's IdP -> the ScienceDirect home page, over the direct line,
830
+ # never back to the article). That is the publisher-site route, not
831
+ # WebVPN; it is tried in phase 1.
832
+ _school = (urllib.parse.urlparse(base).hostname or "").split(".", 1)[-1] # d.buaa.edu.cn -> buaa.edu.cn
833
+ _host_now = urllib.parse.urlparse(url_now).hostname or ""
834
+ if _host_now and _school and not _host_now.endswith(_school) and not any(
835
+ t in url_now for t in _auth_url_signals if t):
836
+ log.info(f" [WebVPN-Browser] Left WebVPN for {_host_now}; giving up this route")
837
+ return None
838
+ if "登录" in title or "身份" in title or "二次认证" in title or "CAS" in title or any(t in url_now for t in _auth_url_signals):
839
+ print(f" 检测到登录页面,请完成登录...")
840
+ # Wait up to 5 minutes, checking title every 3 seconds
841
+ from . import note_user_waiting
842
+ for i in range(100):
843
+ note_user_waiting("login") # the race waits, the app asks the user; stops if the paper is done
844
+ try:
845
+ time.sleep(3) # not page.wait_for_timeout: it hangs on a page stuck loading
846
+ # Post-login SAML/redirect hops also pass through interstitials
847
+ # without auth keywords; wait for them before declaring success.
848
+ title, url_now = wait_page_settled(page, max_wait_s=15)
849
+ except Exception:
850
+ return None
851
+ if i % 10 == 0:
852
+ log.info(f" [WebVPN-Browser] Waiting... title='{title}' url={url_now[:60]}")
853
+ # Detect login success: no longer on auth pages
854
+ is_auth = "登录" in title or "身份" in title or "二次认证" in title or "CAS" in title
855
+ is_auth_url = any(t in url_now for t in _auth_url_signals)
856
+ if not is_auth and not is_auth_url:
857
+ print(f" 登录成功!正在保存 cookies...")
858
+ # Save cookies immediately after login
859
+ try:
860
+ cookies = context.cookies()
861
+ from ..config import DATA_DIR
862
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
863
+ cache_dir.mkdir(parents=True, exist_ok=True)
864
+ cookie_data = [
865
+ {"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
866
+ for c in cookies
867
+ ]
868
+ from ..institutional.session_store import keep_private
869
+ (cache_dir / "instsci-cookies.json").write_text(
870
+ json.dumps(cookie_data, indent=2, ensure_ascii=False), encoding="utf-8")
871
+ keep_private(cache_dir / "instsci-cookies.json")
872
+ lines = ["# Netscape HTTP Cookie File\n"]
873
+ for c in cookies:
874
+ d = c.get("domain", "")
875
+ flag = "TRUE" if d.startswith(".") else "FALSE"
876
+ p = c.get("path", "/")
877
+ sec = "TRUE" if c.get("secure") else "FALSE"
878
+ exp = str(int(c.get("expires", 0)))
879
+ lines.append(f"{d}\t{flag}\t{p}\t{sec}\t{exp}\t{c['name']}\t{c['value']}\n")
880
+ (cache_dir / "instsci-cookies.txt").write_text("".join(lines), encoding="utf-8")
881
+ keep_private(cache_dir / "instsci-cookies.txt")
882
+ log.info(f" [WebVPN-Browser] Saved {len(cookies)} cookies")
883
+ except Exception as e:
884
+ log.info(f" [WebVPN-Browser] Cookie save warning: {e}")
885
+ break
886
+ else:
887
+ print(" 登录超时。")
888
+ return None
889
+
890
+ # Try HTTP download with fresh browser cookies (bypasses JS-heavy pages)
891
+ pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
892
+ if pdf_url:
893
+ time.sleep(2)
894
+ result = _download_pdf_with_browser_cookies(pdf_url, output_path, config, doi, context)
895
+ if result:
896
+ return result
897
+
898
+ # Fall back to browser-based PDF extraction
899
+ time.sleep(2)
900
+ try:
901
+ page.goto(webvpn_url, wait_until="domcontentloaded", timeout=60000)
902
+ except Exception:
903
+ pass
904
+ for _w in range(20):
905
+ time.sleep(2)
906
+ try:
907
+ _t = (page.title() or "").lower()
908
+ _b = (page.evaluate("document.body?.innerText?.length || 0") or 0)
909
+ if _b > 200 and not any(x in _t for x in ("请稍候", "loading", "please wait", "just a moment")):
910
+ break
911
+ except Exception:
912
+ break
913
+ else:
914
+ # Already authenticated — try HTTP download with browser cookies
915
+ pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
916
+ if pdf_url:
917
+ time.sleep(2)
918
+ result = _download_pdf_with_browser_cookies(pdf_url, output_path, config, doi, context)
919
+ if result:
920
+ return result
921
+ time.sleep(5)
922
+
923
+ # Helper: try to save captured PDF
924
+ def _save_captured():
925
+ if captured_pdf:
926
+ output_path.parent.mkdir(parents=True, exist_ok=True)
927
+ output_path.write_bytes(captured_pdf[-1])
928
+ if is_pdf_file(output_path):
929
+ return success(doi, output_path, "WebVPN(Browser)")
930
+ return None
931
+
932
+ def _save_bytes(pdf_bytes: bytes | None) -> dict[str, Any] | None:
933
+ if pdf_bytes:
934
+ output_path.parent.mkdir(parents=True, exist_ok=True)
935
+ output_path.write_bytes(pdf_bytes)
936
+ if is_pdf_file(output_path):
937
+ return success(doi, output_path, "WebVPN(Browser)")
938
+ return None
939
+
940
+ # Check if page itself is a PDF (inline viewer)
941
+ page_title, page_url = wait_page_settled(page)
942
+
943
+ # Some publishers are not proxied by the WebVPN: the school routes them to
944
+ # federated SSO (e.g. Elsevier via Shibboleth), and after login the IdP
945
+ # returns to the publisher's *homepage* outside the VPN. The session is then
946
+ # valid on the publisher itself, so open the article there directly.
947
+ base_host = urllib.parse.urlparse(base).netloc
948
+ if urllib.parse.urlparse(page_url).netloc not in ("", base_host):
949
+ direct_url = resolved_url
950
+ _lh = re.search(r"linkinghub\.elsevier\.com/retrieve/pii/([A-Za-z0-9]+)", direct_url)
951
+ if _lh:
952
+ direct_url = f"https://www.sciencedirect.com/science/article/pii/{_lh.group(1)}"
953
+ if urllib.parse.urlparse(direct_url).path.rstrip("/") != urllib.parse.urlparse(page_url).path.rstrip("/"):
954
+ log.info(f" [WebVPN-Browser] Left the VPN for {urllib.parse.urlparse(page_url).netloc}; opening article directly: {direct_url[:80]}")
955
+ try:
956
+ page.goto(direct_url, wait_until="domcontentloaded", timeout=60000)
957
+ except Exception:
958
+ pass
959
+ page_title, page_url = wait_page_settled(page)
960
+
961
+ log.info(f" [WebVPN-Browser] On page: title='{page_title[:40]}' url={page_url[:60]}")
962
+
963
+ # A Cloudflare challenge served *through* the WebVPN cannot pass: the
964
+ # gateway rewrites the challenge's scripts ("browser not supported").
965
+ # Report it as a block so the (WebVPN, publisher) negative cache skips
966
+ # this lane for the publisher's next papers instead of opening another
967
+ # doomed window each time.
968
+ from ..network import is_cloudflare_challenge
969
+ from ..download.pdf import fail
970
+ if is_cloudflare_challenge(page_title) and urllib.parse.urlparse(page_url).netloc == base_host:
971
+ log.info(" [WebVPN-Browser] Cloudflare challenge inside the WebVPN — publisher unusable via VPN")
972
+ return fail(doi, "Cloudflare challenge inside WebVPN",
973
+ error_type="cloudflare_blocked", action="use_direct_or_carsi")
974
+
975
+ # Check network-captured PDF
976
+ result = _save_captured()
977
+ if result:
978
+ return result
979
+
980
+ # Strategy 0: the article page's own PDF link (carries page-issued tokens),
981
+ # opened by the browser itself and read via CDP
982
+ page_pdf_href = find_page_pdf_link(page, doi=doi)
983
+ if not page_pdf_href:
984
+ try: # what the page offered instead: the next look at a miss starts here
985
+ links = page.evaluate("Array.from(document.querySelectorAll('a[href]')).filter(a => /pdf/i.test(a.innerText + a.href)).slice(0, 5).map(a => (a.innerText || '').trim().slice(0, 30) + ' -> ' + a.href.slice(0, 90))")
986
+ log.info(f" [WebVPN-Browser] No PDF link on the page; PDF-ish links: {links}")
987
+ except Exception:
988
+ pass
989
+ if page_pdf_href:
990
+ log.info(f" [WebVPN-Browser] Page PDF link: {page_pdf_href[:80]}")
991
+ result = _save_bytes(capture_pdf_via_cdp(context, page, page_pdf_href))
992
+ if result:
993
+ return result
994
+
995
+ # If page looks like inline PDF viewer, try to get the PDF bytes
996
+ if _is_inline_pdf_page(page):
997
+ pdf_bytes = _extract_inline_pdf(page)
998
+ if pdf_bytes:
999
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1000
+ output_path.write_bytes(pdf_bytes)
1001
+ if is_pdf_file(output_path):
1002
+ return success(doi, output_path, "WebVPN(Browser)")
1003
+
1004
+ # Strategy 1: Try direct publisher PDF URL via browser-cookie HTTP first
1005
+ resolved_for_pdf = _resolve_doi_url(doi) or f"https://doi.org/{doi}"
1006
+ pdf_url = _construct_publisher_pdf_url(doi, resolved_for_pdf)
1007
+ if pdf_url:
1008
+ result = _download_pdf_with_browser_cookies(pdf_url, output_path, config, doi, context)
1009
+ if result:
1010
+ return result
1011
+
1012
+ # Fallback: open in browser, read the body via CDP, then expect_download.
1013
+ # Off the VPN (federated login to the publisher itself) the session
1014
+ # is on the publisher, so open its own URL, not the WebVPN one.
1015
+ off_vpn = urllib.parse.urlparse(page_url).netloc not in ("", base_host)
1016
+ pdf_webvpn = pdf_url if off_vpn else convert_url(pdf_url, base, config)
1017
+ log.info(f" [WebVPN-Browser] Trying direct PDF via browser: {pdf_webvpn[:80]}")
1018
+ result = _save_bytes(capture_pdf_via_cdp(context, page, pdf_webvpn))
1019
+ if result:
1020
+ return result
1021
+ captured_pdf.clear()
1022
+ try:
1023
+ with page.expect_download(timeout=30000) as download_info:
1024
+ page.goto(pdf_webvpn, wait_until="commit", timeout=30000)
1025
+ download = download_info.value
1026
+ tmp = download.path()
1027
+ pdf_bytes = tmp.read_bytes() if tmp else None
1028
+ if pdf_bytes and pdf_bytes[:4] == b"%PDF-" and len(pdf_bytes) > 5000:
1029
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1030
+ output_path.write_bytes(pdf_bytes)
1031
+ if is_pdf_file(output_path):
1032
+ return success(doi, output_path, "WebVPN(Browser)")
1033
+ except Exception as dl_exc:
1034
+ log.info(f" [WebVPN-Browser] Download event not triggered: {dl_exc}")
1035
+ time.sleep(5)
1036
+ result = _save_captured()
1037
+ if result:
1038
+ return result
1039
+ if _is_inline_pdf_page(page):
1040
+ pdf_bytes = _extract_inline_pdf(page)
1041
+ if pdf_bytes:
1042
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1043
+ output_path.write_bytes(pdf_bytes)
1044
+ if is_pdf_file(output_path):
1045
+ return success(doi, output_path, "WebVPN(Browser)")
1046
+
1047
+ # Strategy 2: Find PDF link in HTML, try browser-cookie HTTP first
1048
+ wait_page_settled(page, max_wait_s=15)
1049
+ try:
1050
+ html = page.content()
1051
+ except Exception as exc:
1052
+ log.info(f" [WebVPN-Browser] page content unavailable: {exc}")
1053
+ html = ""
1054
+ found_pdf_url = extract_pdf_url_from_html(html, page.url) if html else None
1055
+ if found_pdf_url:
1056
+ log.info(f" [WebVPN-Browser] Found PDF link: {found_pdf_url[:80]}")
1057
+ result = _download_pdf_with_browser_cookies(found_pdf_url, output_path, config, doi, context)
1058
+ if result:
1059
+ return result
1060
+
1061
+ # Fallback: expect_download in browser
1062
+ captured_pdf.clear()
1063
+ try:
1064
+ with page.expect_download(timeout=30000) as download_info:
1065
+ page.goto(found_pdf_url, wait_until="commit", timeout=30000)
1066
+ download = download_info.value
1067
+ tmp = download.path()
1068
+ pdf_bytes = tmp.read_bytes() if tmp else None
1069
+ if pdf_bytes and pdf_bytes[:4] == b"%PDF-" and len(pdf_bytes) > 5000:
1070
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1071
+ output_path.write_bytes(pdf_bytes)
1072
+ if is_pdf_file(output_path):
1073
+ return success(doi, output_path, "WebVPN(Browser)")
1074
+ except Exception:
1075
+ time.sleep(5)
1076
+ result = _save_captured()
1077
+ if result:
1078
+ return result
1079
+ if _is_inline_pdf_page(page):
1080
+ pdf_bytes = _extract_inline_pdf(page)
1081
+ if pdf_bytes:
1082
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1083
+ output_path.write_bytes(pdf_bytes)
1084
+ if is_pdf_file(output_path):
1085
+ return success(doi, output_path, "WebVPN(Browser)")
1086
+
1087
+ log.info(f" [WebVPN-Browser] No PDF found. Title: {page.title()[:40]} URL: {page.url[:60]}")
1088
+ return None
1089
+
1090
+ except Exception as e:
1091
+ log.info(f" [WebVPN-Browser] Error: {e}")
1092
+ return None
1093
+ finally:
1094
+ try:
1095
+ browser.close()
1096
+ except Exception:
1097
+ pass
1098
+
1099
+
1100
+ def try_instsci(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
1101
+ """Try downloading paper through institutional access.
1102
+
1103
+ Strategy:
1104
+ 1. If SOCKS5 campus connector (EasyConnect/aTrust) is configured, try direct access
1105
+ 2. Try CloakBrowser download (handles CAS auth + Cloudflare)
1106
+ 3. Try WebVPN HTTP approach (if session cookies valid)
1107
+
1108
+ Note: CARSI is now a standalone source tier (sources.carsi.try_carsi),
1109
+ called independently from the download orchestrator.
1110
+ """
1111
+ if not _cfg(config, "enabled", False):
1112
+ return None
1113
+
1114
+ # Step 0: SOCKS5 campus connector mode (EasyConnect/aTrust) — direct access
1115
+ if _is_campus_connector_mode(config):
1116
+ result = _try_instsci_socks5(doi, output_path, config)
1117
+ if result:
1118
+ return result
1119
+
1120
+ # Step 1: Try stealth browser download (handles CAS auth + Cloudflare)
1121
+ result = _try_instsci_browser(doi, output_path, config)
1122
+ if result:
1123
+ return result
1124
+
1125
+ # Step 2: Try WebVPN HTTP approach (use any saved cookies, even if
1126
+ # _validate_session fails — the stealth browser may have just logged in
1127
+ # and saved fresh cookies that work for the target paper but fail
1128
+ # validation's unrelated test URL)
1129
+ if instsci_cookie_path(config).exists():
1130
+ result = _try_instsci_http(doi, output_path, config)
1131
+ if result:
1132
+ return result
1133
+ log.info(" [WebVPN] No PDF through WebVPN for this paper")
1134
+ return None
1135
+
1136
+ log.info(" [WebVPN] No valid session. Use instsci_login or carsi_login tool first.")
1137
+ return None
1138
+
1139
+
1140
+ def _try_instsci_http(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
1141
+ """Try downloading via HTTP with saved cookies."""
1142
+
1143
+ _instsci_rate_limit()
1144
+
1145
+ log.info(f" [WebVPN] Trying {doi}")
1146
+
1147
+ # Step 1: Resolve DOI to get publisher URL
1148
+ resolved_url = _resolve_doi_url(doi)
1149
+ if not resolved_url:
1150
+ resolved_url = f"https://doi.org/{doi}"
1151
+
1152
+ # Step 2: Try direct publisher PDF URL
1153
+ pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
1154
+ if pdf_url:
1155
+ log.info(f" [WebVPN] Trying publisher PDF: {pdf_url[:80]}...")
1156
+ result = _download_pdf_instsci(pdf_url, output_path, config, doi)
1157
+ if result:
1158
+ return result
1159
+
1160
+ # Step 3: Fetch via WebVPN and look for PDF link in HTML
1161
+ try:
1162
+ doi_url = f"https://doi.org/{doi}"
1163
+ resp = _fetch_via_webvpn(doi_url, config, stream=True)
1164
+ if resp.status_code >= 400:
1165
+ return None
1166
+
1167
+ iterator = resp.iter_content(chunk_size=8192)
1168
+ first = next(iterator, b"")
1169
+
1170
+ # Direct PDF response
1171
+ if _response_looks_pdf(resp, first):
1172
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1173
+ tmp_path = output_path.with_suffix(output_path.suffix + ".part")
1174
+ try:
1175
+ with tmp_path.open("wb") as fh:
1176
+ fh.write(first)
1177
+ for chunk in iterator:
1178
+ if chunk:
1179
+ fh.write(chunk)
1180
+ tmp_path.replace(output_path)
1181
+ except Exception:
1182
+ tmp_path.unlink(missing_ok=True)
1183
+ raise
1184
+ if is_pdf_file(output_path):
1185
+ return success(doi, output_path, "WebVPN")
1186
+
1187
+ # HTML response - extract PDF link
1188
+ html = first + resp.raw.read(512_000, decode_content=True)
1189
+ html_str = html.decode("utf-8", errors="ignore")
1190
+
1191
+ # Check for Cloudflare block
1192
+ from ..network import _is_cloudflare_block
1193
+ if any(sig in html_str.lower() for sig in ("cf-browser-verification", "challenge-platform", "just a moment", "请稍候", "正在验证", "checking your browser")):
1194
+ log.info(" [WebVPN] Cloudflare detected, trying browser...")
1195
+ browser_html = _try_browser_via_webvpn(doi_url, config)
1196
+ if browser_html:
1197
+ html_str = browser_html
1198
+
1199
+ # Taylor & Francis books: the landing page tells authored from edited
1200
+ # (the DOI alone does not); the download endpoint follows from it.
1201
+ tf = _taylorfrancis_book_pdf_url(doi, resp.url)
1202
+ if tf:
1203
+ log.info(f" [WebVPN] Taylor & Francis book PDF: {tf[:80]}...")
1204
+ result = _download_pdf_instsci(tf, output_path, config, doi)
1205
+ if result:
1206
+ return result
1207
+ chapter = _taylorfrancis_chapter_book(doi, resp.url)
1208
+ if chapter:
1209
+ book_doi, form = chapter
1210
+ for kind in dict.fromkeys((form, "mono", "edit")): # the chapter's form first
1211
+ url = (f"https://api.taylorfrancis.com/content/books/{kind}/download?identifierName=doi"
1212
+ f"&identifierValue={book_doi}&type=googlepdf")
1213
+ log.info(f" [WebVPN] Taylor & Francis chapter: trying its book {book_doi} ({kind})")
1214
+ result = _download_pdf_instsci(url, output_path, config, doi)
1215
+ if result:
1216
+ result.update(whole_book=True, book_doi=book_doi, note=_chapter_note(doi, config))
1217
+ return result
1218
+
1219
+ # Try _find_pdf_link (more thorough)
1220
+ found_pdf = _find_pdf_link(html_str, resp.url)
1221
+ if found_pdf:
1222
+ log.info(f" [WebVPN] Found PDF link in HTML: {found_pdf[:80]}...")
1223
+ result = _download_pdf_instsci(found_pdf, output_path, config, doi)
1224
+ if result:
1225
+ return result
1226
+
1227
+ # Fallback to extract_pdf_url_from_html
1228
+ pdf_url = extract_pdf_url_from_html(html_str, resp.url)
1229
+ if pdf_url:
1230
+ return _download_pdf_instsci(pdf_url, output_path, config, doi)
1231
+
1232
+ except Exception as e:
1233
+ log.info(f" [WebVPN] {e}")
1234
+
1235
+ return None
1236
+
1237
+
1238
+ def _try_browser_via_webvpn(url: str, config: dict[str, Any]) -> str | None:
1239
+ """Try fetching a URL through CloakBrowser, using WebVPN proxy."""
1240
+ base = _get_webvpn_base(config)
1241
+ proxied_url = convert_url(url, base, config)
1242
+ try:
1243
+ from ..browser.engine import is_available as browser_avail, get_html as browser_html
1244
+ if browser_avail(config):
1245
+ result = browser_html(proxied_url, config)
1246
+ if result:
1247
+ return result
1248
+ except Exception as e:
1249
+ log.info(f" [browser] {e}")
1250
+ return None
1251
+
1252
+
1253
+ def _download_pdf_instsci(
1254
+ url: str,
1255
+ output_path: Path,
1256
+ config: dict[str, Any],
1257
+ doi: str,
1258
+ ) -> dict[str, Any] | None:
1259
+ if not is_plausible_pdf_url(url):
1260
+ log.info(f" [WebVPN] Not a PDF link, skipped: {url[:80]}")
1261
+ return None
1262
+ try:
1263
+ _instsci_rate_limit()
1264
+ resp = _fetch_via_webvpn(url, config, stream=True)
1265
+ if resp.status_code >= 400:
1266
+ log.info(f" [WebVPN] HTTP {resp.status_code}: {url[:80]}")
1267
+ return None
1268
+
1269
+ iterator = resp.iter_content(chunk_size=8192)
1270
+ first_chunk = next(iterator, b"")
1271
+ if not _response_looks_pdf(resp, first_chunk):
1272
+ log.info(f" [WebVPN] Not a PDF ({resp.headers.get('content-type', '?')}): {url[:80]}")
1273
+ return None
1274
+
1275
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1276
+ tmp_path = output_path.with_suffix(output_path.suffix + ".part")
1277
+ try:
1278
+ with tmp_path.open("wb") as fh:
1279
+ fh.write(first_chunk)
1280
+ for chunk in iterator:
1281
+ if chunk:
1282
+ fh.write(chunk)
1283
+ tmp_path.replace(output_path)
1284
+ except Exception:
1285
+ tmp_path.unlink(missing_ok=True)
1286
+ raise
1287
+
1288
+ if is_pdf_file(output_path):
1289
+ result = success(doi, output_path, "WebVPN")
1290
+ result["doi"] = doi
1291
+ result["identifier"] = doi
1292
+ return result
1293
+ except Exception as e:
1294
+ log.info(f" [WebVPN] Download failed: {e}")
1295
+ return None