litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1295 @@
|
|
|
1
|
+
"""WebVPN institutional proxy source (multi-university support).
|
|
2
|
+
|
|
3
|
+
Uses AES-CFB encrypted URL conversion to access papers through
|
|
4
|
+
Chinese university WebVPN systems. Supports 100+ schools with
|
|
5
|
+
per-school encryption keys.
|
|
6
|
+
|
|
7
|
+
Password safety: Login happens in your browser via CAS.
|
|
8
|
+
The code only stores session cookies, never your password.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import binascii
|
|
14
|
+
import json
|
|
15
|
+
import re
|
|
16
|
+
import time
|
|
17
|
+
import urllib.parse
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import requests
|
|
22
|
+
from bs4 import BeautifulSoup
|
|
23
|
+
|
|
24
|
+
from ..browser.pdf import capture_pdf_via_cdp, find_page_pdf_link, wait_page_settled
|
|
25
|
+
from ..log import get_logger
|
|
26
|
+
from ..download.pdf import (
|
|
27
|
+
_response_looks_pdf,
|
|
28
|
+
extract_pdf_url_from_html,
|
|
29
|
+
is_pdf_file,
|
|
30
|
+
is_plausible_pdf_url,
|
|
31
|
+
success,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
log = get_logger()
|
|
35
|
+
|
|
36
|
+
# Config key migration: v1.7.0 renamed instsci_* → vpnsci_*. Prefer the new
|
|
37
|
+
# vpnsci_* keys, fall back to legacy instsci_* so existing user configs keep
|
|
38
|
+
# working. Both naming conventions resolve to the same value.
|
|
39
|
+
def _cfg(config: dict[str, Any], suffix: str, default: Any = None) -> Any:
|
|
40
|
+
"""Read a config value trying vpnsci_<suffix> then instsci_<suffix>."""
|
|
41
|
+
if "vpnsci_" + suffix in config:
|
|
42
|
+
return config["vpnsci_" + suffix]
|
|
43
|
+
return config.get("instsci_" + suffix, default)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# Rate limiting between WebVPN requests
|
|
47
|
+
_last_instsci_time = 0.0
|
|
48
|
+
_INSTSCI_DELAY_MIN = 2.0
|
|
49
|
+
_INSTSCI_DELAY_MAX = 5.0
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _instsci_rate_limit() -> None:
|
|
53
|
+
global _last_instsci_time
|
|
54
|
+
now = time.time()
|
|
55
|
+
elapsed = now - _last_instsci_time
|
|
56
|
+
delay = __import__("random").uniform(_INSTSCI_DELAY_MIN, _INSTSCI_DELAY_MAX)
|
|
57
|
+
if elapsed < delay:
|
|
58
|
+
time.sleep(delay - elapsed)
|
|
59
|
+
_last_instsci_time = time.time()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def instsci_cookie_path(config: dict[str, Any]) -> Path:
|
|
63
|
+
configured = _cfg(config, "cookie_file")
|
|
64
|
+
if configured:
|
|
65
|
+
return Path(configured).expanduser()
|
|
66
|
+
from ..config import DEFAULT_CONFIG
|
|
67
|
+
return Path(config.get("cache_dir", DEFAULT_CONFIG["cache_dir"])).expanduser() / "instsci-cookies.json"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def instsci_is_configured(config: dict[str, Any]) -> bool:
|
|
71
|
+
return bool(_cfg(config, "enabled", False) and _get_webvpn_base(config))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _get_webvpn_base(config: dict[str, Any]) -> str:
|
|
75
|
+
"""Get WebVPN base URL, resolving from school if needed."""
|
|
76
|
+
base = _cfg(config, "base_url", "").strip()
|
|
77
|
+
if base:
|
|
78
|
+
return base.rstrip("/")
|
|
79
|
+
school = _cfg(config, "school", "")
|
|
80
|
+
if school:
|
|
81
|
+
try:
|
|
82
|
+
from ..institutional.schools import get_school
|
|
83
|
+
entry = get_school(school)
|
|
84
|
+
return entry.host.rstrip("/")
|
|
85
|
+
except ValueError:
|
|
86
|
+
pass
|
|
87
|
+
return ""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _get_aes():
|
|
91
|
+
"""Lazy import AES (pycryptodome may not be installed)."""
|
|
92
|
+
try:
|
|
93
|
+
from Crypto.Cipher import AES
|
|
94
|
+
return AES
|
|
95
|
+
except ImportError:
|
|
96
|
+
try:
|
|
97
|
+
from Cryptodome.Cipher import AES
|
|
98
|
+
return AES
|
|
99
|
+
except ImportError:
|
|
100
|
+
raise ImportError(
|
|
101
|
+
"pycryptodome required for WebVPN. Install: pip install pycryptodome"
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _get_school_keys(config: dict[str, Any]) -> tuple[bytes, bytes]:
|
|
106
|
+
"""Get AES key and IV for the configured school."""
|
|
107
|
+
default_key = b"wrdvpnisthebest!"
|
|
108
|
+
school = _cfg(config, "school", "")
|
|
109
|
+
if school:
|
|
110
|
+
try:
|
|
111
|
+
from ..institutional.schools import get_school
|
|
112
|
+
entry = get_school(school)
|
|
113
|
+
return entry.key, entry.iv
|
|
114
|
+
except ValueError:
|
|
115
|
+
pass
|
|
116
|
+
return default_key, default_key
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def convert_url(url: str, webvpn_base: str, config: dict[str, Any] | None = None) -> str:
|
|
120
|
+
"""Convert a regular URL to a WebVPN URL using AES-CFB encryption.
|
|
121
|
+
|
|
122
|
+
Encrypts only the hostname; path and query are kept as-is.
|
|
123
|
+
Uses per-school encryption keys when config is provided.
|
|
124
|
+
"""
|
|
125
|
+
key, iv = _get_school_keys(config) if config else (b"wrdvpnisthebest!", b"wrdvpnisthebest!")
|
|
126
|
+
|
|
127
|
+
parsed = urllib.parse.urlparse(url)
|
|
128
|
+
scheme = parsed.scheme.lower()
|
|
129
|
+
hostname = parsed.hostname
|
|
130
|
+
port = parsed.port
|
|
131
|
+
path = parsed.path
|
|
132
|
+
query = parsed.query
|
|
133
|
+
|
|
134
|
+
if not hostname:
|
|
135
|
+
return url
|
|
136
|
+
|
|
137
|
+
AES = _get_aes()
|
|
138
|
+
cipher = AES.new(key, AES.MODE_CFB, iv, segment_size=128)
|
|
139
|
+
encrypted = cipher.encrypt(hostname.encode("utf-8"))
|
|
140
|
+
|
|
141
|
+
encrypted_hex = binascii.hexlify(iv).decode() + binascii.hexlify(encrypted).decode()
|
|
142
|
+
|
|
143
|
+
scheme_part = scheme
|
|
144
|
+
if port:
|
|
145
|
+
scheme_part = f"{scheme}-{port}"
|
|
146
|
+
|
|
147
|
+
result = f"{webvpn_base.rstrip('/')}/{scheme_part}/{encrypted_hex}{path}"
|
|
148
|
+
if query:
|
|
149
|
+
result += f"?{query}"
|
|
150
|
+
return result
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _load_cookies(config: dict[str, Any]) -> requests.cookies.RequestsCookieJar:
|
|
154
|
+
path = instsci_cookie_path(config)
|
|
155
|
+
jar = requests.cookies.RequestsCookieJar()
|
|
156
|
+
if not path.exists():
|
|
157
|
+
# Legacy underscore variant written by older browser_login versions.
|
|
158
|
+
legacy = Path(str(path).replace("instsci-cookies.json", "instsci_cookies.json"))
|
|
159
|
+
if legacy.exists():
|
|
160
|
+
path = legacy
|
|
161
|
+
else:
|
|
162
|
+
return jar
|
|
163
|
+
try:
|
|
164
|
+
cookies = json.loads(path.read_text(encoding="utf-8"))
|
|
165
|
+
for c in cookies:
|
|
166
|
+
name = c.get("name")
|
|
167
|
+
value = c.get("value")
|
|
168
|
+
if name and value is not None:
|
|
169
|
+
kwargs: dict[str, Any] = {}
|
|
170
|
+
if c.get("domain"):
|
|
171
|
+
kwargs["domain"] = c["domain"]
|
|
172
|
+
if c.get("path"):
|
|
173
|
+
kwargs["path"] = c["path"]
|
|
174
|
+
jar.set(name, value, **kwargs)
|
|
175
|
+
except Exception:
|
|
176
|
+
pass
|
|
177
|
+
return jar
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def session_status(config: dict[str, Any]) -> str:
|
|
181
|
+
"""Classify the WebVPN session: ``none`` | ``valid`` | ``expired`` | ``unreachable``.
|
|
182
|
+
|
|
183
|
+
``none``: no cookies or no base configured (fresh user — never auto-login).
|
|
184
|
+
``expired``: cookies exist but the probe lands on a CAS/login page.
|
|
185
|
+
``unreachable``: the probe failed for network reasons — do not treat as
|
|
186
|
+
expired, so auto re-login is not triggered during an outage.
|
|
187
|
+
"""
|
|
188
|
+
jar = _load_cookies(config)
|
|
189
|
+
if not jar:
|
|
190
|
+
return "none"
|
|
191
|
+
base = _get_webvpn_base(config)
|
|
192
|
+
if not base:
|
|
193
|
+
return "none"
|
|
194
|
+
from ..network import USER_AGENT
|
|
195
|
+
# Probe the gateway root, not a proxied publisher: an expired session is
|
|
196
|
+
# redirected to the gateway's /login (then the school SSO) either way, but a
|
|
197
|
+
# proxied site adds its own latency/failures and WebVPN remembers it as the
|
|
198
|
+
# post-login target (users landed on nature.com after logging in).
|
|
199
|
+
test_url = base.rstrip("/") + "/"
|
|
200
|
+
try:
|
|
201
|
+
s = requests.Session()
|
|
202
|
+
s.trust_env = False
|
|
203
|
+
s.cookies.update(jar)
|
|
204
|
+
resp = s.get(test_url, timeout=15, allow_redirects=True,
|
|
205
|
+
headers={"User-Agent": USER_AGENT})
|
|
206
|
+
if "cas" in resp.url.lower() or "login" in resp.url.lower():
|
|
207
|
+
return "expired"
|
|
208
|
+
return "valid" if resp.status_code == 200 else "unreachable"
|
|
209
|
+
except Exception:
|
|
210
|
+
return "unreachable"
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _validate_session(config: dict[str, Any]) -> bool:
|
|
214
|
+
"""Check if saved cookies still work."""
|
|
215
|
+
return session_status(config) == "valid"
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def instsci_login(config: dict[str, Any]) -> bool:
|
|
219
|
+
"""Open browser for CAS login. Called from MCP tool, not interactively."""
|
|
220
|
+
return _browser_login(config)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _browser_login(config: dict[str, Any]) -> bool:
|
|
224
|
+
"""Open browser for CAS login via CloakBrowser."""
|
|
225
|
+
try:
|
|
226
|
+
from ..browser.login import webvpn_login
|
|
227
|
+
if webvpn_login(config):
|
|
228
|
+
return True
|
|
229
|
+
except Exception as exc:
|
|
230
|
+
log.info(f" [WebVPN] stealth browser login failed: {exc}")
|
|
231
|
+
return False
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _get_socks5_proxy(config: dict[str, Any]) -> str:
|
|
235
|
+
"""Get SOCKS5 proxy URL from config (for EasyConnect/aTrust campus connectors)."""
|
|
236
|
+
proxy = config.get("network_proxy", "").strip()
|
|
237
|
+
if proxy and proxy.lower().startswith("socks5://"):
|
|
238
|
+
return proxy
|
|
239
|
+
return ""
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _is_campus_connector_mode(config: dict[str, Any]) -> bool:
|
|
243
|
+
"""Check if we should use direct access via SOCKS5 campus connector instead of WebVPN."""
|
|
244
|
+
proxy = _get_socks5_proxy(config)
|
|
245
|
+
if not proxy:
|
|
246
|
+
return False
|
|
247
|
+
# If school type is easyconnect or atrust, use direct access via SOCKS5
|
|
248
|
+
school = _cfg(config, "school", "")
|
|
249
|
+
if school:
|
|
250
|
+
try:
|
|
251
|
+
from ..institutional.schools import get_school
|
|
252
|
+
entry = get_school(school)
|
|
253
|
+
if entry.school_type in ("easyconnect", "atrust"):
|
|
254
|
+
return True
|
|
255
|
+
except (ValueError, AttributeError):
|
|
256
|
+
pass
|
|
257
|
+
return False
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _fetch_via_webvpn(url: str, config: dict[str, Any], *, stream: bool = False) -> requests.Response:
|
|
261
|
+
from ..network import USER_AGENT, request_timeout
|
|
262
|
+
base = _get_webvpn_base(config)
|
|
263
|
+
proxied = convert_url(url, base, config)
|
|
264
|
+
|
|
265
|
+
s = requests.Session()
|
|
266
|
+
s.trust_env = False
|
|
267
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
268
|
+
s.cookies.update(_load_cookies(config))
|
|
269
|
+
|
|
270
|
+
# Use SOCKS5 proxy if configured (for campus connectors like EasyConnect/aTrust)
|
|
271
|
+
socks5 = _get_socks5_proxy(config)
|
|
272
|
+
if socks5:
|
|
273
|
+
s.proxies = {"http": socks5, "https": socks5}
|
|
274
|
+
s.verify = False # Campus connectors often use self-signed certs
|
|
275
|
+
|
|
276
|
+
return s.get(proxied, timeout=request_timeout(config), allow_redirects=True, stream=stream)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _fetch_direct_via_socks5(url: str, config: dict[str, Any], *, stream: bool = False) -> requests.Response:
|
|
280
|
+
"""Fetch a URL directly through SOCKS5 campus connector (no WebVPN URL conversion)."""
|
|
281
|
+
from ..network import USER_AGENT, request_timeout
|
|
282
|
+
socks5 = _get_socks5_proxy(config)
|
|
283
|
+
|
|
284
|
+
s = requests.Session()
|
|
285
|
+
s.trust_env = False
|
|
286
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
287
|
+
s.proxies = {"http": socks5, "https": socks5}
|
|
288
|
+
s.verify = False
|
|
289
|
+
|
|
290
|
+
return s.get(url, timeout=request_timeout(config), allow_redirects=True, stream=stream)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _resolve_doi_url(doi: str) -> str | None:
|
|
294
|
+
"""Resolve DOI to get the publisher URL."""
|
|
295
|
+
try:
|
|
296
|
+
from ..config import load_config
|
|
297
|
+
from ..network import apply_host_proxy
|
|
298
|
+
session = requests.Session()
|
|
299
|
+
session.trust_env = False
|
|
300
|
+
apply_host_proxy(session, load_config())
|
|
301
|
+
resp = session.get(
|
|
302
|
+
f"https://doi.org/{doi}",
|
|
303
|
+
allow_redirects=True,
|
|
304
|
+
timeout=15,
|
|
305
|
+
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"},
|
|
306
|
+
stream=True,
|
|
307
|
+
verify=False,
|
|
308
|
+
)
|
|
309
|
+
resp.close()
|
|
310
|
+
if resp.url and resp.url != f"https://doi.org/{doi}":
|
|
311
|
+
return resp.url
|
|
312
|
+
except Exception:
|
|
313
|
+
pass
|
|
314
|
+
return None
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
# taylorfrancis.com itself, or its WebVPN-rewritten form (/https/<hex>/books/...)
|
|
318
|
+
_TF_BOOK = re.compile(r"(?:taylorfrancis\.com|/https?/[0-9a-f]{16,})/books/(mono|edit)/10\.", re.I)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _taylorfrancis_book_pdf_url(doi: str, resolved_url: str) -> str | None:
|
|
322
|
+
"""Whole Taylor & Francis / CRC / Routledge books: the PDF comes from one
|
|
323
|
+
download endpoint. Its link is on authored books' pages but not on edited
|
|
324
|
+
ones (chapter PDFs only), though the endpoint serves both (checked through
|
|
325
|
+
WebVPN 2026-10-17). Single chapters are not covered: the same endpoint with
|
|
326
|
+
chapters/ returned a page, not the PDF."""
|
|
327
|
+
m = _TF_BOOK.search(resolved_url or "")
|
|
328
|
+
if not m:
|
|
329
|
+
return None
|
|
330
|
+
return (f"https://api.taylorfrancis.com/content/books/{m.group(1).lower()}/download?identifierName=doi"
|
|
331
|
+
f"&identifierValue={doi}&type=googlepdf")
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
_TF_CHAPTER = re.compile(r"(?:taylorfrancis\.com|/https?/[0-9a-f]{16,})/chapters/(mono|edit)/10\.", re.I)
|
|
335
|
+
_TF_CHAPTER_DOI = re.compile(r"^(10\.\d{4,9}/\d{10,13})-\d+$")
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _taylorfrancis_chapter_book(doi: str, landing_url: str) -> tuple[str, str] | None:
|
|
339
|
+
"""A Taylor & Francis chapter (DOI "<book DOI>-<n>"): (book DOI, mono|edit).
|
|
340
|
+
|
|
341
|
+
T&F serves no PDF of a single chapter, only of the whole book (checked
|
|
342
|
+
through WebVPN 2026-10-17), so a chapter is downloaded as its book.
|
|
343
|
+
"""
|
|
344
|
+
m = _TF_CHAPTER.search(landing_url or "")
|
|
345
|
+
b = _TF_CHAPTER_DOI.match((doi or "").strip())
|
|
346
|
+
return (b.group(1), m.group(1).lower()) if m and b else None
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _chapter_note(doi: str, config: dict[str, Any]) -> str:
|
|
350
|
+
"""Where the chapter is in the book, for the user: from Crossref's page range."""
|
|
351
|
+
try:
|
|
352
|
+
from ..citations.formats import fetch_metadata
|
|
353
|
+
meta = fetch_metadata(doi, config) or {}
|
|
354
|
+
page = str(meta.get("page") or "").strip()
|
|
355
|
+
except Exception:
|
|
356
|
+
page = ""
|
|
357
|
+
where = f",本章在第 {page} 页" if page else ""
|
|
358
|
+
return f"T&F 只提供整本书的 PDF:下载的是本章所在的整本书{where}"
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _construct_publisher_pdf_url(doi: str, resolved_url: str) -> str | None:
|
|
362
|
+
"""Try to construct a direct publisher PDF URL from the resolved URL."""
|
|
363
|
+
tf = _taylorfrancis_book_pdf_url(doi, resolved_url)
|
|
364
|
+
if tf:
|
|
365
|
+
return tf
|
|
366
|
+
|
|
367
|
+
parsed = urllib.parse.urlparse(resolved_url)
|
|
368
|
+
hostname = parsed.netloc.lower()
|
|
369
|
+
doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
|
|
370
|
+
|
|
371
|
+
if "pubs.acs.org" in hostname:
|
|
372
|
+
return f"https://pubs.acs.org/doi/pdf/{doi}"
|
|
373
|
+
elif "onlinelibrary.wiley.com" in hostname:
|
|
374
|
+
return f"https://onlinelibrary.wiley.com/doi/pdfdirect/{doi}"
|
|
375
|
+
elif "tandfonline.com" in hostname:
|
|
376
|
+
return f"https://www.tandfonline.com/doi/pdf/{doi}?needAccess=true"
|
|
377
|
+
elif "nature.com" in hostname:
|
|
378
|
+
return f"https://www.nature.com/articles/{doi_suffix}.pdf"
|
|
379
|
+
elif "link.springer.com" in hostname:
|
|
380
|
+
return f"https://link.springer.com/content/pdf/{doi}.pdf"
|
|
381
|
+
elif "pubs.rsc.org" in hostname:
|
|
382
|
+
pdf_url = resolved_url.replace("/articlelanding/", "/articlepdf/")
|
|
383
|
+
return pdf_url if pdf_url != resolved_url else None
|
|
384
|
+
elif "pnas.org" in hostname:
|
|
385
|
+
return f"https://www.pnas.org/doi/pdf/{doi}"
|
|
386
|
+
elif "science.org" in hostname or "sciencemag.org" in hostname:
|
|
387
|
+
return f"https://www.science.org/doi/pdf/{doi}"
|
|
388
|
+
elif "elsevier.com" in hostname or "sciencedirect.com" in hostname:
|
|
389
|
+
pii_match = re.search(r"pii/([A-Z0-9]+)", resolved_url)
|
|
390
|
+
if pii_match:
|
|
391
|
+
return f"https://www.sciencedirect.com/science/article/pii/{pii_match.group(1)}/pdfft"
|
|
392
|
+
|
|
393
|
+
return None
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _find_pdf_link(html: str, base_url: str) -> str | None:
|
|
397
|
+
"""Find a PDF download link in an HTML page.
|
|
398
|
+
|
|
399
|
+
Tries: citation_pdf_url meta, <a> tags with PDF text/class,
|
|
400
|
+
and publisher-specific URL patterns.
|
|
401
|
+
"""
|
|
402
|
+
try:
|
|
403
|
+
soup = BeautifulSoup(html, "lxml")
|
|
404
|
+
except Exception:
|
|
405
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
406
|
+
parsed = urllib.parse.urlparse(base_url)
|
|
407
|
+
base = f"{parsed.scheme}://{parsed.netloc}"
|
|
408
|
+
hostname = parsed.netloc.lower()
|
|
409
|
+
|
|
410
|
+
# Strategy 1: <meta name="citation_pdf_url">
|
|
411
|
+
meta_pdf = soup.find("meta", attrs={"name": "citation_pdf_url"})
|
|
412
|
+
if meta_pdf and meta_pdf.get("content"):
|
|
413
|
+
pdf_url = meta_pdf["content"]
|
|
414
|
+
if pdf_url.startswith("http"):
|
|
415
|
+
return pdf_url
|
|
416
|
+
return base + pdf_url
|
|
417
|
+
|
|
418
|
+
# Strategy 2: <a> tags with PDF-related text/class/href
|
|
419
|
+
for a in soup.find_all("a", href=True):
|
|
420
|
+
href = a["href"]
|
|
421
|
+
text = a.get_text(strip=True).lower()
|
|
422
|
+
classes = " ".join(a.get("class", []))
|
|
423
|
+
|
|
424
|
+
if any(kw in text for kw in ["pdf", "download pdf", "full text pdf", "view pdf", "get pdf"]):
|
|
425
|
+
return _resolve_href(href, base)
|
|
426
|
+
if any(kw in classes for kw in ["pdf", "download-pdf", "pdf-download", "article-pdf"]):
|
|
427
|
+
return _resolve_href(href, base)
|
|
428
|
+
if href.endswith(".pdf"):
|
|
429
|
+
return _resolve_href(href, base)
|
|
430
|
+
if "/doi/pdf/" in href or "/doi/pdfdirect/" in href:
|
|
431
|
+
return _resolve_href(href, base)
|
|
432
|
+
|
|
433
|
+
# Strategy 3: Publisher-specific URL patterns
|
|
434
|
+
path = parsed.path
|
|
435
|
+
if "pubs.acs.org" in hostname and "/doi/" in path and "/pdf/" not in path:
|
|
436
|
+
doi_part = path.split("/doi/")[-1]
|
|
437
|
+
if doi_part:
|
|
438
|
+
return f"{base}/doi/pdf/{doi_part}"
|
|
439
|
+
|
|
440
|
+
if "onlinelibrary.wiley.com" in hostname and "/doi/" in path and "/pdfdirect/" not in path:
|
|
441
|
+
doi_part = path.split("/doi/")[-1]
|
|
442
|
+
if doi_part:
|
|
443
|
+
return f"{base}/doi/pdfdirect/{doi_part}"
|
|
444
|
+
|
|
445
|
+
if "pubs.rsc.org" in hostname and "/articlelanding/" in path:
|
|
446
|
+
return base_url.replace("/articlelanding/", "/articlepdf/")
|
|
447
|
+
|
|
448
|
+
if "tandfonline.com" in hostname and "/doi/" in path and "/pdf/" not in path:
|
|
449
|
+
doi_part = re.sub(r"/doi/(?:full|abs)/", "/doi/pdf/", path)
|
|
450
|
+
if doi_part != path:
|
|
451
|
+
return f"{base}{doi_part}"
|
|
452
|
+
|
|
453
|
+
if "pnas.org" in hostname and "/doi/" in path and "/pdf/" not in path:
|
|
454
|
+
doi_part = path.split("/doi/")[-1]
|
|
455
|
+
if doi_part:
|
|
456
|
+
return f"{base}/doi/pdf/{doi_part}"
|
|
457
|
+
|
|
458
|
+
if "science.org" in hostname and "/doi/" in path and "/pdf/" not in path:
|
|
459
|
+
doi_part = path.split("/doi/")[-1]
|
|
460
|
+
if doi_part:
|
|
461
|
+
return f"{base}/doi/pdf/{doi_part}"
|
|
462
|
+
|
|
463
|
+
if ("elsevier.com" in hostname or "sciencedirect.com" in hostname):
|
|
464
|
+
pii_match = re.search(r"pii/([A-Z0-9]+)", path)
|
|
465
|
+
if pii_match:
|
|
466
|
+
return f"https://www.sciencedirect.com/science/article/pii/{pii_match.group(1)}/pdfft"
|
|
467
|
+
|
|
468
|
+
return None
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _resolve_href(href: str, base: str) -> str:
|
|
472
|
+
if href.startswith("http"):
|
|
473
|
+
return href
|
|
474
|
+
if href.startswith("//"):
|
|
475
|
+
return "https:" + href
|
|
476
|
+
if href.startswith("/"):
|
|
477
|
+
return base + href
|
|
478
|
+
return base + "/" + href
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _is_inline_pdf_page(page: Any) -> bool:
|
|
482
|
+
"""Check if the page is displaying an inline PDF."""
|
|
483
|
+
try:
|
|
484
|
+
url = page.url.lower()
|
|
485
|
+
if url.endswith(".pdf"):
|
|
486
|
+
return True
|
|
487
|
+
# Check for PDF embed/object
|
|
488
|
+
has_embed = page.evaluate("""
|
|
489
|
+
(() => {
|
|
490
|
+
const e = document.querySelector('embed[type="application/pdf"], object[type="application/pdf"], iframe[src*=".pdf"]');
|
|
491
|
+
return !!e;
|
|
492
|
+
})()
|
|
493
|
+
""")
|
|
494
|
+
if has_embed:
|
|
495
|
+
return True
|
|
496
|
+
# Check if page content starts with %PDF
|
|
497
|
+
content = page.evaluate("document.contentType || ''")
|
|
498
|
+
if "pdf" in content.lower():
|
|
499
|
+
return True
|
|
500
|
+
except Exception:
|
|
501
|
+
pass
|
|
502
|
+
return False
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _extract_inline_pdf(page: Any) -> bytes | None:
|
|
506
|
+
"""Extract PDF bytes from an inline PDF page via JS fetch."""
|
|
507
|
+
try:
|
|
508
|
+
result = page.evaluate("""
|
|
509
|
+
(() => {
|
|
510
|
+
// Try embed/object src first
|
|
511
|
+
const embed = document.querySelector('embed[type="application/pdf"], object[type="application/pdf"]');
|
|
512
|
+
if (embed) {
|
|
513
|
+
const src = embed.src || embed.data;
|
|
514
|
+
if (src) return src;
|
|
515
|
+
}
|
|
516
|
+
// Try iframe
|
|
517
|
+
const iframe = document.querySelector('iframe[src*=".pdf"]');
|
|
518
|
+
if (iframe) return iframe.src;
|
|
519
|
+
// Use current URL
|
|
520
|
+
return window.location.href;
|
|
521
|
+
})()
|
|
522
|
+
""")
|
|
523
|
+
if not result:
|
|
524
|
+
return None
|
|
525
|
+
|
|
526
|
+
# Fetch the PDF bytes using JS fetch in the page context
|
|
527
|
+
pdf_bytes = page.evaluate(f"""
|
|
528
|
+
(async () => {{
|
|
529
|
+
try {{
|
|
530
|
+
const resp = await fetch({json.dumps(result)});
|
|
531
|
+
if (!resp.ok) return null;
|
|
532
|
+
const buf = await resp.arrayBuffer();
|
|
533
|
+
const bytes = new Uint8Array(buf);
|
|
534
|
+
// Check PDF magic
|
|
535
|
+
if (bytes[0] !== 0x25 || bytes[1] !== 0x50 || bytes[2] !== 0x44 || bytes[3] !== 0x46) return null;
|
|
536
|
+
if (bytes.length < 5000) return null;
|
|
537
|
+
// Convert to base64
|
|
538
|
+
let binary = '';
|
|
539
|
+
for (let i = 0; i < bytes.length; i++) {{
|
|
540
|
+
binary += String.fromCharCode(bytes[i]);
|
|
541
|
+
}}
|
|
542
|
+
return btoa(binary);
|
|
543
|
+
}} catch(e) {{
|
|
544
|
+
return null;
|
|
545
|
+
}}
|
|
546
|
+
}})()
|
|
547
|
+
""")
|
|
548
|
+
if pdf_bytes:
|
|
549
|
+
import base64
|
|
550
|
+
return base64.b64decode(pdf_bytes)
|
|
551
|
+
except Exception:
|
|
552
|
+
pass
|
|
553
|
+
return None
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _download_pdf_with_browser_cookies(
|
|
557
|
+
pdf_url: str, output_path: Path, config: dict[str, Any], doi: str, context: Any
|
|
558
|
+
) -> dict[str, Any] | None:
|
|
559
|
+
"""Download PDF via WebVPN using cookies from a live browser context.
|
|
560
|
+
|
|
561
|
+
Bypasses stale cookie files by pulling cookies directly from the
|
|
562
|
+
Playwright browser context that was just used for login/page-navigation.
|
|
563
|
+
"""
|
|
564
|
+
if not is_plausible_pdf_url(pdf_url):
|
|
565
|
+
return None
|
|
566
|
+
try:
|
|
567
|
+
from ..network import USER_AGENT, request_timeout
|
|
568
|
+
|
|
569
|
+
base = _get_webvpn_base(config)
|
|
570
|
+
if not base:
|
|
571
|
+
return None
|
|
572
|
+
proxied = convert_url(pdf_url, base, config)
|
|
573
|
+
|
|
574
|
+
s = requests.Session()
|
|
575
|
+
s.trust_env = False
|
|
576
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
577
|
+
for c in context.cookies():
|
|
578
|
+
name = c.get("name")
|
|
579
|
+
value = c.get("value")
|
|
580
|
+
if name and value is not None:
|
|
581
|
+
s.cookies.set(name, value, domain=c.get("domain", ""), path=c.get("path", "/"))
|
|
582
|
+
|
|
583
|
+
resp = s.get(proxied, timeout=request_timeout(config), allow_redirects=True, stream=True)
|
|
584
|
+
if resp.status_code >= 400:
|
|
585
|
+
log.info(f" [WebVPN-Browser] Browser-cookie HTTP status={resp.status_code} for PDF URL")
|
|
586
|
+
return None
|
|
587
|
+
|
|
588
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
589
|
+
first = next(iterator, b"")
|
|
590
|
+
if not _response_looks_pdf(resp, first):
|
|
591
|
+
return None
|
|
592
|
+
|
|
593
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
594
|
+
tmp = output_path.with_suffix(output_path.suffix + ".part")
|
|
595
|
+
try:
|
|
596
|
+
with tmp.open("wb") as fh:
|
|
597
|
+
fh.write(first)
|
|
598
|
+
for chunk in iterator:
|
|
599
|
+
if chunk:
|
|
600
|
+
fh.write(chunk)
|
|
601
|
+
tmp.replace(output_path)
|
|
602
|
+
except Exception:
|
|
603
|
+
tmp.unlink(missing_ok=True)
|
|
604
|
+
raise
|
|
605
|
+
|
|
606
|
+
if is_pdf_file(output_path):
|
|
607
|
+
log.info(f" [WebVPN-Browser] PDF downloaded via browser-cookie HTTP")
|
|
608
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
609
|
+
except Exception as e:
|
|
610
|
+
log.info(f" [WebVPN-Browser] Browser-cookie HTTP error: {e}")
|
|
611
|
+
return None
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
def _try_instsci_socks5(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
615
|
+
"""Try downloading via SOCKS5 campus connector (EasyConnect/aTrust).
|
|
616
|
+
|
|
617
|
+
The SOCKS5 connector handles authentication at the network level,
|
|
618
|
+
so no WebVPN URL conversion or CAS login is needed — just fetch
|
|
619
|
+
the publisher URL directly through the proxy.
|
|
620
|
+
"""
|
|
621
|
+
_instsci_rate_limit()
|
|
622
|
+
socks5 = _get_socks5_proxy(config)
|
|
623
|
+
log.info(f" [CampusConnector] Trying {doi} via {socks5}")
|
|
624
|
+
|
|
625
|
+
# Step 1: Resolve DOI to publisher URL
|
|
626
|
+
resolved_url = _resolve_doi_url(doi)
|
|
627
|
+
if not resolved_url:
|
|
628
|
+
resolved_url = f"https://doi.org/{doi}"
|
|
629
|
+
|
|
630
|
+
# Step 2: Try direct publisher PDF URL
|
|
631
|
+
pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
|
|
632
|
+
if pdf_url:
|
|
633
|
+
log.info(f" [CampusConnector] Trying publisher PDF: {pdf_url[:80]}")
|
|
634
|
+
result = _download_pdf_socks5(pdf_url, output_path, config, doi)
|
|
635
|
+
if result:
|
|
636
|
+
return result
|
|
637
|
+
|
|
638
|
+
# Step 3: Fetch landing page via SOCKS5 and find PDF link
|
|
639
|
+
try:
|
|
640
|
+
resp = _fetch_direct_via_socks5(resolved_url, config, stream=True)
|
|
641
|
+
if resp.status_code >= 400:
|
|
642
|
+
log.info(f" [CampusConnector] HTTP {resp.status_code} for {resolved_url[:60]}")
|
|
643
|
+
return None
|
|
644
|
+
|
|
645
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
646
|
+
first = next(iterator, b"")
|
|
647
|
+
|
|
648
|
+
# Direct PDF response
|
|
649
|
+
if _response_looks_pdf(resp, first):
|
|
650
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
651
|
+
tmp_path = output_path.with_suffix(output_path.suffix + ".part")
|
|
652
|
+
try:
|
|
653
|
+
with tmp_path.open("wb") as fh:
|
|
654
|
+
fh.write(first)
|
|
655
|
+
for chunk in iterator:
|
|
656
|
+
if chunk:
|
|
657
|
+
fh.write(chunk)
|
|
658
|
+
tmp_path.replace(output_path)
|
|
659
|
+
except Exception:
|
|
660
|
+
tmp_path.unlink(missing_ok=True)
|
|
661
|
+
raise
|
|
662
|
+
if is_pdf_file(output_path):
|
|
663
|
+
log.info(" [CampusConnector] PDF downloaded directly")
|
|
664
|
+
return success(doi, output_path, "CampusConnector")
|
|
665
|
+
|
|
666
|
+
# HTML response - look for PDF link
|
|
667
|
+
html = first + resp.raw.read(512_000, decode_content=True)
|
|
668
|
+
html_str = html.decode("utf-8", errors="ignore")
|
|
669
|
+
|
|
670
|
+
found_pdf = _find_pdf_link(html_str, resp.url)
|
|
671
|
+
if found_pdf:
|
|
672
|
+
log.info(f" [CampusConnector] Found PDF link: {found_pdf[:80]}")
|
|
673
|
+
result = _download_pdf_socks5(found_pdf, output_path, config, doi)
|
|
674
|
+
if result:
|
|
675
|
+
return result
|
|
676
|
+
|
|
677
|
+
pdf_url = extract_pdf_url_from_html(html_str, resp.url)
|
|
678
|
+
if pdf_url:
|
|
679
|
+
result = _download_pdf_socks5(pdf_url, output_path, config, doi)
|
|
680
|
+
if result:
|
|
681
|
+
return result
|
|
682
|
+
|
|
683
|
+
except Exception as e:
|
|
684
|
+
log.info(f" [CampusConnector] {e}")
|
|
685
|
+
|
|
686
|
+
return None
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _download_pdf_socks5(
|
|
690
|
+
url: str,
|
|
691
|
+
output_path: Path,
|
|
692
|
+
config: dict[str, Any],
|
|
693
|
+
doi: str,
|
|
694
|
+
) -> dict[str, Any] | None:
|
|
695
|
+
"""Download a PDF URL directly through SOCKS5 campus connector."""
|
|
696
|
+
if not is_plausible_pdf_url(url):
|
|
697
|
+
return None
|
|
698
|
+
try:
|
|
699
|
+
_instsci_rate_limit()
|
|
700
|
+
resp = _fetch_direct_via_socks5(url, config, stream=True)
|
|
701
|
+
if resp.status_code >= 400:
|
|
702
|
+
return None
|
|
703
|
+
|
|
704
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
705
|
+
first_chunk = next(iterator, b"")
|
|
706
|
+
if not _response_looks_pdf(resp, first_chunk):
|
|
707
|
+
return None
|
|
708
|
+
|
|
709
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
710
|
+
tmp_path = output_path.with_suffix(output_path.suffix + ".part")
|
|
711
|
+
try:
|
|
712
|
+
with tmp_path.open("wb") as fh:
|
|
713
|
+
fh.write(first_chunk)
|
|
714
|
+
for chunk in iterator:
|
|
715
|
+
if chunk:
|
|
716
|
+
fh.write(chunk)
|
|
717
|
+
tmp_path.replace(output_path)
|
|
718
|
+
except Exception:
|
|
719
|
+
tmp_path.unlink(missing_ok=True)
|
|
720
|
+
raise
|
|
721
|
+
|
|
722
|
+
if is_pdf_file(output_path):
|
|
723
|
+
log.info(f" [CampusConnector] PDF downloaded: {doi}")
|
|
724
|
+
return success(doi, output_path, "CampusConnector")
|
|
725
|
+
except Exception as e:
|
|
726
|
+
log.info(f" [CampusConnector] Download error: {e}")
|
|
727
|
+
return None
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
def _try_instsci_browser(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
731
|
+
"""Download via visible stealth browser browser. Login + download in same session."""
|
|
732
|
+
try:
|
|
733
|
+
from ..browser.backend import launch
|
|
734
|
+
from ..browser.backend import is_available as _browser_backend_available
|
|
735
|
+
if not _browser_backend_available():
|
|
736
|
+
log.info(" [WebVPN-Browser] no browser backend available")
|
|
737
|
+
return None
|
|
738
|
+
except ImportError:
|
|
739
|
+
log.info(" [WebVPN-Browser] no browser backend available")
|
|
740
|
+
return None
|
|
741
|
+
|
|
742
|
+
base = _get_webvpn_base(config)
|
|
743
|
+
if not base:
|
|
744
|
+
return None
|
|
745
|
+
|
|
746
|
+
resolved_url = _resolve_doi_url(doi)
|
|
747
|
+
if not resolved_url:
|
|
748
|
+
resolved_url = f"https://doi.org/{doi}"
|
|
749
|
+
|
|
750
|
+
webvpn_url = convert_url(resolved_url, base, config)
|
|
751
|
+
log.info(f" [WebVPN-Browser] Target: {webvpn_url[:80]}")
|
|
752
|
+
print(f"\n [WebVPN] 正在打开浏览器,请在浏览器中登录 WebVPN...")
|
|
753
|
+
print(f" 登录完成后等待 5 秒,程序会自动继续下载。\n")
|
|
754
|
+
|
|
755
|
+
try:
|
|
756
|
+
browser = launch(headless=False, humanize=True,
|
|
757
|
+
args=["--disable-features=CrossOriginOpenerPolicy"])
|
|
758
|
+
context = browser.new_context()
|
|
759
|
+
page = context.new_page()
|
|
760
|
+
|
|
761
|
+
# Restore saved cookies before navigating
|
|
762
|
+
cookie_path = instsci_cookie_path(config)
|
|
763
|
+
if cookie_path.exists():
|
|
764
|
+
try:
|
|
765
|
+
saved_cookies = json.loads(cookie_path.read_text(encoding="utf-8"))
|
|
766
|
+
if saved_cookies:
|
|
767
|
+
# Convert to Playwright cookie format
|
|
768
|
+
pw_cookies = []
|
|
769
|
+
for c in saved_cookies:
|
|
770
|
+
pw_c = {
|
|
771
|
+
"name": c["name"],
|
|
772
|
+
"value": c["value"],
|
|
773
|
+
"domain": c.get("domain", ""),
|
|
774
|
+
"path": c.get("path", "/"),
|
|
775
|
+
}
|
|
776
|
+
if c.get("secure"):
|
|
777
|
+
pw_c["secure"] = True
|
|
778
|
+
if c.get("httpOnly"):
|
|
779
|
+
pw_c["httpOnly"] = True
|
|
780
|
+
if pw_c["domain"]:
|
|
781
|
+
pw_cookies.append(pw_c)
|
|
782
|
+
if pw_cookies:
|
|
783
|
+
context.add_cookies(pw_cookies)
|
|
784
|
+
log.info(f" [WebVPN-Browser] Restored {len(pw_cookies)} cookies")
|
|
785
|
+
except Exception:
|
|
786
|
+
pass
|
|
787
|
+
|
|
788
|
+
# Capture PDF from network responses
|
|
789
|
+
captured_pdf = []
|
|
790
|
+
|
|
791
|
+
def on_response(response):
|
|
792
|
+
try:
|
|
793
|
+
ct = response.headers.get("content-type", "")
|
|
794
|
+
url = response.url
|
|
795
|
+
# Capture PDF responses (any content type that's actually a PDF)
|
|
796
|
+
is_pdf_ct = "pdf" in ct.lower() or "octet-stream" in ct.lower()
|
|
797
|
+
is_pdf_url = url.lower().endswith(".pdf") or "/pdfdirect/" in url or "/doi/pdf/" in url
|
|
798
|
+
if not (is_pdf_ct or is_pdf_url):
|
|
799
|
+
return
|
|
800
|
+
if response.status >= 400:
|
|
801
|
+
return
|
|
802
|
+
body = response.body()
|
|
803
|
+
if len(body) > 5000 and body[:4] == b"%PDF-":
|
|
804
|
+
captured_pdf.append(body)
|
|
805
|
+
log.info(f" [WebVPN-Browser] PDF captured: {len(body)} bytes from {url[:60]}")
|
|
806
|
+
except Exception:
|
|
807
|
+
pass
|
|
808
|
+
|
|
809
|
+
page.on("response", on_response)
|
|
810
|
+
|
|
811
|
+
# Navigate to paper URL directly via WebVPN
|
|
812
|
+
# If not logged in, will redirect to login page
|
|
813
|
+
try:
|
|
814
|
+
page.goto(webvpn_url, wait_until="domcontentloaded", timeout=60000)
|
|
815
|
+
except Exception:
|
|
816
|
+
pass
|
|
817
|
+
# WebVPN first shows a "Loading https://..." interstitial, then redirects
|
|
818
|
+
# (possibly to the school's SSO); judge login state only once it settles.
|
|
819
|
+
title, url_now = wait_page_settled(page)
|
|
820
|
+
|
|
821
|
+
# If on login page, wait for user to login then retry
|
|
822
|
+
from ..publishers.strategies import _school_auth_patterns
|
|
823
|
+
_stoks = _school_auth_patterns(config)
|
|
824
|
+
_auth_url_signals = list(_stoks) + ["/do/off/ui/auth"]
|
|
825
|
+
|
|
826
|
+
log.info(f" [WebVPN-Browser] Page title: '{title}' URL: {url_now[:80]}")
|
|
827
|
+
# The publisher sent the browser out of WebVPN to its own institution
|
|
828
|
+
# login (2026-10-17: ScienceDirect via WebVPN -> auth.elsevier.com ->
|
|
829
|
+
# the school's IdP -> the ScienceDirect home page, over the direct line,
|
|
830
|
+
# never back to the article). That is the publisher-site route, not
|
|
831
|
+
# WebVPN; it is tried in phase 1.
|
|
832
|
+
_school = (urllib.parse.urlparse(base).hostname or "").split(".", 1)[-1] # d.buaa.edu.cn -> buaa.edu.cn
|
|
833
|
+
_host_now = urllib.parse.urlparse(url_now).hostname or ""
|
|
834
|
+
if _host_now and _school and not _host_now.endswith(_school) and not any(
|
|
835
|
+
t in url_now for t in _auth_url_signals if t):
|
|
836
|
+
log.info(f" [WebVPN-Browser] Left WebVPN for {_host_now}; giving up this route")
|
|
837
|
+
return None
|
|
838
|
+
if "登录" in title or "身份" in title or "二次认证" in title or "CAS" in title or any(t in url_now for t in _auth_url_signals):
|
|
839
|
+
print(f" 检测到登录页面,请完成登录...")
|
|
840
|
+
# Wait up to 5 minutes, checking title every 3 seconds
|
|
841
|
+
from . import note_user_waiting
|
|
842
|
+
for i in range(100):
|
|
843
|
+
note_user_waiting("login") # the race waits, the app asks the user; stops if the paper is done
|
|
844
|
+
try:
|
|
845
|
+
time.sleep(3) # not page.wait_for_timeout: it hangs on a page stuck loading
|
|
846
|
+
# Post-login SAML/redirect hops also pass through interstitials
|
|
847
|
+
# without auth keywords; wait for them before declaring success.
|
|
848
|
+
title, url_now = wait_page_settled(page, max_wait_s=15)
|
|
849
|
+
except Exception:
|
|
850
|
+
return None
|
|
851
|
+
if i % 10 == 0:
|
|
852
|
+
log.info(f" [WebVPN-Browser] Waiting... title='{title}' url={url_now[:60]}")
|
|
853
|
+
# Detect login success: no longer on auth pages
|
|
854
|
+
is_auth = "登录" in title or "身份" in title or "二次认证" in title or "CAS" in title
|
|
855
|
+
is_auth_url = any(t in url_now for t in _auth_url_signals)
|
|
856
|
+
if not is_auth and not is_auth_url:
|
|
857
|
+
print(f" 登录成功!正在保存 cookies...")
|
|
858
|
+
# Save cookies immediately after login
|
|
859
|
+
try:
|
|
860
|
+
cookies = context.cookies()
|
|
861
|
+
from ..config import DATA_DIR
|
|
862
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
863
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
864
|
+
cookie_data = [
|
|
865
|
+
{"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
|
|
866
|
+
for c in cookies
|
|
867
|
+
]
|
|
868
|
+
from ..institutional.session_store import keep_private
|
|
869
|
+
(cache_dir / "instsci-cookies.json").write_text(
|
|
870
|
+
json.dumps(cookie_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
871
|
+
keep_private(cache_dir / "instsci-cookies.json")
|
|
872
|
+
lines = ["# Netscape HTTP Cookie File\n"]
|
|
873
|
+
for c in cookies:
|
|
874
|
+
d = c.get("domain", "")
|
|
875
|
+
flag = "TRUE" if d.startswith(".") else "FALSE"
|
|
876
|
+
p = c.get("path", "/")
|
|
877
|
+
sec = "TRUE" if c.get("secure") else "FALSE"
|
|
878
|
+
exp = str(int(c.get("expires", 0)))
|
|
879
|
+
lines.append(f"{d}\t{flag}\t{p}\t{sec}\t{exp}\t{c['name']}\t{c['value']}\n")
|
|
880
|
+
(cache_dir / "instsci-cookies.txt").write_text("".join(lines), encoding="utf-8")
|
|
881
|
+
keep_private(cache_dir / "instsci-cookies.txt")
|
|
882
|
+
log.info(f" [WebVPN-Browser] Saved {len(cookies)} cookies")
|
|
883
|
+
except Exception as e:
|
|
884
|
+
log.info(f" [WebVPN-Browser] Cookie save warning: {e}")
|
|
885
|
+
break
|
|
886
|
+
else:
|
|
887
|
+
print(" 登录超时。")
|
|
888
|
+
return None
|
|
889
|
+
|
|
890
|
+
# Try HTTP download with fresh browser cookies (bypasses JS-heavy pages)
|
|
891
|
+
pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
|
|
892
|
+
if pdf_url:
|
|
893
|
+
time.sleep(2)
|
|
894
|
+
result = _download_pdf_with_browser_cookies(pdf_url, output_path, config, doi, context)
|
|
895
|
+
if result:
|
|
896
|
+
return result
|
|
897
|
+
|
|
898
|
+
# Fall back to browser-based PDF extraction
|
|
899
|
+
time.sleep(2)
|
|
900
|
+
try:
|
|
901
|
+
page.goto(webvpn_url, wait_until="domcontentloaded", timeout=60000)
|
|
902
|
+
except Exception:
|
|
903
|
+
pass
|
|
904
|
+
for _w in range(20):
|
|
905
|
+
time.sleep(2)
|
|
906
|
+
try:
|
|
907
|
+
_t = (page.title() or "").lower()
|
|
908
|
+
_b = (page.evaluate("document.body?.innerText?.length || 0") or 0)
|
|
909
|
+
if _b > 200 and not any(x in _t for x in ("请稍候", "loading", "please wait", "just a moment")):
|
|
910
|
+
break
|
|
911
|
+
except Exception:
|
|
912
|
+
break
|
|
913
|
+
else:
|
|
914
|
+
# Already authenticated — try HTTP download with browser cookies
|
|
915
|
+
pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
|
|
916
|
+
if pdf_url:
|
|
917
|
+
time.sleep(2)
|
|
918
|
+
result = _download_pdf_with_browser_cookies(pdf_url, output_path, config, doi, context)
|
|
919
|
+
if result:
|
|
920
|
+
return result
|
|
921
|
+
time.sleep(5)
|
|
922
|
+
|
|
923
|
+
# Helper: try to save captured PDF
|
|
924
|
+
def _save_captured():
|
|
925
|
+
if captured_pdf:
|
|
926
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
927
|
+
output_path.write_bytes(captured_pdf[-1])
|
|
928
|
+
if is_pdf_file(output_path):
|
|
929
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
930
|
+
return None
|
|
931
|
+
|
|
932
|
+
def _save_bytes(pdf_bytes: bytes | None) -> dict[str, Any] | None:
|
|
933
|
+
if pdf_bytes:
|
|
934
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
935
|
+
output_path.write_bytes(pdf_bytes)
|
|
936
|
+
if is_pdf_file(output_path):
|
|
937
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
938
|
+
return None
|
|
939
|
+
|
|
940
|
+
# Check if page itself is a PDF (inline viewer)
|
|
941
|
+
page_title, page_url = wait_page_settled(page)
|
|
942
|
+
|
|
943
|
+
# Some publishers are not proxied by the WebVPN: the school routes them to
|
|
944
|
+
# federated SSO (e.g. Elsevier via Shibboleth), and after login the IdP
|
|
945
|
+
# returns to the publisher's *homepage* outside the VPN. The session is then
|
|
946
|
+
# valid on the publisher itself, so open the article there directly.
|
|
947
|
+
base_host = urllib.parse.urlparse(base).netloc
|
|
948
|
+
if urllib.parse.urlparse(page_url).netloc not in ("", base_host):
|
|
949
|
+
direct_url = resolved_url
|
|
950
|
+
_lh = re.search(r"linkinghub\.elsevier\.com/retrieve/pii/([A-Za-z0-9]+)", direct_url)
|
|
951
|
+
if _lh:
|
|
952
|
+
direct_url = f"https://www.sciencedirect.com/science/article/pii/{_lh.group(1)}"
|
|
953
|
+
if urllib.parse.urlparse(direct_url).path.rstrip("/") != urllib.parse.urlparse(page_url).path.rstrip("/"):
|
|
954
|
+
log.info(f" [WebVPN-Browser] Left the VPN for {urllib.parse.urlparse(page_url).netloc}; opening article directly: {direct_url[:80]}")
|
|
955
|
+
try:
|
|
956
|
+
page.goto(direct_url, wait_until="domcontentloaded", timeout=60000)
|
|
957
|
+
except Exception:
|
|
958
|
+
pass
|
|
959
|
+
page_title, page_url = wait_page_settled(page)
|
|
960
|
+
|
|
961
|
+
log.info(f" [WebVPN-Browser] On page: title='{page_title[:40]}' url={page_url[:60]}")
|
|
962
|
+
|
|
963
|
+
# A Cloudflare challenge served *through* the WebVPN cannot pass: the
|
|
964
|
+
# gateway rewrites the challenge's scripts ("browser not supported").
|
|
965
|
+
# Report it as a block so the (WebVPN, publisher) negative cache skips
|
|
966
|
+
# this lane for the publisher's next papers instead of opening another
|
|
967
|
+
# doomed window each time.
|
|
968
|
+
from ..network import is_cloudflare_challenge
|
|
969
|
+
from ..download.pdf import fail
|
|
970
|
+
if is_cloudflare_challenge(page_title) and urllib.parse.urlparse(page_url).netloc == base_host:
|
|
971
|
+
log.info(" [WebVPN-Browser] Cloudflare challenge inside the WebVPN — publisher unusable via VPN")
|
|
972
|
+
return fail(doi, "Cloudflare challenge inside WebVPN",
|
|
973
|
+
error_type="cloudflare_blocked", action="use_direct_or_carsi")
|
|
974
|
+
|
|
975
|
+
# Check network-captured PDF
|
|
976
|
+
result = _save_captured()
|
|
977
|
+
if result:
|
|
978
|
+
return result
|
|
979
|
+
|
|
980
|
+
# Strategy 0: the article page's own PDF link (carries page-issued tokens),
|
|
981
|
+
# opened by the browser itself and read via CDP
|
|
982
|
+
page_pdf_href = find_page_pdf_link(page, doi=doi)
|
|
983
|
+
if not page_pdf_href:
|
|
984
|
+
try: # what the page offered instead: the next look at a miss starts here
|
|
985
|
+
links = page.evaluate("Array.from(document.querySelectorAll('a[href]')).filter(a => /pdf/i.test(a.innerText + a.href)).slice(0, 5).map(a => (a.innerText || '').trim().slice(0, 30) + ' -> ' + a.href.slice(0, 90))")
|
|
986
|
+
log.info(f" [WebVPN-Browser] No PDF link on the page; PDF-ish links: {links}")
|
|
987
|
+
except Exception:
|
|
988
|
+
pass
|
|
989
|
+
if page_pdf_href:
|
|
990
|
+
log.info(f" [WebVPN-Browser] Page PDF link: {page_pdf_href[:80]}")
|
|
991
|
+
result = _save_bytes(capture_pdf_via_cdp(context, page, page_pdf_href))
|
|
992
|
+
if result:
|
|
993
|
+
return result
|
|
994
|
+
|
|
995
|
+
# If page looks like inline PDF viewer, try to get the PDF bytes
|
|
996
|
+
if _is_inline_pdf_page(page):
|
|
997
|
+
pdf_bytes = _extract_inline_pdf(page)
|
|
998
|
+
if pdf_bytes:
|
|
999
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1000
|
+
output_path.write_bytes(pdf_bytes)
|
|
1001
|
+
if is_pdf_file(output_path):
|
|
1002
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
1003
|
+
|
|
1004
|
+
# Strategy 1: Try direct publisher PDF URL via browser-cookie HTTP first
|
|
1005
|
+
resolved_for_pdf = _resolve_doi_url(doi) or f"https://doi.org/{doi}"
|
|
1006
|
+
pdf_url = _construct_publisher_pdf_url(doi, resolved_for_pdf)
|
|
1007
|
+
if pdf_url:
|
|
1008
|
+
result = _download_pdf_with_browser_cookies(pdf_url, output_path, config, doi, context)
|
|
1009
|
+
if result:
|
|
1010
|
+
return result
|
|
1011
|
+
|
|
1012
|
+
# Fallback: open in browser, read the body via CDP, then expect_download.
|
|
1013
|
+
# Off the VPN (federated login to the publisher itself) the session
|
|
1014
|
+
# is on the publisher, so open its own URL, not the WebVPN one.
|
|
1015
|
+
off_vpn = urllib.parse.urlparse(page_url).netloc not in ("", base_host)
|
|
1016
|
+
pdf_webvpn = pdf_url if off_vpn else convert_url(pdf_url, base, config)
|
|
1017
|
+
log.info(f" [WebVPN-Browser] Trying direct PDF via browser: {pdf_webvpn[:80]}")
|
|
1018
|
+
result = _save_bytes(capture_pdf_via_cdp(context, page, pdf_webvpn))
|
|
1019
|
+
if result:
|
|
1020
|
+
return result
|
|
1021
|
+
captured_pdf.clear()
|
|
1022
|
+
try:
|
|
1023
|
+
with page.expect_download(timeout=30000) as download_info:
|
|
1024
|
+
page.goto(pdf_webvpn, wait_until="commit", timeout=30000)
|
|
1025
|
+
download = download_info.value
|
|
1026
|
+
tmp = download.path()
|
|
1027
|
+
pdf_bytes = tmp.read_bytes() if tmp else None
|
|
1028
|
+
if pdf_bytes and pdf_bytes[:4] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
1029
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1030
|
+
output_path.write_bytes(pdf_bytes)
|
|
1031
|
+
if is_pdf_file(output_path):
|
|
1032
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
1033
|
+
except Exception as dl_exc:
|
|
1034
|
+
log.info(f" [WebVPN-Browser] Download event not triggered: {dl_exc}")
|
|
1035
|
+
time.sleep(5)
|
|
1036
|
+
result = _save_captured()
|
|
1037
|
+
if result:
|
|
1038
|
+
return result
|
|
1039
|
+
if _is_inline_pdf_page(page):
|
|
1040
|
+
pdf_bytes = _extract_inline_pdf(page)
|
|
1041
|
+
if pdf_bytes:
|
|
1042
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1043
|
+
output_path.write_bytes(pdf_bytes)
|
|
1044
|
+
if is_pdf_file(output_path):
|
|
1045
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
1046
|
+
|
|
1047
|
+
# Strategy 2: Find PDF link in HTML, try browser-cookie HTTP first
|
|
1048
|
+
wait_page_settled(page, max_wait_s=15)
|
|
1049
|
+
try:
|
|
1050
|
+
html = page.content()
|
|
1051
|
+
except Exception as exc:
|
|
1052
|
+
log.info(f" [WebVPN-Browser] page content unavailable: {exc}")
|
|
1053
|
+
html = ""
|
|
1054
|
+
found_pdf_url = extract_pdf_url_from_html(html, page.url) if html else None
|
|
1055
|
+
if found_pdf_url:
|
|
1056
|
+
log.info(f" [WebVPN-Browser] Found PDF link: {found_pdf_url[:80]}")
|
|
1057
|
+
result = _download_pdf_with_browser_cookies(found_pdf_url, output_path, config, doi, context)
|
|
1058
|
+
if result:
|
|
1059
|
+
return result
|
|
1060
|
+
|
|
1061
|
+
# Fallback: expect_download in browser
|
|
1062
|
+
captured_pdf.clear()
|
|
1063
|
+
try:
|
|
1064
|
+
with page.expect_download(timeout=30000) as download_info:
|
|
1065
|
+
page.goto(found_pdf_url, wait_until="commit", timeout=30000)
|
|
1066
|
+
download = download_info.value
|
|
1067
|
+
tmp = download.path()
|
|
1068
|
+
pdf_bytes = tmp.read_bytes() if tmp else None
|
|
1069
|
+
if pdf_bytes and pdf_bytes[:4] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
1070
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1071
|
+
output_path.write_bytes(pdf_bytes)
|
|
1072
|
+
if is_pdf_file(output_path):
|
|
1073
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
1074
|
+
except Exception:
|
|
1075
|
+
time.sleep(5)
|
|
1076
|
+
result = _save_captured()
|
|
1077
|
+
if result:
|
|
1078
|
+
return result
|
|
1079
|
+
if _is_inline_pdf_page(page):
|
|
1080
|
+
pdf_bytes = _extract_inline_pdf(page)
|
|
1081
|
+
if pdf_bytes:
|
|
1082
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1083
|
+
output_path.write_bytes(pdf_bytes)
|
|
1084
|
+
if is_pdf_file(output_path):
|
|
1085
|
+
return success(doi, output_path, "WebVPN(Browser)")
|
|
1086
|
+
|
|
1087
|
+
log.info(f" [WebVPN-Browser] No PDF found. Title: {page.title()[:40]} URL: {page.url[:60]}")
|
|
1088
|
+
return None
|
|
1089
|
+
|
|
1090
|
+
except Exception as e:
|
|
1091
|
+
log.info(f" [WebVPN-Browser] Error: {e}")
|
|
1092
|
+
return None
|
|
1093
|
+
finally:
|
|
1094
|
+
try:
|
|
1095
|
+
browser.close()
|
|
1096
|
+
except Exception:
|
|
1097
|
+
pass
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def try_instsci(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
1101
|
+
"""Try downloading paper through institutional access.
|
|
1102
|
+
|
|
1103
|
+
Strategy:
|
|
1104
|
+
1. If SOCKS5 campus connector (EasyConnect/aTrust) is configured, try direct access
|
|
1105
|
+
2. Try CloakBrowser download (handles CAS auth + Cloudflare)
|
|
1106
|
+
3. Try WebVPN HTTP approach (if session cookies valid)
|
|
1107
|
+
|
|
1108
|
+
Note: CARSI is now a standalone source tier (sources.carsi.try_carsi),
|
|
1109
|
+
called independently from the download orchestrator.
|
|
1110
|
+
"""
|
|
1111
|
+
if not _cfg(config, "enabled", False):
|
|
1112
|
+
return None
|
|
1113
|
+
|
|
1114
|
+
# Step 0: SOCKS5 campus connector mode (EasyConnect/aTrust) — direct access
|
|
1115
|
+
if _is_campus_connector_mode(config):
|
|
1116
|
+
result = _try_instsci_socks5(doi, output_path, config)
|
|
1117
|
+
if result:
|
|
1118
|
+
return result
|
|
1119
|
+
|
|
1120
|
+
# Step 1: Try stealth browser download (handles CAS auth + Cloudflare)
|
|
1121
|
+
result = _try_instsci_browser(doi, output_path, config)
|
|
1122
|
+
if result:
|
|
1123
|
+
return result
|
|
1124
|
+
|
|
1125
|
+
# Step 2: Try WebVPN HTTP approach (use any saved cookies, even if
|
|
1126
|
+
# _validate_session fails — the stealth browser may have just logged in
|
|
1127
|
+
# and saved fresh cookies that work for the target paper but fail
|
|
1128
|
+
# validation's unrelated test URL)
|
|
1129
|
+
if instsci_cookie_path(config).exists():
|
|
1130
|
+
result = _try_instsci_http(doi, output_path, config)
|
|
1131
|
+
if result:
|
|
1132
|
+
return result
|
|
1133
|
+
log.info(" [WebVPN] No PDF through WebVPN for this paper")
|
|
1134
|
+
return None
|
|
1135
|
+
|
|
1136
|
+
log.info(" [WebVPN] No valid session. Use instsci_login or carsi_login tool first.")
|
|
1137
|
+
return None
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def _try_instsci_http(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
1141
|
+
"""Try downloading via HTTP with saved cookies."""
|
|
1142
|
+
|
|
1143
|
+
_instsci_rate_limit()
|
|
1144
|
+
|
|
1145
|
+
log.info(f" [WebVPN] Trying {doi}")
|
|
1146
|
+
|
|
1147
|
+
# Step 1: Resolve DOI to get publisher URL
|
|
1148
|
+
resolved_url = _resolve_doi_url(doi)
|
|
1149
|
+
if not resolved_url:
|
|
1150
|
+
resolved_url = f"https://doi.org/{doi}"
|
|
1151
|
+
|
|
1152
|
+
# Step 2: Try direct publisher PDF URL
|
|
1153
|
+
pdf_url = _construct_publisher_pdf_url(doi, resolved_url)
|
|
1154
|
+
if pdf_url:
|
|
1155
|
+
log.info(f" [WebVPN] Trying publisher PDF: {pdf_url[:80]}...")
|
|
1156
|
+
result = _download_pdf_instsci(pdf_url, output_path, config, doi)
|
|
1157
|
+
if result:
|
|
1158
|
+
return result
|
|
1159
|
+
|
|
1160
|
+
# Step 3: Fetch via WebVPN and look for PDF link in HTML
|
|
1161
|
+
try:
|
|
1162
|
+
doi_url = f"https://doi.org/{doi}"
|
|
1163
|
+
resp = _fetch_via_webvpn(doi_url, config, stream=True)
|
|
1164
|
+
if resp.status_code >= 400:
|
|
1165
|
+
return None
|
|
1166
|
+
|
|
1167
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
1168
|
+
first = next(iterator, b"")
|
|
1169
|
+
|
|
1170
|
+
# Direct PDF response
|
|
1171
|
+
if _response_looks_pdf(resp, first):
|
|
1172
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1173
|
+
tmp_path = output_path.with_suffix(output_path.suffix + ".part")
|
|
1174
|
+
try:
|
|
1175
|
+
with tmp_path.open("wb") as fh:
|
|
1176
|
+
fh.write(first)
|
|
1177
|
+
for chunk in iterator:
|
|
1178
|
+
if chunk:
|
|
1179
|
+
fh.write(chunk)
|
|
1180
|
+
tmp_path.replace(output_path)
|
|
1181
|
+
except Exception:
|
|
1182
|
+
tmp_path.unlink(missing_ok=True)
|
|
1183
|
+
raise
|
|
1184
|
+
if is_pdf_file(output_path):
|
|
1185
|
+
return success(doi, output_path, "WebVPN")
|
|
1186
|
+
|
|
1187
|
+
# HTML response - extract PDF link
|
|
1188
|
+
html = first + resp.raw.read(512_000, decode_content=True)
|
|
1189
|
+
html_str = html.decode("utf-8", errors="ignore")
|
|
1190
|
+
|
|
1191
|
+
# Check for Cloudflare block
|
|
1192
|
+
from ..network import _is_cloudflare_block
|
|
1193
|
+
if any(sig in html_str.lower() for sig in ("cf-browser-verification", "challenge-platform", "just a moment", "请稍候", "正在验证", "checking your browser")):
|
|
1194
|
+
log.info(" [WebVPN] Cloudflare detected, trying browser...")
|
|
1195
|
+
browser_html = _try_browser_via_webvpn(doi_url, config)
|
|
1196
|
+
if browser_html:
|
|
1197
|
+
html_str = browser_html
|
|
1198
|
+
|
|
1199
|
+
# Taylor & Francis books: the landing page tells authored from edited
|
|
1200
|
+
# (the DOI alone does not); the download endpoint follows from it.
|
|
1201
|
+
tf = _taylorfrancis_book_pdf_url(doi, resp.url)
|
|
1202
|
+
if tf:
|
|
1203
|
+
log.info(f" [WebVPN] Taylor & Francis book PDF: {tf[:80]}...")
|
|
1204
|
+
result = _download_pdf_instsci(tf, output_path, config, doi)
|
|
1205
|
+
if result:
|
|
1206
|
+
return result
|
|
1207
|
+
chapter = _taylorfrancis_chapter_book(doi, resp.url)
|
|
1208
|
+
if chapter:
|
|
1209
|
+
book_doi, form = chapter
|
|
1210
|
+
for kind in dict.fromkeys((form, "mono", "edit")): # the chapter's form first
|
|
1211
|
+
url = (f"https://api.taylorfrancis.com/content/books/{kind}/download?identifierName=doi"
|
|
1212
|
+
f"&identifierValue={book_doi}&type=googlepdf")
|
|
1213
|
+
log.info(f" [WebVPN] Taylor & Francis chapter: trying its book {book_doi} ({kind})")
|
|
1214
|
+
result = _download_pdf_instsci(url, output_path, config, doi)
|
|
1215
|
+
if result:
|
|
1216
|
+
result.update(whole_book=True, book_doi=book_doi, note=_chapter_note(doi, config))
|
|
1217
|
+
return result
|
|
1218
|
+
|
|
1219
|
+
# Try _find_pdf_link (more thorough)
|
|
1220
|
+
found_pdf = _find_pdf_link(html_str, resp.url)
|
|
1221
|
+
if found_pdf:
|
|
1222
|
+
log.info(f" [WebVPN] Found PDF link in HTML: {found_pdf[:80]}...")
|
|
1223
|
+
result = _download_pdf_instsci(found_pdf, output_path, config, doi)
|
|
1224
|
+
if result:
|
|
1225
|
+
return result
|
|
1226
|
+
|
|
1227
|
+
# Fallback to extract_pdf_url_from_html
|
|
1228
|
+
pdf_url = extract_pdf_url_from_html(html_str, resp.url)
|
|
1229
|
+
if pdf_url:
|
|
1230
|
+
return _download_pdf_instsci(pdf_url, output_path, config, doi)
|
|
1231
|
+
|
|
1232
|
+
except Exception as e:
|
|
1233
|
+
log.info(f" [WebVPN] {e}")
|
|
1234
|
+
|
|
1235
|
+
return None
|
|
1236
|
+
|
|
1237
|
+
|
|
1238
|
+
def _try_browser_via_webvpn(url: str, config: dict[str, Any]) -> str | None:
|
|
1239
|
+
"""Try fetching a URL through CloakBrowser, using WebVPN proxy."""
|
|
1240
|
+
base = _get_webvpn_base(config)
|
|
1241
|
+
proxied_url = convert_url(url, base, config)
|
|
1242
|
+
try:
|
|
1243
|
+
from ..browser.engine import is_available as browser_avail, get_html as browser_html
|
|
1244
|
+
if browser_avail(config):
|
|
1245
|
+
result = browser_html(proxied_url, config)
|
|
1246
|
+
if result:
|
|
1247
|
+
return result
|
|
1248
|
+
except Exception as e:
|
|
1249
|
+
log.info(f" [browser] {e}")
|
|
1250
|
+
return None
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
def _download_pdf_instsci(
|
|
1254
|
+
url: str,
|
|
1255
|
+
output_path: Path,
|
|
1256
|
+
config: dict[str, Any],
|
|
1257
|
+
doi: str,
|
|
1258
|
+
) -> dict[str, Any] | None:
|
|
1259
|
+
if not is_plausible_pdf_url(url):
|
|
1260
|
+
log.info(f" [WebVPN] Not a PDF link, skipped: {url[:80]}")
|
|
1261
|
+
return None
|
|
1262
|
+
try:
|
|
1263
|
+
_instsci_rate_limit()
|
|
1264
|
+
resp = _fetch_via_webvpn(url, config, stream=True)
|
|
1265
|
+
if resp.status_code >= 400:
|
|
1266
|
+
log.info(f" [WebVPN] HTTP {resp.status_code}: {url[:80]}")
|
|
1267
|
+
return None
|
|
1268
|
+
|
|
1269
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
1270
|
+
first_chunk = next(iterator, b"")
|
|
1271
|
+
if not _response_looks_pdf(resp, first_chunk):
|
|
1272
|
+
log.info(f" [WebVPN] Not a PDF ({resp.headers.get('content-type', '?')}): {url[:80]}")
|
|
1273
|
+
return None
|
|
1274
|
+
|
|
1275
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1276
|
+
tmp_path = output_path.with_suffix(output_path.suffix + ".part")
|
|
1277
|
+
try:
|
|
1278
|
+
with tmp_path.open("wb") as fh:
|
|
1279
|
+
fh.write(first_chunk)
|
|
1280
|
+
for chunk in iterator:
|
|
1281
|
+
if chunk:
|
|
1282
|
+
fh.write(chunk)
|
|
1283
|
+
tmp_path.replace(output_path)
|
|
1284
|
+
except Exception:
|
|
1285
|
+
tmp_path.unlink(missing_ok=True)
|
|
1286
|
+
raise
|
|
1287
|
+
|
|
1288
|
+
if is_pdf_file(output_path):
|
|
1289
|
+
result = success(doi, output_path, "WebVPN")
|
|
1290
|
+
result["doi"] = doi
|
|
1291
|
+
result["identifier"] = doi
|
|
1292
|
+
return result
|
|
1293
|
+
except Exception as e:
|
|
1294
|
+
log.info(f" [WebVPN] Download failed: {e}")
|
|
1295
|
+
return None
|