litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2340 @@
|
|
|
1
|
+
"""Publisher-specific browser download strategies.
|
|
2
|
+
|
|
3
|
+
Implements tailored download paths for major academic publishers,
|
|
4
|
+
handling anti-bot challenges, PDF viewers, and publisher-specific DOM structures.
|
|
5
|
+
Inspired by ref-downloader's publisher strategy registry.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import contextlib
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
from ...browser.backend import launch, launch_persistent_context
|
|
21
|
+
from ...browser.backend import is_available as _browser_backend_available
|
|
22
|
+
_HAS_CLOAKBROWSER = _browser_backend_available()
|
|
23
|
+
except ImportError:
|
|
24
|
+
launch = None # type: ignore[assignment]
|
|
25
|
+
launch_persistent_context = None # type: ignore[assignment]
|
|
26
|
+
_HAS_CLOAKBROWSER = False
|
|
27
|
+
from ...log import get_logger
|
|
28
|
+
|
|
29
|
+
log = get_logger()
|
|
30
|
+
|
|
31
|
+
# Per-thread error tracking for caller introspection: racing lanes run in
|
|
32
|
+
# parallel threads, so a module-global would let one lane's error leak into
|
|
33
|
+
# another lane's fallback decision.
|
|
34
|
+
_ERR_TLS = threading.local()
|
|
35
|
+
|
|
36
|
+
# Page titles of human checks (Cloudflare and the like), lower case.
|
|
37
|
+
_CHALLENGE_TITLES = ("just a moment", "attention required", "verify", "security check",
|
|
38
|
+
"请稍候", "正在验证", "checking", "cloudflare")
|
|
39
|
+
|
|
40
|
+
# DOIs whose publisher site was logged into with the school's identity and then
|
|
41
|
+
# still offered the paper for sale: the school does not subscribe, and the
|
|
42
|
+
# school routes (WebVPN, CARSI) end at the same publisher with the same identity
|
|
43
|
+
# (2026-10-18: a Springer chapter, 3 minutes and a WebVPN login for nothing).
|
|
44
|
+
# DOI -> when it was seen. Kept NO_ACCESS_HOURS (自定值) or until the paper is
|
|
45
|
+
# downloaded, so a retry does not walk the school routes again.
|
|
46
|
+
LOGIN_NO_ACCESS: dict[str, float] = {}
|
|
47
|
+
NO_ACCESS_HOURS = 6
|
|
48
|
+
# What an article page says about access, read from the article itself (side
|
|
49
|
+
# panels, recommendations, navigation and cookie banners removed: their "Buy
|
|
50
|
+
# eBook" / "Purchase PDF" belong to other papers). Narrower than _detect_paywall:
|
|
51
|
+
# "(Get) access through your institution" stays on entitled pages too, so it is
|
|
52
|
+
# no cue; a page offering "View PDF" / "Download PDF" is never "no access".
|
|
53
|
+
_SALE_CUES = ("buy article", "buy this article", "buy chapter", "buy this chapter", "purchase pdf",
|
|
54
|
+
"purchase this article", "purchase access", "rent this article", "subscribe to access",
|
|
55
|
+
"you do not have access", "you don't have access")
|
|
56
|
+
_BUY_WORDS = ("buy", "purchase", "rent")
|
|
57
|
+
_PRICE = re.compile(r"(?:[$€£¥]\s?\d|\b\d[\d.,]*\s?(?:usd|eur|gbp)\b)", re.I)
|
|
58
|
+
# A login prompt in place of the full text. Before the school login it only says
|
|
59
|
+
# "not logged in"; after it, "the school has no access".
|
|
60
|
+
_LOGIN_CUES = ("log in via an institution", "access this chapter", "access this article", "access this book")
|
|
61
|
+
_ARTICLE_TEXT_JS = r"""() => {
|
|
62
|
+
const root = document.querySelector('main') || document.querySelector('article') || document.body;
|
|
63
|
+
if (!root) return {text: '', pdf: false};
|
|
64
|
+
const pdf = /\b(view|download) pdf\b/i.test(root.innerText || '');
|
|
65
|
+
const clone = root.cloneNode(true);
|
|
66
|
+
clone.querySelectorAll('aside, nav, footer, script, style, noscript, [class*="recommend" i], [id*="recommend" i],'
|
|
67
|
+
+ ' [class*="related" i], [id*="related" i], [class*="cookie" i], [id*="cookie" i], [class*="consent" i]')
|
|
68
|
+
.forEach((e) => e.remove());
|
|
69
|
+
return {text: (clone.textContent || '').replace(/\s+/g, ' '), pdf};
|
|
70
|
+
}"""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def note_no_access(doi: str | None) -> None:
|
|
74
|
+
LOGIN_NO_ACCESS[(doi or "").strip().lower()] = time.time()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def no_access(doi: str | None) -> bool:
|
|
78
|
+
key = (doi or "").strip().lower()
|
|
79
|
+
seen = LOGIN_NO_ACCESS.get(key)
|
|
80
|
+
if seen is not None and time.time() - seen >= NO_ACCESS_HOURS * 3600:
|
|
81
|
+
LOGIN_NO_ACCESS.pop(key, None) # expired: drop it
|
|
82
|
+
return False
|
|
83
|
+
return seen is not None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def clear_no_access(doi: str | None) -> None:
|
|
87
|
+
LOGIN_NO_ACCESS.pop((doi or "").strip().lower(), None)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _page_access(page: Any) -> dict[str, bool]:
|
|
91
|
+
"""{"pdf": offers View/Download PDF, "sells": sells the paper, "asks_login": asks to log in}."""
|
|
92
|
+
try:
|
|
93
|
+
got = page.evaluate(_ARTICLE_TEXT_JS) or {}
|
|
94
|
+
except Exception:
|
|
95
|
+
return {"pdf": False, "sells": False, "asks_login": False}
|
|
96
|
+
text = str(got.get("text") or "").lower()
|
|
97
|
+
if got.get("pdf"):
|
|
98
|
+
return {"pdf": True, "sells": False, "asks_login": False}
|
|
99
|
+
sells = any(c in text for c in _SALE_CUES) or (
|
|
100
|
+
bool(_PRICE.search(text)) and any(w in text for w in _BUY_WORDS))
|
|
101
|
+
return {"pdf": False, "sells": sells, "asks_login": any(c in text for c in _LOGIN_CUES)}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _page_sells_paper(page: Any) -> bool:
|
|
105
|
+
"""After the school login: the page still sells the paper or asks to log in."""
|
|
106
|
+
state = _page_access(page)
|
|
107
|
+
return state["sells"] or state["asks_login"]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _wait_for_pdf_link(page: Any, doi: str, publisher: str) -> tuple[str | None, str]:
|
|
111
|
+
"""The article page's own PDF link, by what the page says once it is quiet.
|
|
112
|
+
|
|
113
|
+
Returns (href, state), state being "pdf", "no_access" or "unclear":
|
|
114
|
+
- a PDF link or button: take it;
|
|
115
|
+
- the page sells the paper or asks to log in: no link will come, so say so
|
|
116
|
+
at once (before the school login: go and log in; after it: not
|
|
117
|
+
subscribed) instead of looking for 30 s and reloading (2026-10-19,
|
|
118
|
+
Springer: 80 s lost on a page that said "Log in via an institution");
|
|
119
|
+
- neither: the access check may still be running ("View PDF" came after
|
|
120
|
+
15 s on ScienceDirect, 2026-10-18), so look longer and reload once.
|
|
121
|
+
"""
|
|
122
|
+
from ...browser.pdf import find_page_pdf_link
|
|
123
|
+
for attempt in range(2):
|
|
124
|
+
try:
|
|
125
|
+
page.wait_for_load_state("networkidle", timeout=10000)
|
|
126
|
+
except Exception:
|
|
127
|
+
pass
|
|
128
|
+
href = find_page_pdf_link(page, max_wait_s=2.0, doi=doi)
|
|
129
|
+
if href:
|
|
130
|
+
return href, "pdf"
|
|
131
|
+
state = _page_access(page)
|
|
132
|
+
if state["sells"] or state["asks_login"]:
|
|
133
|
+
time.sleep(3) # the cue must still be there, not a flash before the access check
|
|
134
|
+
href = find_page_pdf_link(page, max_wait_s=1.0, doi=doi)
|
|
135
|
+
if href:
|
|
136
|
+
return href, "pdf"
|
|
137
|
+
again = _page_access(page)
|
|
138
|
+
if again["sells"] or again["asks_login"]:
|
|
139
|
+
log.info(f" [{publisher}] the page {'sells the paper' if again['sells'] else 'asks to log in'}; "
|
|
140
|
+
"not waiting for a PDF link")
|
|
141
|
+
return None, "no_access"
|
|
142
|
+
href = find_page_pdf_link(page, max_wait_s=18.0 if attempt == 0 else 8.0, doi=doi)
|
|
143
|
+
if href:
|
|
144
|
+
return href, "pdf"
|
|
145
|
+
if attempt:
|
|
146
|
+
break
|
|
147
|
+
log.info(f" [{publisher}] no PDF link yet; reloading the page once")
|
|
148
|
+
try:
|
|
149
|
+
page.reload(wait_until="domcontentloaded", timeout=30000)
|
|
150
|
+
except Exception:
|
|
151
|
+
pass
|
|
152
|
+
return None, "unclear"
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def get_last_error() -> tuple[str, str]:
|
|
156
|
+
"""(error_type, action) of the last browser strategy run in this thread.
|
|
157
|
+
|
|
158
|
+
Only this thread's: the download's own thread reads its lanes' errors from
|
|
159
|
+
its DownloadAttempt (``sources._try_source`` copies them there), never a
|
|
160
|
+
process-wide value another paper's lane may have written.
|
|
161
|
+
"""
|
|
162
|
+
return getattr(_ERR_TLS, "type", ""), getattr(_ERR_TLS, "action", "")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _set_error(error_type: str, action: str = "") -> None:
|
|
166
|
+
_ERR_TLS.type = error_type
|
|
167
|
+
_ERR_TLS.action = action
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _is_browser_error_page(page: Any) -> bool:
|
|
171
|
+
"""Chrome's own "This site can't be reached" / timeout page, not the site's."""
|
|
172
|
+
try:
|
|
173
|
+
url = page.url or ""
|
|
174
|
+
if url.startswith("chrome-error://"):
|
|
175
|
+
return True
|
|
176
|
+
title = (page.title() or "").lower()
|
|
177
|
+
except Exception:
|
|
178
|
+
return False
|
|
179
|
+
return any(s in title for s in ("无法访问此网站", "this site can’t be reached",
|
|
180
|
+
"this site can't be reached", "err_timed_out", "err_connection"))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _clear_error() -> None:
|
|
184
|
+
_set_error("", "")
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# Click the institution in a search-result list. Containers (the whole result
|
|
188
|
+
# panel, the "Access content through your institution" heading block) also
|
|
189
|
+
# contain the name, so take the shortest visible match: the result row itself.
|
|
190
|
+
_PICK_INSTITUTION_JS = """
|
|
191
|
+
(name) => {
|
|
192
|
+
const items = document.querySelectorAll('[role="option"], [class*="result"], [class*="suggestion"], [class*="federation"], li, a, button');
|
|
193
|
+
let best = null;
|
|
194
|
+
for (const el of items) {
|
|
195
|
+
const text = (el.textContent || '').trim();
|
|
196
|
+
if (!text.toLowerCase().includes(name.toLowerCase()) || el.offsetParent === null) continue;
|
|
197
|
+
if (!best || text.length < best.text.length) best = {el, text};
|
|
198
|
+
}
|
|
199
|
+
if (!best) return null;
|
|
200
|
+
best.el.click();
|
|
201
|
+
return best.text.substring(0, 60);
|
|
202
|
+
}
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# Publishers' own PDF links the generic finder (find_page_pdf_link) may miss;
|
|
207
|
+
# moved from the shared-browser first pass (2026-10-20): Cell Press showPdf,
|
|
208
|
+
# ScienceDirect's View PDF button and pdfft, Wiley pdfdirect, IEEE stamp, and
|
|
209
|
+
# SAGE's China platform, whose PDF button has no href (the file's path sits in
|
|
210
|
+
# the page data as /storage/.../<doi with _ for />.pdf, seen 2026-10-17).
|
|
211
|
+
_PUBLISHER_PDF_LINK_JS = r"""
|
|
212
|
+
(doi) => {
|
|
213
|
+
const first = (sel) => { for (const a of document.querySelectorAll(sel)) if (a.href) return a.href; return null; };
|
|
214
|
+
const hit = first('a[href*="showPdf"]') || first('a[aria-label*="View PDF" i], a.pdf-download-btn-link')
|
|
215
|
+
|| first('a[href*="pdfft"]') || first('a[href*="pdfdirect"]')
|
|
216
|
+
|| first('a[href*="/stamp/"], a[href*="getPDF"], a.xpl-pdf-btn');
|
|
217
|
+
if (hit) return hit;
|
|
218
|
+
for (const a of document.querySelectorAll('a[href*="/doi/pdf/"]')) {
|
|
219
|
+
const h = a.href.toLowerCase();
|
|
220
|
+
if (!h.includes('supplement') && !h.includes('supporting')) return a.href;
|
|
221
|
+
}
|
|
222
|
+
if (location.hostname.endsWith('cnpereading.com')) {
|
|
223
|
+
const m = document.documentElement.outerHTML.match(/\/storage\/[^"'\\\s<>]+?\.pdf/);
|
|
224
|
+
if (m && m[0].toLowerCase().includes(doi.replace(/\//g, '_').toLowerCase())) return location.origin + m[0];
|
|
225
|
+
}
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _known_landing_url(doi: str) -> str:
|
|
232
|
+
"""Article URL built from the DOI, for publishers whose pattern is fixed.
|
|
233
|
+
|
|
234
|
+
Saves the doi.org hop (and its occasional timeouts) when opening the page
|
|
235
|
+
in a browser. Publishers whose URL needs data the DOI lacks (AIP, RSC)
|
|
236
|
+
keep going through doi.org.
|
|
237
|
+
"""
|
|
238
|
+
d = doi.strip()
|
|
239
|
+
low = d.lower()
|
|
240
|
+
suffix = d.split("/", 1)[1] if "/" in d else ""
|
|
241
|
+
if low.startswith("10.1126/"):
|
|
242
|
+
return f"https://www.science.org/doi/{d}"
|
|
243
|
+
if low.startswith("10.1021/"):
|
|
244
|
+
return f"https://pubs.acs.org/doi/{d}"
|
|
245
|
+
if low.startswith("10.1038/") and suffix:
|
|
246
|
+
return f"https://www.nature.com/articles/{suffix}"
|
|
247
|
+
if low.startswith("10.1007/") and suffix:
|
|
248
|
+
kind = "chapter" if suffix.startswith("978-") else "article"
|
|
249
|
+
return f"https://link.springer.com/{kind}/{d}"
|
|
250
|
+
return ""
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _resolve_doi_landing(doi: str) -> str:
|
|
254
|
+
"""Publisher landing URL for a DOI via the doi.org handle API ("" if unknown)."""
|
|
255
|
+
import requests
|
|
256
|
+
try:
|
|
257
|
+
from ...config import load_config
|
|
258
|
+
from ...network import apply_host_proxy
|
|
259
|
+
s = requests.Session()
|
|
260
|
+
s.trust_env = False
|
|
261
|
+
apply_host_proxy(s, load_config())
|
|
262
|
+
r = s.get(f"https://doi.org/api/handles/{doi}", params={"type": "URL"}, timeout=15)
|
|
263
|
+
for v in r.json().get("values", []):
|
|
264
|
+
if v.get("type") == "URL":
|
|
265
|
+
return str(v.get("data", {}).get("value", ""))
|
|
266
|
+
except Exception:
|
|
267
|
+
pass
|
|
268
|
+
return ""
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _on_login_page(where: Any) -> bool:
|
|
272
|
+
"""Whether a page (or any page of a browser context) shows a login."""
|
|
273
|
+
pages = list(getattr(where, "pages", None) or [where])
|
|
274
|
+
for pg in pages:
|
|
275
|
+
try:
|
|
276
|
+
title, url = pg.title() or "", (pg.url or "").lower()
|
|
277
|
+
except Exception:
|
|
278
|
+
continue
|
|
279
|
+
if any(x in title for x in _AUTH_TITLES) or any(x in url for x in _AUTH_KEYWORDS + ("carsi",)):
|
|
280
|
+
return True
|
|
281
|
+
return False
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _user_waiting(what: str = "verify", where: Any = None) -> None:
|
|
285
|
+
"""Heartbeat while the user is logging in or passing a check, so racing does
|
|
286
|
+
not give up on us; the app shows what the user should do.
|
|
287
|
+
|
|
288
|
+
With ``where`` (a page or browser context) the heartbeat is sent only while
|
|
289
|
+
a login page is actually open: the steps before it (finding the
|
|
290
|
+
institution's link, an SSO click that goes nowhere) are not the user's to
|
|
291
|
+
wait for, and saying so would show a notice with nothing to log into."""
|
|
292
|
+
try:
|
|
293
|
+
from ...sources import SourceWaived, _source_may_continue, note_user_waiting
|
|
294
|
+
except Exception:
|
|
295
|
+
return
|
|
296
|
+
if where is not None and not _on_login_page(where):
|
|
297
|
+
if not _source_may_continue():
|
|
298
|
+
raise SourceWaived("download already finished; stop waiting")
|
|
299
|
+
return
|
|
300
|
+
try:
|
|
301
|
+
note_user_waiting(what)
|
|
302
|
+
except SourceWaived:
|
|
303
|
+
raise
|
|
304
|
+
except Exception:
|
|
305
|
+
pass
|
|
306
|
+
_show_window_for_user(what)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
# The publisher window of this thread (``_visible_browser``): it starts
|
|
310
|
+
# minimized and comes up when the user has to act (2026-10-20, user's choice).
|
|
311
|
+
_WINDOW_TLS = threading.local()
|
|
312
|
+
_VERIFY_GRACE_S = 8 # most human checks pass by themselves within seconds (own value)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _set_window_state(page: Any, state: str) -> bool:
|
|
316
|
+
"""Minimize the window of ``page`` or bring it back ("minimized" / "normal")."""
|
|
317
|
+
try:
|
|
318
|
+
cdp = page.context.new_cdp_session(page)
|
|
319
|
+
win = cdp.send("Browser.getWindowForTarget")["windowId"]
|
|
320
|
+
cdp.send("Browser.setWindowBounds", {"windowId": win, "bounds": {"windowState": state}})
|
|
321
|
+
if state == "normal":
|
|
322
|
+
page.bring_to_front()
|
|
323
|
+
return True
|
|
324
|
+
except Exception as exc:
|
|
325
|
+
log.info(f" publisher window: cannot set it {state} ({type(exc).__name__})")
|
|
326
|
+
return False
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _show_window_for_user(what: str) -> None:
|
|
330
|
+
"""Bring this thread's publisher window up, once: at once for a login, after
|
|
331
|
+
a few seconds for a human check (one that passes by itself needs no one)."""
|
|
332
|
+
page = getattr(_WINDOW_TLS, "page", None)
|
|
333
|
+
if page is None or getattr(_WINDOW_TLS, "shown", False):
|
|
334
|
+
return
|
|
335
|
+
if what == "verify":
|
|
336
|
+
since = getattr(_WINDOW_TLS, "verify_since", None)
|
|
337
|
+
if since is None:
|
|
338
|
+
_WINDOW_TLS.verify_since = since = time.time()
|
|
339
|
+
if time.time() - since < _VERIFY_GRACE_S:
|
|
340
|
+
return
|
|
341
|
+
_WINDOW_TLS.shown = _set_window_state(page, "normal")
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _get_profile_dir(config: dict[str, Any], publisher: str = "shared") -> Path:
|
|
345
|
+
from ...config import DATA_DIR
|
|
346
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
347
|
+
profile_dir = cache_dir / f"publisher_profile_{publisher.lower()}"
|
|
348
|
+
profile_dir.mkdir(parents=True, exist_ok=True)
|
|
349
|
+
return profile_dir
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _restore_cookies_to_context(context: Any, config: dict[str, Any]) -> None:
|
|
353
|
+
try:
|
|
354
|
+
from ...browser.cookies import load_saved_cookies
|
|
355
|
+
saved = load_saved_cookies(config)
|
|
356
|
+
if saved:
|
|
357
|
+
pw_cookies = []
|
|
358
|
+
for c in saved:
|
|
359
|
+
pw_c = {"name": c.get("name", ""), "value": c.get("value", ""),
|
|
360
|
+
"domain": c.get("domain", ""), "path": c.get("path", "/")}
|
|
361
|
+
if pw_c["domain"]:
|
|
362
|
+
pw_cookies.append(pw_c)
|
|
363
|
+
if pw_cookies:
|
|
364
|
+
context.add_cookies(pw_cookies)
|
|
365
|
+
except Exception:
|
|
366
|
+
pass
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
@contextlib.contextmanager
|
|
370
|
+
def _visible_browser(config: dict[str, Any], publisher: str, *, viewport: dict | None = None):
|
|
371
|
+
"""Open visible stealth browser with persistent profile. Falls back to ephemeral."""
|
|
372
|
+
from ...browser.engine import close_shared_browser, is_available
|
|
373
|
+
|
|
374
|
+
# Any resolved backend works (camoufox included); the old _HAS_CLOAKBROWSER
|
|
375
|
+
# gate wrongly required the cloakbrowser package.
|
|
376
|
+
if not is_available(config):
|
|
377
|
+
raise RuntimeError(
|
|
378
|
+
"no browser backend available. Run: pip install patchright / cloakbrowser / camoufox")
|
|
379
|
+
# The shared headless browser holds this thread's sync-API loop — a second
|
|
380
|
+
# (visible) launch on the same thread dies with "Sync API inside the
|
|
381
|
+
# asyncio loop". Tear it down first; it relaunches lazily when needed.
|
|
382
|
+
close_shared_browser(config)
|
|
383
|
+
profile_dir = _get_profile_dir(config, publisher)
|
|
384
|
+
browser = None
|
|
385
|
+
|
|
386
|
+
try:
|
|
387
|
+
ctx = launch_persistent_context(
|
|
388
|
+
str(profile_dir),
|
|
389
|
+
headless=False, humanize=True,
|
|
390
|
+
args=["--disable-features=CrossOriginOpenerPolicy"],
|
|
391
|
+
config=config,
|
|
392
|
+
)
|
|
393
|
+
page = ctx.new_page()
|
|
394
|
+
log.info(f" [{publisher}] persistent browser profile: {profile_dir}")
|
|
395
|
+
# Ensure cookies are loaded from saved file
|
|
396
|
+
_restore_cookies_to_context(ctx, config)
|
|
397
|
+
except Exception as _e:
|
|
398
|
+
log.info(f" [{publisher}] persistent context unavailable ({_e}), using ephemeral")
|
|
399
|
+
_vp = viewport or {"width": 1440, "height": 900}
|
|
400
|
+
browser = launch(headless=False, humanize=True,
|
|
401
|
+
args=["--disable-features=CrossOriginOpenerPolicy"],
|
|
402
|
+
config=config)
|
|
403
|
+
ctx = browser.new_context(viewport=_vp)
|
|
404
|
+
_restore_cookies_to_context(ctx, config)
|
|
405
|
+
page = ctx.new_page()
|
|
406
|
+
|
|
407
|
+
_WINDOW_TLS.page, _WINDOW_TLS.shown, _WINDOW_TLS.verify_since = page, False, None
|
|
408
|
+
if config.get("publisher_window_minimized", True):
|
|
409
|
+
_set_window_state(page, "minimized")
|
|
410
|
+
try:
|
|
411
|
+
yield ctx, page
|
|
412
|
+
finally:
|
|
413
|
+
_WINDOW_TLS.page = None
|
|
414
|
+
try:
|
|
415
|
+
if browser:
|
|
416
|
+
browser.close()
|
|
417
|
+
else:
|
|
418
|
+
ctx.close()
|
|
419
|
+
except Exception:
|
|
420
|
+
pass
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _save_all_cookie_formats(
|
|
424
|
+
cookies: list[dict[str, Any]],
|
|
425
|
+
publisher: str,
|
|
426
|
+
config: dict[str, Any],
|
|
427
|
+
) -> None:
|
|
428
|
+
"""Save cookies in all formats: JSON, Netscape, publisher subset, + bridge to browser-engine."""
|
|
429
|
+
from ...config import DATA_DIR
|
|
430
|
+
from ...browser.cookies import cookies_to_netscape, _save_cookies_json
|
|
431
|
+
|
|
432
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
433
|
+
|
|
434
|
+
# 1. Save full cookies to carsi_cookies/{publisher}.json
|
|
435
|
+
carsi_dir = cache_dir / "carsi_cookies"
|
|
436
|
+
carsi_dir.mkdir(parents=True, exist_ok=True)
|
|
437
|
+
cookie_data = [
|
|
438
|
+
{"name": c.get("name", ""), "value": c.get("value", ""),
|
|
439
|
+
"domain": c.get("domain", ""), "path": c.get("path", "/"),
|
|
440
|
+
"secure": c.get("secure", False), "expires": c.get("expires", 0),
|
|
441
|
+
"httpOnly": c.get("httpOnly", False)}
|
|
442
|
+
for c in cookies
|
|
443
|
+
]
|
|
444
|
+
(carsi_dir / f"{publisher.lower()}.json").write_text(
|
|
445
|
+
json.dumps(cookie_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
446
|
+
|
|
447
|
+
# 2. Save Netscape format (for browser-engine import)
|
|
448
|
+
netscape_text = cookies_to_netscape(cookies)
|
|
449
|
+
netscape_file = cache_dir / "publisher_cookies.txt"
|
|
450
|
+
netscape_file.write_text(netscape_text, encoding="utf-8")
|
|
451
|
+
|
|
452
|
+
# 3. Save publisher-domain subset
|
|
453
|
+
_pub_domains = [
|
|
454
|
+
"wiley.com", "onlinelibrary.wiley.com", "elsevier.com", "sciencedirect.com",
|
|
455
|
+
"cell.com", "springer.com", "link.springer.com", "nature.com",
|
|
456
|
+
"ieee.org", "ieeexplore.ieee.org", "acs.org", "pubs.acs.org",
|
|
457
|
+
"rsc.org", "pubs.rsc.org", "tandfonline.com", "oup.com",
|
|
458
|
+
"academic.oup.com", "iop.org", "iopscience.iop.org",
|
|
459
|
+
"aps.org", "journals.aps.org", "aip.org", "pubs.aip.org",
|
|
460
|
+
"dl.acm.org", "acm.org", "science.org", "sciencemag.org",
|
|
461
|
+
"ascelibrary.org", "sagepub.com", "journals.sagepub.com",
|
|
462
|
+
"sage.cnpereading.com",
|
|
463
|
+
"royalsocietypublishing.org", "copernicus.org",
|
|
464
|
+
]
|
|
465
|
+
pub_cookies = [c for c in cookie_data
|
|
466
|
+
if any(c.get("domain", "").endswith(d) for d in _pub_domains)]
|
|
467
|
+
if pub_cookies:
|
|
468
|
+
_save_cookies_json(pub_cookies, cache_dir / "publisher_cookies.json")
|
|
469
|
+
|
|
470
|
+
# 4. Bridge to browser-engine headless service
|
|
471
|
+
try:
|
|
472
|
+
from ...browser.engine import import_cookies, is_available
|
|
473
|
+
if is_available(config):
|
|
474
|
+
imported = import_cookies(str(netscape_file), config)
|
|
475
|
+
log.info(f" [{publisher}] bridged {imported} cookies to browser-engine service")
|
|
476
|
+
except Exception as _e:
|
|
477
|
+
log.info(f" [{publisher}] browser-engine bridge note: {_e}")
|
|
478
|
+
|
|
479
|
+
log.info(f" [{publisher}] saved {len(cookies)} cookies in all formats")
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _detect_paywall(html: str, status_code: int = 0) -> bool:
|
|
483
|
+
"""Detect if the page is a paywall/login wall rather than anti-bot challenge."""
|
|
484
|
+
lower = html.lower()
|
|
485
|
+
# Paywall indicators
|
|
486
|
+
paywall_signals = [
|
|
487
|
+
"sign in to access", "login to access", "institutional access",
|
|
488
|
+
"purchase access", "subscribe to access", "buy article",
|
|
489
|
+
"access through your institution", "get access",
|
|
490
|
+
"this article is behind a paywall", "subscription required",
|
|
491
|
+
"please log in", "register to continue",
|
|
492
|
+
"access denied", "you do not have access",
|
|
493
|
+
"institutional login", "shibboleth", "openathens",
|
|
494
|
+
"check access", # Science (AAAS): the only access cue on its article page
|
|
495
|
+
]
|
|
496
|
+
if any(sig in lower for sig in paywall_signals):
|
|
497
|
+
return True
|
|
498
|
+
# 403 with article content (not Cloudflare challenge) = paywall
|
|
499
|
+
if status_code == 403 and not _is_challenge_page(html):
|
|
500
|
+
return True
|
|
501
|
+
return False
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _has_publisher_cookies(config: dict[str, Any]) -> bool:
|
|
505
|
+
"""Check if we have saved publisher cookies for the current session."""
|
|
506
|
+
try:
|
|
507
|
+
from ...browser.cookies import load_saved_cookies
|
|
508
|
+
if len(load_saved_cookies(config)) > 0:
|
|
509
|
+
return True
|
|
510
|
+
except Exception:
|
|
511
|
+
pass
|
|
512
|
+
# Also check CARSI cookies
|
|
513
|
+
try:
|
|
514
|
+
from ...config import DATA_DIR
|
|
515
|
+
carsi_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "carsi_cookies"
|
|
516
|
+
if carsi_dir.is_dir() and any(carsi_dir.glob("*.json")):
|
|
517
|
+
return True
|
|
518
|
+
except Exception:
|
|
519
|
+
pass
|
|
520
|
+
return False
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _inject_cookies_to_tab(tab_id: str, config: dict[str, Any], publisher: str) -> None:
|
|
524
|
+
"""Inject saved publisher + CARSI cookies into browser-engine session."""
|
|
525
|
+
from ...browser.engine import import_cookies
|
|
526
|
+
from ...config import DATA_DIR
|
|
527
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
528
|
+
total = 0
|
|
529
|
+
|
|
530
|
+
# Import publisher cookies (Netscape format if exists)
|
|
531
|
+
netscape_file = cache_dir / "publisher_cookies.txt"
|
|
532
|
+
if netscape_file.exists():
|
|
533
|
+
try:
|
|
534
|
+
total += import_cookies(str(netscape_file), config)
|
|
535
|
+
except Exception:
|
|
536
|
+
pass
|
|
537
|
+
|
|
538
|
+
# Import CARSI cookies
|
|
539
|
+
carsi_dir = cache_dir / "carsi_cookies"
|
|
540
|
+
if carsi_dir.is_dir():
|
|
541
|
+
for cf in carsi_dir.glob("*.json"):
|
|
542
|
+
try:
|
|
543
|
+
# Convert JSON → Netscape format
|
|
544
|
+
from ...browser.cookies import cookies_to_netscape
|
|
545
|
+
cookies = json.loads(cf.read_text(encoding="utf-8"))
|
|
546
|
+
tmp = cache_dir / f"_tmp_{cf.stem}.txt"
|
|
547
|
+
tmp.write_text(cookies_to_netscape(cookies), encoding="utf-8")
|
|
548
|
+
total += import_cookies(str(tmp), config)
|
|
549
|
+
tmp.unlink(missing_ok=True)
|
|
550
|
+
except Exception:
|
|
551
|
+
pass
|
|
552
|
+
|
|
553
|
+
if total > 0:
|
|
554
|
+
log.info(f" [{publisher}] imported {total} cookies into browser session")
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
# ============================================================
|
|
558
|
+
# Publisher direct PDF URL patterns
|
|
559
|
+
# ============================================================
|
|
560
|
+
|
|
561
|
+
def _visible_browser_download(
|
|
562
|
+
doi: str,
|
|
563
|
+
article_url: str,
|
|
564
|
+
output_path: Path,
|
|
565
|
+
config: dict[str, Any],
|
|
566
|
+
publisher: str,
|
|
567
|
+
*,
|
|
568
|
+
wait_for_loading: float = 0,
|
|
569
|
+
) -> dict[str, Any] | None:
|
|
570
|
+
"""Open the publisher's window: the page's own PDF, else SSO login, then the PDF.
|
|
571
|
+
|
|
572
|
+
Works for any publisher. Uses _PUBLISHER_SSO_CONFIG for publisher-specific
|
|
573
|
+
SSO link patterns, institution search selectors, and PDF fetch paths.
|
|
574
|
+
Without a school set (carsi_idp_name) the login step is skipped: the window
|
|
575
|
+
still passes human checks and uses the campus network's access.
|
|
576
|
+
"""
|
|
577
|
+
from ...download.pdf import is_pdf_file, success
|
|
578
|
+
import base64
|
|
579
|
+
|
|
580
|
+
idp_name = config.get("carsi_idp_name", "")
|
|
581
|
+
idp_en = _IDP_MAP.get(idp_name, idp_name)
|
|
582
|
+
|
|
583
|
+
# Publisher-specific SSO config
|
|
584
|
+
sso_cfg = _PUBLISHER_SSO_CONFIG.get(publisher, _PUBLISHER_SSO_CONFIG["_default"])
|
|
585
|
+
sso_link_js = sso_cfg["sso_link_js"]
|
|
586
|
+
# Publisher-specific selectors first, then the shared generic ones.
|
|
587
|
+
search_selectors = list(sso_cfg["search_selectors"]) + [
|
|
588
|
+
s for s in _INSTITUTION_SEARCH_SELECTORS if s not in sso_cfg["search_selectors"]]
|
|
589
|
+
pdf_paths = sso_cfg["pdf_paths"](doi)
|
|
590
|
+
|
|
591
|
+
with _visible_browser(config, publisher) as (context, page):
|
|
592
|
+
|
|
593
|
+
# For Elsevier DOIs, avoid linkinghub.elsevier.com by using direct URL
|
|
594
|
+
if publisher == "Elsevier" and ("doi.org/" in article_url or "linkinghub" in article_url):
|
|
595
|
+
cell_url = _build_cell_press_url(doi)
|
|
596
|
+
if cell_url:
|
|
597
|
+
log.info(f" [{publisher}] bypassing linkinghub, using direct cell.com URL")
|
|
598
|
+
article_url = cell_url
|
|
599
|
+
else:
|
|
600
|
+
# Resolve DOI → sciencedirect.com PII URL via HTTP
|
|
601
|
+
sd_url = _resolve_elsevier_pii(doi, config)
|
|
602
|
+
if sd_url:
|
|
603
|
+
log.info(f" [{publisher}] bypassing linkinghub, using resolved URL")
|
|
604
|
+
article_url = sd_url
|
|
605
|
+
|
|
606
|
+
# Navigate to article page
|
|
607
|
+
log.info(f" [{publisher}] visible browser: opening {article_url[:60]}")
|
|
608
|
+
try:
|
|
609
|
+
page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
|
|
610
|
+
time.sleep(5)
|
|
611
|
+
except Exception as exc:
|
|
612
|
+
log.info(f" [{publisher}] page load warning: {exc}")
|
|
613
|
+
if "doi.org/" in article_url and _is_browser_error_page(page):
|
|
614
|
+
# doi.org itself sometimes times out from here: ask its API for the
|
|
615
|
+
# publisher URL and go there directly.
|
|
616
|
+
landing = _resolve_doi_landing(doi)
|
|
617
|
+
if landing:
|
|
618
|
+
log.info(f" [{publisher}] doi.org did not load, opening {landing[:80]}")
|
|
619
|
+
article_url = landing
|
|
620
|
+
try:
|
|
621
|
+
page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
|
|
622
|
+
time.sleep(5)
|
|
623
|
+
except Exception as exc:
|
|
624
|
+
log.info(f" [{publisher}] page load warning: {exc}")
|
|
625
|
+
if wait_for_loading > 0: # AIP/AVS "Loading" pages
|
|
626
|
+
time.sleep(wait_for_loading)
|
|
627
|
+
|
|
628
|
+
# If stuck on linkinghub redirect, extract target and navigate directly
|
|
629
|
+
url = page.url
|
|
630
|
+
if "linkinghub" in url or ("retrieve/pii" in url and "Loading" in (page.title() or "")):
|
|
631
|
+
log.info(f" [{publisher}] stuck on linkinghub, extracting direct URL...")
|
|
632
|
+
pii_match = re.search(r"pii/([A-Z0-9]+)", url)
|
|
633
|
+
if pii_match:
|
|
634
|
+
pii = pii_match.group(1)
|
|
635
|
+
# Try cell.com only if it's a known Cell Press journal, else sciencedirect.com
|
|
636
|
+
cell_url = _build_cell_press_url(doi) if publisher == "Elsevier" else None
|
|
637
|
+
direct_urls = []
|
|
638
|
+
if cell_url:
|
|
639
|
+
direct_urls.append(cell_url)
|
|
640
|
+
direct_urls.append(f"https://www.sciencedirect.com/science/article/pii/{pii}")
|
|
641
|
+
for direct_url in direct_urls:
|
|
642
|
+
try:
|
|
643
|
+
page.goto(direct_url, wait_until="domcontentloaded", timeout=30000)
|
|
644
|
+
time.sleep(5)
|
|
645
|
+
if "linkinghub" not in page.url:
|
|
646
|
+
log.info(f" [{publisher}] navigated to {page.url[:60]}")
|
|
647
|
+
break
|
|
648
|
+
except Exception:
|
|
649
|
+
pass
|
|
650
|
+
|
|
651
|
+
# Wait for Cloudflare challenge to resolve (visible browser can pass it).
|
|
652
|
+
# The user may have to tick it: say so (the app shows a notice, the race
|
|
653
|
+
# keeps waiting) and give them up to 5 minutes, not one.
|
|
654
|
+
for _cf_wait in range(60):
|
|
655
|
+
_cf_title = (page.title() or "").lower()
|
|
656
|
+
if any(_sig in _cf_title for _sig in _CHALLENGE_TITLES):
|
|
657
|
+
if _cf_wait % 6 == 0:
|
|
658
|
+
log.info(f" [{publisher}] Cloudflare challenge detected, waiting for it or the user... ({_cf_wait * 5}s)")
|
|
659
|
+
try:
|
|
660
|
+
if _is_challenge_dead_end(page.content()):
|
|
661
|
+
log.info(f" [{publisher}] the check cannot run in this browser; giving up on the site")
|
|
662
|
+
break
|
|
663
|
+
except Exception:
|
|
664
|
+
pass
|
|
665
|
+
_user_waiting("verify")
|
|
666
|
+
time.sleep(5)
|
|
667
|
+
else:
|
|
668
|
+
break
|
|
669
|
+
else:
|
|
670
|
+
log.info(f" [{publisher}] Cloudflare challenge did not resolve")
|
|
671
|
+
|
|
672
|
+
# Let interstitials ("Loading https://...") and auth hops finish before
|
|
673
|
+
# reading the page; evaluating mid-navigation raises.
|
|
674
|
+
from ...browser.pdf import capture_pdf_via_cdp, find_page_pdf_link, wait_page_settled, wait_url_stable
|
|
675
|
+
wait_url_stable(page)
|
|
676
|
+
title, url = wait_page_settled(page)
|
|
677
|
+
log.info(f" [{publisher}] page: '{title[:40]}' {url[:60]}")
|
|
678
|
+
if _is_browser_error_page(page):
|
|
679
|
+
# 2026-10-18: SSRN timed out; the lane waited on it for 16 minutes.
|
|
680
|
+
log.info(f" [{publisher}] the site did not load (browser error page); giving up on it")
|
|
681
|
+
_set_error("navigate_failed", "site_unreachable")
|
|
682
|
+
return None
|
|
683
|
+
try:
|
|
684
|
+
html = page.content()
|
|
685
|
+
if publisher == "Elsevier" and _is_elsevier_crasolve(page.url, html):
|
|
686
|
+
log.info(f" [{publisher}] crasolve shell detected, waiting...")
|
|
687
|
+
time.sleep(5)
|
|
688
|
+
elif publisher == "AIP" and _is_aip_loading_page(html):
|
|
689
|
+
log.info(f" [{publisher}] loading page detected, waiting...")
|
|
690
|
+
time.sleep(10)
|
|
691
|
+
except Exception:
|
|
692
|
+
pass
|
|
693
|
+
|
|
694
|
+
def _page_pdf_via_cdp() -> dict[str, Any] | None:
|
|
695
|
+
"""Open the page's own PDF link in the browser and read the body via CDP."""
|
|
696
|
+
href, state = _wait_for_pdf_link(page, doi, publisher)
|
|
697
|
+
if not href:
|
|
698
|
+
try: # publishers' own link patterns the generic finder misses
|
|
699
|
+
href = page.evaluate(_PUBLISHER_PDF_LINK_JS, doi) or None
|
|
700
|
+
except Exception:
|
|
701
|
+
href = None
|
|
702
|
+
if not href:
|
|
703
|
+
return None
|
|
704
|
+
log.info(f" [{publisher}] page PDF link: {href[:80]}")
|
|
705
|
+
try: # as the headless path says it (2026-10-19: this one stayed on "opening the window")
|
|
706
|
+
from ...download import progress as _pr
|
|
707
|
+
_pr.item_stage(doi, f"在 {publisher} 找到 PDF 链接,正在打开" if publisher not in ("", "Unknown")
|
|
708
|
+
else "在出版商官网找到 PDF 链接,正在打开")
|
|
709
|
+
except Exception:
|
|
710
|
+
pass
|
|
711
|
+
body = capture_pdf_via_cdp(context, page, href)
|
|
712
|
+
if body:
|
|
713
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
714
|
+
output_path.write_bytes(body)
|
|
715
|
+
if is_pdf_file(output_path):
|
|
716
|
+
return success(doi, output_path, f"{publisher}(Visible)")
|
|
717
|
+
return None
|
|
718
|
+
|
|
719
|
+
# Check if already on auth page (previous cookies caused redirect)
|
|
720
|
+
already_on_auth = any(x in (title or "") for x in _AUTH_TITLES) or \
|
|
721
|
+
any(x in (url or "").lower() for x in _AUTH_KEYWORDS)
|
|
722
|
+
|
|
723
|
+
# Try fetching PDF first — if it works, no login needed
|
|
724
|
+
pdf_fetched = False
|
|
725
|
+
if not already_on_auth:
|
|
726
|
+
result = _page_pdf_via_cdp()
|
|
727
|
+
if result:
|
|
728
|
+
return result
|
|
729
|
+
log.info(f" [{publisher}] trying direct PDF fetch...")
|
|
730
|
+
fetch_result = _try_browser_fetch_pdf(page, pdf_paths)
|
|
731
|
+
if fetch_result:
|
|
732
|
+
pdf_bytes = fetch_result
|
|
733
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
734
|
+
output_path.write_bytes(pdf_bytes)
|
|
735
|
+
log.info(f" [{publisher}] downloaded {len(pdf_bytes)} bytes via direct fetch")
|
|
736
|
+
if is_pdf_file(output_path):
|
|
737
|
+
return success(doi, output_path, f"{publisher}(Visible)")
|
|
738
|
+
pdf_fetched = True
|
|
739
|
+
else:
|
|
740
|
+
log.info(f" [{publisher}] direct PDF fetch failed, need SSO login")
|
|
741
|
+
|
|
742
|
+
if not idp_name and not pdf_fetched:
|
|
743
|
+
# No school set: nobody to log in as. Say why there is no PDF.
|
|
744
|
+
log.info(f" [{publisher}] no PDF and no school set (carsi_idp_name): not logging in")
|
|
745
|
+
if _page_sells_paper(page) or already_on_auth:
|
|
746
|
+
_set_error("paywall", "login_required")
|
|
747
|
+
elif not get_last_error()[0]:
|
|
748
|
+
_set_error("no_pdf_found", "try_other_source")
|
|
749
|
+
return None
|
|
750
|
+
|
|
751
|
+
# SSO login needed
|
|
752
|
+
if not already_on_auth and not pdf_fetched:
|
|
753
|
+
log.info(f" [{publisher}] starting SSO login...")
|
|
754
|
+
page.evaluate(sso_link_js)
|
|
755
|
+
time.sleep(8)
|
|
756
|
+
if not any(any(x in (pg.url or "").lower() for x in _AUTH_KEYWORDS) for pg in context.pages):
|
|
757
|
+
# The click went nowhere (e.g. Springer's "Log in via an
|
|
758
|
+
# institution" is a JS widget): follow the page's SSO/WAYF link.
|
|
759
|
+
try:
|
|
760
|
+
sso_href = page.evaluate(_SSO_LINK_FINDER_JS)
|
|
761
|
+
if sso_href:
|
|
762
|
+
log.info(f" [{publisher}] opening SSO link: {sso_href[:80]}")
|
|
763
|
+
page.goto(sso_href, wait_until="domcontentloaded", timeout=30000)
|
|
764
|
+
time.sleep(5)
|
|
765
|
+
except Exception as e:
|
|
766
|
+
log.info(f" [{publisher}] SSO link fallback: {e}")
|
|
767
|
+
|
|
768
|
+
# SSO click may have navigated the page or opened a popup
|
|
769
|
+
# Check all pages in the context for the SSO/IDP page
|
|
770
|
+
sso_page = page
|
|
771
|
+
for ctx_page in context.pages:
|
|
772
|
+
try:
|
|
773
|
+
p_url = ctx_page.url
|
|
774
|
+
p_title = ctx_page.title()
|
|
775
|
+
if any(x in p_url.lower() for x in _AUTH_KEYWORDS) or \
|
|
776
|
+
any(x in p_title for x in _AUTH_TITLES):
|
|
777
|
+
sso_page = ctx_page
|
|
778
|
+
log.info(f" [{publisher}] SSO detected on tab: '{p_title[:30]}' {p_url[:60]}")
|
|
779
|
+
break
|
|
780
|
+
except Exception:
|
|
781
|
+
pass
|
|
782
|
+
|
|
783
|
+
log.info(f" [{publisher}] after SSO click: '{sso_page.title()[:30] if sso_page else '?'}' {sso_page.url[:60] if sso_page else '?'}")
|
|
784
|
+
|
|
785
|
+
# Search for institution on the SSO page. Pick as soon as the
|
|
786
|
+
# results show (2026-10-19, Springer's WAYF: a fixed 3 s per
|
|
787
|
+
# selector added up to 31 s before the school was picked).
|
|
788
|
+
t_pick = time.time()
|
|
789
|
+
for sel in search_selectors:
|
|
790
|
+
si = sso_page.query_selector(sel)
|
|
791
|
+
if not si:
|
|
792
|
+
continue
|
|
793
|
+
# fill() first waits for the box to take input, by default up
|
|
794
|
+
# to 30 s, outside the polling below (2026-10-19: Springer's
|
|
795
|
+
# WAYF still took 30.5 s after the polling fix).
|
|
796
|
+
# Not ready in 5 s (covered by a banner, still animating): type
|
|
797
|
+
# into it anyway, which skips those checks.
|
|
798
|
+
try:
|
|
799
|
+
si.fill(idp_en, timeout=5000)
|
|
800
|
+
except Exception:
|
|
801
|
+
try:
|
|
802
|
+
si.fill(idp_en, force=True, timeout=3000)
|
|
803
|
+
log.info(f" [{publisher}] institution search via {sel!r}: box not ready, typed in anyway")
|
|
804
|
+
except Exception as exc:
|
|
805
|
+
log.info(f" [{publisher}] institution search via {sel!r}: cannot type "
|
|
806
|
+
f"({type(exc).__name__}) after {time.time() - t_pick:.1f}s")
|
|
807
|
+
continue
|
|
808
|
+
clicked = False
|
|
809
|
+
for _ in range(16): # up to 8 s for the results to appear
|
|
810
|
+
time.sleep(0.5)
|
|
811
|
+
try:
|
|
812
|
+
clicked = sso_page.evaluate(_PICK_INSTITUTION_JS, idp_en)
|
|
813
|
+
except Exception:
|
|
814
|
+
clicked = False
|
|
815
|
+
if clicked:
|
|
816
|
+
break
|
|
817
|
+
log.info(f" [{publisher}] institution search via {sel!r}: "
|
|
818
|
+
f"{'picked' if clicked else 'no match'} after {time.time() - t_pick:.1f}s")
|
|
819
|
+
if clicked:
|
|
820
|
+
log.info(f" [{publisher}] selected institution '{idp_en}'")
|
|
821
|
+
time.sleep(5)
|
|
822
|
+
break
|
|
823
|
+
|
|
824
|
+
# Wait for user to complete CAS login
|
|
825
|
+
print(f"\n 请在浏览器中完成机构登录 ({idp_name})")
|
|
826
|
+
print(" 登录成功后会自动下载 PDF\n")
|
|
827
|
+
|
|
828
|
+
login_ok = False
|
|
829
|
+
# "No tab is on a login page" only means success once one was: if the
|
|
830
|
+
# SSO click never left the article, it is not a login.
|
|
831
|
+
seen_auth = already_on_auth
|
|
832
|
+
for i in range(100):
|
|
833
|
+
# A human check on the way to the login (2026-10-18: Wiley's
|
|
834
|
+
# Cloudflare page after the SSO click) is the user's to pass: say
|
|
835
|
+
# "verify", not "login", and keep waiting while it is shown.
|
|
836
|
+
challenge = False
|
|
837
|
+
for ctx_page in context.pages:
|
|
838
|
+
try:
|
|
839
|
+
if any(s in (ctx_page.title() or "").lower() for s in _CHALLENGE_TITLES):
|
|
840
|
+
challenge = True
|
|
841
|
+
break
|
|
842
|
+
except Exception:
|
|
843
|
+
pass
|
|
844
|
+
_user_waiting("verify" if challenge else "login", None if challenge else context)
|
|
845
|
+
time.sleep(3)
|
|
846
|
+
if not seen_auth and i >= 10 and not challenge:
|
|
847
|
+
break # ~30 s and the login page never appeared
|
|
848
|
+
# Check ALL pages in context — SSO may happen in any tab
|
|
849
|
+
any_auth = False
|
|
850
|
+
for ctx_page in context.pages:
|
|
851
|
+
try:
|
|
852
|
+
p_title = ctx_page.title()
|
|
853
|
+
p_url = ctx_page.url
|
|
854
|
+
is_auth = any(x in p_title for x in _AUTH_TITLES)
|
|
855
|
+
is_auth_url = any(x in p_url.lower() for x in _AUTH_KEYWORDS)
|
|
856
|
+
if is_auth or is_auth_url:
|
|
857
|
+
any_auth = True
|
|
858
|
+
break
|
|
859
|
+
except Exception:
|
|
860
|
+
pass
|
|
861
|
+
if any_auth:
|
|
862
|
+
seen_auth = True
|
|
863
|
+
elif seen_auth:
|
|
864
|
+
login_ok = True
|
|
865
|
+
break
|
|
866
|
+
|
|
867
|
+
if not login_ok and seen_auth:
|
|
868
|
+
log.info(f" [{publisher}] visible login timed out")
|
|
869
|
+
_set_error("login_abandoned", "retry_login")
|
|
870
|
+
return None
|
|
871
|
+
if not login_ok:
|
|
872
|
+
# The SSO entry led back to the article, not to a login: the school's
|
|
873
|
+
# exit may need none (2026-10-17, Wiley after a late Cloudflare
|
|
874
|
+
# check). Try the article page as it is rather than giving up.
|
|
875
|
+
log.info(f" [{publisher}] SSO entry did not reach a login page: trying the article page as it is")
|
|
876
|
+
else:
|
|
877
|
+
log.info(f" [{publisher}] login successful, downloading PDF...")
|
|
878
|
+
time.sleep(3)
|
|
879
|
+
|
|
880
|
+
# Save cookies for future use (all formats + bridge to browser-engine)
|
|
881
|
+
try:
|
|
882
|
+
cookies = context.cookies()
|
|
883
|
+
_save_all_cookie_formats(cookies, publisher, config)
|
|
884
|
+
except Exception as e:
|
|
885
|
+
log.info(f" [{publisher}] cookie save note: {e}")
|
|
886
|
+
|
|
887
|
+
# Navigate back to the article page and reload to pick up auth cookies
|
|
888
|
+
log.info(f" [{publisher}] reloading article page with new auth cookies...")
|
|
889
|
+
try:
|
|
890
|
+
page.goto(article_url, wait_until="domcontentloaded", timeout=30000)
|
|
891
|
+
time.sleep(5)
|
|
892
|
+
except Exception:
|
|
893
|
+
pass
|
|
894
|
+
wait_url_stable(page)
|
|
895
|
+
wait_page_settled(page)
|
|
896
|
+
result = _page_pdf_via_cdp()
|
|
897
|
+
if result:
|
|
898
|
+
return result
|
|
899
|
+
|
|
900
|
+
# Try downloading PDF via in-browser fetch (post-login)
|
|
901
|
+
fetch_result = _try_browser_fetch_pdf(page, pdf_paths)
|
|
902
|
+
if fetch_result:
|
|
903
|
+
pdf_bytes = fetch_result
|
|
904
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
905
|
+
output_path.write_bytes(pdf_bytes)
|
|
906
|
+
log.info(f" [{publisher}] downloaded {len(pdf_bytes)} bytes via post-login fetch")
|
|
907
|
+
if is_pdf_file(output_path):
|
|
908
|
+
return success(doi, output_path, f"{publisher}(Visible)")
|
|
909
|
+
|
|
910
|
+
# Fallback: look for PDF link on the page and fetch it
|
|
911
|
+
log.info(f" [{publisher}] fetch failed, trying page PDF links...")
|
|
912
|
+
pdf_link = page.evaluate("""
|
|
913
|
+
(() => {
|
|
914
|
+
for (const a of document.querySelectorAll('a')) {
|
|
915
|
+
const href = (a.href || '').toLowerCase();
|
|
916
|
+
if (href.includes('pdfdirect') || href.includes('/pdf/') || href.includes('.pdf')) {
|
|
917
|
+
if (!href.includes('supplement') && !href.includes('supporting')) return a.href;
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
for (const el of document.querySelectorAll('iframe, embed, object')) {
|
|
921
|
+
const src = el.src || el.data || '';
|
|
922
|
+
if (src.includes('.pdf') || src.includes('pdfdirect')) return src;
|
|
923
|
+
}
|
|
924
|
+
return null;
|
|
925
|
+
})()
|
|
926
|
+
""")
|
|
927
|
+
|
|
928
|
+
if pdf_link and isinstance(pdf_link, str):
|
|
929
|
+
log.info(f" [{publisher}] found PDF link: {pdf_link[:60]}")
|
|
930
|
+
try:
|
|
931
|
+
page.goto(pdf_link, wait_until="domcontentloaded", timeout=30000)
|
|
932
|
+
time.sleep(3)
|
|
933
|
+
except Exception:
|
|
934
|
+
pass
|
|
935
|
+
|
|
936
|
+
# Try fetch from current page context
|
|
937
|
+
current_url = page.url
|
|
938
|
+
from urllib.parse import urlparse
|
|
939
|
+
parsed = urlparse(current_url)
|
|
940
|
+
# Try absolute fetch
|
|
941
|
+
abs_paths = [parsed.path]
|
|
942
|
+
fetch_result2 = _try_browser_fetch_pdf(page, abs_paths)
|
|
943
|
+
if fetch_result2:
|
|
944
|
+
pdf_bytes = fetch_result2
|
|
945
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
946
|
+
output_path.write_bytes(pdf_bytes)
|
|
947
|
+
log.info(f" [{publisher}] downloaded {len(pdf_bytes)} bytes via PDF link")
|
|
948
|
+
if is_pdf_file(output_path):
|
|
949
|
+
return success(doi, output_path, f"{publisher}(Visible)")
|
|
950
|
+
|
|
951
|
+
log.info(f" [{publisher}] visible browser download failed")
|
|
952
|
+
if login_ok and _page_sells_paper(page):
|
|
953
|
+
# Logged in as the school and the page still sells it: not subscribed.
|
|
954
|
+
note_no_access(doi)
|
|
955
|
+
_set_error("not_entitled", "no_subscription")
|
|
956
|
+
elif login_ok:
|
|
957
|
+
# Logged in, no PDF, no sign of a missing subscription: the school
|
|
958
|
+
# routes may still get it, so this says nothing about entitlement.
|
|
959
|
+
_set_error("no_pdf_found", "try_other_source")
|
|
960
|
+
return None
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
def _try_browser_fetch_pdf(page: Any, paths: list[str]) -> bytes | None:
|
|
964
|
+
"""Try fetching PDF via browser JS fetch for given paths. Returns PDF bytes or None."""
|
|
965
|
+
import base64
|
|
966
|
+
paths_js = json.dumps(paths)
|
|
967
|
+
pdf_b64 = page.evaluate(f"""
|
|
968
|
+
(async () => {{
|
|
969
|
+
const paths = {paths_js};
|
|
970
|
+
for (const p of paths) {{
|
|
971
|
+
try {{
|
|
972
|
+
const resp = await fetch(p, {{credentials: 'include', headers: {{'Accept': 'application/pdf,*/*'}}}});
|
|
973
|
+
if (!resp.ok) return 'status:' + resp.status;
|
|
974
|
+
const ct = resp.headers.get('content-type') || '';
|
|
975
|
+
if (!ct.includes('pdf') && !ct.includes('octet')) return 'ct:' + ct;
|
|
976
|
+
const blob = await resp.blob();
|
|
977
|
+
return await new Promise((resolve) => {{
|
|
978
|
+
const reader = new FileReader();
|
|
979
|
+
reader.onload = () => resolve(reader.result);
|
|
980
|
+
reader.readAsDataURL(blob);
|
|
981
|
+
}});
|
|
982
|
+
}} catch(e) {{ return 'error:' + e.message; }}
|
|
983
|
+
}}
|
|
984
|
+
return null;
|
|
985
|
+
}})()
|
|
986
|
+
""")
|
|
987
|
+
|
|
988
|
+
if isinstance(pdf_b64, str) and pdf_b64.startswith("data:"):
|
|
989
|
+
_, data = pdf_b64.split(",", 1)
|
|
990
|
+
pdf_bytes = base64.b64decode(data)
|
|
991
|
+
if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
992
|
+
return pdf_bytes
|
|
993
|
+
return None
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
# Sites whose bot wall answers plain HTTP clients with 403 whatever cookies
|
|
997
|
+
# they carry (the clearance is bound to the browser fingerprint). The direct
|
|
998
|
+
# HTTP try only delayed the browser: pubs.acs.org takes ~12.6 s to say 403.
|
|
999
|
+
# Publishers whose PDF URLs never answer a plain HTTP request (Cloudflare and
|
|
1000
|
+
# the like): skip straight to the browser. SAGE added 2026-10-19: two direct
|
|
1001
|
+
# tries at journals.sagepub.com hung 20.8 + 31.6 s before failing, every paper.
|
|
1002
|
+
_HTTP_DIRECT_BLOCKED = frozenset({"ACS", "Wiley", "RSC", "IOP", "SAGE"})
|
|
1003
|
+
|
|
1004
|
+
# Hosts whose direct PDF request timed out: skipped for _STALLED_HOST_SECONDS
|
|
1005
|
+
# (自定值), so a site that hangs costs one timeout per session, not one per paper.
|
|
1006
|
+
_STALLED_HOSTS: dict[str, float] = {}
|
|
1007
|
+
_STALLED_HOST_SECONDS = 1800
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def _direct_pdf_urls(doi: str, publisher: str) -> list[str]:
|
|
1011
|
+
"""Build direct PDF URL candidates for a publisher (deduplicated)."""
|
|
1012
|
+
if publisher in _HTTP_DIRECT_BLOCKED:
|
|
1013
|
+
return []
|
|
1014
|
+
return list(dict.fromkeys(_direct_pdf_url_candidates(doi, publisher)))
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
def _direct_pdf_url_candidates(doi: str, publisher: str) -> list[str]:
|
|
1018
|
+
urls: list[str] = []
|
|
1019
|
+
|
|
1020
|
+
if publisher == "Elsevier":
|
|
1021
|
+
# ScienceDirect doesn't have a simple direct PDF URL pattern
|
|
1022
|
+
# Need to go through the article page
|
|
1023
|
+
pass
|
|
1024
|
+
|
|
1025
|
+
elif publisher == "Wiley":
|
|
1026
|
+
# PDFDirect is the fastest path
|
|
1027
|
+
urls.append(f"https://onlinelibrary.wiley.com/doi/pdfdirect/{doi}")
|
|
1028
|
+
urls.append(f"https://onlinelibrary.wiley.com/doi/pdf/{doi}")
|
|
1029
|
+
|
|
1030
|
+
elif publisher == "IEEE":
|
|
1031
|
+
# IEEE stamp URL — classic redirect-based PDF
|
|
1032
|
+
# Need to resolve DOI first to get the article number
|
|
1033
|
+
doi_suffix = doi.split("/", 1)[1] if "/" in doi else doi
|
|
1034
|
+
urls.append(f"https://ieeexplore.ieee.org/stampPDF/getPDF.jsp?tp=&arnumber={doi_suffix}")
|
|
1035
|
+
|
|
1036
|
+
elif publisher == "ACS":
|
|
1037
|
+
urls.append(f"https://pubs.acs.org/doi/pdf/{doi}")
|
|
1038
|
+
urls.append(f"https://pubs.acs.org/doi/pdf/10.1021/{doi.split('10.1021/')[-1]}")
|
|
1039
|
+
|
|
1040
|
+
elif publisher == "RSC":
|
|
1041
|
+
urls.append(f"https://pubs.rsc.org/en/content/articlepdf/{doi}")
|
|
1042
|
+
|
|
1043
|
+
elif publisher == "AIP":
|
|
1044
|
+
# AIP has a loading page, direct URL may not work
|
|
1045
|
+
urls.append(f"https://pubs.aip.org/aip/apl/article-pdf/{doi}")
|
|
1046
|
+
|
|
1047
|
+
elif publisher == "Springer":
|
|
1048
|
+
urls.append(f"https://link.springer.com/content/pdf/{doi}.pdf")
|
|
1049
|
+
|
|
1050
|
+
elif publisher == "APS":
|
|
1051
|
+
# Physical Review journals
|
|
1052
|
+
doi_suffix = doi.split("10.1103/")[-1] if "10.1103/" in doi else doi
|
|
1053
|
+
urls.append(f"https://journals.aps.org/prl/pdf/{doi}")
|
|
1054
|
+
urls.append(f"https://journals.aps.org/prb/pdf/{doi}")
|
|
1055
|
+
urls.append(f"https://journals.aps.org/pdf/{doi}")
|
|
1056
|
+
|
|
1057
|
+
elif publisher == "Tandfonline":
|
|
1058
|
+
urls.append(f"https://www.tandfonline.com/doi/pdf/{doi}")
|
|
1059
|
+
|
|
1060
|
+
elif publisher == "OSA":
|
|
1061
|
+
urls.append(f"https://opg.optica.org/viewmedia.cfm?r=1&uri={doi}")
|
|
1062
|
+
|
|
1063
|
+
elif publisher == "IOP":
|
|
1064
|
+
urls.append(f"https://iopscience.iop.org/article/{doi}/pdf")
|
|
1065
|
+
|
|
1066
|
+
elif publisher == "Oxford":
|
|
1067
|
+
urls.append(f"https://academic.oup.com/downloadpdf/{doi}")
|
|
1068
|
+
|
|
1069
|
+
elif publisher == "ACM":
|
|
1070
|
+
urls.append(f"https://dl.acm.org/doi/pdf/{doi}")
|
|
1071
|
+
|
|
1072
|
+
elif publisher == "SAGE":
|
|
1073
|
+
# DOI-resolved host may be journals.sagepub.com or sage.cnpereading.com
|
|
1074
|
+
urls.append(f"https://journals.sagepub.com/doi/pdf/{doi}?download=true")
|
|
1075
|
+
urls.append(f"https://journals.sagepub.com/doi/pdf/{doi}")
|
|
1076
|
+
|
|
1077
|
+
elif publisher == "ASCE":
|
|
1078
|
+
doi_upper = doi.upper()
|
|
1079
|
+
urls.append(f"https://ascelibrary.org/doi/pdf/{doi_upper}?download=true")
|
|
1080
|
+
urls.append(f"https://ascelibrary.org/doi/pdf/{doi_upper}")
|
|
1081
|
+
urls.append(f"https://ascelibrary.org/doi/pdf/{doi}?download=true")
|
|
1082
|
+
|
|
1083
|
+
elif publisher == "Royal Society":
|
|
1084
|
+
urls.append(f"https://royalsocietypublishing.org/doi/pdf/{doi}")
|
|
1085
|
+
urls.append(f"https://royalsocietypublishing.org/doi/epdf/{doi}")
|
|
1086
|
+
|
|
1087
|
+
elif publisher == "Copernicus":
|
|
1088
|
+
# Copernicus uses journal-specific subdomains; resolve DOI first
|
|
1089
|
+
# Typical DOI: 10.5194/hess-22-3433-2018
|
|
1090
|
+
urls.append(f"https://www.copernicus.org/publications/{doi}")
|
|
1091
|
+
urls.append(f"https://www.copernicus.org/articles/{doi}")
|
|
1092
|
+
|
|
1093
|
+
return urls
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
# ============================================================
|
|
1097
|
+
# Publisher-specific article page selectors
|
|
1098
|
+
# ============================================================
|
|
1099
|
+
|
|
1100
|
+
# CSS selectors to find PDF links on article landing pages
|
|
1101
|
+
_PDF_LINK_SELECTORS: dict[str, list[str]] = {
|
|
1102
|
+
"Elsevier": [
|
|
1103
|
+
'a[aria-label*="View PDF" i]',
|
|
1104
|
+
'a[aria-label*="pdf" i]',
|
|
1105
|
+
'a.pdf-download-btn-link',
|
|
1106
|
+
'a[data-aa-name="download-pdf"]',
|
|
1107
|
+
'a[href*="/pdf/"]',
|
|
1108
|
+
'a[href*="pdfft"]',
|
|
1109
|
+
'a[href*=".pdf"]',
|
|
1110
|
+
'a[href*="showPdf"]',
|
|
1111
|
+
'a[href*="pdfExtended"]',
|
|
1112
|
+
],
|
|
1113
|
+
"Wiley": [
|
|
1114
|
+
'a[href*="pdfdirect"]',
|
|
1115
|
+
'a[href*="/doi/pdf/"]',
|
|
1116
|
+
'a[href*="/doi/epdf/"]',
|
|
1117
|
+
'a.c-pdf-download__link',
|
|
1118
|
+
'a[title*="PDF" i]',
|
|
1119
|
+
],
|
|
1120
|
+
"IEEE": [
|
|
1121
|
+
'a[href*="/stamp/"]',
|
|
1122
|
+
'a[href*="getPDF"]',
|
|
1123
|
+
'a[title*="PDF" i]',
|
|
1124
|
+
'a.xpl-pdf-btn',
|
|
1125
|
+
],
|
|
1126
|
+
"ACS": [
|
|
1127
|
+
'a[href*="/doi/pdf/"]',
|
|
1128
|
+
'a[title*="Download PDF" i]',
|
|
1129
|
+
'a.nav-item-link-download',
|
|
1130
|
+
],
|
|
1131
|
+
"RSC": [
|
|
1132
|
+
'a[href*="/articlepdf/"]',
|
|
1133
|
+
'a[title*="PDF" i]',
|
|
1134
|
+
],
|
|
1135
|
+
"AIP": [
|
|
1136
|
+
'a[href*="/article-pdf/"]',
|
|
1137
|
+
'a[title*="PDF" i]',
|
|
1138
|
+
'a[href*=".pdf"]',
|
|
1139
|
+
],
|
|
1140
|
+
"Springer": [
|
|
1141
|
+
'a[href*="/content/pdf/"]',
|
|
1142
|
+
'a[data-track-action="download pdf"]',
|
|
1143
|
+
'a.c-pdf-download__link',
|
|
1144
|
+
],
|
|
1145
|
+
"Nature": [
|
|
1146
|
+
'a.c-pdf-download__link',
|
|
1147
|
+
'a[data-track-action="download pdf"]',
|
|
1148
|
+
'a[href*=".pdf"]',
|
|
1149
|
+
],
|
|
1150
|
+
"Science": [
|
|
1151
|
+
'a[href*="/doi/pdf/"]',
|
|
1152
|
+
'a[title*="PDF" i]',
|
|
1153
|
+
'a.article-pdf',
|
|
1154
|
+
],
|
|
1155
|
+
"APS": [
|
|
1156
|
+
'a[href*="/pdf/"]',
|
|
1157
|
+
'a[title*="PDF" i]',
|
|
1158
|
+
],
|
|
1159
|
+
"Tandfonline": [
|
|
1160
|
+
'a[href*="/doi/pdf/"]',
|
|
1161
|
+
'a[title*="PDF" i]',
|
|
1162
|
+
],
|
|
1163
|
+
"IOP": [
|
|
1164
|
+
'a[href*="/article/"][href$="/pdf"]',
|
|
1165
|
+
'a[title*="PDF" i]',
|
|
1166
|
+
],
|
|
1167
|
+
"Oxford": [
|
|
1168
|
+
'a[href*="/downloadpdf/"]',
|
|
1169
|
+
'a[title*="PDF" i]',
|
|
1170
|
+
],
|
|
1171
|
+
"ACM": [
|
|
1172
|
+
'a[href*="/doi/pdf/"]',
|
|
1173
|
+
'a[title*="PDF" i]',
|
|
1174
|
+
],
|
|
1175
|
+
"SAGE": [
|
|
1176
|
+
'a[href*="/doi/pdf/"]',
|
|
1177
|
+
'a[title*="PDF" i]',
|
|
1178
|
+
'a[href*=".pdf"]',
|
|
1179
|
+
],
|
|
1180
|
+
"ASCE": [
|
|
1181
|
+
'a[href*="/doi/pdf/"]',
|
|
1182
|
+
'a[title*="PDF" i]',
|
|
1183
|
+
'a[href*="download=true"]',
|
|
1184
|
+
],
|
|
1185
|
+
"Royal Society": [
|
|
1186
|
+
'a[href*="/doi/pdf/"]',
|
|
1187
|
+
'a[href*="/doi/epdf/"]',
|
|
1188
|
+
'a[title*="PDF" i]',
|
|
1189
|
+
],
|
|
1190
|
+
"Copernicus": [
|
|
1191
|
+
'a[href*="/pdf"]',
|
|
1192
|
+
'a[href*=".pdf"]',
|
|
1193
|
+
'a[title*="PDF" i]',
|
|
1194
|
+
'a[class*="pdf"]',
|
|
1195
|
+
],
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
# ============================================================
|
|
1199
|
+
# Anti-bot challenge detection
|
|
1200
|
+
# ============================================================
|
|
1201
|
+
|
|
1202
|
+
# Auth detection keywords for institutional login flows
|
|
1203
|
+
_AUTH_KEYWORDS = ("cas", "idp", "saml", "wayf", "sso", "passport", "accounts", "oauth", "/login", "/signin")
|
|
1204
|
+
_AUTH_TITLES = ("登录", "身份", "二次认证", "Login", "Sign in", "Log in")
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
def _school_auth_patterns(config: dict[str, Any]) -> tuple[str, ...]:
|
|
1208
|
+
"""Return school-specific URL auth patterns derived from carsi_idp_name.
|
|
1209
|
+
|
|
1210
|
+
Uses _IDP_MAP to translate the configured Chinese institution name to
|
|
1211
|
+
English, then lowercases it so the caller can match it against IDP URLs
|
|
1212
|
+
(e.g. 'tsinghua' matches 'id.tsinghua.edu.cn').
|
|
1213
|
+
|
|
1214
|
+
For universities not in _IDP_MAP, falls back to pinyin conversion
|
|
1215
|
+
so that e.g. "兰州大学" → "lanzhou" can match "lzu.edu.cn" IDP URLs.
|
|
1216
|
+
"""
|
|
1217
|
+
name = (config.get("carsi_idp_name", "") or "").strip()
|
|
1218
|
+
if not name:
|
|
1219
|
+
return ()
|
|
1220
|
+
en = _IDP_MAP.get(name, "").lower()
|
|
1221
|
+
if not en:
|
|
1222
|
+
# Dynamic fallback: try pinyin conversion for Chinese names
|
|
1223
|
+
en = _chinese_to_pinyin_token(name).lower()
|
|
1224
|
+
if not en:
|
|
1225
|
+
return ()
|
|
1226
|
+
# Split multi-word names into individual tokens for broad matching
|
|
1227
|
+
tokens = tuple(t for t in en.replace("-", " ").split() if len(t) > 2)
|
|
1228
|
+
return tokens
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
def _chinese_to_pinyin_token(name: str) -> str:
|
|
1232
|
+
"""Convert a Chinese university name to a pinyin token for URL matching.
|
|
1233
|
+
|
|
1234
|
+
E.g. "兰州大学" → "lanzhou", "哈尔滨工业大学" → "haerbingongye".
|
|
1235
|
+
Falls back to empty string if pypinyin is not installed.
|
|
1236
|
+
"""
|
|
1237
|
+
try:
|
|
1238
|
+
from pypinyin import lazy_pinyin
|
|
1239
|
+
return "".join(lazy_pinyin(name))
|
|
1240
|
+
except ImportError:
|
|
1241
|
+
pass
|
|
1242
|
+
# Strip common suffixes and return what we have
|
|
1243
|
+
for suffix in ("大学", "学院", "研究院"):
|
|
1244
|
+
name = name.replace(suffix, "")
|
|
1245
|
+
return name
|
|
1246
|
+
|
|
1247
|
+
# Chinese → English university name mapping for institution search (CARSI/OpenAthens WAYF).
|
|
1248
|
+
# This is a lookup table used by _translate_institution_name(). Order is not significant —
|
|
1249
|
+
# the tool does NOT default to any school; the user must configure their own institution.
|
|
1250
|
+
_IDP_MAP: dict[str, str] = {
|
|
1251
|
+
"清华大学": "Tsinghua", "北京大学": "Peking", "浙江大学": "Zhejiang",
|
|
1252
|
+
"复旦大学": "Fudan", "上海交通大学": "Shanghai Jiao Tong",
|
|
1253
|
+
"南京大学": "Nanjing", "中国科学技术大学": "USTC",
|
|
1254
|
+
"武汉大学": "Wuhan", "中山大学": "Sun Yat-sen",
|
|
1255
|
+
"华中科技大学": "Huazhong", "哈尔滨工业大学": "Harbin Institute",
|
|
1256
|
+
"西安交通大学": "Xi'an Jiaotong", "同济大学": "Tongji",
|
|
1257
|
+
"东南大学": "Southeast", "厦门大学": "Xiamen",
|
|
1258
|
+
"北京师范大学": "Beijing Normal", "南开大学": "Nankai",
|
|
1259
|
+
"四川大学": "Sichuan", "天津大学": "Tianjin",
|
|
1260
|
+
"中国人民大学": "Renmin", "北京航空航天大学": "Beihang",
|
|
1261
|
+
"山东大学": "Shandong", "吉林大学": "Jilin",
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
# Institution search input selectors, tried in order on WAYF pages
|
|
1265
|
+
_INSTITUTION_SEARCH_SELECTORS: list[str] = [
|
|
1266
|
+
"#searchInstitution", "#bdd-email", 'input[name="institution"]',
|
|
1267
|
+
"#institution-search", 'input[placeholder*="institution"]',
|
|
1268
|
+
'input[placeholder*="University"]',
|
|
1269
|
+
# Generic fallbacks (e.g. wayf.springernature.com renders an unlabelled
|
|
1270
|
+
# text box under "Find your institution").
|
|
1271
|
+
'input[id*="institution" i]', 'input[name*="institution" i]',
|
|
1272
|
+
'input[aria-label*="institution" i]', 'input[placeholder*="institution" i]',
|
|
1273
|
+
'input[type="search"]', 'main input[type="text"]', 'form input[type="text"]',
|
|
1274
|
+
]
|
|
1275
|
+
|
|
1276
|
+
# Shared JS snippets for SSO/institutional login interaction.
|
|
1277
|
+
# These are Playwright-style (parametrized) — callers using string interpolation
|
|
1278
|
+
# (e.g. evaluate_js) must wrap them in an IIFE and inject params via json.dumps.
|
|
1279
|
+
|
|
1280
|
+
# Find and click SSO/institutional login link on article page
|
|
1281
|
+
_SSO_LINK_FINDER_JS: str = (
|
|
1282
|
+
"() => {"
|
|
1283
|
+
" const links = [...document.querySelectorAll('a')];"
|
|
1284
|
+
" const sso = links.find(a => a.href &&"
|
|
1285
|
+
" (a.href.includes('ssostart') || a.href.includes('shibboleth')"
|
|
1286
|
+
" || a.href.includes('saml') || a.href.includes('institutional-login')"
|
|
1287
|
+
" || a.href.includes('federation') || a.href.includes('/action/showLogin')"
|
|
1288
|
+
" || a.href.includes('/institutional-access') || a.href.includes('wayf')));"
|
|
1289
|
+
" if (sso) { return sso.href; }"
|
|
1290
|
+
" return false;"
|
|
1291
|
+
"}"
|
|
1292
|
+
)
|
|
1293
|
+
|
|
1294
|
+
# Click institution result that matches the given name (shortest match, see
|
|
1295
|
+
# _PICK_INSTITUTION_JS: the first match used to be a container or heading)
|
|
1296
|
+
_INSTITUTION_CLICK_JS: str = _PICK_INSTITUTION_JS
|
|
1297
|
+
|
|
1298
|
+
# Publisher-specific SSO configuration for visible browser download
|
|
1299
|
+
# Each entry: sso_link_js (JS to click institutional login), search_selectors, pdf_paths(doi)
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
def _make_sso_click_js(*href_patterns: str, text_patterns: tuple[str, ...] = ()) -> str:
|
|
1303
|
+
"""Generate a JS IIFE that clicks the first <a> matching the given patterns.
|
|
1304
|
+
|
|
1305
|
+
Args:
|
|
1306
|
+
href_patterns: Substrings to match against a.href (lowercased).
|
|
1307
|
+
text_patterns: Substrings to match against a.innerText (lowercased).
|
|
1308
|
+
"""
|
|
1309
|
+
conditions = " || ".join(f"href.includes('{p}')" for p in href_patterns)
|
|
1310
|
+
if text_patterns:
|
|
1311
|
+
text_conds = " || ".join(f"text.includes('{p}')" for p in text_patterns)
|
|
1312
|
+
conditions = f"({conditions}) || ({text_conds})"
|
|
1313
|
+
return f"""
|
|
1314
|
+
(() => {{
|
|
1315
|
+
const a = [...document.querySelectorAll('a')].find(a => {{
|
|
1316
|
+
const href = (a.href || '').toLowerCase();
|
|
1317
|
+
const text = (a.innerText || '').toLowerCase();
|
|
1318
|
+
return {conditions};
|
|
1319
|
+
}});
|
|
1320
|
+
if (a) a.click();
|
|
1321
|
+
}})()
|
|
1322
|
+
"""
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _elsevier_pdf_paths(doi: str) -> list[str]:
|
|
1326
|
+
"""Build Elsevier/Cell Press PDF fetch paths from DOI."""
|
|
1327
|
+
paths: list[str] = []
|
|
1328
|
+
doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
|
|
1329
|
+
# Cell Press showPdf uses PII — try common Cell journal DOI patterns
|
|
1330
|
+
import re
|
|
1331
|
+
cell_url = _build_cell_press_url(doi)
|
|
1332
|
+
if cell_url and "/abstract/" in cell_url:
|
|
1333
|
+
pii = cell_url.split("/abstract/")[-1]
|
|
1334
|
+
paths.append(f"/action/showPdf?pii={pii}")
|
|
1335
|
+
paths.append(f"/science/article/pii/{pii}/pdfft")
|
|
1336
|
+
# General ScienceDirect path with DOI
|
|
1337
|
+
paths.append(f"/doi/pdfdirect/{doi}")
|
|
1338
|
+
paths.append(f"/science/article/pii/{doi_suffix.replace('j.', 'j.')}/pdfft")
|
|
1339
|
+
return paths
|
|
1340
|
+
|
|
1341
|
+
|
|
1342
|
+
_PUBLISHER_SSO_CONFIG: dict[str, dict[str, Any]] = {
|
|
1343
|
+
"Wiley": {
|
|
1344
|
+
"sso_link_js": _make_sso_click_js("ssostart"),
|
|
1345
|
+
"search_selectors": ["#searchInstitution"],
|
|
1346
|
+
"pdf_paths": lambda doi: [f"/doi/pdfdirect/{doi}", f"/doi/pdf/{doi}"],
|
|
1347
|
+
},
|
|
1348
|
+
"Elsevier": {
|
|
1349
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional",
|
|
1350
|
+
text_patterns=("access through your institution", "institutional access")),
|
|
1351
|
+
"search_selectors": ["#institution-search", "input[name='query']", "#bdd-email"],
|
|
1352
|
+
"pdf_paths": lambda doi: _elsevier_pdf_paths(doi),
|
|
1353
|
+
},
|
|
1354
|
+
"Springer": {
|
|
1355
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional-login"),
|
|
1356
|
+
"search_selectors": ["#idp-search", "input[name='idpSearch']", "#searchInstitution"],
|
|
1357
|
+
"pdf_paths": lambda doi: [f"/content/pdf/{doi}.pdf"],
|
|
1358
|
+
},
|
|
1359
|
+
"ACS": {
|
|
1360
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1361
|
+
"search_selectors": ["input[name='search']", "#searchInstitution"],
|
|
1362
|
+
"pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
|
|
1363
|
+
},
|
|
1364
|
+
"IEEE": {
|
|
1365
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional",
|
|
1366
|
+
text_patterns=("institutional sign in",)),
|
|
1367
|
+
"search_selectors": ["input[name='idpSearch']", "#searchInstitution"],
|
|
1368
|
+
"pdf_paths": lambda doi: [], # IEEE uses stamp URL, needs page-level extraction
|
|
1369
|
+
},
|
|
1370
|
+
"Tandfonline": {
|
|
1371
|
+
"sso_link_js": _make_sso_click_js("ssostart", "shibboleth", "institutional",
|
|
1372
|
+
text_patterns=("access through your institution",)),
|
|
1373
|
+
"search_selectors": [
|
|
1374
|
+
'input[placeholder*="institution"]',
|
|
1375
|
+
'input[placeholder*="Type the name"]',
|
|
1376
|
+
"#searchInstitution",
|
|
1377
|
+
"input[name='query']",
|
|
1378
|
+
],
|
|
1379
|
+
"pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
|
|
1380
|
+
},
|
|
1381
|
+
"Oxford": {
|
|
1382
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1383
|
+
"search_selectors": ["#searchInstitution", "input[name='query']"],
|
|
1384
|
+
"pdf_paths": lambda doi: [f"/downloadpdf/{doi}"],
|
|
1385
|
+
},
|
|
1386
|
+
"RSC": {
|
|
1387
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1388
|
+
"search_selectors": ["input[name='search']", "#searchInstitution"],
|
|
1389
|
+
"pdf_paths": lambda doi: [f"/content/articlepdf/{doi}"],
|
|
1390
|
+
},
|
|
1391
|
+
"IOP": {
|
|
1392
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1393
|
+
"search_selectors": ["input[name='search']", "#searchInstitution"],
|
|
1394
|
+
"pdf_paths": lambda doi: [f"/article/{doi}/pdf"],
|
|
1395
|
+
},
|
|
1396
|
+
"APS": {
|
|
1397
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1398
|
+
"search_selectors": ["input[name='search']", "#searchInstitution"],
|
|
1399
|
+
"pdf_paths": lambda doi: [f"/pdf/{doi}"],
|
|
1400
|
+
},
|
|
1401
|
+
"AIP": {
|
|
1402
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1403
|
+
"search_selectors": ["input[name='search']", "#searchInstitution"],
|
|
1404
|
+
"pdf_paths": lambda doi: [],
|
|
1405
|
+
},
|
|
1406
|
+
"Nature": {
|
|
1407
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1408
|
+
"search_selectors": ["input[name='idpSearch']", "#searchInstitution"],
|
|
1409
|
+
"pdf_paths": lambda doi: [f"/articles/{doi}.pdf"],
|
|
1410
|
+
},
|
|
1411
|
+
"Science": {
|
|
1412
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1413
|
+
"search_selectors": ["input[name='search']", "#searchInstitution"],
|
|
1414
|
+
"pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
|
|
1415
|
+
},
|
|
1416
|
+
"ACM": {
|
|
1417
|
+
"sso_link_js": _make_sso_click_js("shibboleth", "institutional"),
|
|
1418
|
+
"search_selectors": ["input[name='search']", "#searchInstitution"],
|
|
1419
|
+
"pdf_paths": lambda doi: [f"/doi/pdf/{doi}"],
|
|
1420
|
+
},
|
|
1421
|
+
"_default": {
|
|
1422
|
+
"sso_link_js": """
|
|
1423
|
+
(() => {
|
|
1424
|
+
const links = [...document.querySelectorAll('a')];
|
|
1425
|
+
for (const a of links) {
|
|
1426
|
+
const href = (a.href || '').toLowerCase();
|
|
1427
|
+
const text = (a.innerText || '').toLowerCase();
|
|
1428
|
+
if (href.includes('ssostart') || href.includes('shibboleth') ||
|
|
1429
|
+
href.includes('saml') || href.includes('institutional-login') ||
|
|
1430
|
+
href.includes('federation') ||
|
|
1431
|
+
(text.includes('institutional') && text.includes('login')) ||
|
|
1432
|
+
text.includes('access through your institution')) {
|
|
1433
|
+
a.click();
|
|
1434
|
+
return;
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
})()
|
|
1438
|
+
""",
|
|
1439
|
+
"search_selectors": ["#searchInstitution", "input[name='query']", "input[name='search']", "#institution-search"],
|
|
1440
|
+
"pdf_paths": lambda doi: [f"/doi/pdf/{doi}", f"/doi/pdfdirect/{doi}", f"/content/pdf/{doi}.pdf"],
|
|
1441
|
+
},
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
_CHALLENGE_SIGNATURES = [
|
|
1445
|
+
"cf-browser-verification",
|
|
1446
|
+
"challenge-platform",
|
|
1447
|
+
"just a moment",
|
|
1448
|
+
"attention required",
|
|
1449
|
+
"security check",
|
|
1450
|
+
"verify you are human",
|
|
1451
|
+
"access to this page has been denied",
|
|
1452
|
+
"please complete the security check",
|
|
1453
|
+
"px-captcha",
|
|
1454
|
+
"hcaptcha",
|
|
1455
|
+
"recaptcha",
|
|
1456
|
+
"radware",
|
|
1457
|
+
"request rejected",
|
|
1458
|
+
"crasolve",
|
|
1459
|
+
]
|
|
1460
|
+
|
|
1461
|
+
|
|
1462
|
+
# Challenge pages that cannot be passed in this browser at all: the check did
|
|
1463
|
+
# not even run ("extension or network configuration incompatible"). Waiting
|
|
1464
|
+
# for the user to click is pointless here: nothing to click.
|
|
1465
|
+
_CHALLENGE_DEAD_END = (
|
|
1466
|
+
"浏览器扩展或网络配置不兼容",
|
|
1467
|
+
"已阻止",
|
|
1468
|
+
"browser extensions or network settings have blocked",
|
|
1469
|
+
"incompatible browser extension or network configuration",
|
|
1470
|
+
)
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
def _is_challenge_dead_end(html: str) -> bool:
|
|
1474
|
+
lower = html[:20000].lower()
|
|
1475
|
+
return any(sig.lower() in lower for sig in _CHALLENGE_DEAD_END)
|
|
1476
|
+
|
|
1477
|
+
|
|
1478
|
+
def _is_challenge_page(html: str) -> bool:
|
|
1479
|
+
"""Detect if the page is an anti-bot challenge."""
|
|
1480
|
+
lower = html[:5000].lower()
|
|
1481
|
+
return any(sig in lower for sig in _CHALLENGE_SIGNATURES)
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
# ============================================================
|
|
1485
|
+
# Elsevier-specific detection
|
|
1486
|
+
# ============================================================
|
|
1487
|
+
|
|
1488
|
+
def _is_elsevier_crasolve(url: str, html: str) -> bool:
|
|
1489
|
+
"""Detect Elsevier's crasolve anti-bot shell."""
|
|
1490
|
+
if "crasolve=1" in url.lower():
|
|
1491
|
+
return True
|
|
1492
|
+
lower = html[:3000].lower()
|
|
1493
|
+
return "crasolve" in lower and "sciencedirect" in url.lower()
|
|
1494
|
+
|
|
1495
|
+
|
|
1496
|
+
# ============================================================
|
|
1497
|
+
# AIP/AVS loading page detection
|
|
1498
|
+
# ============================================================
|
|
1499
|
+
|
|
1500
|
+
def _is_aip_loading_page(html: str) -> bool:
|
|
1501
|
+
"""Detect AIP's loading/waiting page."""
|
|
1502
|
+
lower = html[:3000].lower()
|
|
1503
|
+
return "loading" in lower and ("aip" in lower or "pubs.aip.org" in lower)
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
# ============================================================
|
|
1507
|
+
# Core browser strategy
|
|
1508
|
+
# ============================================================
|
|
1509
|
+
|
|
1510
|
+
def _try_http_download(
|
|
1511
|
+
pdf_url: str,
|
|
1512
|
+
output_path: Path,
|
|
1513
|
+
config: dict[str, Any],
|
|
1514
|
+
cookies: dict[str, str] | None = None,
|
|
1515
|
+
) -> bool:
|
|
1516
|
+
"""Try direct HTTP download of a PDF URL. Returns True on success.
|
|
1517
|
+
|
|
1518
|
+
A host that timed out is skipped for a while (``_STALLED_HOSTS``); every
|
|
1519
|
+
failure says why and how long it took (2026-10-19: SAGE's two silent
|
|
1520
|
+
timeouts cost 52 s a paper and showed nowhere in the log)."""
|
|
1521
|
+
import requests
|
|
1522
|
+
from urllib.parse import urlparse
|
|
1523
|
+
from ...network import USER_AGENT
|
|
1524
|
+
from ...download.pdf import _response_looks_pdf, is_pdf_file
|
|
1525
|
+
from ...sources.publishers import _write_pdf_atomic, _load_publisher_cookies
|
|
1526
|
+
|
|
1527
|
+
host = (urlparse(pdf_url).hostname or "").lower()
|
|
1528
|
+
stalled_at = _STALLED_HOSTS.get(host)
|
|
1529
|
+
if stalled_at and time.time() - stalled_at < _STALLED_HOST_SECONDS:
|
|
1530
|
+
log.info(f" direct PDF: skipped {host} (it timed out {time.time() - stalled_at:.0f}s ago)")
|
|
1531
|
+
return False
|
|
1532
|
+
t_start = time.time()
|
|
1533
|
+
try:
|
|
1534
|
+
s = requests.Session()
|
|
1535
|
+
s.trust_env = False
|
|
1536
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
1537
|
+
_load_publisher_cookies(s, config)
|
|
1538
|
+
if cookies:
|
|
1539
|
+
for name, value in cookies.items():
|
|
1540
|
+
s.cookies.set(name, value)
|
|
1541
|
+
|
|
1542
|
+
resp = s.get(pdf_url, timeout=20, stream=True,
|
|
1543
|
+
headers={"Accept": "application/pdf,*/*"},
|
|
1544
|
+
allow_redirects=True)
|
|
1545
|
+
|
|
1546
|
+
if resp.status_code >= 400:
|
|
1547
|
+
log.info(f" direct PDF: HTTP {resp.status_code} from {host} ({time.time() - t_start:.1f}s)")
|
|
1548
|
+
return False
|
|
1549
|
+
|
|
1550
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
1551
|
+
first = next(iterator, b"")
|
|
1552
|
+
if not _response_looks_pdf(resp, first):
|
|
1553
|
+
log.info(f" direct PDF: not a PDF from {host} ({time.time() - t_start:.1f}s)")
|
|
1554
|
+
return False
|
|
1555
|
+
|
|
1556
|
+
# timeout=20 bounds each read, not the transfer: a server that trickles
|
|
1557
|
+
# bytes (seen on link.springer.com) kept this lane — and the whole race —
|
|
1558
|
+
# waiting for minutes. Same wall-clock bound as pdf_utils' streams.
|
|
1559
|
+
deadline = time.monotonic() + float(config.get("download_deadline_seconds", 60))
|
|
1560
|
+
|
|
1561
|
+
def _bounded(chunks):
|
|
1562
|
+
for chunk in chunks:
|
|
1563
|
+
if time.monotonic() > deadline:
|
|
1564
|
+
log.info(f" HTTP PDF download too slow, giving up: {pdf_url[:80]}")
|
|
1565
|
+
raise TimeoutError("stream deadline exceeded")
|
|
1566
|
+
yield chunk
|
|
1567
|
+
|
|
1568
|
+
if not _write_pdf_atomic(output_path, first, _bounded(iterator)):
|
|
1569
|
+
return False
|
|
1570
|
+
return is_pdf_file(output_path)
|
|
1571
|
+
except (requests.Timeout, requests.ConnectionError) as exc:
|
|
1572
|
+
if isinstance(exc, requests.Timeout):
|
|
1573
|
+
_STALLED_HOSTS[host] = time.time()
|
|
1574
|
+
log.info(f" direct PDF: {type(exc).__name__} from {host} after {time.time() - t_start:.1f}s")
|
|
1575
|
+
return False
|
|
1576
|
+
except Exception as exc:
|
|
1577
|
+
log.info(f" direct PDF: {type(exc).__name__} from {host} after {time.time() - t_start:.1f}s")
|
|
1578
|
+
return False
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def _is_campus_network(config: dict[str, Any]) -> bool:
|
|
1582
|
+
"""Check if user is on campus network (IP-authenticated access)."""
|
|
1583
|
+
return config.get("is_campus_network", False)
|
|
1584
|
+
|
|
1585
|
+
|
|
1586
|
+
def _find_pdf_link(page: Any, publisher: str) -> str | None:
|
|
1587
|
+
"""Find PDF download link on article page using CSS selectors and JS."""
|
|
1588
|
+
from urllib.parse import urljoin
|
|
1589
|
+
|
|
1590
|
+
selectors = _PDF_LINK_SELECTORS.get(publisher, _PDF_LINK_SELECTORS.get("Elsevier", []))
|
|
1591
|
+
for sel in selectors:
|
|
1592
|
+
try:
|
|
1593
|
+
el = page.query_selector(sel)
|
|
1594
|
+
if el:
|
|
1595
|
+
href = el.get_attribute("href")
|
|
1596
|
+
if href:
|
|
1597
|
+
if href.startswith("/"):
|
|
1598
|
+
href = urljoin(page.url, href)
|
|
1599
|
+
return href
|
|
1600
|
+
except Exception:
|
|
1601
|
+
continue
|
|
1602
|
+
|
|
1603
|
+
# JS extraction fallback
|
|
1604
|
+
try:
|
|
1605
|
+
url = page.evaluate("""
|
|
1606
|
+
(() => {
|
|
1607
|
+
const meta = document.querySelector('meta[name="citation_pdf_url"]');
|
|
1608
|
+
if (meta) return meta.content;
|
|
1609
|
+
const links = [...document.querySelectorAll('a')];
|
|
1610
|
+
for (const a of links) {
|
|
1611
|
+
const href = (a.href || '').toLowerCase();
|
|
1612
|
+
if (href.includes('/pdf/') || href.includes('.pdf') || href.includes('pdfdirect')) {
|
|
1613
|
+
return a.href;
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
return null;
|
|
1617
|
+
})()
|
|
1618
|
+
""")
|
|
1619
|
+
return url
|
|
1620
|
+
except Exception:
|
|
1621
|
+
return None
|
|
1622
|
+
|
|
1623
|
+
|
|
1624
|
+
def _browser_download_with_fallback(
|
|
1625
|
+
doi: str,
|
|
1626
|
+
article_url: str,
|
|
1627
|
+
output_path: Path,
|
|
1628
|
+
config: dict[str, Any],
|
|
1629
|
+
publisher: str,
|
|
1630
|
+
*,
|
|
1631
|
+
wait_for_loading: float = 0,
|
|
1632
|
+
) -> bool:
|
|
1633
|
+
"""The publisher's site, in its own window (persistent profile per publisher).
|
|
1634
|
+
|
|
1635
|
+
2026-10-20: the first pass in the shared browser is gone. It judged pages
|
|
1636
|
+
too early, failed to open some sites the window opens, and could not keep a
|
|
1637
|
+
passed human check between papers (40-60 s per paper on T&F and ACS); the
|
|
1638
|
+
window's profile keeps it. The window starts minimized and comes up only
|
|
1639
|
+
when the user has to log in or pass a check (``_show_window_for_user``).
|
|
1640
|
+
"""
|
|
1641
|
+
from ...browser.engine import _is_pdf_url, is_available
|
|
1642
|
+
from ...download.pdf import is_pdf_file, success
|
|
1643
|
+
_clear_error()
|
|
1644
|
+
if article_url.startswith("https://doi.org/"):
|
|
1645
|
+
article_url = _known_landing_url(doi) or article_url
|
|
1646
|
+
# A PDF URL and saved publisher cookies: try it without a browser first.
|
|
1647
|
+
if _has_publisher_cookies(config) and _is_pdf_url(article_url):
|
|
1648
|
+
log.info(f" [{publisher}] trying HTTP with cookies first (fast-path)")
|
|
1649
|
+
if _try_http_download(article_url, output_path, config) and is_pdf_file(output_path):
|
|
1650
|
+
return success(doi, output_path, f"{publisher}(HTTP)")
|
|
1651
|
+
if not is_available(config):
|
|
1652
|
+
log.info(f" [{publisher}] browser not available, skipping browser strategy")
|
|
1653
|
+
_set_error("browser_unavailable", "start_browser")
|
|
1654
|
+
return False
|
|
1655
|
+
try:
|
|
1656
|
+
from ...download import progress as _pr
|
|
1657
|
+
_pr.item_stage(doi, f"正在打开 {publisher} 的页面" if publisher not in ("", "Unknown") else "正在打开出版商官网")
|
|
1658
|
+
except Exception:
|
|
1659
|
+
pass
|
|
1660
|
+
try:
|
|
1661
|
+
from ...sources import SourceWaived
|
|
1662
|
+
except Exception: # pragma: no cover
|
|
1663
|
+
SourceWaived = () # type: ignore[assignment]
|
|
1664
|
+
try:
|
|
1665
|
+
return bool(_visible_browser_download(doi, article_url, output_path, config, publisher,
|
|
1666
|
+
wait_for_loading=wait_for_loading))
|
|
1667
|
+
except SourceWaived:
|
|
1668
|
+
raise
|
|
1669
|
+
except Exception as e:
|
|
1670
|
+
log.info(f" [{publisher}] publisher window error: {e}")
|
|
1671
|
+
if not get_last_error()[0]:
|
|
1672
|
+
_set_error("navigate_failed", "site_unreachable")
|
|
1673
|
+
return False
|
|
1674
|
+
|
|
1675
|
+
|
|
1676
|
+
# ============================================================
|
|
1677
|
+
# Elsevier API (no browser needed)
|
|
1678
|
+
# ============================================================
|
|
1679
|
+
|
|
1680
|
+
_CLOUDFLARE_COOKIE_NAMES = {"__cf_bm", "_cfuvid", "cf_clearance", "cf_chl_rc_ni"}
|
|
1681
|
+
|
|
1682
|
+
|
|
1683
|
+
def _persist_api_cookies(session: Any, config: dict[str, Any]) -> int:
|
|
1684
|
+
"""Extract cookies from a requests.Session and merge into publisher_cookies.json.
|
|
1685
|
+
|
|
1686
|
+
Skips cookies that are only Cloudflare bot-management cookies (__cf_bm, _cfuvid, etc.)
|
|
1687
|
+
since those don't carry authentication and don't help the browser strategy.
|
|
1688
|
+
|
|
1689
|
+
Returns the number of cookies persisted.
|
|
1690
|
+
"""
|
|
1691
|
+
from ...browser.cookies import merge_cookies
|
|
1692
|
+
|
|
1693
|
+
raw = []
|
|
1694
|
+
meaningful = 0
|
|
1695
|
+
for c in session.cookies:
|
|
1696
|
+
cookie = {
|
|
1697
|
+
"name": c.name,
|
|
1698
|
+
"value": c.value,
|
|
1699
|
+
"domain": c.domain,
|
|
1700
|
+
"path": c.path,
|
|
1701
|
+
"secure": c.secure,
|
|
1702
|
+
"expires": c.expires or 0,
|
|
1703
|
+
}
|
|
1704
|
+
raw.append(cookie)
|
|
1705
|
+
if c.name not in _CLOUDFLARE_COOKIE_NAMES:
|
|
1706
|
+
meaningful += 1
|
|
1707
|
+
|
|
1708
|
+
if not raw:
|
|
1709
|
+
return 0
|
|
1710
|
+
|
|
1711
|
+
if meaningful == 0:
|
|
1712
|
+
log.info(f" [ElsevierAPI] {len(raw)} cookies are Cloudflare-only, skipping persist")
|
|
1713
|
+
return 0
|
|
1714
|
+
|
|
1715
|
+
merged = merge_cookies(raw, config)
|
|
1716
|
+
log.info(f" [ElsevierAPI] persisted {meaningful} meaningful cookies "
|
|
1717
|
+
f"({len(raw)} total, merged: {len(merged)})")
|
|
1718
|
+
return meaningful
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
from ._elsevier_api import ( # noqa: F401 — re-exported, callers import from here
|
|
1722
|
+
_elsevier_header,
|
|
1723
|
+
_elsevier_response_is_pdf,
|
|
1724
|
+
_elsevier_response_is_xml,
|
|
1725
|
+
_elsevier_response_text,
|
|
1726
|
+
_elsevier_xml_local_name,
|
|
1727
|
+
_elsevier_element_text,
|
|
1728
|
+
_elsevier_looks_like_pdf_eid,
|
|
1729
|
+
_elsevier_article_eid_to_main_pdf,
|
|
1730
|
+
_elsevier_attachment_container,
|
|
1731
|
+
_elsevier_attachment_metadata,
|
|
1732
|
+
_elsevier_attachment_score,
|
|
1733
|
+
_extract_elsevier_pdf_attachment_eids,
|
|
1734
|
+
_elsevier_pdf_page_count,
|
|
1735
|
+
_save_elsevier_pdf_content,
|
|
1736
|
+
_try_elsevier_object_pdf_from_xml,
|
|
1737
|
+
_try_elsevier_text_fallback,
|
|
1738
|
+
try_elsevier_api,
|
|
1739
|
+
_try_elsevier_api,
|
|
1740
|
+
)
|
|
1741
|
+
# ============================================================
|
|
1742
|
+
# Publisher entry points
|
|
1743
|
+
# ============================================================
|
|
1744
|
+
|
|
1745
|
+
def try_elsevier_browser(
|
|
1746
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
1747
|
+
) -> dict[str, Any] | None:
|
|
1748
|
+
"""Elsevier/ScienceDirect/Cell Press browser strategy."""
|
|
1749
|
+
from ...download.pdf import is_pdf_file
|
|
1750
|
+
from ...download.pdf import success
|
|
1751
|
+
from ...browser.engine import is_available as browser_available, download_pdf_via_browser
|
|
1752
|
+
|
|
1753
|
+
# Campus network fast-path: skip HTTP, go directly to CloakBrowser
|
|
1754
|
+
if _is_campus_network(config) and browser_available(config):
|
|
1755
|
+
cell_url = _build_cell_press_url(doi)
|
|
1756
|
+
if cell_url:
|
|
1757
|
+
log.info(f" [Elsevier] campus network detected, trying CloakBrowser directly: {cell_url[:80]}")
|
|
1758
|
+
if download_pdf_via_browser(cell_url, output_path, config):
|
|
1759
|
+
if is_pdf_file(output_path):
|
|
1760
|
+
log.info(f" [Elsevier] campus network download succeeded")
|
|
1761
|
+
return success(doi, output_path, "CellPress(Campus)")
|
|
1762
|
+
|
|
1763
|
+
# Fast-path: if we have publisher cookies, try HTTP download first
|
|
1764
|
+
# This avoids 15-30s browser startup overhead
|
|
1765
|
+
if _has_publisher_cookies(config):
|
|
1766
|
+
# Try Cell Press showPdf with PII first (most reliable for Cell Press)
|
|
1767
|
+
cell_url = _build_cell_press_url(doi)
|
|
1768
|
+
if cell_url and "/abstract/" in cell_url:
|
|
1769
|
+
pii = cell_url.split("/abstract/")[-1]
|
|
1770
|
+
show_pdf_url = f"https://www.cell.com/action/showPdf?pii={pii}"
|
|
1771
|
+
log.info(f" [Elsevier] trying Cell Press showPdf HTTP with cookies: {show_pdf_url[:80]}")
|
|
1772
|
+
if _try_http_download(show_pdf_url, output_path, config):
|
|
1773
|
+
if is_pdf_file(output_path):
|
|
1774
|
+
log.info(f" [Elsevier] Cell Press HTTP download succeeded with cached cookies")
|
|
1775
|
+
return success(doi, output_path, "CellPress(HTTP)")
|
|
1776
|
+
|
|
1777
|
+
# For Cell Press DOIs, try direct showPdf download first (bypasses linkinghub)
|
|
1778
|
+
cell_url = _build_cell_press_url(doi)
|
|
1779
|
+
if cell_url and "/abstract/" in cell_url:
|
|
1780
|
+
pii = cell_url.split("/abstract/")[-1]
|
|
1781
|
+
show_pdf_url = f"https://www.cell.com/action/showPdf?pii={pii}"
|
|
1782
|
+
log.info(f" [CellPress] trying showPdf via network capture: {show_pdf_url[:80]}")
|
|
1783
|
+
result = _cell_press_showpdf_download(show_pdf_url, output_path, config)
|
|
1784
|
+
if result and is_pdf_file(output_path):
|
|
1785
|
+
return result
|
|
1786
|
+
|
|
1787
|
+
# If showPdf failed, navigate to the abstract page directly
|
|
1788
|
+
log.info(f" [CellPress] trying abstract page: {cell_url[:80]}")
|
|
1789
|
+
if _browser_download_with_fallback(doi, cell_url, output_path, config, "Elsevier"):
|
|
1790
|
+
if is_pdf_file(output_path):
|
|
1791
|
+
return success(doi, output_path, "CellPress(Browser)")
|
|
1792
|
+
|
|
1793
|
+
# Fallback to general browser download for ScienceDirect articles
|
|
1794
|
+
# Only try if we haven't already detected a paywall
|
|
1795
|
+
if not get_last_error()[0]:
|
|
1796
|
+
# Resolve DOI to direct sciencedirect.com URL (avoid linkinghub)
|
|
1797
|
+
article_url = _resolve_elsevier_pii(doi, config) or f"https://doi.org/{doi}"
|
|
1798
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Elsevier"):
|
|
1799
|
+
if is_pdf_file(output_path):
|
|
1800
|
+
return success(doi, output_path, "Elsevier(Browser)")
|
|
1801
|
+
return None
|
|
1802
|
+
|
|
1803
|
+
|
|
1804
|
+
def _cell_press_showpdf_download(
|
|
1805
|
+
show_pdf_url: str,
|
|
1806
|
+
output_path: Path,
|
|
1807
|
+
config: dict[str, Any],
|
|
1808
|
+
) -> dict[str, Any] | None:
|
|
1809
|
+
"""Download a Cell Press showPdf URL via browser network capture."""
|
|
1810
|
+
from ...browser.engine import create_tab, close_tab, fetch_url
|
|
1811
|
+
from ...download.pdf import success
|
|
1812
|
+
|
|
1813
|
+
tab_id = create_tab("https://example.com", config, timeout=15.0)
|
|
1814
|
+
if not tab_id:
|
|
1815
|
+
return None
|
|
1816
|
+
|
|
1817
|
+
try:
|
|
1818
|
+
_inject_cookies_to_tab(tab_id, config, "Elsevier")
|
|
1819
|
+
result = fetch_url(tab_id, show_pdf_url, config, timeout=30.0)
|
|
1820
|
+
if result and result.get("data"):
|
|
1821
|
+
pdf_bytes = result["data"]
|
|
1822
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1823
|
+
output_path.write_bytes(pdf_bytes)
|
|
1824
|
+
log.info(f" [CellPress] downloaded {len(pdf_bytes)} bytes via showPdf")
|
|
1825
|
+
return success(show_pdf_url, output_path, "CellPress(Network)")
|
|
1826
|
+
except Exception as e:
|
|
1827
|
+
log.info(f" [CellPress] showPdf failed: {e}")
|
|
1828
|
+
finally:
|
|
1829
|
+
close_tab(tab_id, config)
|
|
1830
|
+
return None
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
# Cell Press journal mapping from DOI prefix to cell.com path
|
|
1834
|
+
_CELL_JOURNAL_MAP = {
|
|
1835
|
+
"j.oneear": "one-earth",
|
|
1836
|
+
"j.cell": "cell",
|
|
1837
|
+
"j.cels": "cancer-cell",
|
|
1838
|
+
"j.cub": "current-biology",
|
|
1839
|
+
"j.neuron": "neuron",
|
|
1840
|
+
"j.molcel": "molecular-cell",
|
|
1841
|
+
"j.devcel": "developmental-cell",
|
|
1842
|
+
"j.immuni": "immunity",
|
|
1843
|
+
"j.chom": "cell-host-microbe",
|
|
1844
|
+
"j.cmet": "cell-metabolism",
|
|
1845
|
+
"j.stem": "cell-stem-cell",
|
|
1846
|
+
"j.celrep": "cell-reports",
|
|
1847
|
+
"j.isci": "iscience",
|
|
1848
|
+
"j.xcr": "cell-reports-physical-science",
|
|
1849
|
+
"j.heliyon": "heliyon",
|
|
1850
|
+
"j.applanim": "ajhg",
|
|
1851
|
+
"j.ajhg": "ajhg",
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
|
|
1855
|
+
def _build_cell_press_url(doi: str) -> str | None:
|
|
1856
|
+
"""Build direct cell.com abstract URL from DOI if it's a Cell Press journal."""
|
|
1857
|
+
import re
|
|
1858
|
+
doi_suffix = doi.split("/", 1)[-1] if "/" in doi else ""
|
|
1859
|
+
if not doi_suffix.startswith("j."):
|
|
1860
|
+
return None
|
|
1861
|
+
|
|
1862
|
+
parts = doi_suffix.split(".")
|
|
1863
|
+
if len(parts) < 3:
|
|
1864
|
+
return None
|
|
1865
|
+
|
|
1866
|
+
# Try to match journal: j.oneear, j.cell, etc.
|
|
1867
|
+
journal_key = f"{parts[0]}.{parts[1]}"
|
|
1868
|
+
journal_path = _CELL_JOURNAL_MAP.get(journal_key)
|
|
1869
|
+
if not journal_path:
|
|
1870
|
+
return None
|
|
1871
|
+
|
|
1872
|
+
# Try to get PII from Crossref alternative-id
|
|
1873
|
+
pii = _get_elsevier_pii(doi)
|
|
1874
|
+
if pii:
|
|
1875
|
+
return f"https://www.cell.com/{journal_path}/abstract/{pii}"
|
|
1876
|
+
|
|
1877
|
+
# Fallback: use /doi/ path (may not have showPdf links)
|
|
1878
|
+
return f"https://www.cell.com/{journal_path}/doi/{doi}"
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
def _get_elsevier_pii(doi: str) -> str | None:
|
|
1882
|
+
"""Get PII (Publisher Item Identifier) from Crossref API."""
|
|
1883
|
+
try:
|
|
1884
|
+
import requests
|
|
1885
|
+
from ...network import USER_AGENT
|
|
1886
|
+
resp = requests.get(
|
|
1887
|
+
f"https://api.crossref.org/works/{doi}",
|
|
1888
|
+
timeout=10, verify=False,
|
|
1889
|
+
headers={"User-Agent": USER_AGENT},
|
|
1890
|
+
)
|
|
1891
|
+
if resp.status_code != 200:
|
|
1892
|
+
return None
|
|
1893
|
+
data = resp.json().get("message", {})
|
|
1894
|
+
alt_ids = data.get("alternative-id", [])
|
|
1895
|
+
for aid in alt_ids:
|
|
1896
|
+
if aid.startswith("S") and len(aid) >= 16:
|
|
1897
|
+
return aid
|
|
1898
|
+
except Exception:
|
|
1899
|
+
pass
|
|
1900
|
+
return None
|
|
1901
|
+
|
|
1902
|
+
|
|
1903
|
+
def _resolve_elsevier_pii(doi: str, config: dict[str, Any]) -> str | None:
|
|
1904
|
+
"""Resolve Elsevier DOI to direct article URL, bypassing linkinghub.elsevier.com.
|
|
1905
|
+
|
|
1906
|
+
Uses HTTP to get the PII from linkinghub, then uses Crossref to determine the journal.
|
|
1907
|
+
Returns a direct cell.com/sciencedirect.com URL, or None.
|
|
1908
|
+
"""
|
|
1909
|
+
import re
|
|
1910
|
+
try:
|
|
1911
|
+
import requests
|
|
1912
|
+
from ...network import USER_AGENT
|
|
1913
|
+
s = requests.Session()
|
|
1914
|
+
s.trust_env = False
|
|
1915
|
+
s.headers.update({"User-Agent": USER_AGENT, "Accept": "text/html"})
|
|
1916
|
+
from ...network import apply_host_proxy
|
|
1917
|
+
apply_host_proxy(s, config)
|
|
1918
|
+
resp = s.get(f"https://doi.org/{doi}", timeout=10, allow_redirects=True)
|
|
1919
|
+
final_url = str(resp.url)
|
|
1920
|
+
html = resp.text
|
|
1921
|
+
|
|
1922
|
+
# Extract PII from URL
|
|
1923
|
+
pii_match = re.search(r'pii/(S[0-9A-Za-z]+)', final_url) # PII may end in check char 'X'
|
|
1924
|
+
if not pii_match:
|
|
1925
|
+
return None
|
|
1926
|
+
pii = pii_match.group(1)
|
|
1927
|
+
|
|
1928
|
+
# Check if meta-refresh points to sciencedirect.com or cell.com
|
|
1929
|
+
if "sciencedirect.com" in html:
|
|
1930
|
+
return f"https://www.sciencedirect.com/science/article/pii/{pii}"
|
|
1931
|
+
|
|
1932
|
+
# Check for cell.com in meta-refresh
|
|
1933
|
+
if "cell.com" in html:
|
|
1934
|
+
cell_match = re.search(r'cell\.com/([^/]+)/', html)
|
|
1935
|
+
if cell_match:
|
|
1936
|
+
journal = cell_match.group(1)
|
|
1937
|
+
return f"https://www.cell.com/{journal}/abstract/{pii}"
|
|
1938
|
+
|
|
1939
|
+
return None
|
|
1940
|
+
except Exception:
|
|
1941
|
+
return None
|
|
1942
|
+
|
|
1943
|
+
|
|
1944
|
+
def try_wiley_browser(
|
|
1945
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
1946
|
+
) -> dict[str, Any] | None:
|
|
1947
|
+
"""Wiley browser strategy with PDFDirect."""
|
|
1948
|
+
from ...download.pdf import is_pdf_file
|
|
1949
|
+
from ...download.pdf import success
|
|
1950
|
+
from ...browser.engine import is_available as browser_available, download_pdf_via_browser
|
|
1951
|
+
|
|
1952
|
+
# Campus network fast-path: skip HTTP, go directly to CloakBrowser
|
|
1953
|
+
if _is_campus_network(config) and browser_available(config):
|
|
1954
|
+
article_url = f"https://doi.org/{doi}"
|
|
1955
|
+
log.info(f" [Wiley] campus network detected, trying CloakBrowser directly")
|
|
1956
|
+
if download_pdf_via_browser(article_url, output_path, config):
|
|
1957
|
+
if is_pdf_file(output_path):
|
|
1958
|
+
log.info(f" [Wiley] campus network download succeeded")
|
|
1959
|
+
return success(doi, output_path, "Wiley(Campus)")
|
|
1960
|
+
|
|
1961
|
+
# Try direct PDF URLs first
|
|
1962
|
+
for url in _direct_pdf_urls(doi, "Wiley"):
|
|
1963
|
+
if _try_http_download(url, output_path, config):
|
|
1964
|
+
if is_pdf_file(output_path):
|
|
1965
|
+
return success(doi, output_path, "Wiley(PDFDirect)")
|
|
1966
|
+
|
|
1967
|
+
# Fall back to browser — use direct Wiley URL to avoid slow DOI redirect
|
|
1968
|
+
article_url = f"https://onlinelibrary.wiley.com/doi/{doi}"
|
|
1969
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Wiley"):
|
|
1970
|
+
if is_pdf_file(output_path):
|
|
1971
|
+
return success(doi, output_path, "Wiley(Browser)")
|
|
1972
|
+
return None
|
|
1973
|
+
|
|
1974
|
+
|
|
1975
|
+
def try_ieee_browser(
|
|
1976
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
1977
|
+
) -> dict[str, Any] | None:
|
|
1978
|
+
"""IEEE browser strategy."""
|
|
1979
|
+
from ...download.pdf import is_pdf_file
|
|
1980
|
+
from ...download.pdf import success
|
|
1981
|
+
|
|
1982
|
+
article_url = f"https://doi.org/{doi}"
|
|
1983
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "IEEE"):
|
|
1984
|
+
if is_pdf_file(output_path):
|
|
1985
|
+
return success(doi, output_path, "IEEE(Browser)")
|
|
1986
|
+
return None
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
def try_acs_browser(
|
|
1990
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
1991
|
+
) -> dict[str, Any] | None:
|
|
1992
|
+
"""ACS browser strategy."""
|
|
1993
|
+
from ...download.pdf import is_pdf_file
|
|
1994
|
+
from ...download.pdf import success
|
|
1995
|
+
|
|
1996
|
+
# Try direct PDF URL first
|
|
1997
|
+
for url in _direct_pdf_urls(doi, "ACS"):
|
|
1998
|
+
if _try_http_download(url, output_path, config):
|
|
1999
|
+
if is_pdf_file(output_path):
|
|
2000
|
+
return success(doi, output_path, "ACS(Direct)")
|
|
2001
|
+
|
|
2002
|
+
article_url = f"https://doi.org/{doi}"
|
|
2003
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "ACS"):
|
|
2004
|
+
if is_pdf_file(output_path):
|
|
2005
|
+
return success(doi, output_path, "ACS(Browser)")
|
|
2006
|
+
return None
|
|
2007
|
+
|
|
2008
|
+
|
|
2009
|
+
def try_rsc_browser(
|
|
2010
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2011
|
+
) -> dict[str, Any] | None:
|
|
2012
|
+
"""RSC browser strategy."""
|
|
2013
|
+
from ...download.pdf import is_pdf_file
|
|
2014
|
+
from ...download.pdf import success
|
|
2015
|
+
|
|
2016
|
+
for url in _direct_pdf_urls(doi, "RSC"):
|
|
2017
|
+
if _try_http_download(url, output_path, config):
|
|
2018
|
+
if is_pdf_file(output_path):
|
|
2019
|
+
return success(doi, output_path, "RSC(Direct)")
|
|
2020
|
+
|
|
2021
|
+
article_url = f"https://doi.org/{doi}"
|
|
2022
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "RSC"):
|
|
2023
|
+
if is_pdf_file(output_path):
|
|
2024
|
+
return success(doi, output_path, "RSC(Browser)")
|
|
2025
|
+
return None
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
def try_aip_browser(
|
|
2029
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2030
|
+
) -> dict[str, Any] | None:
|
|
2031
|
+
"""AIP browser strategy with loading page wait."""
|
|
2032
|
+
from ...download.pdf import is_pdf_file
|
|
2033
|
+
from ...download.pdf import success
|
|
2034
|
+
|
|
2035
|
+
article_url = f"https://doi.org/{doi}"
|
|
2036
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "AIP", wait_for_loading=10):
|
|
2037
|
+
if is_pdf_file(output_path):
|
|
2038
|
+
return success(doi, output_path, "AIP(Browser)")
|
|
2039
|
+
return None
|
|
2040
|
+
|
|
2041
|
+
|
|
2042
|
+
def try_springer_browser(
|
|
2043
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2044
|
+
) -> dict[str, Any] | None:
|
|
2045
|
+
"""Springer browser strategy."""
|
|
2046
|
+
from ...download.pdf import is_pdf_file
|
|
2047
|
+
from ...download.pdf import success
|
|
2048
|
+
|
|
2049
|
+
# Try direct PDF URL first
|
|
2050
|
+
for url in _direct_pdf_urls(doi, "Springer"):
|
|
2051
|
+
if _try_http_download(url, output_path, config):
|
|
2052
|
+
if is_pdf_file(output_path):
|
|
2053
|
+
return success(doi, output_path, "Springer(Direct)")
|
|
2054
|
+
|
|
2055
|
+
article_url = f"https://doi.org/{doi}"
|
|
2056
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Springer"):
|
|
2057
|
+
if is_pdf_file(output_path):
|
|
2058
|
+
return success(doi, output_path, "Springer(Browser)")
|
|
2059
|
+
return None
|
|
2060
|
+
|
|
2061
|
+
|
|
2062
|
+
def try_aps_browser(
|
|
2063
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2064
|
+
) -> dict[str, Any] | None:
|
|
2065
|
+
"""APS (Physical Review) browser strategy."""
|
|
2066
|
+
from ...download.pdf import is_pdf_file
|
|
2067
|
+
from ...download.pdf import success
|
|
2068
|
+
|
|
2069
|
+
for url in _direct_pdf_urls(doi, "APS"):
|
|
2070
|
+
if _try_http_download(url, output_path, config):
|
|
2071
|
+
if is_pdf_file(output_path):
|
|
2072
|
+
return success(doi, output_path, "APS(Direct)")
|
|
2073
|
+
|
|
2074
|
+
article_url = f"https://doi.org/{doi}"
|
|
2075
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "APS"):
|
|
2076
|
+
if is_pdf_file(output_path):
|
|
2077
|
+
return success(doi, output_path, "APS(Browser)")
|
|
2078
|
+
return None
|
|
2079
|
+
|
|
2080
|
+
|
|
2081
|
+
def try_tandfonline_browser(
|
|
2082
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2083
|
+
) -> dict[str, Any] | None:
|
|
2084
|
+
"""Taylor & Francis browser strategy."""
|
|
2085
|
+
from ...download.pdf import is_pdf_file
|
|
2086
|
+
from ...download.pdf import success
|
|
2087
|
+
|
|
2088
|
+
for url in _direct_pdf_urls(doi, "Tandfonline"):
|
|
2089
|
+
if _try_http_download(url, output_path, config):
|
|
2090
|
+
if is_pdf_file(output_path):
|
|
2091
|
+
return success(doi, output_path, "T&F(Direct)")
|
|
2092
|
+
|
|
2093
|
+
article_url = f"https://doi.org/{doi}"
|
|
2094
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Tandfonline"):
|
|
2095
|
+
if is_pdf_file(output_path):
|
|
2096
|
+
return success(doi, output_path, "T&F(Browser)")
|
|
2097
|
+
return None
|
|
2098
|
+
|
|
2099
|
+
|
|
2100
|
+
def try_iop_browser(
|
|
2101
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2102
|
+
) -> dict[str, Any] | None:
|
|
2103
|
+
"""IOP browser strategy."""
|
|
2104
|
+
from ...download.pdf import is_pdf_file
|
|
2105
|
+
from ...download.pdf import success
|
|
2106
|
+
|
|
2107
|
+
for url in _direct_pdf_urls(doi, "IOP"):
|
|
2108
|
+
if _try_http_download(url, output_path, config):
|
|
2109
|
+
if is_pdf_file(output_path):
|
|
2110
|
+
return success(doi, output_path, "IOP(Direct)")
|
|
2111
|
+
|
|
2112
|
+
article_url = f"https://doi.org/{doi}"
|
|
2113
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "IOP"):
|
|
2114
|
+
if is_pdf_file(output_path):
|
|
2115
|
+
return success(doi, output_path, "IOP(Browser)")
|
|
2116
|
+
return None
|
|
2117
|
+
|
|
2118
|
+
|
|
2119
|
+
def try_oxford_browser(
|
|
2120
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2121
|
+
) -> dict[str, Any] | None:
|
|
2122
|
+
"""Oxford Academic browser strategy."""
|
|
2123
|
+
from ...download.pdf import is_pdf_file
|
|
2124
|
+
from ...download.pdf import success
|
|
2125
|
+
|
|
2126
|
+
for url in _direct_pdf_urls(doi, "Oxford"):
|
|
2127
|
+
if _try_http_download(url, output_path, config):
|
|
2128
|
+
if is_pdf_file(output_path):
|
|
2129
|
+
return success(doi, output_path, "Oxford(Direct)")
|
|
2130
|
+
|
|
2131
|
+
article_url = f"https://doi.org/{doi}"
|
|
2132
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Oxford"):
|
|
2133
|
+
if is_pdf_file(output_path):
|
|
2134
|
+
return success(doi, output_path, "Oxford(Browser)")
|
|
2135
|
+
return None
|
|
2136
|
+
|
|
2137
|
+
|
|
2138
|
+
def try_acm_browser(
|
|
2139
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2140
|
+
) -> dict[str, Any] | None:
|
|
2141
|
+
"""ACM browser strategy."""
|
|
2142
|
+
from ...download.pdf import is_pdf_file
|
|
2143
|
+
from ...download.pdf import success
|
|
2144
|
+
|
|
2145
|
+
for url in _direct_pdf_urls(doi, "ACM"):
|
|
2146
|
+
if _try_http_download(url, output_path, config):
|
|
2147
|
+
if is_pdf_file(output_path):
|
|
2148
|
+
return success(doi, output_path, "ACM(Direct)")
|
|
2149
|
+
|
|
2150
|
+
article_url = f"https://doi.org/{doi}"
|
|
2151
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "ACM"):
|
|
2152
|
+
if is_pdf_file(output_path):
|
|
2153
|
+
return success(doi, output_path, "ACM(Browser)")
|
|
2154
|
+
return None
|
|
2155
|
+
|
|
2156
|
+
|
|
2157
|
+
def try_nature_browser(
|
|
2158
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2159
|
+
) -> dict[str, Any] | None:
|
|
2160
|
+
"""Nature browser strategy (fallback when direct fails)."""
|
|
2161
|
+
from ...download.pdf import is_pdf_file
|
|
2162
|
+
from ...download.pdf import success
|
|
2163
|
+
|
|
2164
|
+
article_url = f"https://doi.org/{doi}"
|
|
2165
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Nature"):
|
|
2166
|
+
if is_pdf_file(output_path):
|
|
2167
|
+
return success(doi, output_path, "Nature(Browser)")
|
|
2168
|
+
return None
|
|
2169
|
+
|
|
2170
|
+
|
|
2171
|
+
def try_science_browser(
|
|
2172
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2173
|
+
) -> dict[str, Any] | None:
|
|
2174
|
+
"""Science/AAAS browser strategy."""
|
|
2175
|
+
from ...download.pdf import is_pdf_file
|
|
2176
|
+
from ...download.pdf import success
|
|
2177
|
+
|
|
2178
|
+
article_url = f"https://doi.org/{doi}"
|
|
2179
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Science"):
|
|
2180
|
+
if is_pdf_file(output_path):
|
|
2181
|
+
return success(doi, output_path, "Science(Browser)")
|
|
2182
|
+
return None
|
|
2183
|
+
|
|
2184
|
+
|
|
2185
|
+
def try_sage_browser(
|
|
2186
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2187
|
+
) -> dict[str, Any] | None:
|
|
2188
|
+
"""SAGE Journals browser strategy.
|
|
2189
|
+
|
|
2190
|
+
Supports journals.sagepub.com and sage.cnpereading.com regional frontends
|
|
2191
|
+
by resolving DOI first and using host-relative PDF candidates.
|
|
2192
|
+
"""
|
|
2193
|
+
from ...download.pdf import is_pdf_file, success
|
|
2194
|
+
from ...sources.sage_cn import try_sage_cn_authorized
|
|
2195
|
+
|
|
2196
|
+
if try_sage_cn_authorized(doi, output_path, config):
|
|
2197
|
+
return success(doi, output_path, "SAGE-CN(CARSI)")
|
|
2198
|
+
|
|
2199
|
+
for url in _direct_pdf_urls(doi, "SAGE"):
|
|
2200
|
+
if _try_http_download(url, output_path, config):
|
|
2201
|
+
if is_pdf_file(output_path):
|
|
2202
|
+
return success(doi, output_path, "SAGE(Direct)")
|
|
2203
|
+
|
|
2204
|
+
article_url = f"https://doi.org/{doi}"
|
|
2205
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "SAGE"):
|
|
2206
|
+
if is_pdf_file(output_path):
|
|
2207
|
+
return success(doi, output_path, "SAGE(Browser)")
|
|
2208
|
+
return None
|
|
2209
|
+
|
|
2210
|
+
|
|
2211
|
+
def try_asce_browser(
|
|
2212
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2213
|
+
) -> dict[str, Any] | None:
|
|
2214
|
+
"""ASCE Library browser strategy.
|
|
2215
|
+
|
|
2216
|
+
ASCE challenges plain HTTP requests with Cloudflare, so browser path
|
|
2217
|
+
is the primary strategy. DOI case must be uppercased for ASCE URLs.
|
|
2218
|
+
"""
|
|
2219
|
+
from ...download.pdf import is_pdf_file, success
|
|
2220
|
+
|
|
2221
|
+
for url in _direct_pdf_urls(doi, "ASCE"):
|
|
2222
|
+
if _try_http_download(url, output_path, config):
|
|
2223
|
+
if is_pdf_file(output_path):
|
|
2224
|
+
return success(doi, output_path, "ASCE(Direct)")
|
|
2225
|
+
|
|
2226
|
+
article_url = f"https://ascelibrary.org/doi/{doi.upper()}"
|
|
2227
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "ASCE"):
|
|
2228
|
+
if is_pdf_file(output_path):
|
|
2229
|
+
return success(doi, output_path, "ASCE(Browser)")
|
|
2230
|
+
return None
|
|
2231
|
+
|
|
2232
|
+
|
|
2233
|
+
# ============================================================
|
|
2234
|
+
# Royal Society browser
|
|
2235
|
+
# ============================================================
|
|
2236
|
+
|
|
2237
|
+
def try_royalsociety_browser(
|
|
2238
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2239
|
+
) -> dict[str, Any] | None:
|
|
2240
|
+
"""Royal Society Publishing browser strategy.
|
|
2241
|
+
|
|
2242
|
+
Tries direct HTTP PDF URLs first (royalsocietypublishing.org serves PDFs
|
|
2243
|
+
without anti-bot for many articles), then falls back to browser for
|
|
2244
|
+
paywalled or Cloudflare-protected content.
|
|
2245
|
+
"""
|
|
2246
|
+
from ...download.pdf import is_pdf_file, success
|
|
2247
|
+
|
|
2248
|
+
for url in _direct_pdf_urls(doi, "Royal Society"):
|
|
2249
|
+
if _try_http_download(url, output_path, config):
|
|
2250
|
+
if is_pdf_file(output_path):
|
|
2251
|
+
return success(doi, output_path, "Royal Society(Direct)")
|
|
2252
|
+
|
|
2253
|
+
article_url = f"https://doi.org/{doi}"
|
|
2254
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Royal Society"):
|
|
2255
|
+
if is_pdf_file(output_path):
|
|
2256
|
+
return success(doi, output_path, "Royal Society(Browser)")
|
|
2257
|
+
return None
|
|
2258
|
+
|
|
2259
|
+
|
|
2260
|
+
# ============================================================
|
|
2261
|
+
# Copernicus direct download
|
|
2262
|
+
# ============================================================
|
|
2263
|
+
|
|
2264
|
+
def try_copernicus_direct(
|
|
2265
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2266
|
+
) -> dict[str, Any] | None:
|
|
2267
|
+
"""Copernicus Publications direct download strategy.
|
|
2268
|
+
|
|
2269
|
+
Copernicus (EGU journals) provides open-access PDFs at predictable URLs.
|
|
2270
|
+
Resolves DOI to article page, then constructs PDF URL from article path.
|
|
2271
|
+
"""
|
|
2272
|
+
from ...download.pdf import is_pdf_file, success
|
|
2273
|
+
import requests
|
|
2274
|
+
|
|
2275
|
+
article_url = f"https://doi.org/{doi}"
|
|
2276
|
+
try:
|
|
2277
|
+
resp = requests.head(article_url, allow_redirects=True, timeout=8,
|
|
2278
|
+
headers={"User-Agent": "Mozilla/5.0"})
|
|
2279
|
+
final_url = resp.url
|
|
2280
|
+
|
|
2281
|
+
# Strategy 1: Extract citation_pdf_url from article page HTML
|
|
2282
|
+
try:
|
|
2283
|
+
page_resp = requests.get(final_url, timeout=10,
|
|
2284
|
+
headers={"User-Agent": "Mozilla/5.0"})
|
|
2285
|
+
pdf_match = re.search(
|
|
2286
|
+
r'citation_pdf_url["\s]+content="([^"]+)"', page_resp.text[:20000], re.I)
|
|
2287
|
+
if pdf_match:
|
|
2288
|
+
pdf_url = pdf_match.group(1)
|
|
2289
|
+
if pdf_url.startswith("/"):
|
|
2290
|
+
from urllib.parse import urlparse as _up
|
|
2291
|
+
base = _up(final_url)
|
|
2292
|
+
pdf_url = f"{base.scheme}://{base.netloc}{pdf_url}"
|
|
2293
|
+
if _try_http_download(pdf_url, output_path, config):
|
|
2294
|
+
if is_pdf_file(output_path):
|
|
2295
|
+
return success(doi, output_path, "Copernicus(Direct)")
|
|
2296
|
+
except Exception:
|
|
2297
|
+
pass
|
|
2298
|
+
|
|
2299
|
+
# Strategy 2: Construct PDF URL from article path
|
|
2300
|
+
# Resolved URL: https://www.{journal}.net/articles/{vol}/{id}/{year}/
|
|
2301
|
+
# PDF URL: https://www.{journal}.net/articles/{vol}/{id}/{year}/{slug}.pdf
|
|
2302
|
+
# where slug = journal-abbrev-vol-id-year (from DOI: 10.5194/journal-vol-id-year)
|
|
2303
|
+
doi_parts = doi.split("/")[-1] if "/" in doi else doi # e.g. "hess-26-3433-2024"
|
|
2304
|
+
base_url = final_url.rstrip("/")
|
|
2305
|
+
for pdf_candidate in [
|
|
2306
|
+
f"{base_url}/{doi_parts}.pdf",
|
|
2307
|
+
f"{base_url}.pdf",
|
|
2308
|
+
]:
|
|
2309
|
+
if _try_http_download(pdf_candidate, output_path, config):
|
|
2310
|
+
if is_pdf_file(output_path):
|
|
2311
|
+
return success(doi, output_path, "Copernicus(Direct)")
|
|
2312
|
+
except Exception:
|
|
2313
|
+
pass
|
|
2314
|
+
|
|
2315
|
+
# Fallback to browser
|
|
2316
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, "Copernicus"):
|
|
2317
|
+
if is_pdf_file(output_path):
|
|
2318
|
+
return success(doi, output_path, "Copernicus(Browser)")
|
|
2319
|
+
return None
|
|
2320
|
+
|
|
2321
|
+
|
|
2322
|
+
# ============================================================
|
|
2323
|
+
# Generic publisher browser fallback
|
|
2324
|
+
# ============================================================
|
|
2325
|
+
|
|
2326
|
+
def try_generic_browser(
|
|
2327
|
+
doi: str, output_path: Path, config: dict[str, Any],
|
|
2328
|
+
) -> dict[str, Any] | None:
|
|
2329
|
+
"""Generic browser fallback for unknown publishers."""
|
|
2330
|
+
from ...download.pdf import is_pdf_file
|
|
2331
|
+
from ...sources.publishers import get_publisher
|
|
2332
|
+
from ...download.pdf import success
|
|
2333
|
+
|
|
2334
|
+
publisher = get_publisher(doi) or "Unknown"
|
|
2335
|
+
article_url = f"https://doi.org/{doi}"
|
|
2336
|
+
|
|
2337
|
+
if _browser_download_with_fallback(doi, article_url, output_path, config, publisher):
|
|
2338
|
+
if is_pdf_file(output_path):
|
|
2339
|
+
return success(doi, output_path, f"{publisher}(Browser)")
|
|
2340
|
+
return None
|