litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1379 @@
|
|
|
1
|
+
"""Sci-Hub source with domain rotation, CAPTCHA detection, and Tor support."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import atexit
|
|
6
|
+
import re
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
import urllib.parse
|
|
10
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
from ..config import DEFAULT_SCIHUB_DOMAINS
|
|
17
|
+
from . import scihub_domains as domain_db
|
|
18
|
+
from .scihub_domains import (
|
|
19
|
+
load_stats,
|
|
20
|
+
record_result,
|
|
21
|
+
update_probe,
|
|
22
|
+
set_probe_timestamp,
|
|
23
|
+
get_probe_timestamp,
|
|
24
|
+
get_wall_state,
|
|
25
|
+
set_wall_state,
|
|
26
|
+
)
|
|
27
|
+
from ..log import get_logger
|
|
28
|
+
from . import scihub_speed
|
|
29
|
+
from ..network import fetch, proxy_dict, select_proxy_for_url, _is_cloudflare_block, USER_AGENT
|
|
30
|
+
from ..download.pdf import extract_pdf_url_from_html, is_pdf_file, success, _response_looks_pdf
|
|
31
|
+
|
|
32
|
+
log = get_logger()
|
|
33
|
+
|
|
34
|
+
# Track domains that require browser bypass (in-memory, per session)
|
|
35
|
+
_browser_domains: set[str] = set()
|
|
36
|
+
|
|
37
|
+
def _mark_browser_required(domain: str, config: dict[str, Any]) -> None:
|
|
38
|
+
"""Mark a domain as requiring browser bypass for future ranking."""
|
|
39
|
+
_browser_domains.add(domain)
|
|
40
|
+
|
|
41
|
+
def _is_browser_domain(domain: str) -> bool:
|
|
42
|
+
"""Check if domain requires browser bypass."""
|
|
43
|
+
return domain in _browser_domains
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# ---------------------------------------------------------------------------
|
|
47
|
+
# Persistent browser-worker pool
|
|
48
|
+
# ---------------------------------------------------------------------------
|
|
49
|
+
# Every browser-backed Sci-Hub attempt (browser-first download, tab racing,
|
|
50
|
+
# Cloudflare/ALTCHA bypass) runs on THIS pool instead of ad-hoc per-paper
|
|
51
|
+
# executors. Its threads live for the whole process, so each worker's
|
|
52
|
+
# thread-local browser (browser_engine._get_shared_browser) is created once
|
|
53
|
+
# and reused for every paper: no per-paper window flashing, no per-paper
|
|
54
|
+
# startup cost, and Cloudflare clearance cookies survive between attempts.
|
|
55
|
+
# Workers are closed by _shutdown_scihub_pool (registered below) — never per
|
|
56
|
+
# paper. browser_engine's atexit reaper is the last-resort backstop.
|
|
57
|
+
|
|
58
|
+
_RACE_POOL: ThreadPoolExecutor | None = None
|
|
59
|
+
_RACE_POOL_LOCK = threading.Lock()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_HTTP_POOL: ThreadPoolExecutor | None = None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _submit(pool: ThreadPoolExecutor, fn: Any, *args: Any, **kwargs: Any) -> Any:
|
|
66
|
+
"""Submit to the long-lived Sci-Hub pool.
|
|
67
|
+
|
|
68
|
+
The task knows which paper it works for (``carry_owner``: a waived paper's
|
|
69
|
+
task must not launch a browser), but the browser it opens is the worker's
|
|
70
|
+
own and serves later papers too, so it is registered as SHARED: skipping or
|
|
71
|
+
finishing one paper does not kill it under the next (DeepSeek review
|
|
72
|
+
2026-10-19); cancelling the job does."""
|
|
73
|
+
from ..browser.backend import carry_owner, shared
|
|
74
|
+
|
|
75
|
+
owned = carry_owner(fn)
|
|
76
|
+
|
|
77
|
+
def run(*a: Any, **k: Any) -> Any:
|
|
78
|
+
with shared():
|
|
79
|
+
return owned(*a, **k)
|
|
80
|
+
return pool.submit(run, *args, **kwargs)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _lane_that_cleans_up(fn: Any, doi: str, domain: str, src_output: Path,
|
|
84
|
+
config: dict[str, Any], use_tor: bool,
|
|
85
|
+
fail_notes: list[str] | None, race_over: threading.Event) -> Any:
|
|
86
|
+
"""One mirror lane of the Sci-Hub race.
|
|
87
|
+
|
|
88
|
+
The race returns as soon as a mirror delivers, and the other lanes keep
|
|
89
|
+
running in the background; the cleanup at that moment cannot see a file a
|
|
90
|
+
loser is still writing, so "<doi>_Sci-Hub_scihub_<mirror>.pdf" stayed in
|
|
91
|
+
the folder (2026-11-03). Each lane drops its own copy once the race is
|
|
92
|
+
over. A winner has already renamed its file to the download path, so
|
|
93
|
+
nothing left on the lane's own path is read by anyone.
|
|
94
|
+
"""
|
|
95
|
+
try:
|
|
96
|
+
return fn(doi, domain, src_output, config, use_tor, fail_notes)
|
|
97
|
+
finally:
|
|
98
|
+
if race_over.is_set() and src_output.exists():
|
|
99
|
+
try:
|
|
100
|
+
src_output.unlink(missing_ok=True)
|
|
101
|
+
except OSError:
|
|
102
|
+
pass
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _race_pool(config: dict[str, Any]) -> ThreadPoolExecutor:
|
|
106
|
+
"""Workers for the mirror race. With browsers, few (scihub_browser_workers:
|
|
107
|
+
each may hold a window). Over HTTP only, plenty (scihub_http_workers):
|
|
108
|
+
4 papers x 3 mirrors queued behind 3 workers, and losers still holding them,
|
|
109
|
+
made a short list's first round take minutes (2026-10-17)."""
|
|
110
|
+
global _RACE_POOL, _HTTP_POOL
|
|
111
|
+
if not config.get("scihub_use_browser", True):
|
|
112
|
+
with _RACE_POOL_LOCK:
|
|
113
|
+
if _HTTP_POOL is None:
|
|
114
|
+
workers = max(1, int(config.get("scihub_http_workers", 12) or 12))
|
|
115
|
+
_HTTP_POOL = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="scihub-http")
|
|
116
|
+
return _HTTP_POOL
|
|
117
|
+
with _RACE_POOL_LOCK:
|
|
118
|
+
if _RACE_POOL is None:
|
|
119
|
+
workers = max(1, int(config.get("scihub_browser_workers", 3) or 3))
|
|
120
|
+
_RACE_POOL = ThreadPoolExecutor(
|
|
121
|
+
max_workers=workers, thread_name_prefix="scihub-browser"
|
|
122
|
+
)
|
|
123
|
+
return _RACE_POOL
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _close_my_browser() -> None:
|
|
127
|
+
"""Runs INSIDE a pool worker: closes that worker's own thread-local browser."""
|
|
128
|
+
try:
|
|
129
|
+
from ..browser.engine import shutdown_shared_browser
|
|
130
|
+
shutdown_shared_browser()
|
|
131
|
+
except Exception:
|
|
132
|
+
pass
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _shutdown_scihub_pool() -> None:
|
|
136
|
+
"""Gracefully close every pool worker's browser, then the pool. Idempotent."""
|
|
137
|
+
global _RACE_POOL
|
|
138
|
+
with _RACE_POOL_LOCK:
|
|
139
|
+
pool, _RACE_POOL = _RACE_POOL, None
|
|
140
|
+
if pool is None:
|
|
141
|
+
return
|
|
142
|
+
try:
|
|
143
|
+
# Idle workers each pick up one shutdown task and close their own
|
|
144
|
+
# browser in their own thread (Playwright sync objects are
|
|
145
|
+
# thread-affine). shutdown(wait=False) so a worker stuck in an
|
|
146
|
+
# interactive Cloudflare wait can't hang process exit; anything
|
|
147
|
+
# missed here is tree-killed by the process-exit reaper in
|
|
148
|
+
# browser_engine (issue #57).
|
|
149
|
+
for _ in range(pool._max_workers): # type: ignore[attr-defined]
|
|
150
|
+
pool.submit(_close_my_browser)
|
|
151
|
+
pool.shutdown(wait=False)
|
|
152
|
+
except Exception:
|
|
153
|
+
pass
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
atexit.register(_shutdown_scihub_pool)
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
# Verification-wall pacing (ALTCHA gates, e.g. sci-hub.ru)
|
|
160
|
+
# ---------------------------------------------------------------------------
|
|
161
|
+
# Empirically (2026-08-30 live tests): the gate escalates with request
|
|
162
|
+
# VELOCITY, not per session — solving works once, then rapid repeat
|
|
163
|
+
# verifications earn a standing "你是机器人吗" wall on every request. So the
|
|
164
|
+
# winning strategy is pacing (space verifications out) plus exponential
|
|
165
|
+
# cooldown when a wall persists after a solve, instead of hammering.
|
|
166
|
+
|
|
167
|
+
_WALL_MIN_SPACING_SEC = 25.0 # min gap between verification solves
|
|
168
|
+
_WALL_COOLDOWN_BASE_SEC = 90.0 # first persistent-wall cooldown
|
|
169
|
+
_WALL_COOLDOWN_CAP_SEC = 900.0 # never cool down longer than this
|
|
170
|
+
_MIRROR_STRUCTURAL_COOLDOWN_SEC = 7200.0 # structurally broken mirror: skip 2h
|
|
171
|
+
|
|
172
|
+
# Serialize INTERACTIVE Turnstile solving across pool workers. Each pool
|
|
173
|
+
# worker owns a browser, so without this lock N workers hitting a gate open N
|
|
174
|
+
# "please click the captcha" windows at once. Queued workers re-check after
|
|
175
|
+
# the first solve; the clearance cookie is shared per domain, so they usually
|
|
176
|
+
# pass without a new human challenge.
|
|
177
|
+
_TURNSTILE_INTERACTIVE_LOCK = threading.Lock()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _wall_guard(domain: str, config: dict[str, Any]) -> bool:
|
|
181
|
+
"""True while a domain is cooling down after persistent walls — skip it.
|
|
182
|
+
|
|
183
|
+
State lives in domain_db (wall_state table): persistent across processes
|
|
184
|
+
and sessions. Do NOT reintroduce in-memory health dicts — that split
|
|
185
|
+
caused three generations of mirror-health rework.
|
|
186
|
+
"""
|
|
187
|
+
st = domain_db.get_wall_state(domain, config)
|
|
188
|
+
return time.time() < st["cooldown_until"]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _wall_pace(domain: str, config: dict[str, Any]) -> None:
|
|
192
|
+
"""Sleep so consecutive verification solves stay spaced out."""
|
|
193
|
+
st = domain_db.get_wall_state(domain, config)
|
|
194
|
+
wait = _WALL_MIN_SPACING_SEC - (time.time() - st["last_solve"])
|
|
195
|
+
if wait > 0:
|
|
196
|
+
time.sleep(min(wait, _WALL_MIN_SPACING_SEC))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _note_wall(domain: str, config: dict[str, Any]) -> None:
|
|
200
|
+
"""A wall persisted after a solve attempt — escalate the cooldown."""
|
|
201
|
+
st = domain_db.get_wall_state(domain, config)
|
|
202
|
+
walls = int(st["walls"]) + 1
|
|
203
|
+
cooldown = min(_WALL_COOLDOWN_BASE_SEC * (3 ** (walls - 1)), _WALL_COOLDOWN_CAP_SEC)
|
|
204
|
+
until = time.time() + cooldown
|
|
205
|
+
domain_db.set_wall_state(domain, config, last_solve=time.time(), walls=walls, cooldown_until=until)
|
|
206
|
+
log.info(f" Sci-Hub: {domain} verification wall persists — cooling down {cooldown:.0f}s")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _note_wall_success(domain: str, config: dict[str, Any]) -> None:
|
|
210
|
+
"""A solve cleared the wall — reset escalations, record the solve time."""
|
|
211
|
+
domain_db.set_wall_state(domain, config, last_solve=time.time(), walls=0, cooldown_until=0.0)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _note_structural(domain: str, config: dict[str, Any]) -> None:
|
|
215
|
+
"""Structurally broken response (homepage shell / interactive gate).
|
|
216
|
+
|
|
217
|
+
Unlike transient walls, re-paying the timeout every paper is pure loss:
|
|
218
|
+
skip the mirror for hours. It retries automatically once the cooldown
|
|
219
|
+
lapses, so a mirror that comes back to life is picked up again.
|
|
220
|
+
"""
|
|
221
|
+
until = time.time() + _MIRROR_STRUCTURAL_COOLDOWN_SEC
|
|
222
|
+
domain_db.set_wall_state(domain, config, last_solve=time.time(), walls=1, cooldown_until=until)
|
|
223
|
+
log.info(f" Sci-Hub: {domain} structurally broken — skipping for "
|
|
224
|
+
f"{_MIRROR_STRUCTURAL_COOLDOWN_SEC / 3600:.0f}h")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _classify_mirror_page(html: str) -> str:
|
|
228
|
+
"""Classify a mirror response: turnstile | homepage | other.
|
|
229
|
+
|
|
230
|
+
ALTCHA walls are intentionally NOT classified here — the dedicated ALTCHA
|
|
231
|
+
solver handles them.
|
|
232
|
+
"""
|
|
233
|
+
low = (html or "").lower()
|
|
234
|
+
if "challenges.cloudflare.com/turnstile" in low or "verification - sci-hub" in low:
|
|
235
|
+
return "turnstile"
|
|
236
|
+
if "search proxy to download" in low:
|
|
237
|
+
return "homepage"
|
|
238
|
+
return "other"
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _racing_browser_headless(config: dict[str, Any]) -> bool:
|
|
242
|
+
"""Mirror the headless computation used to launch the racing browser."""
|
|
243
|
+
return bool(config.get("scihub_browser_headless", config.get("browser_headless", False)))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
_PROBE_TTL_HOURS = 4
|
|
247
|
+
_SCIHUB_PROBE_WORKERS = 8
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _probe_single_domain(domain: str, proxy: str | None, timeout: tuple[int, int]) -> tuple[str, bool, float]:
|
|
251
|
+
proxies = proxy_dict(proxy)
|
|
252
|
+
t0 = time.time()
|
|
253
|
+
try:
|
|
254
|
+
s = requests.Session()
|
|
255
|
+
s.trust_env = False
|
|
256
|
+
resp = s.get(domain, timeout=timeout, proxies=proxies, allow_redirects=True,
|
|
257
|
+
headers={"User-Agent": USER_AGENT})
|
|
258
|
+
# Accept 200/302/301 as reachable (302/301 = redirect, still means domain is alive)
|
|
259
|
+
# 403/503 = reachable but blocked (Cloudflare) — still mark reachable for browser bypass
|
|
260
|
+
reachable = resp.status_code in (200, 301, 302, 403, 503)
|
|
261
|
+
ok = resp.status_code == 200 and ("sci-hub" in resp.text[:5000].lower() or "scihub" in resp.text[:5000].lower())
|
|
262
|
+
# 403/503 with Cloudflare signature = reachable via browser
|
|
263
|
+
if resp.status_code in (403, 503) and _is_cloudflare_block(resp):
|
|
264
|
+
_mark_browser_required(domain, None)
|
|
265
|
+
reachable = True
|
|
266
|
+
latency = (time.time() - t0) * 1000
|
|
267
|
+
return (domain, ok if ok else reachable, latency)
|
|
268
|
+
except requests.exceptions.Timeout:
|
|
269
|
+
# Timeout doesn't mean unreachable - just slow
|
|
270
|
+
latency = (time.time() - t0) * 1000
|
|
271
|
+
return (domain, True, latency)
|
|
272
|
+
except Exception as e:
|
|
273
|
+
log.debug(f"Sci-Hub probe {domain}: {type(e).__name__}")
|
|
274
|
+
return (domain, False, 99999.0)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
_PROBE_RUNNING = threading.Lock()
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _probe_in_background(config: dict[str, Any]) -> None:
|
|
281
|
+
"""_probe_scihub_domains off the paper's path; one at a time (papers of a
|
|
282
|
+
round start together and would otherwise all probe)."""
|
|
283
|
+
if not _PROBE_RUNNING.acquire(blocking=False):
|
|
284
|
+
return
|
|
285
|
+
|
|
286
|
+
def run() -> None:
|
|
287
|
+
try:
|
|
288
|
+
_probe_scihub_domains(config)
|
|
289
|
+
except Exception as exc:
|
|
290
|
+
log.info(f" Sci-Hub: background probe failed: {type(exc).__name__}: {exc}")
|
|
291
|
+
finally:
|
|
292
|
+
_PROBE_RUNNING.release()
|
|
293
|
+
threading.Thread(target=run, name="scihub-probe", daemon=True).start()
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _probe_scihub_domains(config: dict[str, Any]) -> None:
|
|
297
|
+
last_probe = get_probe_timestamp(config)
|
|
298
|
+
now = time.time()
|
|
299
|
+
if now - last_probe < _PROBE_TTL_HOURS * 3600:
|
|
300
|
+
return
|
|
301
|
+
|
|
302
|
+
proxy = select_proxy_for_url("https://sci-hub.mksa.top", config)
|
|
303
|
+
domains = config.get("scihub_domains") or DEFAULT_SCIHUB_DOMAINS
|
|
304
|
+
timeout = (5, 10)
|
|
305
|
+
|
|
306
|
+
pool = ThreadPoolExecutor(max_workers=_SCIHUB_PROBE_WORKERS)
|
|
307
|
+
futures = {pool.submit(_probe_single_domain, d, proxy, timeout): d for d in domains}
|
|
308
|
+
pending = set(futures)
|
|
309
|
+
try:
|
|
310
|
+
for future in as_completed(futures, timeout=15):
|
|
311
|
+
pending.discard(future)
|
|
312
|
+
domain, ok, latency = future.result()
|
|
313
|
+
update_probe(domain, ok, round(latency, 1), config)
|
|
314
|
+
except TimeoutError:
|
|
315
|
+
# A mirror that hangs past the probe window is unreachable for now.
|
|
316
|
+
# Raising here used to abort the whole Sci-Hub source on every call
|
|
317
|
+
# (the probe timestamp was never set, so the next call re-probed).
|
|
318
|
+
for future in pending:
|
|
319
|
+
update_probe(futures[future], False, 15.0, config)
|
|
320
|
+
finally:
|
|
321
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
322
|
+
|
|
323
|
+
set_probe_timestamp(config)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _is_browser_available(config: dict[str, Any]) -> bool:
|
|
327
|
+
"""Check if CloakBrowser is available. Returns False in asyncio context.
|
|
328
|
+
|
|
329
|
+
scihub_use_browser=false turns every browser path of the grey sources off:
|
|
330
|
+
ALTCHA mirrors are handled over plain HTTP (sources/altcha.py), and
|
|
331
|
+
Turnstile-only mirrors are skipped instead of opening windows.
|
|
332
|
+
"""
|
|
333
|
+
if not config.get("scihub_use_browser", True):
|
|
334
|
+
return False
|
|
335
|
+
# A running asyncio loop normally means "user asyncio code — the Playwright
|
|
336
|
+
# sync API cannot serve it". Exception: our persistent pool workers, where
|
|
337
|
+
# the loop was created by the sync API itself (browser_engine tracks it).
|
|
338
|
+
try:
|
|
339
|
+
import asyncio
|
|
340
|
+
loop = asyncio.get_running_loop()
|
|
341
|
+
except RuntimeError:
|
|
342
|
+
loop = None
|
|
343
|
+
if loop is not None:
|
|
344
|
+
from ..browser.engine import is_playwright_owned_loop
|
|
345
|
+
if not is_playwright_owned_loop(loop):
|
|
346
|
+
return False
|
|
347
|
+
try:
|
|
348
|
+
from ..browser.engine import is_available
|
|
349
|
+
return is_available(config)
|
|
350
|
+
except Exception:
|
|
351
|
+
return False
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _solve_altcha_and_reload(
|
|
355
|
+
solve_result: dict[str, Any],
|
|
356
|
+
landing_url: str,
|
|
357
|
+
doi: str,
|
|
358
|
+
output_path: Path,
|
|
359
|
+
config: dict[str, Any],
|
|
360
|
+
) -> dict[str, Any] | None:
|
|
361
|
+
"""Solve ALTCHA anti-bot verification on a Sci-Hub page, then reload and extract PDF.
|
|
362
|
+
|
|
363
|
+
ALTCHA is a proof-of-work CAPTCHA used by sci-hub.ru. The page shows a checkbox
|
|
364
|
+
labelled "不是" (No/Not a robot). After clicking, a proof-of-work computation runs,
|
|
365
|
+
then the page shows "您是人类!" (You are human!). Once verified, we reload the page
|
|
366
|
+
to get the actual paper content.
|
|
367
|
+
"""
|
|
368
|
+
import time as _time
|
|
369
|
+
|
|
370
|
+
try:
|
|
371
|
+
from ..browser.engine import get_browser_page
|
|
372
|
+
except ImportError:
|
|
373
|
+
log.info(" [altcha] browser engine not available for ALTCHA bypass")
|
|
374
|
+
return None
|
|
375
|
+
|
|
376
|
+
domain = urllib.parse.urlparse(landing_url).netloc
|
|
377
|
+
if _wall_guard(domain, config):
|
|
378
|
+
log.info(f" [altcha] {domain} in wall cooldown — skipping this attempt")
|
|
379
|
+
return None
|
|
380
|
+
_wall_pace(domain, config)
|
|
381
|
+
|
|
382
|
+
page = None
|
|
383
|
+
try:
|
|
384
|
+
page = get_browser_page(config)
|
|
385
|
+
if not page:
|
|
386
|
+
log.info(" [altcha] could not get browser page")
|
|
387
|
+
return None
|
|
388
|
+
|
|
389
|
+
# Navigate to the landing URL
|
|
390
|
+
page.goto(landing_url, wait_until="domcontentloaded", timeout=20000)
|
|
391
|
+
_time.sleep(2)
|
|
392
|
+
|
|
393
|
+
# Find and click the ALTCHA checkbox
|
|
394
|
+
checkbox_selectors = [
|
|
395
|
+
"input[type='checkbox']",
|
|
396
|
+
"#altcha_checkbox",
|
|
397
|
+
"[id^='altcha_checkbox_']",
|
|
398
|
+
]
|
|
399
|
+
clicked = False
|
|
400
|
+
for selector in checkbox_selectors:
|
|
401
|
+
try:
|
|
402
|
+
el = page.query_selector(selector)
|
|
403
|
+
if el:
|
|
404
|
+
el.click(timeout=5000)
|
|
405
|
+
clicked = True
|
|
406
|
+
log.info(f" [altcha] clicked checkbox: {selector}")
|
|
407
|
+
break
|
|
408
|
+
except Exception:
|
|
409
|
+
continue
|
|
410
|
+
|
|
411
|
+
if not clicked:
|
|
412
|
+
# Try clicking the "不是" div
|
|
413
|
+
try:
|
|
414
|
+
no_btn = page.query_selector("div[onclick='check()']")
|
|
415
|
+
if no_btn:
|
|
416
|
+
no_btn.click(timeout=5000)
|
|
417
|
+
clicked = True
|
|
418
|
+
log.info(" [altcha] clicked '不是' button")
|
|
419
|
+
except Exception:
|
|
420
|
+
pass
|
|
421
|
+
|
|
422
|
+
if not clicked:
|
|
423
|
+
log.info(" [altcha] could not find checkbox/button to click")
|
|
424
|
+
return None
|
|
425
|
+
|
|
426
|
+
# Wait for verification to complete (poll for "您是人类" or "verified")
|
|
427
|
+
for i in range(15):
|
|
428
|
+
_time.sleep(1)
|
|
429
|
+
try:
|
|
430
|
+
body_text = page.evaluate("() => document.body ? document.body.innerText : ''")
|
|
431
|
+
except Exception:
|
|
432
|
+
continue
|
|
433
|
+
if "您是人类" in body_text or "verified" in body_text.lower():
|
|
434
|
+
log.info(f" [altcha] verified after {i + 3}s")
|
|
435
|
+
break
|
|
436
|
+
if i % 5 == 4:
|
|
437
|
+
log.info(f" [altcha] still verifying... ({i + 3}s)")
|
|
438
|
+
|
|
439
|
+
# Reload the page to get the actual paper content
|
|
440
|
+
log.info(" [altcha] reloading page after verification...")
|
|
441
|
+
page.goto(landing_url, wait_until="domcontentloaded", timeout=20000)
|
|
442
|
+
_time.sleep(2)
|
|
443
|
+
|
|
444
|
+
# Now try to extract PDF
|
|
445
|
+
try:
|
|
446
|
+
html = page.content()
|
|
447
|
+
except Exception:
|
|
448
|
+
log.info(" [altcha] failed to get page content after reload")
|
|
449
|
+
return None
|
|
450
|
+
|
|
451
|
+
# Check for a PERSISTENT wall: if the reloaded page is still the
|
|
452
|
+
# "你是机器人吗" verification, solving again would only deepen the
|
|
453
|
+
# rate limit — record it and let the cooldown skip this domain.
|
|
454
|
+
from ..download.pdf import is_pdf_file, success, extract_pdf_url_from_html
|
|
455
|
+
from ..browser.engine import download_pdf_via_browser
|
|
456
|
+
|
|
457
|
+
lower = html.lower()
|
|
458
|
+
if "你是机器人吗" in html or "altcha" in lower and "iframe" not in lower:
|
|
459
|
+
_note_wall(domain, config)
|
|
460
|
+
log.info(" [altcha] verification wall persists after solve")
|
|
461
|
+
return None
|
|
462
|
+
|
|
463
|
+
if any(sig in lower for sig in ["article not found", "статья не найдена", "не найден"]):
|
|
464
|
+
log.info(" [altcha] article not found after verification")
|
|
465
|
+
return None
|
|
466
|
+
|
|
467
|
+
# Extract PDF URL
|
|
468
|
+
pdf_url = extract_pdf_url_from_html(html, landing_url)
|
|
469
|
+
if pdf_url:
|
|
470
|
+
log.info(f" [altcha] found PDF: {pdf_url[:80]}")
|
|
471
|
+
if download_pdf_via_browser(pdf_url, output_path, config):
|
|
472
|
+
if is_pdf_file(output_path):
|
|
473
|
+
_note_wall_success(domain, config)
|
|
474
|
+
return success(doi, output_path, "Sci-Hub(altcha)")
|
|
475
|
+
|
|
476
|
+
log.info(" [altcha] no PDF found after verification")
|
|
477
|
+
return None
|
|
478
|
+
|
|
479
|
+
except Exception as e:
|
|
480
|
+
log.info(f" [altcha] error: {e}")
|
|
481
|
+
return None
|
|
482
|
+
finally:
|
|
483
|
+
if page:
|
|
484
|
+
try:
|
|
485
|
+
page.close()
|
|
486
|
+
except Exception:
|
|
487
|
+
pass
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _browser_first_download(
|
|
491
|
+
landing_url: str,
|
|
492
|
+
doi: str,
|
|
493
|
+
output_path: Path,
|
|
494
|
+
config: dict[str, Any],
|
|
495
|
+
fail_notes: list[str] | None = None,
|
|
496
|
+
) -> dict[str, Any] | None:
|
|
497
|
+
"""Try browser-first download for Sci-Hub. Bypasses Cloudflare/CAPTCHA.
|
|
498
|
+
|
|
499
|
+
fail_notes (optional): appended with a short classification string at
|
|
500
|
+
every failure exit so callers can report WHY Sci-Hub gave up
|
|
501
|
+
(not_in_library / challenge_* / network / http_*).
|
|
502
|
+
"""
|
|
503
|
+
try:
|
|
504
|
+
from ..browser.engine import solve_url, download_pdf_via_browser
|
|
505
|
+
from ..download.pdf import is_pdf_file, success, extract_pdf_url_from_html
|
|
506
|
+
from urllib.parse import urlparse
|
|
507
|
+
domain = urlparse(landing_url).netloc or landing_url[:40]
|
|
508
|
+
|
|
509
|
+
log.info(f" [browser-first] trying {landing_url[:80]}")
|
|
510
|
+
result = solve_url(landing_url, config, max_timeout=30000)
|
|
511
|
+
if not result:
|
|
512
|
+
log.info(f" [browser-first] no response")
|
|
513
|
+
if fail_notes is not None:
|
|
514
|
+
fail_notes.append("network(no response)")
|
|
515
|
+
return None
|
|
516
|
+
|
|
517
|
+
solution = result.get("solution", {})
|
|
518
|
+
html = solution.get("response", "")
|
|
519
|
+
if not html:
|
|
520
|
+
log.info(f" [browser-first] empty response")
|
|
521
|
+
if fail_notes is not None:
|
|
522
|
+
fail_notes.append("network(empty response)")
|
|
523
|
+
return None
|
|
524
|
+
|
|
525
|
+
# Check for article not found
|
|
526
|
+
lower = html.lower()
|
|
527
|
+
if any(sig in lower for sig in ["article not found", "статья не найдена", "не найден"]):
|
|
528
|
+
log.info(f" [browser-first] article not found on {domain}")
|
|
529
|
+
if fail_notes is not None:
|
|
530
|
+
fail_notes.append("not_in_library")
|
|
531
|
+
return None
|
|
532
|
+
|
|
533
|
+
# Structural failures: mirror shell or interactive gate. These cost a
|
|
534
|
+
# fixed timeout every paper until cooled down — note once, skip for hours.
|
|
535
|
+
kind = _classify_mirror_page(html)
|
|
536
|
+
if kind == "homepage":
|
|
537
|
+
_note_structural(domain, config)
|
|
538
|
+
log.info(f" [browser-first] {domain} serves its homepage shell — structural, cooling down")
|
|
539
|
+
if fail_notes is not None:
|
|
540
|
+
fail_notes.append("structural_homepage")
|
|
541
|
+
return None
|
|
542
|
+
if kind == "turnstile":
|
|
543
|
+
if _racing_browser_headless(config) or not config.get("scihub_turnstile_click", True):
|
|
544
|
+
_note_structural(domain, config)
|
|
545
|
+
log.info(f" [browser-first] Turnstile gate on {domain} — headless/no-click session, cooling down")
|
|
546
|
+
return None
|
|
547
|
+
# Interactive mode: surface it on the progress bar, wait for one
|
|
548
|
+
# human click. The clearance cookie then lasts the whole batch.
|
|
549
|
+
# Serialized across pool workers — see _TURNSTILE_INTERACTIVE_LOCK.
|
|
550
|
+
with _TURNSTILE_INTERACTIVE_LOCK:
|
|
551
|
+
from ..browser.engine import get_browser_page
|
|
552
|
+
page = get_browser_page(config)
|
|
553
|
+
if page is None:
|
|
554
|
+
_note_structural(domain, config)
|
|
555
|
+
return None
|
|
556
|
+
attention_key = f"turnstile:{domain}"
|
|
557
|
+
try:
|
|
558
|
+
from ..download import progress as _pr
|
|
559
|
+
_pr.set_attention(attention_key, "请在浏览器窗口完成 Turnstile 人机验证",
|
|
560
|
+
current=doi, phase="人工验证")
|
|
561
|
+
except Exception:
|
|
562
|
+
_pr = None
|
|
563
|
+
passed = False
|
|
564
|
+
try:
|
|
565
|
+
deadline = time.time() + max(30, int(config.get("turnstile_wait_sec", 180)))
|
|
566
|
+
while time.time() < deadline:
|
|
567
|
+
time.sleep(5)
|
|
568
|
+
page.goto(landing_url, wait_until="domcontentloaded", timeout=30000)
|
|
569
|
+
html = page.content()
|
|
570
|
+
if _classify_mirror_page(html) != "turnstile":
|
|
571
|
+
passed = True
|
|
572
|
+
break
|
|
573
|
+
if not passed:
|
|
574
|
+
log.info(f" [browser-first] Turnstile not passed within wait window")
|
|
575
|
+
if fail_notes is not None:
|
|
576
|
+
fail_notes.append("challenge_turnstile_timeout")
|
|
577
|
+
return None
|
|
578
|
+
solution["url"] = page.url
|
|
579
|
+
log.info(f" [browser-first] Turnstile cleared by human — cookie kept for the batch")
|
|
580
|
+
finally:
|
|
581
|
+
if _pr is not None:
|
|
582
|
+
try:
|
|
583
|
+
_pr.clear_attention(attention_key)
|
|
584
|
+
except Exception:
|
|
585
|
+
pass
|
|
586
|
+
try:
|
|
587
|
+
page.close()
|
|
588
|
+
except Exception:
|
|
589
|
+
pass
|
|
590
|
+
lower = html.lower()
|
|
591
|
+
|
|
592
|
+
# Check for ALTCHA anti-bot verification (used by sci-hub.ru and other mirrors)
|
|
593
|
+
if any(sig in lower for sig in ["altcha", "你是机器人吗", "not a robot"]):
|
|
594
|
+
log.info(f" [browser-first] ALTCHA detected on {domain}, attempting bypass...")
|
|
595
|
+
try:
|
|
596
|
+
altcha_result = _solve_altcha_and_reload(result, landing_url, doi, output_path, config)
|
|
597
|
+
if altcha_result:
|
|
598
|
+
return altcha_result
|
|
599
|
+
except Exception as e:
|
|
600
|
+
log.info(f" [browser-first] ALTCHA bypass failed: {e}")
|
|
601
|
+
|
|
602
|
+
# Check for Cloudflare challenge page
|
|
603
|
+
if any(sig in lower for sig in ["checking your browser", "just a moment", "cf-browser-verification"]):
|
|
604
|
+
log.info(f" [browser-first] Cloudflare challenge on {domain}, page may need more time")
|
|
605
|
+
# The solve_url should have waited, but if we still see the challenge,
|
|
606
|
+
# mark this domain as needing browser bypass for future attempts
|
|
607
|
+
if fail_notes is not None:
|
|
608
|
+
fail_notes.append("challenge_cloudflare")
|
|
609
|
+
return None
|
|
610
|
+
|
|
611
|
+
# Check for empty embed (Sci-Hub has no PDF)
|
|
612
|
+
if '<embed' in lower and 'src=""' in lower:
|
|
613
|
+
log.info(f" [browser-first] empty embed — article not in Sci-Hub database")
|
|
614
|
+
if fail_notes is not None:
|
|
615
|
+
fail_notes.append("not_in_library")
|
|
616
|
+
return None
|
|
617
|
+
|
|
618
|
+
# Extract PDF URL from HTML
|
|
619
|
+
pdf_url = extract_pdf_url_from_html(html, solution.get("url", landing_url))
|
|
620
|
+
if pdf_url:
|
|
621
|
+
log.info(f" [browser-first] found PDF: {pdf_url[:80]}")
|
|
622
|
+
# Download via browser (handles Cloudflare on PDF host too)
|
|
623
|
+
if download_pdf_via_browser(pdf_url, output_path, config):
|
|
624
|
+
# Retry is_pdf_file check — browser may still be flushing to disk
|
|
625
|
+
for _retry in range(5):
|
|
626
|
+
if is_pdf_file(output_path):
|
|
627
|
+
return success(doi, output_path, f"Sci-Hub(browser)")
|
|
628
|
+
time.sleep(0.2)
|
|
629
|
+
log.info(f" [browser-first] downloaded but file not recognized as PDF")
|
|
630
|
+
|
|
631
|
+
# Check if the response itself is a PDF
|
|
632
|
+
import base64
|
|
633
|
+
resp_data = solution.get("response", "")
|
|
634
|
+
if isinstance(resp_data, str) and len(resp_data) > 5000:
|
|
635
|
+
try:
|
|
636
|
+
pdf_bytes = base64.b64decode(resp_data) if resp_data.startswith("JVBER") else resp_data.encode("utf-8")
|
|
637
|
+
if pdf_bytes[:5] == b"%PDF-":
|
|
638
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
639
|
+
output_path.write_bytes(pdf_bytes)
|
|
640
|
+
if is_pdf_file(output_path):
|
|
641
|
+
return success(doi, output_path, f"Sci-Hub(browser)")
|
|
642
|
+
except Exception:
|
|
643
|
+
pass
|
|
644
|
+
|
|
645
|
+
log.info(f" [browser-first] no PDF found in response")
|
|
646
|
+
if fail_notes is not None:
|
|
647
|
+
fail_notes.append("not_in_library(no pdf in response)")
|
|
648
|
+
return None
|
|
649
|
+
except Exception as e:
|
|
650
|
+
log.info(f" [browser-first] error: {e}")
|
|
651
|
+
if fail_notes is not None:
|
|
652
|
+
fail_notes.append(f"network({type(e).__name__})")
|
|
653
|
+
return None
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _said_absent(domain: str, fail_notes: list[str]) -> bool:
|
|
657
|
+
"""The domain answered "not in the library": a correct answer, not a failure of the mirror."""
|
|
658
|
+
return any(n.endswith(f"@{domain}") and n.startswith("not_in_library") for n in fail_notes)
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
# _http_altcha_download's answer when the gate was solved and the article page
|
|
662
|
+
# has no PDF: the mirror does not hold the paper, so no browser is opened for it.
|
|
663
|
+
ABSENT: dict[str, Any] = {"success": False, "not_in_library": True}
|
|
664
|
+
|
|
665
|
+
# Papers every answering mirror said it does not have, with when: a later
|
|
666
|
+
# download of the same paper within the hour (a short list's second round)
|
|
667
|
+
# does not ask Sci-Hub again, so the browser slot goes to the publisher's site.
|
|
668
|
+
_ABSENT_AT: dict[str, float] = {}
|
|
669
|
+
_ABSENT_TTL = 3600.0
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def recently_absent(doi: str) -> bool:
|
|
673
|
+
return time.time() - _ABSENT_AT.get((doi or "").strip().lower(), 0.0) < _ABSENT_TTL
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
# Publication years the caller knows (an app job has them), by DOI.
|
|
677
|
+
_YEARS: dict[str, int] = {}
|
|
678
|
+
_DOI_YEAR = re.compile(r"[./](20[0-9]{2})[./]") # 10.1016/j.ceramint.2026.06.274, 10.26599/jac.2025.9221068
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def note_years(years: dict[str, Any]) -> None:
|
|
682
|
+
for doi, year in years.items():
|
|
683
|
+
try:
|
|
684
|
+
_YEARS[(doi or "").strip().lower()] = int(str(year)[:4])
|
|
685
|
+
except (TypeError, ValueError):
|
|
686
|
+
pass
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def publication_year(doi: str) -> int | None:
|
|
690
|
+
"""The year noted by the caller, else one written in the DOI, else None."""
|
|
691
|
+
key = (doi or "").strip().lower()
|
|
692
|
+
if key in _YEARS:
|
|
693
|
+
return _YEARS[key]
|
|
694
|
+
m = _DOI_YEAR.search(key.split("/", 1)[-1])
|
|
695
|
+
return int(m.group(1)) if m else None
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def too_new(doi: str, config: dict[str, Any]) -> bool:
|
|
699
|
+
"""Published in or after scihub_skip_from_year: Sci-Hub has almost none of
|
|
700
|
+
those (2026-10-17: 0 of 19 papers from 2024-2026), so it is not asked."""
|
|
701
|
+
try:
|
|
702
|
+
start = int(config.get("scihub_skip_from_year") or 0)
|
|
703
|
+
except (TypeError, ValueError):
|
|
704
|
+
return False
|
|
705
|
+
year = publication_year(doi) if start else None
|
|
706
|
+
return bool(year and year >= start)
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def _http_altcha_download(landing_url: str, domain: str, doi: str, output_path: Path,
|
|
710
|
+
config: dict[str, Any]) -> dict[str, Any] | None:
|
|
711
|
+
"""Open the mirror page over HTTP, solve an ALTCHA gate if any, fetch the PDF.
|
|
712
|
+
|
|
713
|
+
Returns the download, ABSENT (gate solved, no PDF on the page), or None
|
|
714
|
+
(nothing learned: unreachable, a gate that needs a browser, ...).
|
|
715
|
+
"""
|
|
716
|
+
from ..network import CHROME_UA
|
|
717
|
+
from . import altcha
|
|
718
|
+
|
|
719
|
+
session = requests.Session()
|
|
720
|
+
session.trust_env = False
|
|
721
|
+
session.headers.update({"User-Agent": CHROME_UA,
|
|
722
|
+
"Accept": "text/html,application/xhtml+xml,*/*;q=0.8"})
|
|
723
|
+
session.proxies = proxy_dict(select_proxy_for_url(landing_url, config)) or {}
|
|
724
|
+
timeout = (10, 25)
|
|
725
|
+
try:
|
|
726
|
+
resp = session.get(landing_url, timeout=timeout)
|
|
727
|
+
except Exception as exc:
|
|
728
|
+
log.info(f" [altcha-http] {domain}: {type(exc).__name__}")
|
|
729
|
+
return None
|
|
730
|
+
if resp.status_code != 200:
|
|
731
|
+
return None
|
|
732
|
+
html = resp.text
|
|
733
|
+
gated = altcha.is_altcha_page(html)
|
|
734
|
+
if gated:
|
|
735
|
+
# The article page behind the gate still embeds an ALTCHA widget (for
|
|
736
|
+
# its own buttons), so judge success by the PDF link, not by "altcha".
|
|
737
|
+
html = altcha.pass_gate(session, html, landing_url, timeout) or ""
|
|
738
|
+
if not html:
|
|
739
|
+
return None
|
|
740
|
+
elif _classify_mirror_page(html) == "turnstile":
|
|
741
|
+
return None # needs a real browser: leave it to the browser-first path
|
|
742
|
+
pdf_url = extract_pdf_url_from_html(html, landing_url)
|
|
743
|
+
if not pdf_url:
|
|
744
|
+
return ABSENT if gated else None
|
|
745
|
+
log.info(f" [altcha-http] {domain}: PDF {pdf_url[:80]}")
|
|
746
|
+
from ..download import progress as progress_reporter
|
|
747
|
+
progress_reporter.item_stage(doi, "Sci-Hub 有下载链接,正在核对")
|
|
748
|
+
result = download_pdf_from_scihub(pdf_url, output_path, config, f"Sci-Hub({domain})",
|
|
749
|
+
cookies=session.cookies, referer=landing_url)
|
|
750
|
+
if result:
|
|
751
|
+
result["doi"] = doi
|
|
752
|
+
result["identifier"] = doi
|
|
753
|
+
return result
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def try_scihub_domain(
|
|
757
|
+
doi: str,
|
|
758
|
+
domain: str,
|
|
759
|
+
output_path: Path,
|
|
760
|
+
config: dict[str, Any],
|
|
761
|
+
use_tor: bool = False,
|
|
762
|
+
fail_notes: list[str] | None = None,
|
|
763
|
+
) -> dict[str, Any] | None:
|
|
764
|
+
landing_url = f"{domain.rstrip('/')}/{urllib.parse.quote(doi, safe='/')}"
|
|
765
|
+
|
|
766
|
+
# Plain HTTP first: an ALTCHA gate is a proof-of-work that Python solves in
|
|
767
|
+
# seconds, so mirrors behind it (sci-hub.ru) need no browser window at all.
|
|
768
|
+
# Only gates that need a real browser (Cloudflare Turnstile) fall through.
|
|
769
|
+
if config.get("scihub_http_altcha", True) and not use_tor:
|
|
770
|
+
result = _http_altcha_download(landing_url, domain, doi, output_path, config)
|
|
771
|
+
if result is ABSENT:
|
|
772
|
+
log.info(f" [altcha-http] {domain}: gate solved, the paper is not in this mirror")
|
|
773
|
+
if fail_notes is not None:
|
|
774
|
+
fail_notes.append(f"not_in_library(after altcha)@{domain}")
|
|
775
|
+
return None
|
|
776
|
+
if result:
|
|
777
|
+
return result
|
|
778
|
+
|
|
779
|
+
# Browser-first: bypass Cloudflare/CAPTCHA before HTTP attempt.
|
|
780
|
+
# scihub_browser_first=false forces the pure-HTTP path — when the PDF
|
|
781
|
+
# CDN challenges headless browsers but serves plain requests (observed
|
|
782
|
+
# in the field, 2026-09: sci.bban.top), HTTP is the working lane.
|
|
783
|
+
if (config.get("scihub_browser_first", True)
|
|
784
|
+
and _is_browser_available(config)):
|
|
785
|
+
result = _browser_first_download(landing_url, doi, output_path, config,
|
|
786
|
+
fail_notes=fail_notes)
|
|
787
|
+
if result:
|
|
788
|
+
return result
|
|
789
|
+
|
|
790
|
+
# Sci-Hub pages and the PDF CDN are slow through proxies; racing-grade
|
|
791
|
+
# timeouts (users run connect=3/read=7 for source racing) starve this
|
|
792
|
+
# lane — a 3s read timeout kills every attempt before the page lands.
|
|
793
|
+
# Floor the timeouts for this lane only; other sources keep their pace.
|
|
794
|
+
config = {
|
|
795
|
+
**config,
|
|
796
|
+
"connect_timeout": max(int(config.get("connect_timeout", 3)), 10),
|
|
797
|
+
"read_timeout": max(int(config.get("read_timeout", 7)), 25),
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
try:
|
|
801
|
+
resp = fetch(landing_url, config, stream=True, use_tor=use_tor)
|
|
802
|
+
|
|
803
|
+
# Fallback to browser on Cloudflare/403/CAPTCHA
|
|
804
|
+
if resp.status_code in (403, 503) or _is_cloudflare_block(resp):
|
|
805
|
+
_mark_browser_required(domain, config)
|
|
806
|
+
resp = _try_browser(landing_url, config, resp) if _is_browser_available(config) else None
|
|
807
|
+
if resp is None:
|
|
808
|
+
if fail_notes is not None:
|
|
809
|
+
fail_notes.append("challenge(browser bypass failed)")
|
|
810
|
+
return None
|
|
811
|
+
|
|
812
|
+
if resp.status_code >= 400:
|
|
813
|
+
if fail_notes is not None:
|
|
814
|
+
fail_notes.append(f"http_{resp.status_code}")
|
|
815
|
+
return None
|
|
816
|
+
|
|
817
|
+
first = next(resp.iter_content(chunk_size=8192), b"")
|
|
818
|
+
# Check for CAPTCHA in first chunk
|
|
819
|
+
if resp.status_code == 200 and first:
|
|
820
|
+
content_sample = first[:5000].decode('utf-8', errors='ignore').lower()
|
|
821
|
+
if 'captcha' in content_sample or 'recaptcha' in content_sample:
|
|
822
|
+
log.info(f" CAPTCHA detected, trying browser...")
|
|
823
|
+
# Use browser to bypass CAPTCHA
|
|
824
|
+
browser_resp = _try_browser(landing_url, config, resp) if _is_browser_available(config) else None
|
|
825
|
+
if browser_resp is None:
|
|
826
|
+
log.warning(f" browser bypass failed — is CloakBrowser installed? Run: pip install cloakbrowser")
|
|
827
|
+
if fail_notes is not None:
|
|
828
|
+
fail_notes.append("challenge(captcha bypass failed)")
|
|
829
|
+
return None
|
|
830
|
+
# Get new content from browser response
|
|
831
|
+
resp = browser_resp
|
|
832
|
+
first = resp.content[:8192] if resp.content else b""
|
|
833
|
+
log.info(f" browser bypassed CAPTCHA, content size: {len(first)}")
|
|
834
|
+
|
|
835
|
+
if _response_looks_pdf(resp, first):
|
|
836
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
837
|
+
tmp_path = output_path.with_suffix(output_path.suffix + ".part")
|
|
838
|
+
try:
|
|
839
|
+
with tmp_path.open("wb") as fh:
|
|
840
|
+
fh.write(first)
|
|
841
|
+
for chunk in resp.iter_content(chunk_size=8192):
|
|
842
|
+
if chunk:
|
|
843
|
+
fh.write(chunk)
|
|
844
|
+
tmp_path.replace(output_path)
|
|
845
|
+
except Exception:
|
|
846
|
+
tmp_path.unlink(missing_ok=True)
|
|
847
|
+
raise
|
|
848
|
+
if is_pdf_file(output_path):
|
|
849
|
+
return success(doi, output_path, f"Sci-Hub({domain})")
|
|
850
|
+
|
|
851
|
+
# Collect full HTML content (browser responses have _content, direct responses need raw.read)
|
|
852
|
+
if resp._content:
|
|
853
|
+
html = first + resp._content
|
|
854
|
+
else:
|
|
855
|
+
try:
|
|
856
|
+
html = first + resp.raw.read(512_000, decode_content=True)
|
|
857
|
+
except Exception:
|
|
858
|
+
html = first
|
|
859
|
+
pdf_url = extract_pdf_url_from_html(html.decode("utf-8", errors="ignore"), resp.url)
|
|
860
|
+
if not pdf_url:
|
|
861
|
+
if fail_notes is not None:
|
|
862
|
+
fail_notes.append("not_in_library(no pdf url)")
|
|
863
|
+
return None
|
|
864
|
+
result = download_pdf_from_scihub(pdf_url, output_path, config, f"Sci-Hub({domain})",
|
|
865
|
+
use_tor=use_tor, cookies=resp.cookies, referer=landing_url)
|
|
866
|
+
if result:
|
|
867
|
+
result["doi"] = doi
|
|
868
|
+
result["identifier"] = doi
|
|
869
|
+
elif fail_notes is not None:
|
|
870
|
+
fail_notes.append("not_in_library(pdf download failed)")
|
|
871
|
+
return result
|
|
872
|
+
except Exception as e:
|
|
873
|
+
log.warning(" [scihub] %s HTTP lane failed: %r", domain, e, exc_info=True)
|
|
874
|
+
if fail_notes is not None:
|
|
875
|
+
fail_notes.append(f"network({type(e).__name__})")
|
|
876
|
+
if isinstance(e, (requests.Timeout, requests.ConnectionError)):
|
|
877
|
+
# A mirror that does not answer (2026-10-17: sci-hub.mksa.top now
|
|
878
|
+
# redirects to a host that times out) costs every paper 15-30 s:
|
|
879
|
+
# mark it unreachable, so the next 10 minutes skip it.
|
|
880
|
+
try:
|
|
881
|
+
from .scihub_domains import record_result as _db_result, update_probe
|
|
882
|
+
_db_result(domain, False, config)
|
|
883
|
+
update_probe(domain, False, 0.0, config)
|
|
884
|
+
except Exception:
|
|
885
|
+
pass
|
|
886
|
+
return None
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
def _try_browser(
|
|
890
|
+
url: str,
|
|
891
|
+
config: dict[str, Any],
|
|
892
|
+
original_resp: requests.Response,
|
|
893
|
+
) -> requests.Response | None:
|
|
894
|
+
"""Try CloakBrowser to bypass Cloudflare. Returns Response or None."""
|
|
895
|
+
if not _is_browser_available(config):
|
|
896
|
+
return None
|
|
897
|
+
# solve_url lives in browser_engine; the old import path (the FlareSolverr
|
|
898
|
+
# client module, removed 2026-10-19) broke the challenge bypass with ImportError.
|
|
899
|
+
from ..browser.engine import solve_url
|
|
900
|
+
result = solve_url(url, config)
|
|
901
|
+
if not result:
|
|
902
|
+
return None
|
|
903
|
+
solution = result.get("solution", {})
|
|
904
|
+
status = solution.get("status", 0)
|
|
905
|
+
if status >= 400:
|
|
906
|
+
return None
|
|
907
|
+
# Build a Response from browser solution
|
|
908
|
+
resp = requests.Response()
|
|
909
|
+
resp.status_code = status
|
|
910
|
+
html_content = solution.get("response", "")
|
|
911
|
+
resp._content = html_content.encode("utf-8") if isinstance(html_content, str) else html_content
|
|
912
|
+
resp.url = solution.get("url", url)
|
|
913
|
+
cookies = solution.get("cookies", [])
|
|
914
|
+
if isinstance(cookies, list):
|
|
915
|
+
for c in cookies:
|
|
916
|
+
if "name" in c and "value" in c:
|
|
917
|
+
resp.cookies.set(c["name"], c["value"])
|
|
918
|
+
# Mock raw attribute so downstream code can read content
|
|
919
|
+
class _RawMock:
|
|
920
|
+
def read(self, size=0, decode_content=False):
|
|
921
|
+
return resp._content[size:] if size else resp._content
|
|
922
|
+
resp.raw = _RawMock()
|
|
923
|
+
return resp
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def _await_slow_pdfs(pending: dict[Any, tuple[str, Path]],
|
|
927
|
+
config: dict[str, Any]) -> tuple[Any, dict[str, Any]] | None:
|
|
928
|
+
"""Wait for backup lanes whose PDF is already streaming but missed the race.
|
|
929
|
+
|
|
930
|
+
Only lanes with a growing ``.part`` file are worth waiting for: a lane that
|
|
931
|
+
has not started writing is not about to deliver a PDF. Each waits at most
|
|
932
|
+
``scihub_slow_pdf_seconds`` in total and is dropped after
|
|
933
|
+
``scihub_slow_pdf_idle_seconds`` without new bytes.
|
|
934
|
+
"""
|
|
935
|
+
total = float(config.get("scihub_slow_pdf_seconds", 90) or 0)
|
|
936
|
+
if total <= 0 or not pending:
|
|
937
|
+
return None
|
|
938
|
+
idle = float(config.get("scihub_slow_pdf_idle_seconds", 30) or 30)
|
|
939
|
+
deadline = time.monotonic() + total
|
|
940
|
+
last_size: dict[Any, int] = {}
|
|
941
|
+
grew_at: dict[Any, float] = {}
|
|
942
|
+
while pending:
|
|
943
|
+
now = time.monotonic()
|
|
944
|
+
for future in list(pending):
|
|
945
|
+
if future.done():
|
|
946
|
+
pending.pop(future)
|
|
947
|
+
try:
|
|
948
|
+
result = future.result()
|
|
949
|
+
except Exception:
|
|
950
|
+
result = None
|
|
951
|
+
if result and result.get("success"):
|
|
952
|
+
return future, result
|
|
953
|
+
continue
|
|
954
|
+
src = pending[future][1]
|
|
955
|
+
part = src.with_suffix(src.suffix + ".part")
|
|
956
|
+
try:
|
|
957
|
+
size = part.stat().st_size
|
|
958
|
+
except OSError:
|
|
959
|
+
size = 0
|
|
960
|
+
if size > last_size.get(future, 0):
|
|
961
|
+
last_size[future] = size
|
|
962
|
+
grew_at[future] = now
|
|
963
|
+
elif future not in grew_at or now - grew_at[future] > idle:
|
|
964
|
+
pending.pop(future) # never started writing, or it stalled
|
|
965
|
+
if not pending or time.monotonic() >= deadline:
|
|
966
|
+
break
|
|
967
|
+
time.sleep(0.5)
|
|
968
|
+
return None
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def download_pdf_from_scihub(
|
|
972
|
+
url: str,
|
|
973
|
+
output_path: Path,
|
|
974
|
+
config: dict[str, Any],
|
|
975
|
+
source: str,
|
|
976
|
+
use_tor: bool = False,
|
|
977
|
+
cookies: Any = None,
|
|
978
|
+
referer: str = "",
|
|
979
|
+
) -> dict[str, Any] | None:
|
|
980
|
+
from ..download.pdf import download_pdf
|
|
981
|
+
return download_pdf(url, output_path, config, source, require_pdf_like_url=False,
|
|
982
|
+
use_tor=use_tor, cookies=cookies, referer=referer, browser_ua=True)
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def _race_browser_domains(
|
|
986
|
+
domains: list[str],
|
|
987
|
+
doi: str,
|
|
988
|
+
output_path: Path,
|
|
989
|
+
config: dict[str, Any],
|
|
990
|
+
max_workers: int = 3,
|
|
991
|
+
) -> dict[str, Any] | None:
|
|
992
|
+
"""Race multiple Sci-Hub domains using multiple tabs in ONE browser.
|
|
993
|
+
|
|
994
|
+
Instead of spawning N threads each with their own browser (heavy),
|
|
995
|
+
this opens N tabs in a single browser context and polls them in
|
|
996
|
+
round-robin. The browser's internal network stack handles all tabs
|
|
997
|
+
concurrently, so we get real parallelism without multiple processes.
|
|
998
|
+
|
|
999
|
+
Each tab writes to its own temp output file to avoid conflicts.
|
|
1000
|
+
The winning file is renamed to output_path; loser tabs and temp
|
|
1001
|
+
files are cleaned up.
|
|
1002
|
+
"""
|
|
1003
|
+
from ..browser.engine import _get_shared_browser, is_available as _browser_available
|
|
1004
|
+
|
|
1005
|
+
if not _browser_available(config):
|
|
1006
|
+
return None
|
|
1007
|
+
|
|
1008
|
+
try:
|
|
1009
|
+
browser, context = _get_shared_browser(config)
|
|
1010
|
+
except Exception as e:
|
|
1011
|
+
log.info(f" Sci-Hub: cannot get browser: {e}")
|
|
1012
|
+
return None
|
|
1013
|
+
|
|
1014
|
+
log.info(f" Sci-Hub: racing {len(domains)} browser domains via tabs (1 browser)...")
|
|
1015
|
+
|
|
1016
|
+
# Phase 1: Fire all navigations — open a tab per domain, trigger navigation
|
|
1017
|
+
# without blocking (use JS location assignment, returns immediately)
|
|
1018
|
+
tabs: list[tuple[str, Any, Path]] = [] # (domain, page, temp_output)
|
|
1019
|
+
for domain in domains:
|
|
1020
|
+
landing_url = f"{domain.rstrip('/')}/{urllib.parse.quote(doi, safe='/')}"
|
|
1021
|
+
safe_suffix = domain.split("//")[-1].replace(".", "_").replace("/", "_")[:25]
|
|
1022
|
+
temp_output = output_path.parent / f"{output_path.stem}_browser_{safe_suffix}.pdf"
|
|
1023
|
+
try:
|
|
1024
|
+
page = context.new_page()
|
|
1025
|
+
# Fire-and-forget via JS — navigate without blocking so all tabs load concurrently
|
|
1026
|
+
page.evaluate(f"window.location.href = '{landing_url}'")
|
|
1027
|
+
tabs.append((domain, page, temp_output))
|
|
1028
|
+
except Exception as e:
|
|
1029
|
+
log.info(f" Sci-Hub: tab open failed for {domain}: {e}")
|
|
1030
|
+
try:
|
|
1031
|
+
page.close()
|
|
1032
|
+
except Exception:
|
|
1033
|
+
pass
|
|
1034
|
+
|
|
1035
|
+
if not tabs:
|
|
1036
|
+
return None
|
|
1037
|
+
|
|
1038
|
+
# Wait a moment for all navigations to start, then poll for DOM readiness
|
|
1039
|
+
time.sleep(1.5)
|
|
1040
|
+
|
|
1041
|
+
# Phase 2: Poll tabs in round-robin — wait for DOM, then extract PDF URL
|
|
1042
|
+
deadline = time.time() + 30
|
|
1043
|
+
winner: tuple[dict[str, Any], Path] | None = None
|
|
1044
|
+
|
|
1045
|
+
while time.time() < deadline and winner is None:
|
|
1046
|
+
for domain, page, temp_output in tabs:
|
|
1047
|
+
try:
|
|
1048
|
+
# Wait for this tab's DOM to be ready (short timeout per check)
|
|
1049
|
+
try:
|
|
1050
|
+
page.wait_for_load_state("domcontentloaded", timeout=3000)
|
|
1051
|
+
except Exception:
|
|
1052
|
+
pass # Not ready yet, try next tab
|
|
1053
|
+
|
|
1054
|
+
html = page.content()
|
|
1055
|
+
if len(html) < 5000:
|
|
1056
|
+
continue
|
|
1057
|
+
|
|
1058
|
+
lower = html.lower()
|
|
1059
|
+
# Skip if still on Cloudflare challenge or article not found
|
|
1060
|
+
if any(sig in lower for sig in ["checking your browser", "just a moment"]):
|
|
1061
|
+
continue
|
|
1062
|
+
if any(sig in lower for sig in ["article not found", "статья не найдена"]):
|
|
1063
|
+
log.info(f" Sci-Hub: {domain} — article not found")
|
|
1064
|
+
continue
|
|
1065
|
+
|
|
1066
|
+
# Try to extract PDF URL
|
|
1067
|
+
pdf_url = extract_pdf_url_from_html(html, page.url)
|
|
1068
|
+
if pdf_url:
|
|
1069
|
+
log.info(f" Sci-Hub: {domain} found PDF, downloading...")
|
|
1070
|
+
from ..browser.engine import download_pdf_via_browser
|
|
1071
|
+
if download_pdf_via_browser(pdf_url, temp_output, config):
|
|
1072
|
+
# Verify PDF file — retry with backoff (browser may still flush)
|
|
1073
|
+
for _retry in range(10):
|
|
1074
|
+
if temp_output.exists() and temp_output.stat().st_size > 5000:
|
|
1075
|
+
if is_pdf_file(temp_output):
|
|
1076
|
+
result = success(doi, temp_output, f"Sci-Hub(tab:{domain.split('//')[-1][:15]})")
|
|
1077
|
+
winner = (result, temp_output)
|
|
1078
|
+
break
|
|
1079
|
+
time.sleep(0.3)
|
|
1080
|
+
if winner is not None:
|
|
1081
|
+
break
|
|
1082
|
+
else:
|
|
1083
|
+
log.info(f" Sci-Hub: {domain} download finished but PDF check failed")
|
|
1084
|
+
except Exception:
|
|
1085
|
+
continue
|
|
1086
|
+
if winner is None:
|
|
1087
|
+
time.sleep(0.3)
|
|
1088
|
+
|
|
1089
|
+
# Phase 3: Cleanup — close all tabs, remove loser temp files
|
|
1090
|
+
for domain, page, temp_output in tabs:
|
|
1091
|
+
try:
|
|
1092
|
+
page.close()
|
|
1093
|
+
except Exception:
|
|
1094
|
+
pass
|
|
1095
|
+
if winner is None or temp_output != winner[1]:
|
|
1096
|
+
if temp_output.exists():
|
|
1097
|
+
try:
|
|
1098
|
+
temp_output.unlink(missing_ok=True)
|
|
1099
|
+
except OSError:
|
|
1100
|
+
pass
|
|
1101
|
+
|
|
1102
|
+
if winner is not None:
|
|
1103
|
+
result, temp_output = winner
|
|
1104
|
+
if temp_output != output_path and temp_output.exists():
|
|
1105
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1106
|
+
if output_path.exists():
|
|
1107
|
+
output_path.unlink()
|
|
1108
|
+
temp_output.rename(output_path)
|
|
1109
|
+
result["file"] = str(output_path)
|
|
1110
|
+
return result
|
|
1111
|
+
|
|
1112
|
+
log.info(" Sci-Hub: no tab found a PDF")
|
|
1113
|
+
return None
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
def try_scihub(doi: str, output_path: Path, config: dict[str, Any], use_tor: bool = False) -> dict[str, Any] | None:
|
|
1117
|
+
# All browser-backed attempts below run on the persistent worker pool
|
|
1118
|
+
# (_race_pool): pool threads live for the whole process, so each worker's
|
|
1119
|
+
# thread-local browser is created once and reused across papers — no
|
|
1120
|
+
# per-paper window flashing. Do NOT shut browsers down here; cleanup is
|
|
1121
|
+
# _shutdown_scihub_pool (atexit) plus browser_engine's process-exit reaper.
|
|
1122
|
+
if too_new(doi, config):
|
|
1123
|
+
log.info(f" Sci-Hub: skipped, published {publication_year(doi)} (scihub_skip_from_year)")
|
|
1124
|
+
return None
|
|
1125
|
+
try:
|
|
1126
|
+
return _try_scihub_impl(doi, output_path, config, use_tor)
|
|
1127
|
+
except Exception as e:
|
|
1128
|
+
log.info(f" Sci-Hub: unexpected error: {type(e).__name__}: {e}")
|
|
1129
|
+
# Check if a PDF was written to disk despite the exception
|
|
1130
|
+
if output_path.exists():
|
|
1131
|
+
from ..download.pdf import is_pdf_file as _is_pdf
|
|
1132
|
+
if _is_pdf(output_path):
|
|
1133
|
+
log.info(f" Sci-Hub: recovered PDF after {type(e).__name__}")
|
|
1134
|
+
return {"success": True, "identifier": doi, "doi": doi,
|
|
1135
|
+
"file": str(output_path), "source": "Sci-Hub(recovered)"}
|
|
1136
|
+
return None
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
def _try_scihub_impl(doi: str, output_path: Path, config: dict[str, Any], use_tor: bool = False) -> dict[str, Any] | None:
|
|
1140
|
+
log.info(f" try_scihub called for {doi}")
|
|
1141
|
+
if not config.get("scihub_enabled", False):
|
|
1142
|
+
log.info(f" Sci-Hub disabled")
|
|
1143
|
+
return None
|
|
1144
|
+
|
|
1145
|
+
# Browser-first pass: race configured domains via browser in parallel.
|
|
1146
|
+
# Opt-in (default off) — opening CloakBrowser for every Sci-Hub attempt
|
|
1147
|
+
# is slow and can leave orphan Chromium processes (issue #19). The HTTP
|
|
1148
|
+
# path below still works; browser is only tried when explicitly enabled
|
|
1149
|
+
# or as a Cloudflare/ALTCHA challenge fallback.
|
|
1150
|
+
if config.get("scihub_browser_first_enabled", False) and _is_browser_available(config):
|
|
1151
|
+
configured_domains = config.get("scihub_domains") or DEFAULT_SCIHUB_DOMAINS
|
|
1152
|
+
browser_domains = [d for d in configured_domains[:5] if not _wall_guard(d, config)]
|
|
1153
|
+
if not browser_domains:
|
|
1154
|
+
browser_domains = [d for d in configured_domains[:5]] # all cooling down — try anyway, paced
|
|
1155
|
+
max_workers = min(config.get("scihub_browser_workers", 3), len(browser_domains))
|
|
1156
|
+
|
|
1157
|
+
pool = _race_pool(config)
|
|
1158
|
+
if max_workers <= 1 or len(browser_domains) == 1:
|
|
1159
|
+
# Single worker or single domain: sequential attempts on the pool
|
|
1160
|
+
# (browser lives on the pool worker, not the caller's thread).
|
|
1161
|
+
for domain in browser_domains:
|
|
1162
|
+
landing_url = f"{domain.rstrip('/')}/{urllib.parse.quote(doi, safe='/')}"
|
|
1163
|
+
result = _submit(
|
|
1164
|
+
pool, _browser_first_download, landing_url, doi, output_path, config
|
|
1165
|
+
).result()
|
|
1166
|
+
if result:
|
|
1167
|
+
return result
|
|
1168
|
+
else:
|
|
1169
|
+
result = _submit(
|
|
1170
|
+
pool, _race_browser_domains, browser_domains, doi, output_path, config, max_workers
|
|
1171
|
+
).result()
|
|
1172
|
+
if result:
|
|
1173
|
+
return result
|
|
1174
|
+
|
|
1175
|
+
all_domains = config.get("scihub_domains") or DEFAULT_SCIHUB_DOMAINS
|
|
1176
|
+
fastest = scihub_speed.preferred_mirror()
|
|
1177
|
+
if fastest and fastest in all_domains and not _wall_guard(fastest, config):
|
|
1178
|
+
# The batch's speed test has just reached this mirror end to end: start
|
|
1179
|
+
# with it, and refresh the reachability stats in the background
|
|
1180
|
+
# (2026-10-19: the probe held the first paper 15 s after a 5 s speed test).
|
|
1181
|
+
_probe_in_background(config)
|
|
1182
|
+
else:
|
|
1183
|
+
_probe_scihub_domains(config)
|
|
1184
|
+
stats = load_stats(config)
|
|
1185
|
+
|
|
1186
|
+
# Track failure reason for better diagnostic messages
|
|
1187
|
+
_failure_reason = "all domains unreachable"
|
|
1188
|
+
_any_reachable = False
|
|
1189
|
+
_any_browser_tried = False
|
|
1190
|
+
fail_notes: list[str] = []
|
|
1191
|
+
|
|
1192
|
+
now = time.time()
|
|
1193
|
+
cooldown_domains = []
|
|
1194
|
+
for d in all_domains:
|
|
1195
|
+
if _wall_guard(d, config):
|
|
1196
|
+
continue # persistent verification wall — pacing cooldown
|
|
1197
|
+
d_stats = stats.get(d, {})
|
|
1198
|
+
last_fail = d_stats.get("last_fail_time", 0)
|
|
1199
|
+
fail_streak = d_stats.get("fail_streak", 0)
|
|
1200
|
+
reachable = d_stats.get("reachable")
|
|
1201
|
+
# Skip domains with many consecutive failures
|
|
1202
|
+
if fail_streak >= 10 and (now - last_fail) < 300:
|
|
1203
|
+
continue
|
|
1204
|
+
# Skip domains that are unreachable (from recent probe)
|
|
1205
|
+
if reachable is False and (now - last_fail) < 600:
|
|
1206
|
+
continue
|
|
1207
|
+
cooldown_domains.append(d)
|
|
1208
|
+
|
|
1209
|
+
if not cooldown_domains:
|
|
1210
|
+
for d in all_domains:
|
|
1211
|
+
stats[d] = {"success": 0, "fail": 0, "last_fail_time": 0, "fail_streak": 0}
|
|
1212
|
+
cooldown_domains = all_domains
|
|
1213
|
+
|
|
1214
|
+
def _domain_score(d: str) -> float:
|
|
1215
|
+
s = stats.get(d, {})
|
|
1216
|
+
successes = s.get("success", 0)
|
|
1217
|
+
failures = s.get("fail", 0)
|
|
1218
|
+
reachable = s.get("reachable")
|
|
1219
|
+
total = successes + failures
|
|
1220
|
+
# Unreachable domains get lowest score
|
|
1221
|
+
if reachable is False:
|
|
1222
|
+
return -99999
|
|
1223
|
+
if total == 0:
|
|
1224
|
+
return 0.5
|
|
1225
|
+
success_rate = successes / total
|
|
1226
|
+
avg_latency = s.get("avg_latency_ms", 5000)
|
|
1227
|
+
score = success_rate * 1000 - avg_latency / 1000
|
|
1228
|
+
# Boost browser-accessible domains (bypasses Cloudflare)
|
|
1229
|
+
if _is_browser_available(config) and _is_browser_domain(d):
|
|
1230
|
+
score += 5000
|
|
1231
|
+
return score
|
|
1232
|
+
|
|
1233
|
+
cooldown_domains.sort(key=_domain_score, reverse=True)
|
|
1234
|
+
domains = cooldown_domains[:3]
|
|
1235
|
+
# The batch's speed test picked a mirror: ask it first (the others follow).
|
|
1236
|
+
fastest = scihub_speed.preferred_mirror()
|
|
1237
|
+
if fastest and fastest in all_domains and not _wall_guard(fastest, config):
|
|
1238
|
+
domains = [fastest] + [d for d in domains if d != fastest][:2]
|
|
1239
|
+
log.info(f" Sci-Hub domains to try: {domains}")
|
|
1240
|
+
|
|
1241
|
+
if len(domains) == 1:
|
|
1242
|
+
try:
|
|
1243
|
+
result = try_scihub_domain(doi, domains[0], output_path, config, use_tor=use_tor,
|
|
1244
|
+
fail_notes=fail_notes)
|
|
1245
|
+
if result:
|
|
1246
|
+
record_result(domains[0], True, config)
|
|
1247
|
+
return result
|
|
1248
|
+
if not _said_absent(domains[0], fail_notes):
|
|
1249
|
+
record_result(domains[0], False, config)
|
|
1250
|
+
except Exception:
|
|
1251
|
+
record_result(domains[0], False, config)
|
|
1252
|
+
log.info(f" Sci-Hub: only domain {domains[0]} failed")
|
|
1253
|
+
return None
|
|
1254
|
+
|
|
1255
|
+
# Try best domain first with short timeout
|
|
1256
|
+
best_domain = domains[0]
|
|
1257
|
+
best_tag = re.sub(r"[^A-Za-z0-9_-]+", "_", best_domain.split("//")[-1])
|
|
1258
|
+
best_output = output_path.parent / f"{output_path.stem}_scihub_{best_tag}.pdf"
|
|
1259
|
+
log.info(f" Sci-Hub: trying {best_domain} first...")
|
|
1260
|
+
try:
|
|
1261
|
+
result = _submit(
|
|
1262
|
+
_race_pool(config), try_scihub_domain, doi, best_domain, best_output, config, use_tor, fail_notes
|
|
1263
|
+
).result()
|
|
1264
|
+
if result and result.get("success"):
|
|
1265
|
+
final_path = Path(result.get("file", ""))
|
|
1266
|
+
if final_path != output_path and final_path.exists():
|
|
1267
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1268
|
+
if output_path.exists():
|
|
1269
|
+
output_path.unlink()
|
|
1270
|
+
final_path.rename(output_path)
|
|
1271
|
+
result["file"] = str(output_path)
|
|
1272
|
+
log.info(f" Sci-Hub: OK {best_domain}")
|
|
1273
|
+
record_result(best_domain, True, config)
|
|
1274
|
+
scihub_speed.note_result(best_domain, True)
|
|
1275
|
+
return result
|
|
1276
|
+
scihub_speed.note_result(best_domain, False, absent=_said_absent(best_domain, fail_notes))
|
|
1277
|
+
if not _said_absent(best_domain, fail_notes):
|
|
1278
|
+
record_result(best_domain, False, config)
|
|
1279
|
+
except Exception:
|
|
1280
|
+
record_result(best_domain, False, config)
|
|
1281
|
+
|
|
1282
|
+
# Best domain failed - race remaining domains
|
|
1283
|
+
remaining = domains[1:]
|
|
1284
|
+
if not remaining:
|
|
1285
|
+
return None
|
|
1286
|
+
|
|
1287
|
+
log.info(f" Sci-Hub: racing {len(remaining)} backup domains...")
|
|
1288
|
+
# Persistent pool — workers (and their browsers) are NOT torn down here.
|
|
1289
|
+
# Losers that outlive the 45 s collection window keep streaming in the
|
|
1290
|
+
# background and are simply ignored. Each lane deletes its own temp file
|
|
1291
|
+
# once the race is over: cleanup here only sees the files that exist by
|
|
1292
|
+
# then, and a loser writing its PDF afterwards used to leave
|
|
1293
|
+
# "<doi>_Sci-Hub_scihub_<mirror>.pdf" behind (2026-11-03).
|
|
1294
|
+
pool = _race_pool(config)
|
|
1295
|
+
race_over = threading.Event()
|
|
1296
|
+
|
|
1297
|
+
def _adopt(domain: str, result: dict[str, Any]) -> dict[str, Any]:
|
|
1298
|
+
final_path = Path(result.get("file", ""))
|
|
1299
|
+
if final_path != output_path and final_path.exists():
|
|
1300
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1301
|
+
if output_path.exists():
|
|
1302
|
+
output_path.unlink()
|
|
1303
|
+
final_path.rename(output_path)
|
|
1304
|
+
result["file"] = str(output_path)
|
|
1305
|
+
for _, other_path in futures.values():
|
|
1306
|
+
if other_path != output_path and other_path.exists():
|
|
1307
|
+
try:
|
|
1308
|
+
other_path.unlink(missing_ok=True)
|
|
1309
|
+
except OSError:
|
|
1310
|
+
pass
|
|
1311
|
+
record_result(domain, True, config)
|
|
1312
|
+
log.info(f" Sci-Hub: OK {domain}")
|
|
1313
|
+
race_over.set()
|
|
1314
|
+
return result
|
|
1315
|
+
|
|
1316
|
+
futures = {}
|
|
1317
|
+
for domain in remaining:
|
|
1318
|
+
dom_tag = re.sub(r"[^A-Za-z0-9_-]+", "_", domain.split("//")[-1])
|
|
1319
|
+
src_output = output_path.parent / f"{output_path.stem}_scihub_{dom_tag}.pdf"
|
|
1320
|
+
futures[_submit(pool, _lane_that_cleans_up, try_scihub_domain, doi, domain, src_output,
|
|
1321
|
+
config, use_tor, fail_notes, race_over)] = (domain, src_output)
|
|
1322
|
+
backup_timeout = float(config.get("scihub_backup_race_seconds", 45) or 45)
|
|
1323
|
+
try:
|
|
1324
|
+
# 10 s ended the race while the backups were still streaming PDFs they
|
|
1325
|
+
# had already found (sci.bban.top serves at a few hundred KB/s here).
|
|
1326
|
+
for future in as_completed(futures, timeout=backup_timeout):
|
|
1327
|
+
domain, src_output = futures[future]
|
|
1328
|
+
try:
|
|
1329
|
+
result = future.result(timeout=1)
|
|
1330
|
+
except Exception:
|
|
1331
|
+
result = None
|
|
1332
|
+
if result and result.get("success"):
|
|
1333
|
+
return _adopt(domain, result)
|
|
1334
|
+
else:
|
|
1335
|
+
if not _said_absent(domain, fail_notes):
|
|
1336
|
+
record_result(domain, False, config)
|
|
1337
|
+
if src_output.exists():
|
|
1338
|
+
try:
|
|
1339
|
+
src_output.unlink(missing_ok=True)
|
|
1340
|
+
except OSError:
|
|
1341
|
+
pass
|
|
1342
|
+
except TimeoutError:
|
|
1343
|
+
log.info(" Sci-Hub: backup domains timed out")
|
|
1344
|
+
# A mirror that found the PDF but streams it slowly loses the 45 s race
|
|
1345
|
+
# while its file is still growing on disk (sci.bban.top serves at a few
|
|
1346
|
+
# hundred KB/s). Give those lanes a longer leash instead of calling the
|
|
1347
|
+
# download a failure and fetching the paper again later.
|
|
1348
|
+
pending = {f: v for f, v in futures.items() if not f.done()}
|
|
1349
|
+
slow = _await_slow_pdfs(pending, config)
|
|
1350
|
+
if slow is not None:
|
|
1351
|
+
return _adopt(futures[slow[0]][0], slow[1])
|
|
1352
|
+
for _, src_output in futures.values():
|
|
1353
|
+
if src_output.exists():
|
|
1354
|
+
try:
|
|
1355
|
+
src_output.unlink(missing_ok=True)
|
|
1356
|
+
except OSError:
|
|
1357
|
+
pass
|
|
1358
|
+
# Whatever still runs now deletes its own copy when it ends.
|
|
1359
|
+
race_over.set()
|
|
1360
|
+
|
|
1361
|
+
# Every mirror that answered said the paper is not there: Tor would ask the
|
|
1362
|
+
# same library again, only slower.
|
|
1363
|
+
absent = bool(fail_notes) and all(n.startswith("not_in_library") for n in fail_notes)
|
|
1364
|
+
if absent:
|
|
1365
|
+
log.warning(f" Sci-Hub: {doi} is not in the Sci-Hub library (every mirror that answered said so)")
|
|
1366
|
+
_ABSENT_AT[(doi or "").strip().lower()] = time.time()
|
|
1367
|
+
return None
|
|
1368
|
+
# All clearnet domains failed — auto-retry with Tor + .onion (only if config allows)
|
|
1369
|
+
if not use_tor and config.get("use_tor_for_scihub", True):
|
|
1370
|
+
log.info(" Sci-Hub: all clearnet domains failed, retrying via Tor...")
|
|
1371
|
+
return try_scihub(doi, output_path, config, use_tor=True)
|
|
1372
|
+
|
|
1373
|
+
if fail_notes:
|
|
1374
|
+
from collections import Counter
|
|
1375
|
+
summary = ", ".join(f"{k}×{v}" for k, v in Counter(fail_notes).most_common())
|
|
1376
|
+
log.warning(f" Sci-Hub: all domains failed for {doi} ({summary})")
|
|
1377
|
+
else:
|
|
1378
|
+
log.warning(f" Sci-Hub: all domains failed for {doi}. Check: 1) network connectivity 2) Tor status (litloop tor_start)")
|
|
1379
|
+
return None
|