litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1379 @@
1
+ """Sci-Hub source with domain rotation, CAPTCHA detection, and Tor support."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import atexit
6
+ import re
7
+ import threading
8
+ import time
9
+ import urllib.parse
10
+ from concurrent.futures import ThreadPoolExecutor, as_completed
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ import requests
15
+
16
+ from ..config import DEFAULT_SCIHUB_DOMAINS
17
+ from . import scihub_domains as domain_db
18
+ from .scihub_domains import (
19
+ load_stats,
20
+ record_result,
21
+ update_probe,
22
+ set_probe_timestamp,
23
+ get_probe_timestamp,
24
+ get_wall_state,
25
+ set_wall_state,
26
+ )
27
+ from ..log import get_logger
28
+ from . import scihub_speed
29
+ from ..network import fetch, proxy_dict, select_proxy_for_url, _is_cloudflare_block, USER_AGENT
30
+ from ..download.pdf import extract_pdf_url_from_html, is_pdf_file, success, _response_looks_pdf
31
+
32
+ log = get_logger()
33
+
34
+ # Track domains that require browser bypass (in-memory, per session)
35
+ _browser_domains: set[str] = set()
36
+
37
+ def _mark_browser_required(domain: str, config: dict[str, Any]) -> None:
38
+ """Mark a domain as requiring browser bypass for future ranking."""
39
+ _browser_domains.add(domain)
40
+
41
+ def _is_browser_domain(domain: str) -> bool:
42
+ """Check if domain requires browser bypass."""
43
+ return domain in _browser_domains
44
+
45
+
46
+ # ---------------------------------------------------------------------------
47
+ # Persistent browser-worker pool
48
+ # ---------------------------------------------------------------------------
49
+ # Every browser-backed Sci-Hub attempt (browser-first download, tab racing,
50
+ # Cloudflare/ALTCHA bypass) runs on THIS pool instead of ad-hoc per-paper
51
+ # executors. Its threads live for the whole process, so each worker's
52
+ # thread-local browser (browser_engine._get_shared_browser) is created once
53
+ # and reused for every paper: no per-paper window flashing, no per-paper
54
+ # startup cost, and Cloudflare clearance cookies survive between attempts.
55
+ # Workers are closed by _shutdown_scihub_pool (registered below) — never per
56
+ # paper. browser_engine's atexit reaper is the last-resort backstop.
57
+
58
+ _RACE_POOL: ThreadPoolExecutor | None = None
59
+ _RACE_POOL_LOCK = threading.Lock()
60
+
61
+
62
+ _HTTP_POOL: ThreadPoolExecutor | None = None
63
+
64
+
65
+ def _submit(pool: ThreadPoolExecutor, fn: Any, *args: Any, **kwargs: Any) -> Any:
66
+ """Submit to the long-lived Sci-Hub pool.
67
+
68
+ The task knows which paper it works for (``carry_owner``: a waived paper's
69
+ task must not launch a browser), but the browser it opens is the worker's
70
+ own and serves later papers too, so it is registered as SHARED: skipping or
71
+ finishing one paper does not kill it under the next (DeepSeek review
72
+ 2026-10-19); cancelling the job does."""
73
+ from ..browser.backend import carry_owner, shared
74
+
75
+ owned = carry_owner(fn)
76
+
77
+ def run(*a: Any, **k: Any) -> Any:
78
+ with shared():
79
+ return owned(*a, **k)
80
+ return pool.submit(run, *args, **kwargs)
81
+
82
+
83
+ def _lane_that_cleans_up(fn: Any, doi: str, domain: str, src_output: Path,
84
+ config: dict[str, Any], use_tor: bool,
85
+ fail_notes: list[str] | None, race_over: threading.Event) -> Any:
86
+ """One mirror lane of the Sci-Hub race.
87
+
88
+ The race returns as soon as a mirror delivers, and the other lanes keep
89
+ running in the background; the cleanup at that moment cannot see a file a
90
+ loser is still writing, so "<doi>_Sci-Hub_scihub_<mirror>.pdf" stayed in
91
+ the folder (2026-11-03). Each lane drops its own copy once the race is
92
+ over. A winner has already renamed its file to the download path, so
93
+ nothing left on the lane's own path is read by anyone.
94
+ """
95
+ try:
96
+ return fn(doi, domain, src_output, config, use_tor, fail_notes)
97
+ finally:
98
+ if race_over.is_set() and src_output.exists():
99
+ try:
100
+ src_output.unlink(missing_ok=True)
101
+ except OSError:
102
+ pass
103
+
104
+
105
+ def _race_pool(config: dict[str, Any]) -> ThreadPoolExecutor:
106
+ """Workers for the mirror race. With browsers, few (scihub_browser_workers:
107
+ each may hold a window). Over HTTP only, plenty (scihub_http_workers):
108
+ 4 papers x 3 mirrors queued behind 3 workers, and losers still holding them,
109
+ made a short list's first round take minutes (2026-10-17)."""
110
+ global _RACE_POOL, _HTTP_POOL
111
+ if not config.get("scihub_use_browser", True):
112
+ with _RACE_POOL_LOCK:
113
+ if _HTTP_POOL is None:
114
+ workers = max(1, int(config.get("scihub_http_workers", 12) or 12))
115
+ _HTTP_POOL = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="scihub-http")
116
+ return _HTTP_POOL
117
+ with _RACE_POOL_LOCK:
118
+ if _RACE_POOL is None:
119
+ workers = max(1, int(config.get("scihub_browser_workers", 3) or 3))
120
+ _RACE_POOL = ThreadPoolExecutor(
121
+ max_workers=workers, thread_name_prefix="scihub-browser"
122
+ )
123
+ return _RACE_POOL
124
+
125
+
126
+ def _close_my_browser() -> None:
127
+ """Runs INSIDE a pool worker: closes that worker's own thread-local browser."""
128
+ try:
129
+ from ..browser.engine import shutdown_shared_browser
130
+ shutdown_shared_browser()
131
+ except Exception:
132
+ pass
133
+
134
+
135
+ def _shutdown_scihub_pool() -> None:
136
+ """Gracefully close every pool worker's browser, then the pool. Idempotent."""
137
+ global _RACE_POOL
138
+ with _RACE_POOL_LOCK:
139
+ pool, _RACE_POOL = _RACE_POOL, None
140
+ if pool is None:
141
+ return
142
+ try:
143
+ # Idle workers each pick up one shutdown task and close their own
144
+ # browser in their own thread (Playwright sync objects are
145
+ # thread-affine). shutdown(wait=False) so a worker stuck in an
146
+ # interactive Cloudflare wait can't hang process exit; anything
147
+ # missed here is tree-killed by the process-exit reaper in
148
+ # browser_engine (issue #57).
149
+ for _ in range(pool._max_workers): # type: ignore[attr-defined]
150
+ pool.submit(_close_my_browser)
151
+ pool.shutdown(wait=False)
152
+ except Exception:
153
+ pass
154
+
155
+
156
+ atexit.register(_shutdown_scihub_pool)
157
+
158
+ # ---------------------------------------------------------------------------
159
+ # Verification-wall pacing (ALTCHA gates, e.g. sci-hub.ru)
160
+ # ---------------------------------------------------------------------------
161
+ # Empirically (2026-08-30 live tests): the gate escalates with request
162
+ # VELOCITY, not per session — solving works once, then rapid repeat
163
+ # verifications earn a standing "你是机器人吗" wall on every request. So the
164
+ # winning strategy is pacing (space verifications out) plus exponential
165
+ # cooldown when a wall persists after a solve, instead of hammering.
166
+
167
+ _WALL_MIN_SPACING_SEC = 25.0 # min gap between verification solves
168
+ _WALL_COOLDOWN_BASE_SEC = 90.0 # first persistent-wall cooldown
169
+ _WALL_COOLDOWN_CAP_SEC = 900.0 # never cool down longer than this
170
+ _MIRROR_STRUCTURAL_COOLDOWN_SEC = 7200.0 # structurally broken mirror: skip 2h
171
+
172
+ # Serialize INTERACTIVE Turnstile solving across pool workers. Each pool
173
+ # worker owns a browser, so without this lock N workers hitting a gate open N
174
+ # "please click the captcha" windows at once. Queued workers re-check after
175
+ # the first solve; the clearance cookie is shared per domain, so they usually
176
+ # pass without a new human challenge.
177
+ _TURNSTILE_INTERACTIVE_LOCK = threading.Lock()
178
+
179
+
180
+ def _wall_guard(domain: str, config: dict[str, Any]) -> bool:
181
+ """True while a domain is cooling down after persistent walls — skip it.
182
+
183
+ State lives in domain_db (wall_state table): persistent across processes
184
+ and sessions. Do NOT reintroduce in-memory health dicts — that split
185
+ caused three generations of mirror-health rework.
186
+ """
187
+ st = domain_db.get_wall_state(domain, config)
188
+ return time.time() < st["cooldown_until"]
189
+
190
+
191
+ def _wall_pace(domain: str, config: dict[str, Any]) -> None:
192
+ """Sleep so consecutive verification solves stay spaced out."""
193
+ st = domain_db.get_wall_state(domain, config)
194
+ wait = _WALL_MIN_SPACING_SEC - (time.time() - st["last_solve"])
195
+ if wait > 0:
196
+ time.sleep(min(wait, _WALL_MIN_SPACING_SEC))
197
+
198
+
199
+ def _note_wall(domain: str, config: dict[str, Any]) -> None:
200
+ """A wall persisted after a solve attempt — escalate the cooldown."""
201
+ st = domain_db.get_wall_state(domain, config)
202
+ walls = int(st["walls"]) + 1
203
+ cooldown = min(_WALL_COOLDOWN_BASE_SEC * (3 ** (walls - 1)), _WALL_COOLDOWN_CAP_SEC)
204
+ until = time.time() + cooldown
205
+ domain_db.set_wall_state(domain, config, last_solve=time.time(), walls=walls, cooldown_until=until)
206
+ log.info(f" Sci-Hub: {domain} verification wall persists — cooling down {cooldown:.0f}s")
207
+
208
+
209
+ def _note_wall_success(domain: str, config: dict[str, Any]) -> None:
210
+ """A solve cleared the wall — reset escalations, record the solve time."""
211
+ domain_db.set_wall_state(domain, config, last_solve=time.time(), walls=0, cooldown_until=0.0)
212
+
213
+
214
+ def _note_structural(domain: str, config: dict[str, Any]) -> None:
215
+ """Structurally broken response (homepage shell / interactive gate).
216
+
217
+ Unlike transient walls, re-paying the timeout every paper is pure loss:
218
+ skip the mirror for hours. It retries automatically once the cooldown
219
+ lapses, so a mirror that comes back to life is picked up again.
220
+ """
221
+ until = time.time() + _MIRROR_STRUCTURAL_COOLDOWN_SEC
222
+ domain_db.set_wall_state(domain, config, last_solve=time.time(), walls=1, cooldown_until=until)
223
+ log.info(f" Sci-Hub: {domain} structurally broken — skipping for "
224
+ f"{_MIRROR_STRUCTURAL_COOLDOWN_SEC / 3600:.0f}h")
225
+
226
+
227
+ def _classify_mirror_page(html: str) -> str:
228
+ """Classify a mirror response: turnstile | homepage | other.
229
+
230
+ ALTCHA walls are intentionally NOT classified here — the dedicated ALTCHA
231
+ solver handles them.
232
+ """
233
+ low = (html or "").lower()
234
+ if "challenges.cloudflare.com/turnstile" in low or "verification - sci-hub" in low:
235
+ return "turnstile"
236
+ if "search proxy to download" in low:
237
+ return "homepage"
238
+ return "other"
239
+
240
+
241
+ def _racing_browser_headless(config: dict[str, Any]) -> bool:
242
+ """Mirror the headless computation used to launch the racing browser."""
243
+ return bool(config.get("scihub_browser_headless", config.get("browser_headless", False)))
244
+
245
+
246
+ _PROBE_TTL_HOURS = 4
247
+ _SCIHUB_PROBE_WORKERS = 8
248
+
249
+
250
+ def _probe_single_domain(domain: str, proxy: str | None, timeout: tuple[int, int]) -> tuple[str, bool, float]:
251
+ proxies = proxy_dict(proxy)
252
+ t0 = time.time()
253
+ try:
254
+ s = requests.Session()
255
+ s.trust_env = False
256
+ resp = s.get(domain, timeout=timeout, proxies=proxies, allow_redirects=True,
257
+ headers={"User-Agent": USER_AGENT})
258
+ # Accept 200/302/301 as reachable (302/301 = redirect, still means domain is alive)
259
+ # 403/503 = reachable but blocked (Cloudflare) — still mark reachable for browser bypass
260
+ reachable = resp.status_code in (200, 301, 302, 403, 503)
261
+ ok = resp.status_code == 200 and ("sci-hub" in resp.text[:5000].lower() or "scihub" in resp.text[:5000].lower())
262
+ # 403/503 with Cloudflare signature = reachable via browser
263
+ if resp.status_code in (403, 503) and _is_cloudflare_block(resp):
264
+ _mark_browser_required(domain, None)
265
+ reachable = True
266
+ latency = (time.time() - t0) * 1000
267
+ return (domain, ok if ok else reachable, latency)
268
+ except requests.exceptions.Timeout:
269
+ # Timeout doesn't mean unreachable - just slow
270
+ latency = (time.time() - t0) * 1000
271
+ return (domain, True, latency)
272
+ except Exception as e:
273
+ log.debug(f"Sci-Hub probe {domain}: {type(e).__name__}")
274
+ return (domain, False, 99999.0)
275
+
276
+
277
+ _PROBE_RUNNING = threading.Lock()
278
+
279
+
280
+ def _probe_in_background(config: dict[str, Any]) -> None:
281
+ """_probe_scihub_domains off the paper's path; one at a time (papers of a
282
+ round start together and would otherwise all probe)."""
283
+ if not _PROBE_RUNNING.acquire(blocking=False):
284
+ return
285
+
286
+ def run() -> None:
287
+ try:
288
+ _probe_scihub_domains(config)
289
+ except Exception as exc:
290
+ log.info(f" Sci-Hub: background probe failed: {type(exc).__name__}: {exc}")
291
+ finally:
292
+ _PROBE_RUNNING.release()
293
+ threading.Thread(target=run, name="scihub-probe", daemon=True).start()
294
+
295
+
296
+ def _probe_scihub_domains(config: dict[str, Any]) -> None:
297
+ last_probe = get_probe_timestamp(config)
298
+ now = time.time()
299
+ if now - last_probe < _PROBE_TTL_HOURS * 3600:
300
+ return
301
+
302
+ proxy = select_proxy_for_url("https://sci-hub.mksa.top", config)
303
+ domains = config.get("scihub_domains") or DEFAULT_SCIHUB_DOMAINS
304
+ timeout = (5, 10)
305
+
306
+ pool = ThreadPoolExecutor(max_workers=_SCIHUB_PROBE_WORKERS)
307
+ futures = {pool.submit(_probe_single_domain, d, proxy, timeout): d for d in domains}
308
+ pending = set(futures)
309
+ try:
310
+ for future in as_completed(futures, timeout=15):
311
+ pending.discard(future)
312
+ domain, ok, latency = future.result()
313
+ update_probe(domain, ok, round(latency, 1), config)
314
+ except TimeoutError:
315
+ # A mirror that hangs past the probe window is unreachable for now.
316
+ # Raising here used to abort the whole Sci-Hub source on every call
317
+ # (the probe timestamp was never set, so the next call re-probed).
318
+ for future in pending:
319
+ update_probe(futures[future], False, 15.0, config)
320
+ finally:
321
+ pool.shutdown(wait=False, cancel_futures=True)
322
+
323
+ set_probe_timestamp(config)
324
+
325
+
326
+ def _is_browser_available(config: dict[str, Any]) -> bool:
327
+ """Check if CloakBrowser is available. Returns False in asyncio context.
328
+
329
+ scihub_use_browser=false turns every browser path of the grey sources off:
330
+ ALTCHA mirrors are handled over plain HTTP (sources/altcha.py), and
331
+ Turnstile-only mirrors are skipped instead of opening windows.
332
+ """
333
+ if not config.get("scihub_use_browser", True):
334
+ return False
335
+ # A running asyncio loop normally means "user asyncio code — the Playwright
336
+ # sync API cannot serve it". Exception: our persistent pool workers, where
337
+ # the loop was created by the sync API itself (browser_engine tracks it).
338
+ try:
339
+ import asyncio
340
+ loop = asyncio.get_running_loop()
341
+ except RuntimeError:
342
+ loop = None
343
+ if loop is not None:
344
+ from ..browser.engine import is_playwright_owned_loop
345
+ if not is_playwright_owned_loop(loop):
346
+ return False
347
+ try:
348
+ from ..browser.engine import is_available
349
+ return is_available(config)
350
+ except Exception:
351
+ return False
352
+
353
+
354
+ def _solve_altcha_and_reload(
355
+ solve_result: dict[str, Any],
356
+ landing_url: str,
357
+ doi: str,
358
+ output_path: Path,
359
+ config: dict[str, Any],
360
+ ) -> dict[str, Any] | None:
361
+ """Solve ALTCHA anti-bot verification on a Sci-Hub page, then reload and extract PDF.
362
+
363
+ ALTCHA is a proof-of-work CAPTCHA used by sci-hub.ru. The page shows a checkbox
364
+ labelled "不是" (No/Not a robot). After clicking, a proof-of-work computation runs,
365
+ then the page shows "您是人类!" (You are human!). Once verified, we reload the page
366
+ to get the actual paper content.
367
+ """
368
+ import time as _time
369
+
370
+ try:
371
+ from ..browser.engine import get_browser_page
372
+ except ImportError:
373
+ log.info(" [altcha] browser engine not available for ALTCHA bypass")
374
+ return None
375
+
376
+ domain = urllib.parse.urlparse(landing_url).netloc
377
+ if _wall_guard(domain, config):
378
+ log.info(f" [altcha] {domain} in wall cooldown — skipping this attempt")
379
+ return None
380
+ _wall_pace(domain, config)
381
+
382
+ page = None
383
+ try:
384
+ page = get_browser_page(config)
385
+ if not page:
386
+ log.info(" [altcha] could not get browser page")
387
+ return None
388
+
389
+ # Navigate to the landing URL
390
+ page.goto(landing_url, wait_until="domcontentloaded", timeout=20000)
391
+ _time.sleep(2)
392
+
393
+ # Find and click the ALTCHA checkbox
394
+ checkbox_selectors = [
395
+ "input[type='checkbox']",
396
+ "#altcha_checkbox",
397
+ "[id^='altcha_checkbox_']",
398
+ ]
399
+ clicked = False
400
+ for selector in checkbox_selectors:
401
+ try:
402
+ el = page.query_selector(selector)
403
+ if el:
404
+ el.click(timeout=5000)
405
+ clicked = True
406
+ log.info(f" [altcha] clicked checkbox: {selector}")
407
+ break
408
+ except Exception:
409
+ continue
410
+
411
+ if not clicked:
412
+ # Try clicking the "不是" div
413
+ try:
414
+ no_btn = page.query_selector("div[onclick='check()']")
415
+ if no_btn:
416
+ no_btn.click(timeout=5000)
417
+ clicked = True
418
+ log.info(" [altcha] clicked '不是' button")
419
+ except Exception:
420
+ pass
421
+
422
+ if not clicked:
423
+ log.info(" [altcha] could not find checkbox/button to click")
424
+ return None
425
+
426
+ # Wait for verification to complete (poll for "您是人类" or "verified")
427
+ for i in range(15):
428
+ _time.sleep(1)
429
+ try:
430
+ body_text = page.evaluate("() => document.body ? document.body.innerText : ''")
431
+ except Exception:
432
+ continue
433
+ if "您是人类" in body_text or "verified" in body_text.lower():
434
+ log.info(f" [altcha] verified after {i + 3}s")
435
+ break
436
+ if i % 5 == 4:
437
+ log.info(f" [altcha] still verifying... ({i + 3}s)")
438
+
439
+ # Reload the page to get the actual paper content
440
+ log.info(" [altcha] reloading page after verification...")
441
+ page.goto(landing_url, wait_until="domcontentloaded", timeout=20000)
442
+ _time.sleep(2)
443
+
444
+ # Now try to extract PDF
445
+ try:
446
+ html = page.content()
447
+ except Exception:
448
+ log.info(" [altcha] failed to get page content after reload")
449
+ return None
450
+
451
+ # Check for a PERSISTENT wall: if the reloaded page is still the
452
+ # "你是机器人吗" verification, solving again would only deepen the
453
+ # rate limit — record it and let the cooldown skip this domain.
454
+ from ..download.pdf import is_pdf_file, success, extract_pdf_url_from_html
455
+ from ..browser.engine import download_pdf_via_browser
456
+
457
+ lower = html.lower()
458
+ if "你是机器人吗" in html or "altcha" in lower and "iframe" not in lower:
459
+ _note_wall(domain, config)
460
+ log.info(" [altcha] verification wall persists after solve")
461
+ return None
462
+
463
+ if any(sig in lower for sig in ["article not found", "статья не найдена", "не найден"]):
464
+ log.info(" [altcha] article not found after verification")
465
+ return None
466
+
467
+ # Extract PDF URL
468
+ pdf_url = extract_pdf_url_from_html(html, landing_url)
469
+ if pdf_url:
470
+ log.info(f" [altcha] found PDF: {pdf_url[:80]}")
471
+ if download_pdf_via_browser(pdf_url, output_path, config):
472
+ if is_pdf_file(output_path):
473
+ _note_wall_success(domain, config)
474
+ return success(doi, output_path, "Sci-Hub(altcha)")
475
+
476
+ log.info(" [altcha] no PDF found after verification")
477
+ return None
478
+
479
+ except Exception as e:
480
+ log.info(f" [altcha] error: {e}")
481
+ return None
482
+ finally:
483
+ if page:
484
+ try:
485
+ page.close()
486
+ except Exception:
487
+ pass
488
+
489
+
490
+ def _browser_first_download(
491
+ landing_url: str,
492
+ doi: str,
493
+ output_path: Path,
494
+ config: dict[str, Any],
495
+ fail_notes: list[str] | None = None,
496
+ ) -> dict[str, Any] | None:
497
+ """Try browser-first download for Sci-Hub. Bypasses Cloudflare/CAPTCHA.
498
+
499
+ fail_notes (optional): appended with a short classification string at
500
+ every failure exit so callers can report WHY Sci-Hub gave up
501
+ (not_in_library / challenge_* / network / http_*).
502
+ """
503
+ try:
504
+ from ..browser.engine import solve_url, download_pdf_via_browser
505
+ from ..download.pdf import is_pdf_file, success, extract_pdf_url_from_html
506
+ from urllib.parse import urlparse
507
+ domain = urlparse(landing_url).netloc or landing_url[:40]
508
+
509
+ log.info(f" [browser-first] trying {landing_url[:80]}")
510
+ result = solve_url(landing_url, config, max_timeout=30000)
511
+ if not result:
512
+ log.info(f" [browser-first] no response")
513
+ if fail_notes is not None:
514
+ fail_notes.append("network(no response)")
515
+ return None
516
+
517
+ solution = result.get("solution", {})
518
+ html = solution.get("response", "")
519
+ if not html:
520
+ log.info(f" [browser-first] empty response")
521
+ if fail_notes is not None:
522
+ fail_notes.append("network(empty response)")
523
+ return None
524
+
525
+ # Check for article not found
526
+ lower = html.lower()
527
+ if any(sig in lower for sig in ["article not found", "статья не найдена", "не найден"]):
528
+ log.info(f" [browser-first] article not found on {domain}")
529
+ if fail_notes is not None:
530
+ fail_notes.append("not_in_library")
531
+ return None
532
+
533
+ # Structural failures: mirror shell or interactive gate. These cost a
534
+ # fixed timeout every paper until cooled down — note once, skip for hours.
535
+ kind = _classify_mirror_page(html)
536
+ if kind == "homepage":
537
+ _note_structural(domain, config)
538
+ log.info(f" [browser-first] {domain} serves its homepage shell — structural, cooling down")
539
+ if fail_notes is not None:
540
+ fail_notes.append("structural_homepage")
541
+ return None
542
+ if kind == "turnstile":
543
+ if _racing_browser_headless(config) or not config.get("scihub_turnstile_click", True):
544
+ _note_structural(domain, config)
545
+ log.info(f" [browser-first] Turnstile gate on {domain} — headless/no-click session, cooling down")
546
+ return None
547
+ # Interactive mode: surface it on the progress bar, wait for one
548
+ # human click. The clearance cookie then lasts the whole batch.
549
+ # Serialized across pool workers — see _TURNSTILE_INTERACTIVE_LOCK.
550
+ with _TURNSTILE_INTERACTIVE_LOCK:
551
+ from ..browser.engine import get_browser_page
552
+ page = get_browser_page(config)
553
+ if page is None:
554
+ _note_structural(domain, config)
555
+ return None
556
+ attention_key = f"turnstile:{domain}"
557
+ try:
558
+ from ..download import progress as _pr
559
+ _pr.set_attention(attention_key, "请在浏览器窗口完成 Turnstile 人机验证",
560
+ current=doi, phase="人工验证")
561
+ except Exception:
562
+ _pr = None
563
+ passed = False
564
+ try:
565
+ deadline = time.time() + max(30, int(config.get("turnstile_wait_sec", 180)))
566
+ while time.time() < deadline:
567
+ time.sleep(5)
568
+ page.goto(landing_url, wait_until="domcontentloaded", timeout=30000)
569
+ html = page.content()
570
+ if _classify_mirror_page(html) != "turnstile":
571
+ passed = True
572
+ break
573
+ if not passed:
574
+ log.info(f" [browser-first] Turnstile not passed within wait window")
575
+ if fail_notes is not None:
576
+ fail_notes.append("challenge_turnstile_timeout")
577
+ return None
578
+ solution["url"] = page.url
579
+ log.info(f" [browser-first] Turnstile cleared by human — cookie kept for the batch")
580
+ finally:
581
+ if _pr is not None:
582
+ try:
583
+ _pr.clear_attention(attention_key)
584
+ except Exception:
585
+ pass
586
+ try:
587
+ page.close()
588
+ except Exception:
589
+ pass
590
+ lower = html.lower()
591
+
592
+ # Check for ALTCHA anti-bot verification (used by sci-hub.ru and other mirrors)
593
+ if any(sig in lower for sig in ["altcha", "你是机器人吗", "not a robot"]):
594
+ log.info(f" [browser-first] ALTCHA detected on {domain}, attempting bypass...")
595
+ try:
596
+ altcha_result = _solve_altcha_and_reload(result, landing_url, doi, output_path, config)
597
+ if altcha_result:
598
+ return altcha_result
599
+ except Exception as e:
600
+ log.info(f" [browser-first] ALTCHA bypass failed: {e}")
601
+
602
+ # Check for Cloudflare challenge page
603
+ if any(sig in lower for sig in ["checking your browser", "just a moment", "cf-browser-verification"]):
604
+ log.info(f" [browser-first] Cloudflare challenge on {domain}, page may need more time")
605
+ # The solve_url should have waited, but if we still see the challenge,
606
+ # mark this domain as needing browser bypass for future attempts
607
+ if fail_notes is not None:
608
+ fail_notes.append("challenge_cloudflare")
609
+ return None
610
+
611
+ # Check for empty embed (Sci-Hub has no PDF)
612
+ if '<embed' in lower and 'src=""' in lower:
613
+ log.info(f" [browser-first] empty embed — article not in Sci-Hub database")
614
+ if fail_notes is not None:
615
+ fail_notes.append("not_in_library")
616
+ return None
617
+
618
+ # Extract PDF URL from HTML
619
+ pdf_url = extract_pdf_url_from_html(html, solution.get("url", landing_url))
620
+ if pdf_url:
621
+ log.info(f" [browser-first] found PDF: {pdf_url[:80]}")
622
+ # Download via browser (handles Cloudflare on PDF host too)
623
+ if download_pdf_via_browser(pdf_url, output_path, config):
624
+ # Retry is_pdf_file check — browser may still be flushing to disk
625
+ for _retry in range(5):
626
+ if is_pdf_file(output_path):
627
+ return success(doi, output_path, f"Sci-Hub(browser)")
628
+ time.sleep(0.2)
629
+ log.info(f" [browser-first] downloaded but file not recognized as PDF")
630
+
631
+ # Check if the response itself is a PDF
632
+ import base64
633
+ resp_data = solution.get("response", "")
634
+ if isinstance(resp_data, str) and len(resp_data) > 5000:
635
+ try:
636
+ pdf_bytes = base64.b64decode(resp_data) if resp_data.startswith("JVBER") else resp_data.encode("utf-8")
637
+ if pdf_bytes[:5] == b"%PDF-":
638
+ output_path.parent.mkdir(parents=True, exist_ok=True)
639
+ output_path.write_bytes(pdf_bytes)
640
+ if is_pdf_file(output_path):
641
+ return success(doi, output_path, f"Sci-Hub(browser)")
642
+ except Exception:
643
+ pass
644
+
645
+ log.info(f" [browser-first] no PDF found in response")
646
+ if fail_notes is not None:
647
+ fail_notes.append("not_in_library(no pdf in response)")
648
+ return None
649
+ except Exception as e:
650
+ log.info(f" [browser-first] error: {e}")
651
+ if fail_notes is not None:
652
+ fail_notes.append(f"network({type(e).__name__})")
653
+ return None
654
+
655
+
656
+ def _said_absent(domain: str, fail_notes: list[str]) -> bool:
657
+ """The domain answered "not in the library": a correct answer, not a failure of the mirror."""
658
+ return any(n.endswith(f"@{domain}") and n.startswith("not_in_library") for n in fail_notes)
659
+
660
+
661
+ # _http_altcha_download's answer when the gate was solved and the article page
662
+ # has no PDF: the mirror does not hold the paper, so no browser is opened for it.
663
+ ABSENT: dict[str, Any] = {"success": False, "not_in_library": True}
664
+
665
+ # Papers every answering mirror said it does not have, with when: a later
666
+ # download of the same paper within the hour (a short list's second round)
667
+ # does not ask Sci-Hub again, so the browser slot goes to the publisher's site.
668
+ _ABSENT_AT: dict[str, float] = {}
669
+ _ABSENT_TTL = 3600.0
670
+
671
+
672
+ def recently_absent(doi: str) -> bool:
673
+ return time.time() - _ABSENT_AT.get((doi or "").strip().lower(), 0.0) < _ABSENT_TTL
674
+
675
+
676
+ # Publication years the caller knows (an app job has them), by DOI.
677
+ _YEARS: dict[str, int] = {}
678
+ _DOI_YEAR = re.compile(r"[./](20[0-9]{2})[./]") # 10.1016/j.ceramint.2026.06.274, 10.26599/jac.2025.9221068
679
+
680
+
681
+ def note_years(years: dict[str, Any]) -> None:
682
+ for doi, year in years.items():
683
+ try:
684
+ _YEARS[(doi or "").strip().lower()] = int(str(year)[:4])
685
+ except (TypeError, ValueError):
686
+ pass
687
+
688
+
689
+ def publication_year(doi: str) -> int | None:
690
+ """The year noted by the caller, else one written in the DOI, else None."""
691
+ key = (doi or "").strip().lower()
692
+ if key in _YEARS:
693
+ return _YEARS[key]
694
+ m = _DOI_YEAR.search(key.split("/", 1)[-1])
695
+ return int(m.group(1)) if m else None
696
+
697
+
698
+ def too_new(doi: str, config: dict[str, Any]) -> bool:
699
+ """Published in or after scihub_skip_from_year: Sci-Hub has almost none of
700
+ those (2026-10-17: 0 of 19 papers from 2024-2026), so it is not asked."""
701
+ try:
702
+ start = int(config.get("scihub_skip_from_year") or 0)
703
+ except (TypeError, ValueError):
704
+ return False
705
+ year = publication_year(doi) if start else None
706
+ return bool(year and year >= start)
707
+
708
+
709
+ def _http_altcha_download(landing_url: str, domain: str, doi: str, output_path: Path,
710
+ config: dict[str, Any]) -> dict[str, Any] | None:
711
+ """Open the mirror page over HTTP, solve an ALTCHA gate if any, fetch the PDF.
712
+
713
+ Returns the download, ABSENT (gate solved, no PDF on the page), or None
714
+ (nothing learned: unreachable, a gate that needs a browser, ...).
715
+ """
716
+ from ..network import CHROME_UA
717
+ from . import altcha
718
+
719
+ session = requests.Session()
720
+ session.trust_env = False
721
+ session.headers.update({"User-Agent": CHROME_UA,
722
+ "Accept": "text/html,application/xhtml+xml,*/*;q=0.8"})
723
+ session.proxies = proxy_dict(select_proxy_for_url(landing_url, config)) or {}
724
+ timeout = (10, 25)
725
+ try:
726
+ resp = session.get(landing_url, timeout=timeout)
727
+ except Exception as exc:
728
+ log.info(f" [altcha-http] {domain}: {type(exc).__name__}")
729
+ return None
730
+ if resp.status_code != 200:
731
+ return None
732
+ html = resp.text
733
+ gated = altcha.is_altcha_page(html)
734
+ if gated:
735
+ # The article page behind the gate still embeds an ALTCHA widget (for
736
+ # its own buttons), so judge success by the PDF link, not by "altcha".
737
+ html = altcha.pass_gate(session, html, landing_url, timeout) or ""
738
+ if not html:
739
+ return None
740
+ elif _classify_mirror_page(html) == "turnstile":
741
+ return None # needs a real browser: leave it to the browser-first path
742
+ pdf_url = extract_pdf_url_from_html(html, landing_url)
743
+ if not pdf_url:
744
+ return ABSENT if gated else None
745
+ log.info(f" [altcha-http] {domain}: PDF {pdf_url[:80]}")
746
+ from ..download import progress as progress_reporter
747
+ progress_reporter.item_stage(doi, "Sci-Hub 有下载链接,正在核对")
748
+ result = download_pdf_from_scihub(pdf_url, output_path, config, f"Sci-Hub({domain})",
749
+ cookies=session.cookies, referer=landing_url)
750
+ if result:
751
+ result["doi"] = doi
752
+ result["identifier"] = doi
753
+ return result
754
+
755
+
756
+ def try_scihub_domain(
757
+ doi: str,
758
+ domain: str,
759
+ output_path: Path,
760
+ config: dict[str, Any],
761
+ use_tor: bool = False,
762
+ fail_notes: list[str] | None = None,
763
+ ) -> dict[str, Any] | None:
764
+ landing_url = f"{domain.rstrip('/')}/{urllib.parse.quote(doi, safe='/')}"
765
+
766
+ # Plain HTTP first: an ALTCHA gate is a proof-of-work that Python solves in
767
+ # seconds, so mirrors behind it (sci-hub.ru) need no browser window at all.
768
+ # Only gates that need a real browser (Cloudflare Turnstile) fall through.
769
+ if config.get("scihub_http_altcha", True) and not use_tor:
770
+ result = _http_altcha_download(landing_url, domain, doi, output_path, config)
771
+ if result is ABSENT:
772
+ log.info(f" [altcha-http] {domain}: gate solved, the paper is not in this mirror")
773
+ if fail_notes is not None:
774
+ fail_notes.append(f"not_in_library(after altcha)@{domain}")
775
+ return None
776
+ if result:
777
+ return result
778
+
779
+ # Browser-first: bypass Cloudflare/CAPTCHA before HTTP attempt.
780
+ # scihub_browser_first=false forces the pure-HTTP path — when the PDF
781
+ # CDN challenges headless browsers but serves plain requests (observed
782
+ # in the field, 2026-09: sci.bban.top), HTTP is the working lane.
783
+ if (config.get("scihub_browser_first", True)
784
+ and _is_browser_available(config)):
785
+ result = _browser_first_download(landing_url, doi, output_path, config,
786
+ fail_notes=fail_notes)
787
+ if result:
788
+ return result
789
+
790
+ # Sci-Hub pages and the PDF CDN are slow through proxies; racing-grade
791
+ # timeouts (users run connect=3/read=7 for source racing) starve this
792
+ # lane — a 3s read timeout kills every attempt before the page lands.
793
+ # Floor the timeouts for this lane only; other sources keep their pace.
794
+ config = {
795
+ **config,
796
+ "connect_timeout": max(int(config.get("connect_timeout", 3)), 10),
797
+ "read_timeout": max(int(config.get("read_timeout", 7)), 25),
798
+ }
799
+
800
+ try:
801
+ resp = fetch(landing_url, config, stream=True, use_tor=use_tor)
802
+
803
+ # Fallback to browser on Cloudflare/403/CAPTCHA
804
+ if resp.status_code in (403, 503) or _is_cloudflare_block(resp):
805
+ _mark_browser_required(domain, config)
806
+ resp = _try_browser(landing_url, config, resp) if _is_browser_available(config) else None
807
+ if resp is None:
808
+ if fail_notes is not None:
809
+ fail_notes.append("challenge(browser bypass failed)")
810
+ return None
811
+
812
+ if resp.status_code >= 400:
813
+ if fail_notes is not None:
814
+ fail_notes.append(f"http_{resp.status_code}")
815
+ return None
816
+
817
+ first = next(resp.iter_content(chunk_size=8192), b"")
818
+ # Check for CAPTCHA in first chunk
819
+ if resp.status_code == 200 and first:
820
+ content_sample = first[:5000].decode('utf-8', errors='ignore').lower()
821
+ if 'captcha' in content_sample or 'recaptcha' in content_sample:
822
+ log.info(f" CAPTCHA detected, trying browser...")
823
+ # Use browser to bypass CAPTCHA
824
+ browser_resp = _try_browser(landing_url, config, resp) if _is_browser_available(config) else None
825
+ if browser_resp is None:
826
+ log.warning(f" browser bypass failed — is CloakBrowser installed? Run: pip install cloakbrowser")
827
+ if fail_notes is not None:
828
+ fail_notes.append("challenge(captcha bypass failed)")
829
+ return None
830
+ # Get new content from browser response
831
+ resp = browser_resp
832
+ first = resp.content[:8192] if resp.content else b""
833
+ log.info(f" browser bypassed CAPTCHA, content size: {len(first)}")
834
+
835
+ if _response_looks_pdf(resp, first):
836
+ output_path.parent.mkdir(parents=True, exist_ok=True)
837
+ tmp_path = output_path.with_suffix(output_path.suffix + ".part")
838
+ try:
839
+ with tmp_path.open("wb") as fh:
840
+ fh.write(first)
841
+ for chunk in resp.iter_content(chunk_size=8192):
842
+ if chunk:
843
+ fh.write(chunk)
844
+ tmp_path.replace(output_path)
845
+ except Exception:
846
+ tmp_path.unlink(missing_ok=True)
847
+ raise
848
+ if is_pdf_file(output_path):
849
+ return success(doi, output_path, f"Sci-Hub({domain})")
850
+
851
+ # Collect full HTML content (browser responses have _content, direct responses need raw.read)
852
+ if resp._content:
853
+ html = first + resp._content
854
+ else:
855
+ try:
856
+ html = first + resp.raw.read(512_000, decode_content=True)
857
+ except Exception:
858
+ html = first
859
+ pdf_url = extract_pdf_url_from_html(html.decode("utf-8", errors="ignore"), resp.url)
860
+ if not pdf_url:
861
+ if fail_notes is not None:
862
+ fail_notes.append("not_in_library(no pdf url)")
863
+ return None
864
+ result = download_pdf_from_scihub(pdf_url, output_path, config, f"Sci-Hub({domain})",
865
+ use_tor=use_tor, cookies=resp.cookies, referer=landing_url)
866
+ if result:
867
+ result["doi"] = doi
868
+ result["identifier"] = doi
869
+ elif fail_notes is not None:
870
+ fail_notes.append("not_in_library(pdf download failed)")
871
+ return result
872
+ except Exception as e:
873
+ log.warning(" [scihub] %s HTTP lane failed: %r", domain, e, exc_info=True)
874
+ if fail_notes is not None:
875
+ fail_notes.append(f"network({type(e).__name__})")
876
+ if isinstance(e, (requests.Timeout, requests.ConnectionError)):
877
+ # A mirror that does not answer (2026-10-17: sci-hub.mksa.top now
878
+ # redirects to a host that times out) costs every paper 15-30 s:
879
+ # mark it unreachable, so the next 10 minutes skip it.
880
+ try:
881
+ from .scihub_domains import record_result as _db_result, update_probe
882
+ _db_result(domain, False, config)
883
+ update_probe(domain, False, 0.0, config)
884
+ except Exception:
885
+ pass
886
+ return None
887
+
888
+
889
+ def _try_browser(
890
+ url: str,
891
+ config: dict[str, Any],
892
+ original_resp: requests.Response,
893
+ ) -> requests.Response | None:
894
+ """Try CloakBrowser to bypass Cloudflare. Returns Response or None."""
895
+ if not _is_browser_available(config):
896
+ return None
897
+ # solve_url lives in browser_engine; the old import path (the FlareSolverr
898
+ # client module, removed 2026-10-19) broke the challenge bypass with ImportError.
899
+ from ..browser.engine import solve_url
900
+ result = solve_url(url, config)
901
+ if not result:
902
+ return None
903
+ solution = result.get("solution", {})
904
+ status = solution.get("status", 0)
905
+ if status >= 400:
906
+ return None
907
+ # Build a Response from browser solution
908
+ resp = requests.Response()
909
+ resp.status_code = status
910
+ html_content = solution.get("response", "")
911
+ resp._content = html_content.encode("utf-8") if isinstance(html_content, str) else html_content
912
+ resp.url = solution.get("url", url)
913
+ cookies = solution.get("cookies", [])
914
+ if isinstance(cookies, list):
915
+ for c in cookies:
916
+ if "name" in c and "value" in c:
917
+ resp.cookies.set(c["name"], c["value"])
918
+ # Mock raw attribute so downstream code can read content
919
+ class _RawMock:
920
+ def read(self, size=0, decode_content=False):
921
+ return resp._content[size:] if size else resp._content
922
+ resp.raw = _RawMock()
923
+ return resp
924
+
925
+
926
+ def _await_slow_pdfs(pending: dict[Any, tuple[str, Path]],
927
+ config: dict[str, Any]) -> tuple[Any, dict[str, Any]] | None:
928
+ """Wait for backup lanes whose PDF is already streaming but missed the race.
929
+
930
+ Only lanes with a growing ``.part`` file are worth waiting for: a lane that
931
+ has not started writing is not about to deliver a PDF. Each waits at most
932
+ ``scihub_slow_pdf_seconds`` in total and is dropped after
933
+ ``scihub_slow_pdf_idle_seconds`` without new bytes.
934
+ """
935
+ total = float(config.get("scihub_slow_pdf_seconds", 90) or 0)
936
+ if total <= 0 or not pending:
937
+ return None
938
+ idle = float(config.get("scihub_slow_pdf_idle_seconds", 30) or 30)
939
+ deadline = time.monotonic() + total
940
+ last_size: dict[Any, int] = {}
941
+ grew_at: dict[Any, float] = {}
942
+ while pending:
943
+ now = time.monotonic()
944
+ for future in list(pending):
945
+ if future.done():
946
+ pending.pop(future)
947
+ try:
948
+ result = future.result()
949
+ except Exception:
950
+ result = None
951
+ if result and result.get("success"):
952
+ return future, result
953
+ continue
954
+ src = pending[future][1]
955
+ part = src.with_suffix(src.suffix + ".part")
956
+ try:
957
+ size = part.stat().st_size
958
+ except OSError:
959
+ size = 0
960
+ if size > last_size.get(future, 0):
961
+ last_size[future] = size
962
+ grew_at[future] = now
963
+ elif future not in grew_at or now - grew_at[future] > idle:
964
+ pending.pop(future) # never started writing, or it stalled
965
+ if not pending or time.monotonic() >= deadline:
966
+ break
967
+ time.sleep(0.5)
968
+ return None
969
+
970
+
971
+ def download_pdf_from_scihub(
972
+ url: str,
973
+ output_path: Path,
974
+ config: dict[str, Any],
975
+ source: str,
976
+ use_tor: bool = False,
977
+ cookies: Any = None,
978
+ referer: str = "",
979
+ ) -> dict[str, Any] | None:
980
+ from ..download.pdf import download_pdf
981
+ return download_pdf(url, output_path, config, source, require_pdf_like_url=False,
982
+ use_tor=use_tor, cookies=cookies, referer=referer, browser_ua=True)
983
+
984
+
985
+ def _race_browser_domains(
986
+ domains: list[str],
987
+ doi: str,
988
+ output_path: Path,
989
+ config: dict[str, Any],
990
+ max_workers: int = 3,
991
+ ) -> dict[str, Any] | None:
992
+ """Race multiple Sci-Hub domains using multiple tabs in ONE browser.
993
+
994
+ Instead of spawning N threads each with their own browser (heavy),
995
+ this opens N tabs in a single browser context and polls them in
996
+ round-robin. The browser's internal network stack handles all tabs
997
+ concurrently, so we get real parallelism without multiple processes.
998
+
999
+ Each tab writes to its own temp output file to avoid conflicts.
1000
+ The winning file is renamed to output_path; loser tabs and temp
1001
+ files are cleaned up.
1002
+ """
1003
+ from ..browser.engine import _get_shared_browser, is_available as _browser_available
1004
+
1005
+ if not _browser_available(config):
1006
+ return None
1007
+
1008
+ try:
1009
+ browser, context = _get_shared_browser(config)
1010
+ except Exception as e:
1011
+ log.info(f" Sci-Hub: cannot get browser: {e}")
1012
+ return None
1013
+
1014
+ log.info(f" Sci-Hub: racing {len(domains)} browser domains via tabs (1 browser)...")
1015
+
1016
+ # Phase 1: Fire all navigations — open a tab per domain, trigger navigation
1017
+ # without blocking (use JS location assignment, returns immediately)
1018
+ tabs: list[tuple[str, Any, Path]] = [] # (domain, page, temp_output)
1019
+ for domain in domains:
1020
+ landing_url = f"{domain.rstrip('/')}/{urllib.parse.quote(doi, safe='/')}"
1021
+ safe_suffix = domain.split("//")[-1].replace(".", "_").replace("/", "_")[:25]
1022
+ temp_output = output_path.parent / f"{output_path.stem}_browser_{safe_suffix}.pdf"
1023
+ try:
1024
+ page = context.new_page()
1025
+ # Fire-and-forget via JS — navigate without blocking so all tabs load concurrently
1026
+ page.evaluate(f"window.location.href = '{landing_url}'")
1027
+ tabs.append((domain, page, temp_output))
1028
+ except Exception as e:
1029
+ log.info(f" Sci-Hub: tab open failed for {domain}: {e}")
1030
+ try:
1031
+ page.close()
1032
+ except Exception:
1033
+ pass
1034
+
1035
+ if not tabs:
1036
+ return None
1037
+
1038
+ # Wait a moment for all navigations to start, then poll for DOM readiness
1039
+ time.sleep(1.5)
1040
+
1041
+ # Phase 2: Poll tabs in round-robin — wait for DOM, then extract PDF URL
1042
+ deadline = time.time() + 30
1043
+ winner: tuple[dict[str, Any], Path] | None = None
1044
+
1045
+ while time.time() < deadline and winner is None:
1046
+ for domain, page, temp_output in tabs:
1047
+ try:
1048
+ # Wait for this tab's DOM to be ready (short timeout per check)
1049
+ try:
1050
+ page.wait_for_load_state("domcontentloaded", timeout=3000)
1051
+ except Exception:
1052
+ pass # Not ready yet, try next tab
1053
+
1054
+ html = page.content()
1055
+ if len(html) < 5000:
1056
+ continue
1057
+
1058
+ lower = html.lower()
1059
+ # Skip if still on Cloudflare challenge or article not found
1060
+ if any(sig in lower for sig in ["checking your browser", "just a moment"]):
1061
+ continue
1062
+ if any(sig in lower for sig in ["article not found", "статья не найдена"]):
1063
+ log.info(f" Sci-Hub: {domain} — article not found")
1064
+ continue
1065
+
1066
+ # Try to extract PDF URL
1067
+ pdf_url = extract_pdf_url_from_html(html, page.url)
1068
+ if pdf_url:
1069
+ log.info(f" Sci-Hub: {domain} found PDF, downloading...")
1070
+ from ..browser.engine import download_pdf_via_browser
1071
+ if download_pdf_via_browser(pdf_url, temp_output, config):
1072
+ # Verify PDF file — retry with backoff (browser may still flush)
1073
+ for _retry in range(10):
1074
+ if temp_output.exists() and temp_output.stat().st_size > 5000:
1075
+ if is_pdf_file(temp_output):
1076
+ result = success(doi, temp_output, f"Sci-Hub(tab:{domain.split('//')[-1][:15]})")
1077
+ winner = (result, temp_output)
1078
+ break
1079
+ time.sleep(0.3)
1080
+ if winner is not None:
1081
+ break
1082
+ else:
1083
+ log.info(f" Sci-Hub: {domain} download finished but PDF check failed")
1084
+ except Exception:
1085
+ continue
1086
+ if winner is None:
1087
+ time.sleep(0.3)
1088
+
1089
+ # Phase 3: Cleanup — close all tabs, remove loser temp files
1090
+ for domain, page, temp_output in tabs:
1091
+ try:
1092
+ page.close()
1093
+ except Exception:
1094
+ pass
1095
+ if winner is None or temp_output != winner[1]:
1096
+ if temp_output.exists():
1097
+ try:
1098
+ temp_output.unlink(missing_ok=True)
1099
+ except OSError:
1100
+ pass
1101
+
1102
+ if winner is not None:
1103
+ result, temp_output = winner
1104
+ if temp_output != output_path and temp_output.exists():
1105
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1106
+ if output_path.exists():
1107
+ output_path.unlink()
1108
+ temp_output.rename(output_path)
1109
+ result["file"] = str(output_path)
1110
+ return result
1111
+
1112
+ log.info(" Sci-Hub: no tab found a PDF")
1113
+ return None
1114
+
1115
+
1116
+ def try_scihub(doi: str, output_path: Path, config: dict[str, Any], use_tor: bool = False) -> dict[str, Any] | None:
1117
+ # All browser-backed attempts below run on the persistent worker pool
1118
+ # (_race_pool): pool threads live for the whole process, so each worker's
1119
+ # thread-local browser is created once and reused across papers — no
1120
+ # per-paper window flashing. Do NOT shut browsers down here; cleanup is
1121
+ # _shutdown_scihub_pool (atexit) plus browser_engine's process-exit reaper.
1122
+ if too_new(doi, config):
1123
+ log.info(f" Sci-Hub: skipped, published {publication_year(doi)} (scihub_skip_from_year)")
1124
+ return None
1125
+ try:
1126
+ return _try_scihub_impl(doi, output_path, config, use_tor)
1127
+ except Exception as e:
1128
+ log.info(f" Sci-Hub: unexpected error: {type(e).__name__}: {e}")
1129
+ # Check if a PDF was written to disk despite the exception
1130
+ if output_path.exists():
1131
+ from ..download.pdf import is_pdf_file as _is_pdf
1132
+ if _is_pdf(output_path):
1133
+ log.info(f" Sci-Hub: recovered PDF after {type(e).__name__}")
1134
+ return {"success": True, "identifier": doi, "doi": doi,
1135
+ "file": str(output_path), "source": "Sci-Hub(recovered)"}
1136
+ return None
1137
+
1138
+
1139
+ def _try_scihub_impl(doi: str, output_path: Path, config: dict[str, Any], use_tor: bool = False) -> dict[str, Any] | None:
1140
+ log.info(f" try_scihub called for {doi}")
1141
+ if not config.get("scihub_enabled", False):
1142
+ log.info(f" Sci-Hub disabled")
1143
+ return None
1144
+
1145
+ # Browser-first pass: race configured domains via browser in parallel.
1146
+ # Opt-in (default off) — opening CloakBrowser for every Sci-Hub attempt
1147
+ # is slow and can leave orphan Chromium processes (issue #19). The HTTP
1148
+ # path below still works; browser is only tried when explicitly enabled
1149
+ # or as a Cloudflare/ALTCHA challenge fallback.
1150
+ if config.get("scihub_browser_first_enabled", False) and _is_browser_available(config):
1151
+ configured_domains = config.get("scihub_domains") or DEFAULT_SCIHUB_DOMAINS
1152
+ browser_domains = [d for d in configured_domains[:5] if not _wall_guard(d, config)]
1153
+ if not browser_domains:
1154
+ browser_domains = [d for d in configured_domains[:5]] # all cooling down — try anyway, paced
1155
+ max_workers = min(config.get("scihub_browser_workers", 3), len(browser_domains))
1156
+
1157
+ pool = _race_pool(config)
1158
+ if max_workers <= 1 or len(browser_domains) == 1:
1159
+ # Single worker or single domain: sequential attempts on the pool
1160
+ # (browser lives on the pool worker, not the caller's thread).
1161
+ for domain in browser_domains:
1162
+ landing_url = f"{domain.rstrip('/')}/{urllib.parse.quote(doi, safe='/')}"
1163
+ result = _submit(
1164
+ pool, _browser_first_download, landing_url, doi, output_path, config
1165
+ ).result()
1166
+ if result:
1167
+ return result
1168
+ else:
1169
+ result = _submit(
1170
+ pool, _race_browser_domains, browser_domains, doi, output_path, config, max_workers
1171
+ ).result()
1172
+ if result:
1173
+ return result
1174
+
1175
+ all_domains = config.get("scihub_domains") or DEFAULT_SCIHUB_DOMAINS
1176
+ fastest = scihub_speed.preferred_mirror()
1177
+ if fastest and fastest in all_domains and not _wall_guard(fastest, config):
1178
+ # The batch's speed test has just reached this mirror end to end: start
1179
+ # with it, and refresh the reachability stats in the background
1180
+ # (2026-10-19: the probe held the first paper 15 s after a 5 s speed test).
1181
+ _probe_in_background(config)
1182
+ else:
1183
+ _probe_scihub_domains(config)
1184
+ stats = load_stats(config)
1185
+
1186
+ # Track failure reason for better diagnostic messages
1187
+ _failure_reason = "all domains unreachable"
1188
+ _any_reachable = False
1189
+ _any_browser_tried = False
1190
+ fail_notes: list[str] = []
1191
+
1192
+ now = time.time()
1193
+ cooldown_domains = []
1194
+ for d in all_domains:
1195
+ if _wall_guard(d, config):
1196
+ continue # persistent verification wall — pacing cooldown
1197
+ d_stats = stats.get(d, {})
1198
+ last_fail = d_stats.get("last_fail_time", 0)
1199
+ fail_streak = d_stats.get("fail_streak", 0)
1200
+ reachable = d_stats.get("reachable")
1201
+ # Skip domains with many consecutive failures
1202
+ if fail_streak >= 10 and (now - last_fail) < 300:
1203
+ continue
1204
+ # Skip domains that are unreachable (from recent probe)
1205
+ if reachable is False and (now - last_fail) < 600:
1206
+ continue
1207
+ cooldown_domains.append(d)
1208
+
1209
+ if not cooldown_domains:
1210
+ for d in all_domains:
1211
+ stats[d] = {"success": 0, "fail": 0, "last_fail_time": 0, "fail_streak": 0}
1212
+ cooldown_domains = all_domains
1213
+
1214
+ def _domain_score(d: str) -> float:
1215
+ s = stats.get(d, {})
1216
+ successes = s.get("success", 0)
1217
+ failures = s.get("fail", 0)
1218
+ reachable = s.get("reachable")
1219
+ total = successes + failures
1220
+ # Unreachable domains get lowest score
1221
+ if reachable is False:
1222
+ return -99999
1223
+ if total == 0:
1224
+ return 0.5
1225
+ success_rate = successes / total
1226
+ avg_latency = s.get("avg_latency_ms", 5000)
1227
+ score = success_rate * 1000 - avg_latency / 1000
1228
+ # Boost browser-accessible domains (bypasses Cloudflare)
1229
+ if _is_browser_available(config) and _is_browser_domain(d):
1230
+ score += 5000
1231
+ return score
1232
+
1233
+ cooldown_domains.sort(key=_domain_score, reverse=True)
1234
+ domains = cooldown_domains[:3]
1235
+ # The batch's speed test picked a mirror: ask it first (the others follow).
1236
+ fastest = scihub_speed.preferred_mirror()
1237
+ if fastest and fastest in all_domains and not _wall_guard(fastest, config):
1238
+ domains = [fastest] + [d for d in domains if d != fastest][:2]
1239
+ log.info(f" Sci-Hub domains to try: {domains}")
1240
+
1241
+ if len(domains) == 1:
1242
+ try:
1243
+ result = try_scihub_domain(doi, domains[0], output_path, config, use_tor=use_tor,
1244
+ fail_notes=fail_notes)
1245
+ if result:
1246
+ record_result(domains[0], True, config)
1247
+ return result
1248
+ if not _said_absent(domains[0], fail_notes):
1249
+ record_result(domains[0], False, config)
1250
+ except Exception:
1251
+ record_result(domains[0], False, config)
1252
+ log.info(f" Sci-Hub: only domain {domains[0]} failed")
1253
+ return None
1254
+
1255
+ # Try best domain first with short timeout
1256
+ best_domain = domains[0]
1257
+ best_tag = re.sub(r"[^A-Za-z0-9_-]+", "_", best_domain.split("//")[-1])
1258
+ best_output = output_path.parent / f"{output_path.stem}_scihub_{best_tag}.pdf"
1259
+ log.info(f" Sci-Hub: trying {best_domain} first...")
1260
+ try:
1261
+ result = _submit(
1262
+ _race_pool(config), try_scihub_domain, doi, best_domain, best_output, config, use_tor, fail_notes
1263
+ ).result()
1264
+ if result and result.get("success"):
1265
+ final_path = Path(result.get("file", ""))
1266
+ if final_path != output_path and final_path.exists():
1267
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1268
+ if output_path.exists():
1269
+ output_path.unlink()
1270
+ final_path.rename(output_path)
1271
+ result["file"] = str(output_path)
1272
+ log.info(f" Sci-Hub: OK {best_domain}")
1273
+ record_result(best_domain, True, config)
1274
+ scihub_speed.note_result(best_domain, True)
1275
+ return result
1276
+ scihub_speed.note_result(best_domain, False, absent=_said_absent(best_domain, fail_notes))
1277
+ if not _said_absent(best_domain, fail_notes):
1278
+ record_result(best_domain, False, config)
1279
+ except Exception:
1280
+ record_result(best_domain, False, config)
1281
+
1282
+ # Best domain failed - race remaining domains
1283
+ remaining = domains[1:]
1284
+ if not remaining:
1285
+ return None
1286
+
1287
+ log.info(f" Sci-Hub: racing {len(remaining)} backup domains...")
1288
+ # Persistent pool — workers (and their browsers) are NOT torn down here.
1289
+ # Losers that outlive the 45 s collection window keep streaming in the
1290
+ # background and are simply ignored. Each lane deletes its own temp file
1291
+ # once the race is over: cleanup here only sees the files that exist by
1292
+ # then, and a loser writing its PDF afterwards used to leave
1293
+ # "<doi>_Sci-Hub_scihub_<mirror>.pdf" behind (2026-11-03).
1294
+ pool = _race_pool(config)
1295
+ race_over = threading.Event()
1296
+
1297
+ def _adopt(domain: str, result: dict[str, Any]) -> dict[str, Any]:
1298
+ final_path = Path(result.get("file", ""))
1299
+ if final_path != output_path and final_path.exists():
1300
+ output_path.parent.mkdir(parents=True, exist_ok=True)
1301
+ if output_path.exists():
1302
+ output_path.unlink()
1303
+ final_path.rename(output_path)
1304
+ result["file"] = str(output_path)
1305
+ for _, other_path in futures.values():
1306
+ if other_path != output_path and other_path.exists():
1307
+ try:
1308
+ other_path.unlink(missing_ok=True)
1309
+ except OSError:
1310
+ pass
1311
+ record_result(domain, True, config)
1312
+ log.info(f" Sci-Hub: OK {domain}")
1313
+ race_over.set()
1314
+ return result
1315
+
1316
+ futures = {}
1317
+ for domain in remaining:
1318
+ dom_tag = re.sub(r"[^A-Za-z0-9_-]+", "_", domain.split("//")[-1])
1319
+ src_output = output_path.parent / f"{output_path.stem}_scihub_{dom_tag}.pdf"
1320
+ futures[_submit(pool, _lane_that_cleans_up, try_scihub_domain, doi, domain, src_output,
1321
+ config, use_tor, fail_notes, race_over)] = (domain, src_output)
1322
+ backup_timeout = float(config.get("scihub_backup_race_seconds", 45) or 45)
1323
+ try:
1324
+ # 10 s ended the race while the backups were still streaming PDFs they
1325
+ # had already found (sci.bban.top serves at a few hundred KB/s here).
1326
+ for future in as_completed(futures, timeout=backup_timeout):
1327
+ domain, src_output = futures[future]
1328
+ try:
1329
+ result = future.result(timeout=1)
1330
+ except Exception:
1331
+ result = None
1332
+ if result and result.get("success"):
1333
+ return _adopt(domain, result)
1334
+ else:
1335
+ if not _said_absent(domain, fail_notes):
1336
+ record_result(domain, False, config)
1337
+ if src_output.exists():
1338
+ try:
1339
+ src_output.unlink(missing_ok=True)
1340
+ except OSError:
1341
+ pass
1342
+ except TimeoutError:
1343
+ log.info(" Sci-Hub: backup domains timed out")
1344
+ # A mirror that found the PDF but streams it slowly loses the 45 s race
1345
+ # while its file is still growing on disk (sci.bban.top serves at a few
1346
+ # hundred KB/s). Give those lanes a longer leash instead of calling the
1347
+ # download a failure and fetching the paper again later.
1348
+ pending = {f: v for f, v in futures.items() if not f.done()}
1349
+ slow = _await_slow_pdfs(pending, config)
1350
+ if slow is not None:
1351
+ return _adopt(futures[slow[0]][0], slow[1])
1352
+ for _, src_output in futures.values():
1353
+ if src_output.exists():
1354
+ try:
1355
+ src_output.unlink(missing_ok=True)
1356
+ except OSError:
1357
+ pass
1358
+ # Whatever still runs now deletes its own copy when it ends.
1359
+ race_over.set()
1360
+
1361
+ # Every mirror that answered said the paper is not there: Tor would ask the
1362
+ # same library again, only slower.
1363
+ absent = bool(fail_notes) and all(n.startswith("not_in_library") for n in fail_notes)
1364
+ if absent:
1365
+ log.warning(f" Sci-Hub: {doi} is not in the Sci-Hub library (every mirror that answered said so)")
1366
+ _ABSENT_AT[(doi or "").strip().lower()] = time.time()
1367
+ return None
1368
+ # All clearnet domains failed — auto-retry with Tor + .onion (only if config allows)
1369
+ if not use_tor and config.get("use_tor_for_scihub", True):
1370
+ log.info(" Sci-Hub: all clearnet domains failed, retrying via Tor...")
1371
+ return try_scihub(doi, output_path, config, use_tor=True)
1372
+
1373
+ if fail_notes:
1374
+ from collections import Counter
1375
+ summary = ", ".join(f"{k}×{v}" for k, v in Counter(fail_notes).most_common())
1376
+ log.warning(f" Sci-Hub: all domains failed for {doi} ({summary})")
1377
+ else:
1378
+ log.warning(f" Sci-Hub: all domains failed for {doi}. Check: 1) network connectivity 2) Tor status (litloop tor_start)")
1379
+ return None