litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1339 @@
1
+ """Source registry and download orchestration with tiered parallel racing."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import inspect
6
+ import json
7
+ import threading
8
+ import time
9
+ from concurrent.futures import ThreadPoolExecutor
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from ..cache import cache_get, cache_set
14
+ from ..config import load_config, DATA_DIR
15
+ from ..identifiers import is_arxiv_identifier, normalize_doi, safe_filename
16
+ from ..log import get_logger
17
+ from ..download.pdf import fail
18
+ from ..download.rename import rename_pdf, generate_filename as rename_pdf_generate_filename
19
+
20
+ log = get_logger()
21
+
22
+ from .arxiv import try_arxiv
23
+ from .carsi import try_carsi
24
+ from .core_api import try_core
25
+ from .crossref import try_crossref_page_scrape
26
+ from .europepmc import try_europepmc, try_pmc
27
+ from .libgen import try_libgen
28
+ from .oa_discovery import try_doaj
29
+ from .openalex import try_openalex_oa, try_openalex_content_api
30
+ from . import attempt as _attempt
31
+ from .publishers import get_publisher_fast_sources
32
+ from .scibban import try_scibban
33
+ from .scihub import try_scihub
34
+ from .semantic_scholar import try_semanticscholar
35
+ from .openaire import try_openaire
36
+ from .unpaywall import try_unpaywall
37
+ from .vpnsci import try_vpnsci
38
+ from .ezproxy import try_ezproxy
39
+
40
+ # Global semaphore to limit concurrent browser-based sources across all DOI
41
+ # Prevents batch_workers × browser_sources_per_DOI explosion of Chrome windows
42
+ _browser_semaphore: threading.Semaphore | None = None
43
+ _browser_semaphore_lock = threading.Lock()
44
+
45
+ # Source labels that require launching a browser (CloakBrowser)
46
+ _BROWSER_SOURCE_LABELS = frozenset({
47
+ "ElsevierBrowser", "WileyBrowser", "IEEEBrowser", "ACSBrowser",
48
+ "RSCBrowser", "AIPBrowser", "SpringerBrowser", "APSBrowser",
49
+ "TandFBrowser", "IOPBrowser", "OxfordBrowser", "ACMBrowser",
50
+ "NatureBrowser", "ScienceBrowser", "SAGEBrowser", "ASCEBrowser",
51
+ "RoyalSocietyBrowser", "CopernicusDirect",
52
+ "GenericBrowser", "WebVPN", "CARSI", "EZProxy",
53
+ # Sci-Hub launches CloakBrowser internally (browser-first pass +
54
+ # Cloudflare/ALTCHA challenge solving), so it must participate in the
55
+ # global browser-source concurrency budget.
56
+ "Sci-Hub",
57
+ })
58
+
59
+ # ponytail: negative cache — (source, publisher) -> expiry. A challenge/block
60
+ # on one paper skips that source for the whole publisher prefix for 15 min,
61
+ # saving the doomed timeout on every later paper. Broaden the key or raise
62
+ # the TTL if false-skip reports come in.
63
+ _NEG_TTL = 900.0
64
+ _NEG_CACHE: dict[tuple[str, str], float] = {}
65
+ _NEG_LOCK = threading.Lock()
66
+ _NEG_KEYWORDS = ("cloudflare", "challenge", "captcha", "blocked", "forbidden", "403", "429")
67
+
68
+
69
+ def _publisher_of(doi: str) -> str:
70
+ return doi.split("/")[0].lower() if doi.lower().startswith("10.") else "*"
71
+
72
+
73
+ def _neg_record(label: str, doi: str, result: dict[str, Any]) -> None:
74
+ reason = (str(result.get("error_type", "")) + " " + str(result.get("error", ""))).lower()
75
+ if not any(k in reason for k in _NEG_KEYWORDS):
76
+ return # one-off failures (404, paywall) must not poison the source
77
+ with _NEG_LOCK:
78
+ _NEG_CACHE[(label, _publisher_of(doi))] = time.time() + _NEG_TTL
79
+
80
+
81
+ def _neg_blocked(label: str, doi: str) -> bool:
82
+ now = time.time()
83
+ with _NEG_LOCK:
84
+ expired = [k for k, exp in _NEG_CACHE.items() if exp <= now]
85
+ for k in expired:
86
+ _NEG_CACHE.pop(k, None)
87
+ pub = _publisher_of(doi)
88
+ return (label, pub) in _NEG_CACHE or (label, "*") in _NEG_CACHE
89
+
90
+
91
+ def _get_browser_semaphore(config: dict[str, Any]) -> threading.Semaphore:
92
+ """Get or create the global browser concurrency semaphore."""
93
+ global _browser_semaphore
94
+ max_workers = config.get("max_browser_workers", 1)
95
+ if _browser_semaphore is None:
96
+ with _browser_semaphore_lock:
97
+ if _browser_semaphore is None:
98
+ _browser_semaphore = threading.Semaphore(max_workers)
99
+ return _browser_semaphore
100
+
101
+ __all__ = ["download", "batch_download"]
102
+
103
+ _cleanup_done = False
104
+
105
+
106
+ def _cleanup_stale_files(target_dir: Path) -> None:
107
+ """Remove orphaned .part files and racing temp files from previous runs."""
108
+ global _cleanup_done
109
+ if _cleanup_done:
110
+ return
111
+ _cleanup_done = True
112
+ if not target_dir.exists():
113
+ return
114
+ count = 0
115
+ for f in target_dir.iterdir():
116
+ if not f.is_file():
117
+ continue
118
+ name = f.name
119
+ # Skip hidden files (like .doi_index.json)
120
+ if name.startswith("."):
121
+ continue
122
+ # Always clean up .part files
123
+ if name.endswith(".part"):
124
+ try:
125
+ f.unlink()
126
+ count += 1
127
+ except OSError:
128
+ pass
129
+ continue
130
+ # Clean up racing temp files: DOI-based identifier + source label suffix
131
+ # e.g. 10_1038_nature12373_Unpaywall.pdf, 10_1016_test_scihub_st.pdf
132
+ # Must start with 10_ (DOI prefix) and have at least 4 segments (10_NNNN_suffix_label)
133
+ if name.endswith(".pdf") and name.startswith("10_"):
134
+ stem = name[:-4] # remove .pdf
135
+ # Split from the right to get the last segment as the source label
136
+ parts = stem.rsplit("_", 1)
137
+ if len(parts) == 2:
138
+ base, label = parts
139
+ # Verify: base must look like a DOI (10_NNNN_... with at least 4 segments)
140
+ # and label must be a source name (letters, not just a number)
141
+ base_parts = base.split("_")
142
+ if (len(base_parts) >= 3
143
+ and base_parts[0] == "10"
144
+ and base_parts[1].isdigit()
145
+ and len(base_parts[1]) >= 3
146
+ and any(c.isalpha() for c in label)):
147
+ try:
148
+ f.unlink()
149
+ count += 1
150
+ except OSError:
151
+ pass
152
+ if count > 0:
153
+ log.info(f"Cleaned up {count} stale temp files")
154
+
155
+
156
+ def _is_wrong_paper(path: Path, doi: str, config: dict[str, Any], label: str) -> bool:
157
+ """True when the PDF a source produced is a different paper (download.pdf.paper_check)."""
158
+ from ..download.pdf import paper_check
159
+ t = time.time()
160
+ verdict = paper_check(path, doi, config)
161
+ log.info(f" [timing] {label} paper check: {(time.time() - t) * 1000:.0f}ms ({verdict})")
162
+ if verdict == "wrong":
163
+ log.info(f" WRONG PAPER {label}: neither the DOI nor the title is on its first pages")
164
+ return verdict == "wrong"
165
+
166
+
167
+ def _try_source(
168
+ source_fn: Any, doi: str, output_path: Path, config: dict[str, Any], label: str, use_tor: bool = False
169
+ ) -> dict[str, Any] | None:
170
+ from .scoring import record_result, classify_error, get_user_advice
171
+ t0 = time.time()
172
+ key = normalize_doi(doi) if not is_arxiv_identifier(doi) else doi
173
+ _SOURCE_TLS.doi = key
174
+ # The download this source works for, taken now: by the time a slow source
175
+ # ends, a retry of the same paper may have registered a new attempt, which
176
+ # must not get this one's steps or errors (DeepSeek review 2026-10-19).
177
+ att = _attempt.get(key)
178
+ result = None
179
+ try:
180
+ result = _try_source_inner(source_fn, doi, output_path, config, label, use_tor, t0,
181
+ record_result, classify_error, get_user_advice)
182
+ finally:
183
+ settled = _SOURCE_TLS.doi in _SETTLED or (att is not None and _attempt.get(key) is not att)
184
+ _SOURCE_TLS.doi = None
185
+ ok = bool(result and result.get("success"))
186
+ log.info(f" [timing] {label}: {time.time() - t0:.1f}s, {'PDF' if ok else 'no PDF'}")
187
+ if att is not None:
188
+ from ..publishers.strategies._core import get_last_error
189
+ att.add_step("school" if label in _STEP_TEXT else "free", label, time.time() - t0, result,
190
+ error=get_last_error()) # this thread's: _try_source_inner cleared it first
191
+ if settled:
192
+ # Late finisher of an already-returned download: its copy is a duplicate.
193
+ if output_path.exists():
194
+ try:
195
+ output_path.unlink()
196
+ except OSError:
197
+ pass
198
+ return None
199
+ return result
200
+
201
+
202
+ def _try_source_inner(source_fn: Any, doi: str, output_path: Path, config: dict[str, Any], label: str,
203
+ use_tor: bool, t0: float, record_result: Any, classify_error: Any,
204
+ get_user_advice: Any) -> dict[str, Any] | None:
205
+
206
+ # Limit concurrency for browser-based sources
207
+ is_browser = label in _BROWSER_SOURCE_LABELS
208
+ if label == "Sci-Hub" and not config.get("scihub_use_browser", True):
209
+ is_browser = False # pure-HTTP Sci-Hub must not queue behind the browser lanes
210
+ sem = _get_browser_semaphore(config) if is_browser else None
211
+ if sem:
212
+ t_wait = time.time()
213
+ sem.acquire()
214
+ if time.time() - t_wait > 1:
215
+ log.info(f" [timing] {label}: waited {time.time() - t_wait:.1f}s for the browser")
216
+ try:
217
+ # Each source starts with no browser error of its own: a fresh racing
218
+ # thread otherwise read the latest error any lane reported, and the
219
+ # Elsevier browser skipped its ScienceDirect attempt (0.0 s, no log)
220
+ # because the previous paper had ended on a paywall (2026-10-17).
221
+ from ..publishers.strategies._core import _set_error
222
+ _set_error("", "")
223
+ sig = inspect.signature(source_fn)
224
+ if "use_tor" in sig.parameters:
225
+ result = source_fn(doi, output_path, config, use_tor=use_tor)
226
+ else:
227
+ result = source_fn(doi, output_path, config)
228
+ latency_ms = (time.time() - t0) * 1000
229
+ if result:
230
+ result["doi"] = doi
231
+ result["identifier"] = doi
232
+ if result.get("success"):
233
+ # Check for suspicious PDF (1-page or very small — likely not full text)
234
+ file_path = result.get("file", "")
235
+ if file_path:
236
+ from ..download.pdf import is_suspicious_pdf, suspicious_pdf
237
+ fp = Path(file_path)
238
+ if fp.exists() and is_suspicious_pdf(fp, doi):
239
+ log.info(f" SUSPICIOUS {label}: PDF appears to be a preview/cover page, not full text")
240
+ record_result(label, False, latency_ms, "suspicious_pdf")
241
+ return suspicious_pdf(doi, fp, label)
242
+ if fp.exists() and _is_wrong_paper(fp, doi, config, label):
243
+ record_result(label, False, latency_ms, "wrong_paper")
244
+ from ..download.pdf import wrong_paper
245
+ return wrong_paper(doi, fp, label)
246
+ if fp.exists():
247
+ from ..download.pdf import preprint_mark
248
+ mark = preprint_mark(fp, doi)
249
+ if mark:
250
+ # Still this paper, but not the version of record:
251
+ # the race keeps the copy as a fallback and looks
252
+ # for the published PDF (2026-11-03).
253
+ result["preprint"] = mark
254
+ log.info(f" PREPRINT {label}: {mark} copy, not the published version")
255
+ record_result(label, True, latency_ms)
256
+ else:
257
+ error_type = classify_error(result.get("status_code", 0))
258
+ record_result(label, False, latency_ms, error_type)
259
+ return result
260
+ except SourceWaived:
261
+ log.info(f" {label}: stopped, the paper is already downloaded")
262
+ return None
263
+ except Exception as e:
264
+ latency_ms = (time.time() - t0) * 1000
265
+ error_type = classify_error(exception=e)
266
+ # Check if a valid PDF was actually written to disk despite the exception
267
+ # (common with Sci-Hub: browser download succeeds but post-download logic raises)
268
+ from ..download.pdf import is_pdf_file as _is_pdf
269
+ if output_path.exists() and _is_pdf(output_path) and not _is_wrong_paper(output_path, doi, config, label):
270
+ log.info(f" OK {label} (recovered after {error_type})")
271
+ record_result(label, True, latency_ms)
272
+ return {"success": True, "identifier": doi, "doi": doi,
273
+ "file": str(output_path), "source": label}
274
+ record_result(label, False, latency_ms, error_type)
275
+ advice = get_user_advice(error_type, label)
276
+ log.info(f" FAIL {label}: {error_type} — {advice}")
277
+ return None
278
+ finally:
279
+ if sem:
280
+ sem.release()
281
+
282
+
283
+ def _is_browser_lane(label: str) -> bool:
284
+ """A lane that opens a browser window (Sci-Hub mostly solves its gate over HTTP)."""
285
+ return label != "Sci-Hub" and label in _BROWSER_SOURCE_LABELS
286
+
287
+
288
+ def _wait_any(events: tuple[threading.Event, threading.Event], timeout: float) -> bool:
289
+ """Wait for any event to be set, polling at a fine grain (no wait-any API).
290
+
291
+ Used by the racing engine to react to either the first success or the
292
+ completion of every submitted source, whichever comes first.
293
+ """
294
+ deadline = time.monotonic() + timeout
295
+ while time.monotonic() < deadline:
296
+ if any(e.is_set() for e in events):
297
+ return True
298
+ if _cancelled():
299
+ return False
300
+ time.sleep(0.05)
301
+ return any(e.is_set() for e in events)
302
+
303
+
304
+ def _promote_publisher_browser(launchable: list[tuple[Any, str]], doi: str) -> None:
305
+ """Launch the paper's own publisher browser lane second, not after every cheap lane.
306
+
307
+ The hedged cascade adds one lane per hedge delay; for a paywalled paper the
308
+ publisher's browser lane (the one that can actually get it) used to start
309
+ only after ~5 open-access lanes, 7-8 s in. Second place keeps the best
310
+ cheap lane first, so open-access papers still win in a second or two.
311
+ """
312
+ try:
313
+ from .publishers import get_publisher
314
+ own = f"{get_publisher(doi)}Browser"
315
+ except Exception:
316
+ return
317
+ for i, (_fn, label) in enumerate(launchable):
318
+ if label == own and i > 1:
319
+ launchable.insert(1, launchable.pop(i))
320
+ return
321
+
322
+
323
+ def _run_tiers_parallel(
324
+ tiers: list[tuple[list[tuple[Any, str]], str, int]],
325
+ doi: str,
326
+ target_dir: Path,
327
+ output_path: Path,
328
+ config: dict[str, Any],
329
+ use_tor: bool,
330
+ overall_timeout: int,
331
+ failures: list[dict[str, str]] | None = None,
332
+ ) -> dict[str, Any] | None:
333
+ """Race all tiers in parallel. First successful tier wins.
334
+
335
+ Uses a shared result dict so that any source thread can publish its
336
+ success immediately, even if it's running inside a nested parallel
337
+ call (like Sci-Hub domain racing). `failures` (optional) collects why
338
+ each losing lane failed — pool workers append to it.
339
+ """
340
+ if failures is None:
341
+ failures = []
342
+ if _cancelled():
343
+ return None
344
+ # A new race for this DOI is live again: callers such as batch_download race
345
+ # without going through download(), so clear any earlier "settled" mark here
346
+ # or every result of this race would be discarded as a late finisher.
347
+ _SETTLED.discard(normalize_doi(doi) if not is_arxiv_identifier(doi) else doi)
348
+ if not tiers:
349
+ return None
350
+
351
+ # Flatten all sources across tiers with their labels
352
+ all_sources: list[tuple[Any, str, str, int]] = [] # (fn, label, tier_label, timeout)
353
+ for tier_sources, tier_label, tier_timeout in tiers:
354
+ for fn, label in tier_sources:
355
+ all_sources.append((fn, label, tier_label, tier_timeout))
356
+
357
+ if not all_sources:
358
+ return None
359
+
360
+ # If only one source, run directly
361
+ if len(all_sources) == 1:
362
+ fn, label, tier_label, timeout = all_sources[0]
363
+ src_output = target_dir / f"{safe_filename(doi)}_{label}.pdf"
364
+ result = _try_source(fn, doi, src_output, config, label, use_tor=use_tor)
365
+ if result and result.get("success"):
366
+ final_path = Path(result.get("file", ""))
367
+ if final_path != output_path and final_path.exists():
368
+ output_path.parent.mkdir(parents=True, exist_ok=True)
369
+ if output_path.exists():
370
+ output_path.unlink()
371
+ final_path.rename(output_path)
372
+ result["file"] = str(output_path)
373
+ return result
374
+ _record_source_failure(failures, label, result)
375
+ return None
376
+
377
+ # Shared result: any thread can publish success here, signaled via Event
378
+ result_lock = threading.Lock()
379
+ success_event = threading.Event()
380
+ cancel_event = threading.Event()
381
+ all_done_event = threading.Event() # set when every submitted future returned
382
+ done_counter: list[int] = [0]
383
+ done_counter_lock = threading.Lock()
384
+ submission_done = threading.Event() # staggered scheduler finished launching lanes
385
+ lanes_idle = threading.Event() # every launched-so-far lane has answered
386
+ shared_result: dict[str, Any] = {"result": None}
387
+ # A preprint copy of the right paper: kept, but a version of record wins
388
+ # (2026-10-26: the open-access route served the Research Square preprint
389
+ # for a Scientific Reports DOI and ended the race before the publisher).
390
+ preprint_result: dict[str, Any] = {"result": None}
391
+ race_over = threading.Event() # set when this race returns; waived lanes stop launching
392
+
393
+ def _try_and_publish(fn, label, src_output):
394
+ # Skip if another source already succeeded
395
+ if cancel_event.is_set():
396
+ return None
397
+ _SOURCE_TLS.race = race_over
398
+ try:
399
+ result = _try_source(fn, doi, src_output, config, label, use_tor=use_tor)
400
+ finally:
401
+ _SOURCE_TLS.race = None
402
+ # This pool is created per download (issue #57): once its worker
403
+ # threads die, their thread-local shared browsers are unreachable
404
+ # to close_shared_browser() and the atexit reaper only fires at
405
+ # process exit — a long-lived MCP server accumulates Chromium
406
+ # processes. Each worker closes its own browser before exiting.
407
+ try:
408
+ from ..browser.engine import shutdown_shared_browser
409
+ shutdown_shared_browser()
410
+ except Exception:
411
+ pass
412
+ if result and not result.get("success"):
413
+ # ponytail: (source, publisher) negative cache — a Cloudflare block
414
+ # on one paper must not burn every later paper's timeout; raise TTL
415
+ # or key by full DOI if false-skip reports come in.
416
+ _neg_record(label, doi, result)
417
+ _record_source_failure(failures, label, result)
418
+ if result and result.get("success"):
419
+ if result.get("preprint"):
420
+ # Not the version of record: hold it and keep the other lanes
421
+ # going. It is used only if none of them delivers one.
422
+ with result_lock:
423
+ if preprint_result["result"] is None:
424
+ preprint_result["result"] = (result, label, src_output)
425
+ log.info(f" PREPRINT {label} ({result['preprint']}); still looking for the published version")
426
+ return result
427
+ with result_lock:
428
+ if shared_result["result"] is None:
429
+ shared_result["result"] = (result, label, src_output)
430
+ cancel_event.set()
431
+ success_event.set()
432
+ return result
433
+
434
+ def _take_preprint() -> dict[str, Any] | None:
435
+ """The kept preprint copy, moved into place, when nothing better came back."""
436
+ item = preprint_result["result"]
437
+ if item is None:
438
+ return None
439
+ result, label, src_output = item
440
+ final_path = Path(result.get("file", ""))
441
+ if final_path != output_path and final_path.exists():
442
+ output_path.parent.mkdir(parents=True, exist_ok=True)
443
+ if output_path.exists():
444
+ output_path.unlink()
445
+ final_path.rename(output_path)
446
+ result["file"] = str(output_path)
447
+ log.info(f" OK {label} (preprint only: {result.get('preprint')})")
448
+ return result
449
+
450
+ def _on_future_done(fut) -> None:
451
+ with done_counter_lock:
452
+ done_counter[0] += 1
453
+ if done_counter[0] >= len(futures):
454
+ # Mid-scheduling this wakes the scheduler so a fast-failing
455
+ # lane widens immediately instead of idling out the hedge
456
+ # delay; after submission_done it means the race is over.
457
+ lanes_idle.set()
458
+ if submission_done.is_set():
459
+ all_done_event.set()
460
+
461
+ # Hedged cascade (default): lanes arrive score-ordered by the caller.
462
+ # Launch the best first and only widen when it fails to answer within
463
+ # hedge_delay_seconds — cuts request volume and anti-bot heat vs a flat
464
+ # race while keeping tail latency close. race_mode=full restores the
465
+ # flat all-at-once race.
466
+ race_mode = str(config.get("race_mode", "hedge")).lower()
467
+ hedge_delay = max(0.0, float(config.get("hedge_delay_seconds", 1.5)))
468
+ if race_mode == "full":
469
+ hedge_delay = 0.0
470
+ mode_label = "hedged cascade" if hedge_delay > 0 else "flat race"
471
+
472
+ launchable: list[tuple[Any, str]] = []
473
+ for fn, label, _tier_label, _tier_timeout in all_sources:
474
+ if _neg_blocked(label, doi):
475
+ log.info(f" SKIP {label} (negative cache: recently failed for this publisher)")
476
+ continue
477
+ launchable.append((fn, label))
478
+ _promote_publisher_browser(launchable, doi)
479
+
480
+ log.info(f" Racing {len(all_sources)} sources across {len(tiers)} tiers ({mode_label})...")
481
+ deadline = time.time() + overall_timeout + 5
482
+ pool = ThreadPoolExecutor(max_workers=max(1, len(launchable)))
483
+ futures = {}
484
+ try:
485
+ for i, (fn, label) in enumerate(launchable):
486
+ lanes_idle.clear()
487
+ src_output = target_dir / f"{safe_filename(doi)}_{label}.pdf"
488
+ fut = pool.submit(_try_and_publish, fn, label, src_output)
489
+ futures[fut] = (label, src_output)
490
+ fut.add_done_callback(_on_future_done)
491
+ if hedge_delay <= 0 or i == len(launchable) - 1 or success_event.is_set():
492
+ continue
493
+ # Widen only when needed: another lane succeeded, everything
494
+ # launched so far already answered (fast failure), the hedge
495
+ # delay elapsed, or the overall deadline arrived.
496
+ now = time.time()
497
+ if now >= deadline:
498
+ break
499
+ _wait_any((success_event, lanes_idle),
500
+ min(hedge_delay, deadline - now))
501
+ if success_event.is_set() or time.time() >= deadline:
502
+ break
503
+ submission_done.set()
504
+ with done_counter_lock:
505
+ if done_counter[0] >= len(futures):
506
+ all_done_event.set()
507
+
508
+ # Wait for first success, all sources done, or the overall timeout.
509
+ # all_done lets a full fast-failure return immediately instead of
510
+ # burning the whole timeout + grace period.
511
+ if not success_event.is_set() and not all_done_event.is_set():
512
+ _wait_any((success_event, all_done_event),
513
+ max(0.0, deadline - time.time()))
514
+
515
+ if shared_result["result"] is not None:
516
+ result, label, src_output = shared_result["result"]
517
+ final_path = Path(result.get("file", ""))
518
+ if final_path != output_path and final_path.exists():
519
+ output_path.parent.mkdir(parents=True, exist_ok=True)
520
+ if output_path.exists():
521
+ output_path.unlink()
522
+ final_path.rename(output_path)
523
+ result["file"] = str(output_path)
524
+ log.info(f" OK {label}")
525
+ return result
526
+ if all_done_event.is_set():
527
+ preprint = _take_preprint()
528
+ if preprint is not None:
529
+ return preprint
530
+ log.info(" All sources failed")
531
+ return None
532
+
533
+ # Timeout reached — late results. A browser lane still at work (opening
534
+ # the page, reaching a login) gets up to race_grace_browser_seconds;
535
+ # once none is, only race_grace_seconds. A lane that waits for the user
536
+ # says so (note_user_waiting) and is waited for below.
537
+ grace = float(config.get("race_grace_seconds", 15))
538
+ grace_browser = float(config.get("race_grace_browser_seconds", 180))
539
+ log.info(f" Racing timed out after {overall_timeout + 5}s, waiting for late results...")
540
+ t_grace = time.time()
541
+ quick_until = time.time() + grace
542
+ stop_at = time.time() + max(grace, grace_browser)
543
+ while not success_event.is_set() and not all_done_event.is_set() and not _cancelled():
544
+ now = time.time()
545
+ if now >= stop_at or _user_waiting_for(doi):
546
+ break
547
+ browser_at_work = any(not f.done() and _is_browser_lane(lbl) for f, (lbl, _) in futures.items())
548
+ if not browser_at_work and now >= quick_until:
549
+ break
550
+ _wait_any((success_event, all_done_event), 1.0)
551
+ # A lane is waiting for the user's login: keep waiting while its
552
+ # heartbeat is fresh instead of abandoning a login in progress.
553
+ if _user_waiting_for(doi):
554
+ log.info(" A source is waiting for your login — waiting for it...")
555
+ stop_at = time.time() + _USER_WAIT_MAX_S
556
+ while (not success_event.is_set() and not all_done_event.is_set()
557
+ and not _cancelled() and time.time() < stop_at
558
+ and _user_waiting_for(doi)):
559
+ _wait_any((success_event, all_done_event), 3.0)
560
+ if shared_result["result"] is not None:
561
+ result, label, src_output = shared_result["result"]
562
+ final_path = Path(result.get("file", ""))
563
+ if final_path != output_path and final_path.exists():
564
+ output_path.parent.mkdir(parents=True, exist_ok=True)
565
+ if output_path.exists():
566
+ output_path.unlink()
567
+ final_path.rename(output_path)
568
+ result["file"] = str(output_path)
569
+ log.info(f" OK {label} (late, after {time.time() - t_grace:.1f}s)")
570
+ return result
571
+ if all_done_event.is_set():
572
+ preprint = _take_preprint()
573
+ if preprint is not None:
574
+ return preprint
575
+ log.info(" All sources failed")
576
+ return None
577
+
578
+ # Final scan: check if any source wrote a valid PDF file despite timeout
579
+ from ..download.pdf import is_pdf_file, is_suspicious_pdf, suspicious_pdf
580
+ for label, src_output in futures.values():
581
+ if src_output.exists() and is_pdf_file(src_output):
582
+ if is_suspicious_pdf(src_output, doi):
583
+ log.info(f" SUSPICIOUS {label} (file scan): skipping suspicious PDF")
584
+ continue
585
+ if _is_wrong_paper(src_output, doi, config, f"{label} (file scan)"):
586
+ continue
587
+ output_path.parent.mkdir(parents=True, exist_ok=True)
588
+ if output_path.exists():
589
+ output_path.unlink()
590
+ src_output.rename(output_path)
591
+ log.info(f" OK {label} (file scan)")
592
+ return {"success": True, "identifier": doi, "doi": doi,
593
+ "file": str(output_path), "source": label}
594
+
595
+ preprint = _take_preprint()
596
+ if preprint is not None:
597
+ return preprint
598
+ log.info(f" All sources failed")
599
+ finally:
600
+ # Never block termination on sources that refuse to return: cancel
601
+ # pending futures and waive stragglers. Their own I/O deadlines
602
+ # (connect/read timeouts, stream deadline in pdf_utils) bound how long
603
+ # an abandoned worker thread can outlive the race.
604
+ race_over.set()
605
+ alive = [f for f in futures if not f.done()]
606
+ if alive:
607
+ log.info(f" Racing finished with {len(alive)} source(s) still running (waived)")
608
+ pool.shutdown(wait=False, cancel_futures=True)
609
+ # Cleanup temp files
610
+ for _, other_path in futures.values():
611
+ if other_path != output_path and other_path.exists():
612
+ try:
613
+ other_path.unlink(missing_ok=True)
614
+ except OSError:
615
+ pass
616
+
617
+ return None
618
+
619
+
620
+ # Single-flight guard: at most one download() per identifier per process.
621
+ # Duplicate submissions (e.g. an MCP client retrying after its own 30s client
622
+ # timeout while the server is still racing sources) used to start a second,
623
+ # parallel browser task for the same paper — the "infinite popup" experience.
624
+ _INFLIGHT_LOCK = threading.Lock()
625
+ # The papers in flight are the DownloadAttempts registered in sources.attempt.
626
+
627
+ class _RecentSet:
628
+ """A set that keeps only the latest ``limit`` keys (insertion order).
629
+
630
+ Late finishers only need to know a paper ended while they still run, so a
631
+ long-running server need not remember every paper it ever downloaded.
632
+ """
633
+
634
+ def __init__(self, limit: int = 2000) -> None:
635
+ self._keys: dict[str, None] = {}
636
+ self._limit = limit
637
+ self._lock = threading.Lock()
638
+
639
+ def add(self, key: str) -> None:
640
+ with self._lock:
641
+ self._keys.pop(key, None)
642
+ self._keys[key] = None
643
+ while len(self._keys) > self._limit:
644
+ self._keys.pop(next(iter(self._keys)))
645
+
646
+ def update(self, keys: Any) -> None:
647
+ for k in keys:
648
+ self.add(k)
649
+
650
+ def discard(self, key: str) -> None:
651
+ with self._lock:
652
+ self._keys.pop(key, None)
653
+
654
+ def clear(self) -> None:
655
+ with self._lock:
656
+ self._keys.clear()
657
+
658
+ def __contains__(self, key: object) -> bool:
659
+ return key in self._keys
660
+
661
+ def __len__(self) -> int:
662
+ return len(self._keys)
663
+
664
+
665
+ # DOIs whose download() has returned. Racing returns on the first success while
666
+ # other sources keep running ("waived"); those late finishers check this set to
667
+ # drop their duplicate files and to stop launching browsers.
668
+ _SETTLED = _RecentSet()
669
+ _SOURCE_TLS = threading.local()
670
+
671
+ # Heartbeats from sources that are waiting for the user to log in, by DOI.
672
+ # The race must not give up (and open the next phase's window) while the user
673
+ # is in the middle of a login.
674
+ # Heartbeats stop the moment the login succeeds, but the source still has to
675
+ # reopen the article and capture the PDF: keep waiting this long after the
676
+ # last heartbeat so the race doesn't open the next phase's window meanwhile.
677
+ _USER_WAIT_FRESH_S = 120.0
678
+ _USER_WAIT_MAX_S = 600.0
679
+
680
+
681
+ _WAIT_TEXT = {
682
+ "login": "请在弹出的浏览器窗口中登录学校账号",
683
+ "verify": "请在弹出的浏览器窗口中完成人机验证,出现勾选框时勾选它",
684
+ }
685
+
686
+
687
+ class SourceWaived(RuntimeError):
688
+ """Raised in a source's wait-for-the-user loop once its paper is settled."""
689
+
690
+
691
+ def note_user_waiting(what: str = "") -> None:
692
+ """Called by a source on every poll while it waits for the user (a login, a
693
+ human check). Keeps the race from closing that window, and tells the app:
694
+ a notice on top of the job page and a line under the paper.
695
+
696
+ Raises ``SourceWaived`` when the paper is already done (another source got
697
+ it): the waiting window must not hold the browser or ask for a login."""
698
+ if not _source_may_continue():
699
+ raise SourceWaived("download already finished; stop waiting for the user")
700
+ doi = getattr(_SOURCE_TLS, "doi", None)
701
+ if doi:
702
+ att = _attempt.get(doi)
703
+ if att is not None:
704
+ att.user_wait_at = time.time()
705
+ try:
706
+ from ..download import progress as pr
707
+ text = _WAIT_TEXT.get(what, "请在弹出的浏览器窗口中完成登录或人机验证")
708
+ pr.set_attention(f"wait:{doi}", f"{text}({doi})", current=doi, phase="等待你的操作")
709
+ pr.item_stage(doi, "等待你在浏览器中操作")
710
+ except Exception:
711
+ pass
712
+
713
+
714
+ def _user_waiting_for(doi: str) -> bool:
715
+ key = normalize_doi(doi) if not is_arxiv_identifier(doi) else doi
716
+ att = _attempt.get(key)
717
+ return att is not None and time.time() - att.user_wait_at < _USER_WAIT_FRESH_S
718
+
719
+
720
+ # Cooperative cancellation. cancel_downloads() bumps the generation; every
721
+ # download() remembers the generation it started under (thread-local), so it
722
+ # knows it was cancelled without killing threads. Downloads started afterwards
723
+ # are unaffected.
724
+ _CANCEL_GEN = [0]
725
+ _DL_TLS = threading.local()
726
+ # Papers the user chose to skip while one of their windows waited for them
727
+ # (the app's 跳过这篇): that download stops as if cancelled; the others go on.
728
+ _SKIPPED: set[str] = set()
729
+
730
+
731
+ def _cancelled() -> bool:
732
+ """This thread's paper was cancelled (its job's generation moved on) or skipped."""
733
+ key = getattr(_DL_TLS, "key", None)
734
+ att = _attempt.get(key)
735
+ gen = att.gen if att is not None else getattr(_DL_TLS, "gen", None)
736
+ if gen is not None and gen != _CANCEL_GEN[0]:
737
+ return True
738
+ return key in _SKIPPED
739
+
740
+
741
+ def skip_download(identifier: str) -> bool:
742
+ """Give up on one paper: in flight, its windows stop waiting for the user and
743
+ close and its race returns; not started yet, it is skipped when its turn
744
+ comes (download_jobs clears the requests when a job starts)."""
745
+ key = identifier.strip()
746
+ if not is_arxiv_identifier(key):
747
+ key = normalize_doi(key)
748
+ with _INFLIGHT_LOCK:
749
+ _SKIPPED.add(key)
750
+ if _attempt.get(key) is None:
751
+ log.info(f" Will skip when its turn comes: {key}")
752
+ return True
753
+ _SETTLED.add(key) # its sources stop waiting, launch nothing, drop late results
754
+ try:
755
+ from ..download import progress as _pr
756
+ _pr.clear_attention(f"wait:{key}")
757
+ from ..browser.backend import close_browsers_since
758
+ close_browsers_since(0.0, owner=key) # its own windows only
759
+ except Exception as exc:
760
+ log.info(f" Skip: browser cleanup skipped: {exc}")
761
+ log.info(f" Skipped by the user: {key}")
762
+ return True
763
+
764
+
765
+ def cancel_downloads() -> dict[str, Any]:
766
+ """Cancel every download in flight: stop waiting, drop late results, close windows."""
767
+ with _INFLIGHT_LOCK:
768
+ _CANCEL_GEN[0] += 1
769
+ keys = sorted(_attempt.active_keys())
770
+ _SETTLED.update(keys) # sources still running drop results, launch nothing
771
+ closed = 0
772
+ try:
773
+ from ..browser.backend import close_browsers_since
774
+ closed = close_browsers_since(0.0)
775
+ except Exception as exc:
776
+ log.info(f" Cancel: browser cleanup skipped: {exc}")
777
+ log.info(f" Cancelled {len(keys)} download(s); closed {closed} browser(s)")
778
+ return {"cancelled": keys, "browsers_closed": closed}
779
+
780
+
781
+ def _source_may_continue() -> bool:
782
+ """LAUNCH_GUARD hook: False once this thread's race or download is over."""
783
+ race = getattr(_SOURCE_TLS, "race", None)
784
+ if race is not None and race.is_set():
785
+ return False # waived lane: its race already returned
786
+ doi = getattr(_SOURCE_TLS, "doi", None)
787
+ if not doi: # a pool thread working for a source (Sci-Hub's mirror race)
788
+ from ..browser.backend import current_owner
789
+ doi = current_owner()
790
+ return not (doi and doi in _SETTLED)
791
+
792
+
793
+ # Per-download record of WHY each racing source failed. A channel being
794
+ # unreachable is usually temporary or a missing user decision (e.g. Unpaywall
795
+ # needs the user's email) — surfaced in the final failure result so the
796
+ # user/agent can choose: retry, fix config, or switch channel. Never skip a
797
+ # channel silently. The list is created per download() and threaded through
798
+ # _run_tiers_parallel (pool workers append to it; list.append is atomic).
799
+ def _record_source_failure(failures: list[dict[str, str]], label: str,
800
+ result: dict[str, Any] | None) -> None:
801
+ """Collect structured failure reasons (label, reason, error_type, action)."""
802
+ if not result or result.get("success"):
803
+ return
804
+ try:
805
+ failures.append({
806
+ "source": label,
807
+ "reason": str(result.get("reason") or result.get("error") or "")[:200],
808
+ "error_type": str(result.get("error_type", "")),
809
+ "action": str(result.get("action", "")),
810
+ })
811
+ except Exception:
812
+ pass
813
+
814
+
815
+ def download(
816
+ identifier: str,
817
+ output_dir: str | Path | None = None,
818
+ *,
819
+ scihub_enabled: bool | None = None,
820
+ use_tor: bool = False,
821
+ use_vpnsci: bool = False,
822
+ bibtex: bool = False,
823
+ rename: bool = True,
824
+ _institutional: bool = True,
825
+ strategy: str | None = None,
826
+ _batch: bool = False,
827
+ _scihub_http_only: bool = False,
828
+ _school_only: bool = False,
829
+ ) -> dict[str, Any]:
830
+ key = identifier.strip()
831
+ if not is_arxiv_identifier(key):
832
+ key = normalize_doi(key)
833
+ from ..download.doi_check import registered
834
+ if registered(key) is False: # a typo: no source can have it
835
+ return fail(key, "DOI is not registered at doi.org", error_type="doi_not_found",
836
+ action="check the DOI for typos")
837
+ gen = getattr(_DL_TLS, "gen", None)
838
+ with _INFLIGHT_LOCK:
839
+ if _attempt.get(key) is not None:
840
+ log.info(f" Single-flight: download already in progress for {key}")
841
+ return fail(
842
+ key,
843
+ reason="download already in progress for this identifier",
844
+ error_type="in_progress",
845
+ action="wait for the in-flight attempt to finish (it is still "
846
+ "racing sources); retry in a minute only if it failed",
847
+ )
848
+ if key in _SKIPPED: # the user skipped it while it was queued
849
+ _SKIPPED.discard(key) # used up: a later retry of the paper runs
850
+ return fail(key, "skipped by user", error_type="skipped", action="none")
851
+ _attempt.begin(key, gen if gen is not None else _CANCEL_GEN[0])
852
+ _SETTLED.discard(key)
853
+ t0 = time.perf_counter() # same clock as browser_backend._remember
854
+ prev_key = getattr(_DL_TLS, "key", None)
855
+ _DL_TLS.key = key
856
+ prev_gen = getattr(_DL_TLS, "gen", None)
857
+ if prev_gen is None: # nested calls (batch phase 2) keep the batch's generation
858
+ _DL_TLS.gen = _CANCEL_GEN[0]
859
+ try:
860
+ result = _download_impl(
861
+ identifier,
862
+ output_dir,
863
+ scihub_enabled=scihub_enabled,
864
+ use_tor=use_tor,
865
+ use_vpnsci=use_vpnsci,
866
+ bibtex=bibtex,
867
+ rename=rename,
868
+ _institutional=_institutional,
869
+ strategy=strategy,
870
+ _batch=_batch,
871
+ _scihub_http_only=_scihub_http_only,
872
+ _school_only=_school_only,
873
+ )
874
+ att = _attempt.get(key)
875
+ steps = list(att.steps) if att is not None else []
876
+ if key in _SKIPPED and not result.get("success"):
877
+ _SKIPPED.discard(key) # used up: a later retry of the paper runs
878
+ return {**fail(key, "skipped by user", error_type="skipped", action="none"), "attempts": steps}
879
+ if _cancelled() and not result.get("success"):
880
+ return {**fail(key, "cancelled by user", error_type="cancelled", action="none"), "attempts": steps}
881
+ if steps and isinstance(result, dict):
882
+ result.setdefault("attempts", steps) # which ways were tried, how long, why they failed
883
+ if result.get("success"):
884
+ from ..publishers.strategies._core import clear_no_access
885
+ clear_no_access(key)
886
+ _SKIPPED.discard(key) # a skip that came too late must not skip the next try
887
+ return result
888
+ finally:
889
+ _DL_TLS.key = prev_key
890
+ try: # whatever this paper waited on the user for is over now
891
+ from ..download import progress as _pr
892
+ _pr.clear_attention(f"wait:{normalize_doi(key) if not is_arxiv_identifier(key) else key}")
893
+ except Exception:
894
+ pass
895
+ if prev_gen is None:
896
+ _DL_TLS.gen = None
897
+ with _INFLIGHT_LOCK:
898
+ _attempt.end(key)
899
+ _SETTLED.add(key)
900
+ # Racing returns on the first success but waived sources keep their
901
+ # browser windows open. Close what this download launched (its own
902
+ # browsers only, so a concurrent download keeps its windows).
903
+ try:
904
+ from ..browser.backend import close_browsers_since
905
+ n = close_browsers_since(t0, owner=key)
906
+ if n:
907
+ log.info(f" Closed {n} browser(s) left open by this download")
908
+ except Exception as exc:
909
+ log.info(f" Browser cleanup skipped: {exc}")
910
+
911
+
912
+ def _download_impl(
913
+ identifier: str,
914
+ output_dir: str | Path | None = None,
915
+ *,
916
+ scihub_enabled: bool | None = None,
917
+ use_tor: bool = False,
918
+ use_vpnsci: bool = False,
919
+ bibtex: bool = False,
920
+ rename: bool = True,
921
+ _institutional: bool = True,
922
+ strategy: str | None = None,
923
+ _batch: bool = False,
924
+ _scihub_http_only: bool = False,
925
+ _school_only: bool = False,
926
+ ) -> dict[str, Any]:
927
+ race_failures: list[dict[str, str]] = []
928
+ config = load_config()
929
+ if _scihub_http_only:
930
+ # Sci-Hub's gate solved over HTTP only: no browser, so the paper does
931
+ # not queue for the one browser slot (a batch's first round).
932
+ config["scihub_use_browser"] = False
933
+ DATA_DIR.mkdir(parents=True, exist_ok=True)
934
+
935
+ if scihub_enabled is not None:
936
+ config["scihub_enabled"] = scihub_enabled
937
+
938
+ if strategy is not None:
939
+ config["download_strategy"] = strategy
940
+
941
+ # PMC allows automated retrieval only through its Cloud / OAI-PMH /
942
+ # E-utilities / BioC services (https://pmc.ncbi.nlm.nih.gov/tools/developers/),
943
+ # so its PDF pages are used for single downloads only, never in a batch.
944
+ config["_batch"] = _batch
945
+
946
+ target_dir = Path(output_dir) if output_dir else Path(config["output_dir"])
947
+ target_dir.mkdir(parents=True, exist_ok=True)
948
+
949
+ _cleanup_stale_files(target_dir)
950
+
951
+ identifier = identifier.strip()
952
+ output_path = target_dir / f"{safe_filename(identifier)}.pdf"
953
+
954
+ cached = cache_get(identifier, config)
955
+ if cached and cached.get("success"):
956
+ cached_file = Path(cached.get("file", ""))
957
+ cached_doi = str(cached.get("doi") or "")
958
+ # A result cached before the paper check existed may be a different paper
959
+ # (a JSTOR essay came back from the cache for a Ceramics International DOI, 2026-10-28).
960
+ if cached_file.is_file() and cached_doi and _is_wrong_paper(cached_file, cached_doi, config, "cache"):
961
+ log.info(" The cached file is a different paper; downloading again")
962
+ cached_file = Path("")
963
+ if cached_file.is_file() and not _same_dir(cached_file.parent, target_dir):
964
+ # Got before into another folder (the save location has changed since,
965
+ # 2026-10-19): the user expects it here. Copy it, no new download.
966
+ try:
967
+ import shutil
968
+ copied = target_dir / cached_file.name
969
+ if not copied.exists():
970
+ shutil.copy2(cached_file, copied)
971
+ log.info(f" Cached file is in another folder; copied {cached_file.name} here")
972
+ cached = {**cached, "file": str(copied)}
973
+ cached_file = copied
974
+ if cached.get("doi"):
975
+ _update_doi_index(target_dir, str(cached["doi"]), copied,
976
+ source=str(cached.get("source") or ""), config=config)
977
+ cache_set(identifier, cached, config)
978
+ except OSError as exc:
979
+ log.info(f" Cannot copy the cached file here ({exc}); downloading again")
980
+ cached_file = Path("")
981
+ if cached_file.is_file():
982
+ cached["cached"] = True
983
+ if bibtex:
984
+ from ..citations.bibtex_crossref import fetch_bibtex
985
+ cached["bibtex"] = fetch_bibtex(identifier, config)
986
+ return cached
987
+
988
+ # Scan output dir for existing file with same rename pattern
989
+ doi = normalize_doi(identifier) if not is_arxiv_identifier(identifier) else identifier
990
+
991
+ # Validate DOI before attempting download
992
+ if not is_arxiv_identifier(identifier):
993
+ from ..identifiers import validate_doi
994
+ valid, msg = validate_doi(doi)
995
+ if not valid:
996
+ log.info(f" DOI validation failed: {msg}")
997
+ return {"success": False, "identifier": identifier, "doi": doi, "error": f"Invalid DOI: {msg}"}
998
+
999
+ # Scan output dir for any PDF already downloaded for this DOI.
1000
+ # Uses a DOI→file index to catch renamed files reliably.
1001
+ doi_index = target_dir / ".doi_index.json"
1002
+ if doi_index.exists():
1003
+ try:
1004
+ idx = json.loads(doi_index.read_text(encoding="utf-8"))
1005
+ raw_entry = idx.get(doi)
1006
+ if raw_entry:
1007
+ entry = _index_entry(raw_entry)
1008
+ candidate = Path(entry["file"]) if entry and entry.get("file") else None
1009
+ if entry and candidate and _index_entry_is_fresh(entry, config) and _index_strategy_compatible(entry, config):
1010
+ if candidate.exists():
1011
+ log.info(f" Found existing file (index): {candidate.name}")
1012
+ result = {
1013
+ "success": True, "identifier": identifier,
1014
+ "doi": doi, "file": str(candidate),
1015
+ "source": "local_cache", "cached": True,
1016
+ }
1017
+ # Upgrade legacy entries to records and refresh provenance.
1018
+ idx[doi] = {
1019
+ "file": str(candidate),
1020
+ "source": entry["source"] or "local_cache",
1021
+ "strategy": entry["strategy"] or config.get("download_strategy", ""),
1022
+ "ts": time.time(),
1023
+ }
1024
+ doi_index.write_text(json.dumps(idx, indent=2, ensure_ascii=False), encoding="utf-8")
1025
+ cache_set(identifier, result, config)
1026
+ return result
1027
+ # Stale (file deleted), TTL-expired, or strategy-mismatched entry —
1028
+ # drop it so the download is retried instead of serving stale state.
1029
+ del idx[doi]
1030
+ doi_index.write_text(json.dumps(idx, indent=2, ensure_ascii=False), encoding="utf-8")
1031
+ except Exception:
1032
+ pass
1033
+
1034
+ from ..citations.formats import fetch_metadata
1035
+ metadata = fetch_metadata(doi, config)
1036
+ if metadata:
1037
+ from ..download.rename import rename_format
1038
+ expected_name = rename_pdf_generate_filename(metadata, doi=doi, fmt=rename_format(config))
1039
+ if expected_name:
1040
+ # Generated names can collide across papers, so a name match
1041
+ # alone is not proof: skip files the index gives to
1042
+ # another DOI and require this DOI inside the PDF.
1043
+ claimed = _files_claimed_by_other_dois(target_dir, doi)
1044
+ for suffix in ("", "_2", "_3", "_4", "_5"):
1045
+ candidate = target_dir / f"{expected_name}{suffix}.pdf"
1046
+ if (candidate.exists() and _norm_path(candidate) not in claimed
1047
+ and _pdf_mentions_doi(candidate, doi)):
1048
+ log.info(f" Found existing file: {candidate.name}")
1049
+ result = {
1050
+ "success": True, "identifier": identifier,
1051
+ "doi": doi, "file": str(candidate),
1052
+ "source": "local_cache", "cached": True,
1053
+ }
1054
+ cache_set(identifier, result, config)
1055
+ return result
1056
+
1057
+ log.info(f"LitLoop - {identifier}")
1058
+ if _cancelled(): # skipped or cancelled while the metadata lookup ran
1059
+ return fail(identifier, "skipped by user" if getattr(_DL_TLS, "key", None) in _SKIPPED else "cancelled by user")
1060
+
1061
+ if is_arxiv_identifier(identifier):
1062
+ log.info(" [L0] arXiv direct")
1063
+ result = try_arxiv(identifier, output_path, config)
1064
+ if result:
1065
+ _update_doi_index(
1066
+ target_dir,
1067
+ identifier,
1068
+ Path(result.get("file", "")),
1069
+ source=result.get("source") or "arXiv",
1070
+ strategy=config.get("download_strategy", ""),
1071
+ config=config,
1072
+ )
1073
+ if rename:
1074
+ _auto_rename(result, identifier, config, doi=identifier, target_dir=target_dir)
1075
+ cache_set(identifier, result, config)
1076
+ if bibtex:
1077
+ from ..citations.bibtex_crossref import fetch_bibtex
1078
+ result["bibtex"] = fetch_bibtex(identifier, config)
1079
+ return result
1080
+ return fail(identifier, "arXiv PDF not available")
1081
+
1082
+ doi = normalize_doi(identifier)
1083
+
1084
+ # A preprint is a file for this DOI but not the version of record: keep it
1085
+ # aside and let the later phases look for the published PDF (2026-11-03).
1086
+ preprint_only: dict[str, Any] = {"result": None}
1087
+
1088
+ def _stash_preprint(res: dict[str, Any]) -> bool:
1089
+ if not (res and res.get("preprint")):
1090
+ return False
1091
+ path = Path(res.get("file", ""))
1092
+ marked = path.with_name(f"{path.stem}_preprint{path.suffix}")
1093
+ if path.exists() and marked != path:
1094
+ try:
1095
+ path.replace(marked)
1096
+ res["file"] = str(marked)
1097
+ except OSError:
1098
+ pass
1099
+ preprint_only["result"] = res
1100
+ log.info(f" {res['preprint']} preprint of this paper; still looking for the published version")
1101
+ return True
1102
+
1103
+ def _use_preprint() -> dict[str, Any] | None:
1104
+ res = preprint_only["result"]
1105
+ if res is None:
1106
+ return None
1107
+ if rename:
1108
+ _auto_rename(res, identifier, config, doi=doi, target_dir=target_dir)
1109
+ if bibtex:
1110
+ from ..citations.bibtex_crossref import fetch_bibtex
1111
+ res["bibtex"] = fetch_bibtex(doi, config)
1112
+ log.info(f" No version of record available; keeping the {res.get('preprint')} preprint")
1113
+ return res
1114
+
1115
+ # Phase 1: Free sources (OA + grey) — parallel race
1116
+ t_phase = time.time()
1117
+ p_phase = time.perf_counter() # close_browsers_since's clock
1118
+ # _school_only: a batch's second pass; its first pass already tried these.
1119
+ free_sources = [] if _school_only else _build_free_sources(doi, config)
1120
+ log.info(f" [timing] phase 1 sources built: {time.time() - t_phase:.1f}s")
1121
+ if free_sources:
1122
+ result = _run_tiers_parallel(
1123
+ [(free_sources, "Free", 15)], doi, target_dir, output_path, config, use_tor, 15,
1124
+ failures=race_failures,
1125
+ )
1126
+ log.info(f" [timing] phase 1 (free sources and publisher site): {time.time() - t_phase:.1f}s, "
1127
+ f"{'PDF' if result else 'no PDF'}")
1128
+ if result and _stash_preprint(result):
1129
+ pass
1130
+ elif result:
1131
+ _update_doi_index(
1132
+ target_dir,
1133
+ doi,
1134
+ Path(result.get("file", "")),
1135
+ source=result.get("source") or "",
1136
+ strategy=config.get("download_strategy", ""),
1137
+ config=config,
1138
+ )
1139
+ if rename:
1140
+ _auto_rename(result, identifier, config, doi=doi, target_dir=target_dir)
1141
+ cache_set(identifier, result, config)
1142
+ if bibtex:
1143
+ from ..citations.bibtex_crossref import fetch_bibtex
1144
+ result["bibtex"] = fetch_bibtex(doi, config)
1145
+ return result
1146
+
1147
+ # Phase 2: Institutional access — only when Phase 1 failed
1148
+ # Skip institutional fallback for grey_only/scihub_only strategy
1149
+ from ..publishers.strategies._core import no_access
1150
+ if _institutional and no_access(doi) and not config.get("school_routes_after_publisher_login", False):
1151
+ log.info(" Logged into the publisher as the school and still no full text: skipping the school routes")
1152
+ preprint = _use_preprint()
1153
+ if preprint is not None:
1154
+ return preprint
1155
+ return fail(identifier, "the school does not subscribe: logged into the publisher with the school's identity, still no full text",
1156
+ error_type="not_entitled", action="no_subscription")
1157
+ if _institutional and config.get("download_strategy") not in ("scihub_only", "grey_only") and not _cancelled():
1158
+ # Phase 1's lanes that are still at work (waived) keep their browsers and
1159
+ # the browser slot: on 2026-10-18 a stuck publisher-site lane held it
1160
+ # for 16 minutes and the WebVPN step queued 697 s behind it. Phase 1 is
1161
+ # over, so close what it opened before the school routes need a browser.
1162
+ try:
1163
+ from ..browser.backend import close_browsers_since
1164
+ n = close_browsers_since(p_phase, owner=getattr(_DL_TLS, "key", None) or doi)
1165
+ if n:
1166
+ log.info(f" Closed {n} browser(s) phase 1 left open")
1167
+ except Exception as exc:
1168
+ log.info(f" Phase 1 browser cleanup skipped: {exc}")
1169
+ # Session self-healing: validate and refresh institutional sessions
1170
+ # before racing them (WebVPN only; CARSI self-heals inside login()).
1171
+ t_heal = time.time()
1172
+ try:
1173
+ from ..institutional.session_heal import ensure_institutional_sessions
1174
+ heal_report = ensure_institutional_sessions(
1175
+ config, use_vpnsci=use_vpnsci or bool(config.get("vpnsci_enabled")))
1176
+ for channel, state in heal_report.items():
1177
+ log.info(f" [session-heal] {channel}: {state.get('status')}")
1178
+ except Exception as exc:
1179
+ log.info(f" [session-heal] skipped: {exc}")
1180
+ log.info(f" [timing] session check: {time.time() - t_heal:.1f}s")
1181
+ if _cancelled(): # skipped or cancelled while the session check waited for a login
1182
+ return fail(identifier, "skipped by user" if getattr(_DL_TLS, "key", None) in _SKIPPED else "cancelled by user")
1183
+ inst_sources = _build_institutional_sources(doi, config, use_vpnsci=use_vpnsci)
1184
+ if inst_sources:
1185
+ log.info(" Phase 1 failed, trying institutional access...")
1186
+ result = _run_institutional_sequence(inst_sources, doi, target_dir, output_path, config,
1187
+ failures=race_failures)
1188
+ if result and _stash_preprint(result):
1189
+ pass
1190
+ elif result:
1191
+ _update_doi_index(
1192
+ target_dir,
1193
+ doi,
1194
+ Path(result.get("file", "")),
1195
+ source=result.get("source") or "",
1196
+ strategy=config.get("download_strategy", ""),
1197
+ config=config,
1198
+ )
1199
+ if rename:
1200
+ _auto_rename(result, identifier, config, doi=doi, target_dir=target_dir)
1201
+ cache_set(identifier, result, config)
1202
+ if bibtex:
1203
+ from ..citations.bibtex_crossref import fetch_bibtex
1204
+ result["bibtex"] = fetch_bibtex(doi, config)
1205
+ return result
1206
+
1207
+ # Late capture: wait briefly for browser downloads that complete after race timeout,
1208
+ # then scan for any PDFs that were saved to disk by racing threads.
1209
+ import time as _time
1210
+ _time.sleep(2) # grace period for browser threads to finish writing
1211
+ from ..download.pdf import is_pdf_file
1212
+ held = Path(preprint_only["result"]["file"]) if preprint_only["result"] else None
1213
+ for p in target_dir.glob(f"{safe_filename(identifier)}*.pdf"):
1214
+ if held is not None and p == held:
1215
+ continue # the kept preprint is not a late capture of the published PDF
1216
+ # Only a whole PDF of this very paper: a racing thread may have left a
1217
+ # different paper under this name (Sci-Hub mapped a Ceramics
1218
+ # International DOI to a JSTOR essay; it was taken here, 2026-10-17).
1219
+ if p.stat().st_size > 5000 and is_pdf_file(p) and not _is_wrong_paper(p, doi, config, "late capture"):
1220
+ result = {
1221
+ "success": True, "identifier": identifier, "doi": doi,
1222
+ "file": str(p), "source": "late_capture",
1223
+ }
1224
+ cache_set(identifier, result, config)
1225
+ if rename:
1226
+ _auto_rename(result, identifier, config, doi=doi, target_dir=target_dir)
1227
+ if bibtex:
1228
+ from ..citations.bibtex_crossref import fetch_bibtex
1229
+ result["bibtex"] = fetch_bibtex(identifier, config)
1230
+ return result
1231
+
1232
+ # Nothing delivered the published PDF: a preprint of this paper is better
1233
+ # than reporting failure, and its file name says what it is.
1234
+ preprint = _use_preprint()
1235
+ if preprint is not None:
1236
+ return preprint
1237
+
1238
+ # What this paper's own browser strategies reported (never another paper's).
1239
+ att = _attempt.get(getattr(_DL_TLS, "key", None))
1240
+ error_type, error_action = att.last_error if att is not None else ("", "")
1241
+ guidance = _build_failure_guidance(doi, config, error_type)
1242
+
1243
+ hint: dict[str, Any] = {"manual_url": f"https://sci-hub.ru/{doi}", "guidance": guidance}
1244
+
1245
+ reason = "no PDF found"
1246
+ if error_type in ("paywall", "cloudflare_blocked"):
1247
+ from ..desktop.error_text import raw_for
1248
+ reason = raw_for(error_type)
1249
+
1250
+ # Add agent hint for paywall - clear instruction on what to do next
1251
+ if error_type == "paywall" or error_action == "login_required":
1252
+ result = fail(
1253
+ identifier,
1254
+ reason,
1255
+ hint,
1256
+ error_type=error_type or "paywall",
1257
+ action=error_action or "login_required",
1258
+ )
1259
+ result["agent_hint"] = (
1260
+ f"此论文需要机构登录才能下载。请运行 litloop_login(identifier=\"{doi}\") "
1261
+ "打开浏览器让用户登录机构账号,登录后关闭浏览器,然后重试下载。"
1262
+ )
1263
+ else:
1264
+ result = fail(
1265
+ identifier,
1266
+ reason,
1267
+ hint,
1268
+ error_type=error_type or "not_found",
1269
+ action=error_action or "",
1270
+ )
1271
+ result["source"] = "none"
1272
+
1273
+ # Surface WHY each channel failed so the user can decide (channels are
1274
+ # often only temporarily unreachable; some need a user decision, e.g.
1275
+ # Unpaywall requires the user's real email).
1276
+ source_failures = race_failures
1277
+ if source_failures:
1278
+ result["source_failures"] = source_failures
1279
+ config_needs = [f for f in source_failures if f.get("error_type") == "config_needed"]
1280
+ if config_needs and "agent_hint" not in result:
1281
+ channels = ", ".join(sorted({f["source"] for f in config_needs}))
1282
+ result["agent_hint"] = (
1283
+ f"配置缺失({channels}):Unpaywall 等渠道需要用户真实邮箱。"
1284
+ '请询问用户的邮箱地址,调用 litloop_config(key="email", value="<用户邮箱>") '
1285
+ "后重试下载。"
1286
+ )
1287
+ return result
1288
+
1289
+
1290
+ # Re-export symbols split into internal modules so callers, tests, and
1291
+ # monkeypatches against ``litloop.sources`` keep working unchanged.
1292
+ from ._index import (
1293
+ _EXPLICIT_STRATEGIES,
1294
+ _auto_rename,
1295
+ _files_claimed_by_other_dois,
1296
+ _index_entry,
1297
+ _index_entry_is_fresh,
1298
+ _index_strategy_compatible,
1299
+ _norm_path,
1300
+ _pdf_mentions_doi,
1301
+ _same_dir,
1302
+ _update_doi_index,
1303
+ )
1304
+ from ._routes import (
1305
+ DEFAULT_INSTITUTIONAL_ORDER,
1306
+ _INSTITUTIONAL_STEPS,
1307
+ _OPEN_HOSTS,
1308
+ _STEP_TEXT,
1309
+ _build_failure_guidance,
1310
+ _build_free_sources,
1311
+ _build_institutional_sources,
1312
+ _run_institutional_sequence,
1313
+ _step_watchdog,
1314
+ _webvpn_browser,
1315
+ _webvpn_http,
1316
+ )
1317
+ from ._batch import (
1318
+ _apply_preprint_fallbacks,
1319
+ _batch_institutional_phase,
1320
+ _clear_progress,
1321
+ _close_batch_browsers,
1322
+ _get_progress_file,
1323
+ _load_progress,
1324
+ _progress_lock,
1325
+ _save_progress,
1326
+ _school_routes_each,
1327
+ _write_download_results,
1328
+ batch_download,
1329
+ institutional_download,
1330
+ )
1331
+
1332
+
1333
+ try:
1334
+ from ..browser import backend as _bb
1335
+ _bb.LAUNCH_GUARD = _source_may_continue
1336
+ # A browser belongs to the paper the launching thread works for.
1337
+ _bb.OWNER = lambda: getattr(_SOURCE_TLS, "doi", None) or getattr(_DL_TLS, "key", None)
1338
+ except Exception:
1339
+ pass