litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1339 @@
|
|
|
1
|
+
"""Source registry and download orchestration with tiered parallel racing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
import json
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ..cache import cache_get, cache_set
|
|
14
|
+
from ..config import load_config, DATA_DIR
|
|
15
|
+
from ..identifiers import is_arxiv_identifier, normalize_doi, safe_filename
|
|
16
|
+
from ..log import get_logger
|
|
17
|
+
from ..download.pdf import fail
|
|
18
|
+
from ..download.rename import rename_pdf, generate_filename as rename_pdf_generate_filename
|
|
19
|
+
|
|
20
|
+
log = get_logger()
|
|
21
|
+
|
|
22
|
+
from .arxiv import try_arxiv
|
|
23
|
+
from .carsi import try_carsi
|
|
24
|
+
from .core_api import try_core
|
|
25
|
+
from .crossref import try_crossref_page_scrape
|
|
26
|
+
from .europepmc import try_europepmc, try_pmc
|
|
27
|
+
from .libgen import try_libgen
|
|
28
|
+
from .oa_discovery import try_doaj
|
|
29
|
+
from .openalex import try_openalex_oa, try_openalex_content_api
|
|
30
|
+
from . import attempt as _attempt
|
|
31
|
+
from .publishers import get_publisher_fast_sources
|
|
32
|
+
from .scibban import try_scibban
|
|
33
|
+
from .scihub import try_scihub
|
|
34
|
+
from .semantic_scholar import try_semanticscholar
|
|
35
|
+
from .openaire import try_openaire
|
|
36
|
+
from .unpaywall import try_unpaywall
|
|
37
|
+
from .vpnsci import try_vpnsci
|
|
38
|
+
from .ezproxy import try_ezproxy
|
|
39
|
+
|
|
40
|
+
# Global semaphore to limit concurrent browser-based sources across all DOI
|
|
41
|
+
# Prevents batch_workers × browser_sources_per_DOI explosion of Chrome windows
|
|
42
|
+
_browser_semaphore: threading.Semaphore | None = None
|
|
43
|
+
_browser_semaphore_lock = threading.Lock()
|
|
44
|
+
|
|
45
|
+
# Source labels that require launching a browser (CloakBrowser)
|
|
46
|
+
_BROWSER_SOURCE_LABELS = frozenset({
|
|
47
|
+
"ElsevierBrowser", "WileyBrowser", "IEEEBrowser", "ACSBrowser",
|
|
48
|
+
"RSCBrowser", "AIPBrowser", "SpringerBrowser", "APSBrowser",
|
|
49
|
+
"TandFBrowser", "IOPBrowser", "OxfordBrowser", "ACMBrowser",
|
|
50
|
+
"NatureBrowser", "ScienceBrowser", "SAGEBrowser", "ASCEBrowser",
|
|
51
|
+
"RoyalSocietyBrowser", "CopernicusDirect",
|
|
52
|
+
"GenericBrowser", "WebVPN", "CARSI", "EZProxy",
|
|
53
|
+
# Sci-Hub launches CloakBrowser internally (browser-first pass +
|
|
54
|
+
# Cloudflare/ALTCHA challenge solving), so it must participate in the
|
|
55
|
+
# global browser-source concurrency budget.
|
|
56
|
+
"Sci-Hub",
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
# ponytail: negative cache — (source, publisher) -> expiry. A challenge/block
|
|
60
|
+
# on one paper skips that source for the whole publisher prefix for 15 min,
|
|
61
|
+
# saving the doomed timeout on every later paper. Broaden the key or raise
|
|
62
|
+
# the TTL if false-skip reports come in.
|
|
63
|
+
_NEG_TTL = 900.0
|
|
64
|
+
_NEG_CACHE: dict[tuple[str, str], float] = {}
|
|
65
|
+
_NEG_LOCK = threading.Lock()
|
|
66
|
+
_NEG_KEYWORDS = ("cloudflare", "challenge", "captcha", "blocked", "forbidden", "403", "429")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _publisher_of(doi: str) -> str:
|
|
70
|
+
return doi.split("/")[0].lower() if doi.lower().startswith("10.") else "*"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _neg_record(label: str, doi: str, result: dict[str, Any]) -> None:
|
|
74
|
+
reason = (str(result.get("error_type", "")) + " " + str(result.get("error", ""))).lower()
|
|
75
|
+
if not any(k in reason for k in _NEG_KEYWORDS):
|
|
76
|
+
return # one-off failures (404, paywall) must not poison the source
|
|
77
|
+
with _NEG_LOCK:
|
|
78
|
+
_NEG_CACHE[(label, _publisher_of(doi))] = time.time() + _NEG_TTL
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _neg_blocked(label: str, doi: str) -> bool:
|
|
82
|
+
now = time.time()
|
|
83
|
+
with _NEG_LOCK:
|
|
84
|
+
expired = [k for k, exp in _NEG_CACHE.items() if exp <= now]
|
|
85
|
+
for k in expired:
|
|
86
|
+
_NEG_CACHE.pop(k, None)
|
|
87
|
+
pub = _publisher_of(doi)
|
|
88
|
+
return (label, pub) in _NEG_CACHE or (label, "*") in _NEG_CACHE
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _get_browser_semaphore(config: dict[str, Any]) -> threading.Semaphore:
|
|
92
|
+
"""Get or create the global browser concurrency semaphore."""
|
|
93
|
+
global _browser_semaphore
|
|
94
|
+
max_workers = config.get("max_browser_workers", 1)
|
|
95
|
+
if _browser_semaphore is None:
|
|
96
|
+
with _browser_semaphore_lock:
|
|
97
|
+
if _browser_semaphore is None:
|
|
98
|
+
_browser_semaphore = threading.Semaphore(max_workers)
|
|
99
|
+
return _browser_semaphore
|
|
100
|
+
|
|
101
|
+
__all__ = ["download", "batch_download"]
|
|
102
|
+
|
|
103
|
+
_cleanup_done = False
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _cleanup_stale_files(target_dir: Path) -> None:
|
|
107
|
+
"""Remove orphaned .part files and racing temp files from previous runs."""
|
|
108
|
+
global _cleanup_done
|
|
109
|
+
if _cleanup_done:
|
|
110
|
+
return
|
|
111
|
+
_cleanup_done = True
|
|
112
|
+
if not target_dir.exists():
|
|
113
|
+
return
|
|
114
|
+
count = 0
|
|
115
|
+
for f in target_dir.iterdir():
|
|
116
|
+
if not f.is_file():
|
|
117
|
+
continue
|
|
118
|
+
name = f.name
|
|
119
|
+
# Skip hidden files (like .doi_index.json)
|
|
120
|
+
if name.startswith("."):
|
|
121
|
+
continue
|
|
122
|
+
# Always clean up .part files
|
|
123
|
+
if name.endswith(".part"):
|
|
124
|
+
try:
|
|
125
|
+
f.unlink()
|
|
126
|
+
count += 1
|
|
127
|
+
except OSError:
|
|
128
|
+
pass
|
|
129
|
+
continue
|
|
130
|
+
# Clean up racing temp files: DOI-based identifier + source label suffix
|
|
131
|
+
# e.g. 10_1038_nature12373_Unpaywall.pdf, 10_1016_test_scihub_st.pdf
|
|
132
|
+
# Must start with 10_ (DOI prefix) and have at least 4 segments (10_NNNN_suffix_label)
|
|
133
|
+
if name.endswith(".pdf") and name.startswith("10_"):
|
|
134
|
+
stem = name[:-4] # remove .pdf
|
|
135
|
+
# Split from the right to get the last segment as the source label
|
|
136
|
+
parts = stem.rsplit("_", 1)
|
|
137
|
+
if len(parts) == 2:
|
|
138
|
+
base, label = parts
|
|
139
|
+
# Verify: base must look like a DOI (10_NNNN_... with at least 4 segments)
|
|
140
|
+
# and label must be a source name (letters, not just a number)
|
|
141
|
+
base_parts = base.split("_")
|
|
142
|
+
if (len(base_parts) >= 3
|
|
143
|
+
and base_parts[0] == "10"
|
|
144
|
+
and base_parts[1].isdigit()
|
|
145
|
+
and len(base_parts[1]) >= 3
|
|
146
|
+
and any(c.isalpha() for c in label)):
|
|
147
|
+
try:
|
|
148
|
+
f.unlink()
|
|
149
|
+
count += 1
|
|
150
|
+
except OSError:
|
|
151
|
+
pass
|
|
152
|
+
if count > 0:
|
|
153
|
+
log.info(f"Cleaned up {count} stale temp files")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _is_wrong_paper(path: Path, doi: str, config: dict[str, Any], label: str) -> bool:
|
|
157
|
+
"""True when the PDF a source produced is a different paper (download.pdf.paper_check)."""
|
|
158
|
+
from ..download.pdf import paper_check
|
|
159
|
+
t = time.time()
|
|
160
|
+
verdict = paper_check(path, doi, config)
|
|
161
|
+
log.info(f" [timing] {label} paper check: {(time.time() - t) * 1000:.0f}ms ({verdict})")
|
|
162
|
+
if verdict == "wrong":
|
|
163
|
+
log.info(f" WRONG PAPER {label}: neither the DOI nor the title is on its first pages")
|
|
164
|
+
return verdict == "wrong"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _try_source(
|
|
168
|
+
source_fn: Any, doi: str, output_path: Path, config: dict[str, Any], label: str, use_tor: bool = False
|
|
169
|
+
) -> dict[str, Any] | None:
|
|
170
|
+
from .scoring import record_result, classify_error, get_user_advice
|
|
171
|
+
t0 = time.time()
|
|
172
|
+
key = normalize_doi(doi) if not is_arxiv_identifier(doi) else doi
|
|
173
|
+
_SOURCE_TLS.doi = key
|
|
174
|
+
# The download this source works for, taken now: by the time a slow source
|
|
175
|
+
# ends, a retry of the same paper may have registered a new attempt, which
|
|
176
|
+
# must not get this one's steps or errors (DeepSeek review 2026-10-19).
|
|
177
|
+
att = _attempt.get(key)
|
|
178
|
+
result = None
|
|
179
|
+
try:
|
|
180
|
+
result = _try_source_inner(source_fn, doi, output_path, config, label, use_tor, t0,
|
|
181
|
+
record_result, classify_error, get_user_advice)
|
|
182
|
+
finally:
|
|
183
|
+
settled = _SOURCE_TLS.doi in _SETTLED or (att is not None and _attempt.get(key) is not att)
|
|
184
|
+
_SOURCE_TLS.doi = None
|
|
185
|
+
ok = bool(result and result.get("success"))
|
|
186
|
+
log.info(f" [timing] {label}: {time.time() - t0:.1f}s, {'PDF' if ok else 'no PDF'}")
|
|
187
|
+
if att is not None:
|
|
188
|
+
from ..publishers.strategies._core import get_last_error
|
|
189
|
+
att.add_step("school" if label in _STEP_TEXT else "free", label, time.time() - t0, result,
|
|
190
|
+
error=get_last_error()) # this thread's: _try_source_inner cleared it first
|
|
191
|
+
if settled:
|
|
192
|
+
# Late finisher of an already-returned download: its copy is a duplicate.
|
|
193
|
+
if output_path.exists():
|
|
194
|
+
try:
|
|
195
|
+
output_path.unlink()
|
|
196
|
+
except OSError:
|
|
197
|
+
pass
|
|
198
|
+
return None
|
|
199
|
+
return result
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _try_source_inner(source_fn: Any, doi: str, output_path: Path, config: dict[str, Any], label: str,
|
|
203
|
+
use_tor: bool, t0: float, record_result: Any, classify_error: Any,
|
|
204
|
+
get_user_advice: Any) -> dict[str, Any] | None:
|
|
205
|
+
|
|
206
|
+
# Limit concurrency for browser-based sources
|
|
207
|
+
is_browser = label in _BROWSER_SOURCE_LABELS
|
|
208
|
+
if label == "Sci-Hub" and not config.get("scihub_use_browser", True):
|
|
209
|
+
is_browser = False # pure-HTTP Sci-Hub must not queue behind the browser lanes
|
|
210
|
+
sem = _get_browser_semaphore(config) if is_browser else None
|
|
211
|
+
if sem:
|
|
212
|
+
t_wait = time.time()
|
|
213
|
+
sem.acquire()
|
|
214
|
+
if time.time() - t_wait > 1:
|
|
215
|
+
log.info(f" [timing] {label}: waited {time.time() - t_wait:.1f}s for the browser")
|
|
216
|
+
try:
|
|
217
|
+
# Each source starts with no browser error of its own: a fresh racing
|
|
218
|
+
# thread otherwise read the latest error any lane reported, and the
|
|
219
|
+
# Elsevier browser skipped its ScienceDirect attempt (0.0 s, no log)
|
|
220
|
+
# because the previous paper had ended on a paywall (2026-10-17).
|
|
221
|
+
from ..publishers.strategies._core import _set_error
|
|
222
|
+
_set_error("", "")
|
|
223
|
+
sig = inspect.signature(source_fn)
|
|
224
|
+
if "use_tor" in sig.parameters:
|
|
225
|
+
result = source_fn(doi, output_path, config, use_tor=use_tor)
|
|
226
|
+
else:
|
|
227
|
+
result = source_fn(doi, output_path, config)
|
|
228
|
+
latency_ms = (time.time() - t0) * 1000
|
|
229
|
+
if result:
|
|
230
|
+
result["doi"] = doi
|
|
231
|
+
result["identifier"] = doi
|
|
232
|
+
if result.get("success"):
|
|
233
|
+
# Check for suspicious PDF (1-page or very small — likely not full text)
|
|
234
|
+
file_path = result.get("file", "")
|
|
235
|
+
if file_path:
|
|
236
|
+
from ..download.pdf import is_suspicious_pdf, suspicious_pdf
|
|
237
|
+
fp = Path(file_path)
|
|
238
|
+
if fp.exists() and is_suspicious_pdf(fp, doi):
|
|
239
|
+
log.info(f" SUSPICIOUS {label}: PDF appears to be a preview/cover page, not full text")
|
|
240
|
+
record_result(label, False, latency_ms, "suspicious_pdf")
|
|
241
|
+
return suspicious_pdf(doi, fp, label)
|
|
242
|
+
if fp.exists() and _is_wrong_paper(fp, doi, config, label):
|
|
243
|
+
record_result(label, False, latency_ms, "wrong_paper")
|
|
244
|
+
from ..download.pdf import wrong_paper
|
|
245
|
+
return wrong_paper(doi, fp, label)
|
|
246
|
+
if fp.exists():
|
|
247
|
+
from ..download.pdf import preprint_mark
|
|
248
|
+
mark = preprint_mark(fp, doi)
|
|
249
|
+
if mark:
|
|
250
|
+
# Still this paper, but not the version of record:
|
|
251
|
+
# the race keeps the copy as a fallback and looks
|
|
252
|
+
# for the published PDF (2026-11-03).
|
|
253
|
+
result["preprint"] = mark
|
|
254
|
+
log.info(f" PREPRINT {label}: {mark} copy, not the published version")
|
|
255
|
+
record_result(label, True, latency_ms)
|
|
256
|
+
else:
|
|
257
|
+
error_type = classify_error(result.get("status_code", 0))
|
|
258
|
+
record_result(label, False, latency_ms, error_type)
|
|
259
|
+
return result
|
|
260
|
+
except SourceWaived:
|
|
261
|
+
log.info(f" {label}: stopped, the paper is already downloaded")
|
|
262
|
+
return None
|
|
263
|
+
except Exception as e:
|
|
264
|
+
latency_ms = (time.time() - t0) * 1000
|
|
265
|
+
error_type = classify_error(exception=e)
|
|
266
|
+
# Check if a valid PDF was actually written to disk despite the exception
|
|
267
|
+
# (common with Sci-Hub: browser download succeeds but post-download logic raises)
|
|
268
|
+
from ..download.pdf import is_pdf_file as _is_pdf
|
|
269
|
+
if output_path.exists() and _is_pdf(output_path) and not _is_wrong_paper(output_path, doi, config, label):
|
|
270
|
+
log.info(f" OK {label} (recovered after {error_type})")
|
|
271
|
+
record_result(label, True, latency_ms)
|
|
272
|
+
return {"success": True, "identifier": doi, "doi": doi,
|
|
273
|
+
"file": str(output_path), "source": label}
|
|
274
|
+
record_result(label, False, latency_ms, error_type)
|
|
275
|
+
advice = get_user_advice(error_type, label)
|
|
276
|
+
log.info(f" FAIL {label}: {error_type} — {advice}")
|
|
277
|
+
return None
|
|
278
|
+
finally:
|
|
279
|
+
if sem:
|
|
280
|
+
sem.release()
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _is_browser_lane(label: str) -> bool:
|
|
284
|
+
"""A lane that opens a browser window (Sci-Hub mostly solves its gate over HTTP)."""
|
|
285
|
+
return label != "Sci-Hub" and label in _BROWSER_SOURCE_LABELS
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _wait_any(events: tuple[threading.Event, threading.Event], timeout: float) -> bool:
|
|
289
|
+
"""Wait for any event to be set, polling at a fine grain (no wait-any API).
|
|
290
|
+
|
|
291
|
+
Used by the racing engine to react to either the first success or the
|
|
292
|
+
completion of every submitted source, whichever comes first.
|
|
293
|
+
"""
|
|
294
|
+
deadline = time.monotonic() + timeout
|
|
295
|
+
while time.monotonic() < deadline:
|
|
296
|
+
if any(e.is_set() for e in events):
|
|
297
|
+
return True
|
|
298
|
+
if _cancelled():
|
|
299
|
+
return False
|
|
300
|
+
time.sleep(0.05)
|
|
301
|
+
return any(e.is_set() for e in events)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _promote_publisher_browser(launchable: list[tuple[Any, str]], doi: str) -> None:
|
|
305
|
+
"""Launch the paper's own publisher browser lane second, not after every cheap lane.
|
|
306
|
+
|
|
307
|
+
The hedged cascade adds one lane per hedge delay; for a paywalled paper the
|
|
308
|
+
publisher's browser lane (the one that can actually get it) used to start
|
|
309
|
+
only after ~5 open-access lanes, 7-8 s in. Second place keeps the best
|
|
310
|
+
cheap lane first, so open-access papers still win in a second or two.
|
|
311
|
+
"""
|
|
312
|
+
try:
|
|
313
|
+
from .publishers import get_publisher
|
|
314
|
+
own = f"{get_publisher(doi)}Browser"
|
|
315
|
+
except Exception:
|
|
316
|
+
return
|
|
317
|
+
for i, (_fn, label) in enumerate(launchable):
|
|
318
|
+
if label == own and i > 1:
|
|
319
|
+
launchable.insert(1, launchable.pop(i))
|
|
320
|
+
return
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _run_tiers_parallel(
|
|
324
|
+
tiers: list[tuple[list[tuple[Any, str]], str, int]],
|
|
325
|
+
doi: str,
|
|
326
|
+
target_dir: Path,
|
|
327
|
+
output_path: Path,
|
|
328
|
+
config: dict[str, Any],
|
|
329
|
+
use_tor: bool,
|
|
330
|
+
overall_timeout: int,
|
|
331
|
+
failures: list[dict[str, str]] | None = None,
|
|
332
|
+
) -> dict[str, Any] | None:
|
|
333
|
+
"""Race all tiers in parallel. First successful tier wins.
|
|
334
|
+
|
|
335
|
+
Uses a shared result dict so that any source thread can publish its
|
|
336
|
+
success immediately, even if it's running inside a nested parallel
|
|
337
|
+
call (like Sci-Hub domain racing). `failures` (optional) collects why
|
|
338
|
+
each losing lane failed — pool workers append to it.
|
|
339
|
+
"""
|
|
340
|
+
if failures is None:
|
|
341
|
+
failures = []
|
|
342
|
+
if _cancelled():
|
|
343
|
+
return None
|
|
344
|
+
# A new race for this DOI is live again: callers such as batch_download race
|
|
345
|
+
# without going through download(), so clear any earlier "settled" mark here
|
|
346
|
+
# or every result of this race would be discarded as a late finisher.
|
|
347
|
+
_SETTLED.discard(normalize_doi(doi) if not is_arxiv_identifier(doi) else doi)
|
|
348
|
+
if not tiers:
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
# Flatten all sources across tiers with their labels
|
|
352
|
+
all_sources: list[tuple[Any, str, str, int]] = [] # (fn, label, tier_label, timeout)
|
|
353
|
+
for tier_sources, tier_label, tier_timeout in tiers:
|
|
354
|
+
for fn, label in tier_sources:
|
|
355
|
+
all_sources.append((fn, label, tier_label, tier_timeout))
|
|
356
|
+
|
|
357
|
+
if not all_sources:
|
|
358
|
+
return None
|
|
359
|
+
|
|
360
|
+
# If only one source, run directly
|
|
361
|
+
if len(all_sources) == 1:
|
|
362
|
+
fn, label, tier_label, timeout = all_sources[0]
|
|
363
|
+
src_output = target_dir / f"{safe_filename(doi)}_{label}.pdf"
|
|
364
|
+
result = _try_source(fn, doi, src_output, config, label, use_tor=use_tor)
|
|
365
|
+
if result and result.get("success"):
|
|
366
|
+
final_path = Path(result.get("file", ""))
|
|
367
|
+
if final_path != output_path and final_path.exists():
|
|
368
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
369
|
+
if output_path.exists():
|
|
370
|
+
output_path.unlink()
|
|
371
|
+
final_path.rename(output_path)
|
|
372
|
+
result["file"] = str(output_path)
|
|
373
|
+
return result
|
|
374
|
+
_record_source_failure(failures, label, result)
|
|
375
|
+
return None
|
|
376
|
+
|
|
377
|
+
# Shared result: any thread can publish success here, signaled via Event
|
|
378
|
+
result_lock = threading.Lock()
|
|
379
|
+
success_event = threading.Event()
|
|
380
|
+
cancel_event = threading.Event()
|
|
381
|
+
all_done_event = threading.Event() # set when every submitted future returned
|
|
382
|
+
done_counter: list[int] = [0]
|
|
383
|
+
done_counter_lock = threading.Lock()
|
|
384
|
+
submission_done = threading.Event() # staggered scheduler finished launching lanes
|
|
385
|
+
lanes_idle = threading.Event() # every launched-so-far lane has answered
|
|
386
|
+
shared_result: dict[str, Any] = {"result": None}
|
|
387
|
+
# A preprint copy of the right paper: kept, but a version of record wins
|
|
388
|
+
# (2026-10-26: the open-access route served the Research Square preprint
|
|
389
|
+
# for a Scientific Reports DOI and ended the race before the publisher).
|
|
390
|
+
preprint_result: dict[str, Any] = {"result": None}
|
|
391
|
+
race_over = threading.Event() # set when this race returns; waived lanes stop launching
|
|
392
|
+
|
|
393
|
+
def _try_and_publish(fn, label, src_output):
|
|
394
|
+
# Skip if another source already succeeded
|
|
395
|
+
if cancel_event.is_set():
|
|
396
|
+
return None
|
|
397
|
+
_SOURCE_TLS.race = race_over
|
|
398
|
+
try:
|
|
399
|
+
result = _try_source(fn, doi, src_output, config, label, use_tor=use_tor)
|
|
400
|
+
finally:
|
|
401
|
+
_SOURCE_TLS.race = None
|
|
402
|
+
# This pool is created per download (issue #57): once its worker
|
|
403
|
+
# threads die, their thread-local shared browsers are unreachable
|
|
404
|
+
# to close_shared_browser() and the atexit reaper only fires at
|
|
405
|
+
# process exit — a long-lived MCP server accumulates Chromium
|
|
406
|
+
# processes. Each worker closes its own browser before exiting.
|
|
407
|
+
try:
|
|
408
|
+
from ..browser.engine import shutdown_shared_browser
|
|
409
|
+
shutdown_shared_browser()
|
|
410
|
+
except Exception:
|
|
411
|
+
pass
|
|
412
|
+
if result and not result.get("success"):
|
|
413
|
+
# ponytail: (source, publisher) negative cache — a Cloudflare block
|
|
414
|
+
# on one paper must not burn every later paper's timeout; raise TTL
|
|
415
|
+
# or key by full DOI if false-skip reports come in.
|
|
416
|
+
_neg_record(label, doi, result)
|
|
417
|
+
_record_source_failure(failures, label, result)
|
|
418
|
+
if result and result.get("success"):
|
|
419
|
+
if result.get("preprint"):
|
|
420
|
+
# Not the version of record: hold it and keep the other lanes
|
|
421
|
+
# going. It is used only if none of them delivers one.
|
|
422
|
+
with result_lock:
|
|
423
|
+
if preprint_result["result"] is None:
|
|
424
|
+
preprint_result["result"] = (result, label, src_output)
|
|
425
|
+
log.info(f" PREPRINT {label} ({result['preprint']}); still looking for the published version")
|
|
426
|
+
return result
|
|
427
|
+
with result_lock:
|
|
428
|
+
if shared_result["result"] is None:
|
|
429
|
+
shared_result["result"] = (result, label, src_output)
|
|
430
|
+
cancel_event.set()
|
|
431
|
+
success_event.set()
|
|
432
|
+
return result
|
|
433
|
+
|
|
434
|
+
def _take_preprint() -> dict[str, Any] | None:
|
|
435
|
+
"""The kept preprint copy, moved into place, when nothing better came back."""
|
|
436
|
+
item = preprint_result["result"]
|
|
437
|
+
if item is None:
|
|
438
|
+
return None
|
|
439
|
+
result, label, src_output = item
|
|
440
|
+
final_path = Path(result.get("file", ""))
|
|
441
|
+
if final_path != output_path and final_path.exists():
|
|
442
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
443
|
+
if output_path.exists():
|
|
444
|
+
output_path.unlink()
|
|
445
|
+
final_path.rename(output_path)
|
|
446
|
+
result["file"] = str(output_path)
|
|
447
|
+
log.info(f" OK {label} (preprint only: {result.get('preprint')})")
|
|
448
|
+
return result
|
|
449
|
+
|
|
450
|
+
def _on_future_done(fut) -> None:
|
|
451
|
+
with done_counter_lock:
|
|
452
|
+
done_counter[0] += 1
|
|
453
|
+
if done_counter[0] >= len(futures):
|
|
454
|
+
# Mid-scheduling this wakes the scheduler so a fast-failing
|
|
455
|
+
# lane widens immediately instead of idling out the hedge
|
|
456
|
+
# delay; after submission_done it means the race is over.
|
|
457
|
+
lanes_idle.set()
|
|
458
|
+
if submission_done.is_set():
|
|
459
|
+
all_done_event.set()
|
|
460
|
+
|
|
461
|
+
# Hedged cascade (default): lanes arrive score-ordered by the caller.
|
|
462
|
+
# Launch the best first and only widen when it fails to answer within
|
|
463
|
+
# hedge_delay_seconds — cuts request volume and anti-bot heat vs a flat
|
|
464
|
+
# race while keeping tail latency close. race_mode=full restores the
|
|
465
|
+
# flat all-at-once race.
|
|
466
|
+
race_mode = str(config.get("race_mode", "hedge")).lower()
|
|
467
|
+
hedge_delay = max(0.0, float(config.get("hedge_delay_seconds", 1.5)))
|
|
468
|
+
if race_mode == "full":
|
|
469
|
+
hedge_delay = 0.0
|
|
470
|
+
mode_label = "hedged cascade" if hedge_delay > 0 else "flat race"
|
|
471
|
+
|
|
472
|
+
launchable: list[tuple[Any, str]] = []
|
|
473
|
+
for fn, label, _tier_label, _tier_timeout in all_sources:
|
|
474
|
+
if _neg_blocked(label, doi):
|
|
475
|
+
log.info(f" SKIP {label} (negative cache: recently failed for this publisher)")
|
|
476
|
+
continue
|
|
477
|
+
launchable.append((fn, label))
|
|
478
|
+
_promote_publisher_browser(launchable, doi)
|
|
479
|
+
|
|
480
|
+
log.info(f" Racing {len(all_sources)} sources across {len(tiers)} tiers ({mode_label})...")
|
|
481
|
+
deadline = time.time() + overall_timeout + 5
|
|
482
|
+
pool = ThreadPoolExecutor(max_workers=max(1, len(launchable)))
|
|
483
|
+
futures = {}
|
|
484
|
+
try:
|
|
485
|
+
for i, (fn, label) in enumerate(launchable):
|
|
486
|
+
lanes_idle.clear()
|
|
487
|
+
src_output = target_dir / f"{safe_filename(doi)}_{label}.pdf"
|
|
488
|
+
fut = pool.submit(_try_and_publish, fn, label, src_output)
|
|
489
|
+
futures[fut] = (label, src_output)
|
|
490
|
+
fut.add_done_callback(_on_future_done)
|
|
491
|
+
if hedge_delay <= 0 or i == len(launchable) - 1 or success_event.is_set():
|
|
492
|
+
continue
|
|
493
|
+
# Widen only when needed: another lane succeeded, everything
|
|
494
|
+
# launched so far already answered (fast failure), the hedge
|
|
495
|
+
# delay elapsed, or the overall deadline arrived.
|
|
496
|
+
now = time.time()
|
|
497
|
+
if now >= deadline:
|
|
498
|
+
break
|
|
499
|
+
_wait_any((success_event, lanes_idle),
|
|
500
|
+
min(hedge_delay, deadline - now))
|
|
501
|
+
if success_event.is_set() or time.time() >= deadline:
|
|
502
|
+
break
|
|
503
|
+
submission_done.set()
|
|
504
|
+
with done_counter_lock:
|
|
505
|
+
if done_counter[0] >= len(futures):
|
|
506
|
+
all_done_event.set()
|
|
507
|
+
|
|
508
|
+
# Wait for first success, all sources done, or the overall timeout.
|
|
509
|
+
# all_done lets a full fast-failure return immediately instead of
|
|
510
|
+
# burning the whole timeout + grace period.
|
|
511
|
+
if not success_event.is_set() and not all_done_event.is_set():
|
|
512
|
+
_wait_any((success_event, all_done_event),
|
|
513
|
+
max(0.0, deadline - time.time()))
|
|
514
|
+
|
|
515
|
+
if shared_result["result"] is not None:
|
|
516
|
+
result, label, src_output = shared_result["result"]
|
|
517
|
+
final_path = Path(result.get("file", ""))
|
|
518
|
+
if final_path != output_path and final_path.exists():
|
|
519
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
520
|
+
if output_path.exists():
|
|
521
|
+
output_path.unlink()
|
|
522
|
+
final_path.rename(output_path)
|
|
523
|
+
result["file"] = str(output_path)
|
|
524
|
+
log.info(f" OK {label}")
|
|
525
|
+
return result
|
|
526
|
+
if all_done_event.is_set():
|
|
527
|
+
preprint = _take_preprint()
|
|
528
|
+
if preprint is not None:
|
|
529
|
+
return preprint
|
|
530
|
+
log.info(" All sources failed")
|
|
531
|
+
return None
|
|
532
|
+
|
|
533
|
+
# Timeout reached — late results. A browser lane still at work (opening
|
|
534
|
+
# the page, reaching a login) gets up to race_grace_browser_seconds;
|
|
535
|
+
# once none is, only race_grace_seconds. A lane that waits for the user
|
|
536
|
+
# says so (note_user_waiting) and is waited for below.
|
|
537
|
+
grace = float(config.get("race_grace_seconds", 15))
|
|
538
|
+
grace_browser = float(config.get("race_grace_browser_seconds", 180))
|
|
539
|
+
log.info(f" Racing timed out after {overall_timeout + 5}s, waiting for late results...")
|
|
540
|
+
t_grace = time.time()
|
|
541
|
+
quick_until = time.time() + grace
|
|
542
|
+
stop_at = time.time() + max(grace, grace_browser)
|
|
543
|
+
while not success_event.is_set() and not all_done_event.is_set() and not _cancelled():
|
|
544
|
+
now = time.time()
|
|
545
|
+
if now >= stop_at or _user_waiting_for(doi):
|
|
546
|
+
break
|
|
547
|
+
browser_at_work = any(not f.done() and _is_browser_lane(lbl) for f, (lbl, _) in futures.items())
|
|
548
|
+
if not browser_at_work and now >= quick_until:
|
|
549
|
+
break
|
|
550
|
+
_wait_any((success_event, all_done_event), 1.0)
|
|
551
|
+
# A lane is waiting for the user's login: keep waiting while its
|
|
552
|
+
# heartbeat is fresh instead of abandoning a login in progress.
|
|
553
|
+
if _user_waiting_for(doi):
|
|
554
|
+
log.info(" A source is waiting for your login — waiting for it...")
|
|
555
|
+
stop_at = time.time() + _USER_WAIT_MAX_S
|
|
556
|
+
while (not success_event.is_set() and not all_done_event.is_set()
|
|
557
|
+
and not _cancelled() and time.time() < stop_at
|
|
558
|
+
and _user_waiting_for(doi)):
|
|
559
|
+
_wait_any((success_event, all_done_event), 3.0)
|
|
560
|
+
if shared_result["result"] is not None:
|
|
561
|
+
result, label, src_output = shared_result["result"]
|
|
562
|
+
final_path = Path(result.get("file", ""))
|
|
563
|
+
if final_path != output_path and final_path.exists():
|
|
564
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
565
|
+
if output_path.exists():
|
|
566
|
+
output_path.unlink()
|
|
567
|
+
final_path.rename(output_path)
|
|
568
|
+
result["file"] = str(output_path)
|
|
569
|
+
log.info(f" OK {label} (late, after {time.time() - t_grace:.1f}s)")
|
|
570
|
+
return result
|
|
571
|
+
if all_done_event.is_set():
|
|
572
|
+
preprint = _take_preprint()
|
|
573
|
+
if preprint is not None:
|
|
574
|
+
return preprint
|
|
575
|
+
log.info(" All sources failed")
|
|
576
|
+
return None
|
|
577
|
+
|
|
578
|
+
# Final scan: check if any source wrote a valid PDF file despite timeout
|
|
579
|
+
from ..download.pdf import is_pdf_file, is_suspicious_pdf, suspicious_pdf
|
|
580
|
+
for label, src_output in futures.values():
|
|
581
|
+
if src_output.exists() and is_pdf_file(src_output):
|
|
582
|
+
if is_suspicious_pdf(src_output, doi):
|
|
583
|
+
log.info(f" SUSPICIOUS {label} (file scan): skipping suspicious PDF")
|
|
584
|
+
continue
|
|
585
|
+
if _is_wrong_paper(src_output, doi, config, f"{label} (file scan)"):
|
|
586
|
+
continue
|
|
587
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
588
|
+
if output_path.exists():
|
|
589
|
+
output_path.unlink()
|
|
590
|
+
src_output.rename(output_path)
|
|
591
|
+
log.info(f" OK {label} (file scan)")
|
|
592
|
+
return {"success": True, "identifier": doi, "doi": doi,
|
|
593
|
+
"file": str(output_path), "source": label}
|
|
594
|
+
|
|
595
|
+
preprint = _take_preprint()
|
|
596
|
+
if preprint is not None:
|
|
597
|
+
return preprint
|
|
598
|
+
log.info(f" All sources failed")
|
|
599
|
+
finally:
|
|
600
|
+
# Never block termination on sources that refuse to return: cancel
|
|
601
|
+
# pending futures and waive stragglers. Their own I/O deadlines
|
|
602
|
+
# (connect/read timeouts, stream deadline in pdf_utils) bound how long
|
|
603
|
+
# an abandoned worker thread can outlive the race.
|
|
604
|
+
race_over.set()
|
|
605
|
+
alive = [f for f in futures if not f.done()]
|
|
606
|
+
if alive:
|
|
607
|
+
log.info(f" Racing finished with {len(alive)} source(s) still running (waived)")
|
|
608
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
609
|
+
# Cleanup temp files
|
|
610
|
+
for _, other_path in futures.values():
|
|
611
|
+
if other_path != output_path and other_path.exists():
|
|
612
|
+
try:
|
|
613
|
+
other_path.unlink(missing_ok=True)
|
|
614
|
+
except OSError:
|
|
615
|
+
pass
|
|
616
|
+
|
|
617
|
+
return None
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
# Single-flight guard: at most one download() per identifier per process.
|
|
621
|
+
# Duplicate submissions (e.g. an MCP client retrying after its own 30s client
|
|
622
|
+
# timeout while the server is still racing sources) used to start a second,
|
|
623
|
+
# parallel browser task for the same paper — the "infinite popup" experience.
|
|
624
|
+
_INFLIGHT_LOCK = threading.Lock()
|
|
625
|
+
# The papers in flight are the DownloadAttempts registered in sources.attempt.
|
|
626
|
+
|
|
627
|
+
class _RecentSet:
|
|
628
|
+
"""A set that keeps only the latest ``limit`` keys (insertion order).
|
|
629
|
+
|
|
630
|
+
Late finishers only need to know a paper ended while they still run, so a
|
|
631
|
+
long-running server need not remember every paper it ever downloaded.
|
|
632
|
+
"""
|
|
633
|
+
|
|
634
|
+
def __init__(self, limit: int = 2000) -> None:
|
|
635
|
+
self._keys: dict[str, None] = {}
|
|
636
|
+
self._limit = limit
|
|
637
|
+
self._lock = threading.Lock()
|
|
638
|
+
|
|
639
|
+
def add(self, key: str) -> None:
|
|
640
|
+
with self._lock:
|
|
641
|
+
self._keys.pop(key, None)
|
|
642
|
+
self._keys[key] = None
|
|
643
|
+
while len(self._keys) > self._limit:
|
|
644
|
+
self._keys.pop(next(iter(self._keys)))
|
|
645
|
+
|
|
646
|
+
def update(self, keys: Any) -> None:
|
|
647
|
+
for k in keys:
|
|
648
|
+
self.add(k)
|
|
649
|
+
|
|
650
|
+
def discard(self, key: str) -> None:
|
|
651
|
+
with self._lock:
|
|
652
|
+
self._keys.pop(key, None)
|
|
653
|
+
|
|
654
|
+
def clear(self) -> None:
|
|
655
|
+
with self._lock:
|
|
656
|
+
self._keys.clear()
|
|
657
|
+
|
|
658
|
+
def __contains__(self, key: object) -> bool:
|
|
659
|
+
return key in self._keys
|
|
660
|
+
|
|
661
|
+
def __len__(self) -> int:
|
|
662
|
+
return len(self._keys)
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
# DOIs whose download() has returned. Racing returns on the first success while
|
|
666
|
+
# other sources keep running ("waived"); those late finishers check this set to
|
|
667
|
+
# drop their duplicate files and to stop launching browsers.
|
|
668
|
+
_SETTLED = _RecentSet()
|
|
669
|
+
_SOURCE_TLS = threading.local()
|
|
670
|
+
|
|
671
|
+
# Heartbeats from sources that are waiting for the user to log in, by DOI.
|
|
672
|
+
# The race must not give up (and open the next phase's window) while the user
|
|
673
|
+
# is in the middle of a login.
|
|
674
|
+
# Heartbeats stop the moment the login succeeds, but the source still has to
|
|
675
|
+
# reopen the article and capture the PDF: keep waiting this long after the
|
|
676
|
+
# last heartbeat so the race doesn't open the next phase's window meanwhile.
|
|
677
|
+
_USER_WAIT_FRESH_S = 120.0
|
|
678
|
+
_USER_WAIT_MAX_S = 600.0
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
_WAIT_TEXT = {
|
|
682
|
+
"login": "请在弹出的浏览器窗口中登录学校账号",
|
|
683
|
+
"verify": "请在弹出的浏览器窗口中完成人机验证,出现勾选框时勾选它",
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
class SourceWaived(RuntimeError):
|
|
688
|
+
"""Raised in a source's wait-for-the-user loop once its paper is settled."""
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def note_user_waiting(what: str = "") -> None:
|
|
692
|
+
"""Called by a source on every poll while it waits for the user (a login, a
|
|
693
|
+
human check). Keeps the race from closing that window, and tells the app:
|
|
694
|
+
a notice on top of the job page and a line under the paper.
|
|
695
|
+
|
|
696
|
+
Raises ``SourceWaived`` when the paper is already done (another source got
|
|
697
|
+
it): the waiting window must not hold the browser or ask for a login."""
|
|
698
|
+
if not _source_may_continue():
|
|
699
|
+
raise SourceWaived("download already finished; stop waiting for the user")
|
|
700
|
+
doi = getattr(_SOURCE_TLS, "doi", None)
|
|
701
|
+
if doi:
|
|
702
|
+
att = _attempt.get(doi)
|
|
703
|
+
if att is not None:
|
|
704
|
+
att.user_wait_at = time.time()
|
|
705
|
+
try:
|
|
706
|
+
from ..download import progress as pr
|
|
707
|
+
text = _WAIT_TEXT.get(what, "请在弹出的浏览器窗口中完成登录或人机验证")
|
|
708
|
+
pr.set_attention(f"wait:{doi}", f"{text}({doi})", current=doi, phase="等待你的操作")
|
|
709
|
+
pr.item_stage(doi, "等待你在浏览器中操作")
|
|
710
|
+
except Exception:
|
|
711
|
+
pass
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _user_waiting_for(doi: str) -> bool:
|
|
715
|
+
key = normalize_doi(doi) if not is_arxiv_identifier(doi) else doi
|
|
716
|
+
att = _attempt.get(key)
|
|
717
|
+
return att is not None and time.time() - att.user_wait_at < _USER_WAIT_FRESH_S
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
# Cooperative cancellation. cancel_downloads() bumps the generation; every
|
|
721
|
+
# download() remembers the generation it started under (thread-local), so it
|
|
722
|
+
# knows it was cancelled without killing threads. Downloads started afterwards
|
|
723
|
+
# are unaffected.
|
|
724
|
+
_CANCEL_GEN = [0]
|
|
725
|
+
_DL_TLS = threading.local()
|
|
726
|
+
# Papers the user chose to skip while one of their windows waited for them
|
|
727
|
+
# (the app's 跳过这篇): that download stops as if cancelled; the others go on.
|
|
728
|
+
_SKIPPED: set[str] = set()
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _cancelled() -> bool:
|
|
732
|
+
"""This thread's paper was cancelled (its job's generation moved on) or skipped."""
|
|
733
|
+
key = getattr(_DL_TLS, "key", None)
|
|
734
|
+
att = _attempt.get(key)
|
|
735
|
+
gen = att.gen if att is not None else getattr(_DL_TLS, "gen", None)
|
|
736
|
+
if gen is not None and gen != _CANCEL_GEN[0]:
|
|
737
|
+
return True
|
|
738
|
+
return key in _SKIPPED
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def skip_download(identifier: str) -> bool:
|
|
742
|
+
"""Give up on one paper: in flight, its windows stop waiting for the user and
|
|
743
|
+
close and its race returns; not started yet, it is skipped when its turn
|
|
744
|
+
comes (download_jobs clears the requests when a job starts)."""
|
|
745
|
+
key = identifier.strip()
|
|
746
|
+
if not is_arxiv_identifier(key):
|
|
747
|
+
key = normalize_doi(key)
|
|
748
|
+
with _INFLIGHT_LOCK:
|
|
749
|
+
_SKIPPED.add(key)
|
|
750
|
+
if _attempt.get(key) is None:
|
|
751
|
+
log.info(f" Will skip when its turn comes: {key}")
|
|
752
|
+
return True
|
|
753
|
+
_SETTLED.add(key) # its sources stop waiting, launch nothing, drop late results
|
|
754
|
+
try:
|
|
755
|
+
from ..download import progress as _pr
|
|
756
|
+
_pr.clear_attention(f"wait:{key}")
|
|
757
|
+
from ..browser.backend import close_browsers_since
|
|
758
|
+
close_browsers_since(0.0, owner=key) # its own windows only
|
|
759
|
+
except Exception as exc:
|
|
760
|
+
log.info(f" Skip: browser cleanup skipped: {exc}")
|
|
761
|
+
log.info(f" Skipped by the user: {key}")
|
|
762
|
+
return True
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def cancel_downloads() -> dict[str, Any]:
|
|
766
|
+
"""Cancel every download in flight: stop waiting, drop late results, close windows."""
|
|
767
|
+
with _INFLIGHT_LOCK:
|
|
768
|
+
_CANCEL_GEN[0] += 1
|
|
769
|
+
keys = sorted(_attempt.active_keys())
|
|
770
|
+
_SETTLED.update(keys) # sources still running drop results, launch nothing
|
|
771
|
+
closed = 0
|
|
772
|
+
try:
|
|
773
|
+
from ..browser.backend import close_browsers_since
|
|
774
|
+
closed = close_browsers_since(0.0)
|
|
775
|
+
except Exception as exc:
|
|
776
|
+
log.info(f" Cancel: browser cleanup skipped: {exc}")
|
|
777
|
+
log.info(f" Cancelled {len(keys)} download(s); closed {closed} browser(s)")
|
|
778
|
+
return {"cancelled": keys, "browsers_closed": closed}
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def _source_may_continue() -> bool:
|
|
782
|
+
"""LAUNCH_GUARD hook: False once this thread's race or download is over."""
|
|
783
|
+
race = getattr(_SOURCE_TLS, "race", None)
|
|
784
|
+
if race is not None and race.is_set():
|
|
785
|
+
return False # waived lane: its race already returned
|
|
786
|
+
doi = getattr(_SOURCE_TLS, "doi", None)
|
|
787
|
+
if not doi: # a pool thread working for a source (Sci-Hub's mirror race)
|
|
788
|
+
from ..browser.backend import current_owner
|
|
789
|
+
doi = current_owner()
|
|
790
|
+
return not (doi and doi in _SETTLED)
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
# Per-download record of WHY each racing source failed. A channel being
|
|
794
|
+
# unreachable is usually temporary or a missing user decision (e.g. Unpaywall
|
|
795
|
+
# needs the user's email) — surfaced in the final failure result so the
|
|
796
|
+
# user/agent can choose: retry, fix config, or switch channel. Never skip a
|
|
797
|
+
# channel silently. The list is created per download() and threaded through
|
|
798
|
+
# _run_tiers_parallel (pool workers append to it; list.append is atomic).
|
|
799
|
+
def _record_source_failure(failures: list[dict[str, str]], label: str,
|
|
800
|
+
result: dict[str, Any] | None) -> None:
|
|
801
|
+
"""Collect structured failure reasons (label, reason, error_type, action)."""
|
|
802
|
+
if not result or result.get("success"):
|
|
803
|
+
return
|
|
804
|
+
try:
|
|
805
|
+
failures.append({
|
|
806
|
+
"source": label,
|
|
807
|
+
"reason": str(result.get("reason") or result.get("error") or "")[:200],
|
|
808
|
+
"error_type": str(result.get("error_type", "")),
|
|
809
|
+
"action": str(result.get("action", "")),
|
|
810
|
+
})
|
|
811
|
+
except Exception:
|
|
812
|
+
pass
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def download(
|
|
816
|
+
identifier: str,
|
|
817
|
+
output_dir: str | Path | None = None,
|
|
818
|
+
*,
|
|
819
|
+
scihub_enabled: bool | None = None,
|
|
820
|
+
use_tor: bool = False,
|
|
821
|
+
use_vpnsci: bool = False,
|
|
822
|
+
bibtex: bool = False,
|
|
823
|
+
rename: bool = True,
|
|
824
|
+
_institutional: bool = True,
|
|
825
|
+
strategy: str | None = None,
|
|
826
|
+
_batch: bool = False,
|
|
827
|
+
_scihub_http_only: bool = False,
|
|
828
|
+
_school_only: bool = False,
|
|
829
|
+
) -> dict[str, Any]:
|
|
830
|
+
key = identifier.strip()
|
|
831
|
+
if not is_arxiv_identifier(key):
|
|
832
|
+
key = normalize_doi(key)
|
|
833
|
+
from ..download.doi_check import registered
|
|
834
|
+
if registered(key) is False: # a typo: no source can have it
|
|
835
|
+
return fail(key, "DOI is not registered at doi.org", error_type="doi_not_found",
|
|
836
|
+
action="check the DOI for typos")
|
|
837
|
+
gen = getattr(_DL_TLS, "gen", None)
|
|
838
|
+
with _INFLIGHT_LOCK:
|
|
839
|
+
if _attempt.get(key) is not None:
|
|
840
|
+
log.info(f" Single-flight: download already in progress for {key}")
|
|
841
|
+
return fail(
|
|
842
|
+
key,
|
|
843
|
+
reason="download already in progress for this identifier",
|
|
844
|
+
error_type="in_progress",
|
|
845
|
+
action="wait for the in-flight attempt to finish (it is still "
|
|
846
|
+
"racing sources); retry in a minute only if it failed",
|
|
847
|
+
)
|
|
848
|
+
if key in _SKIPPED: # the user skipped it while it was queued
|
|
849
|
+
_SKIPPED.discard(key) # used up: a later retry of the paper runs
|
|
850
|
+
return fail(key, "skipped by user", error_type="skipped", action="none")
|
|
851
|
+
_attempt.begin(key, gen if gen is not None else _CANCEL_GEN[0])
|
|
852
|
+
_SETTLED.discard(key)
|
|
853
|
+
t0 = time.perf_counter() # same clock as browser_backend._remember
|
|
854
|
+
prev_key = getattr(_DL_TLS, "key", None)
|
|
855
|
+
_DL_TLS.key = key
|
|
856
|
+
prev_gen = getattr(_DL_TLS, "gen", None)
|
|
857
|
+
if prev_gen is None: # nested calls (batch phase 2) keep the batch's generation
|
|
858
|
+
_DL_TLS.gen = _CANCEL_GEN[0]
|
|
859
|
+
try:
|
|
860
|
+
result = _download_impl(
|
|
861
|
+
identifier,
|
|
862
|
+
output_dir,
|
|
863
|
+
scihub_enabled=scihub_enabled,
|
|
864
|
+
use_tor=use_tor,
|
|
865
|
+
use_vpnsci=use_vpnsci,
|
|
866
|
+
bibtex=bibtex,
|
|
867
|
+
rename=rename,
|
|
868
|
+
_institutional=_institutional,
|
|
869
|
+
strategy=strategy,
|
|
870
|
+
_batch=_batch,
|
|
871
|
+
_scihub_http_only=_scihub_http_only,
|
|
872
|
+
_school_only=_school_only,
|
|
873
|
+
)
|
|
874
|
+
att = _attempt.get(key)
|
|
875
|
+
steps = list(att.steps) if att is not None else []
|
|
876
|
+
if key in _SKIPPED and not result.get("success"):
|
|
877
|
+
_SKIPPED.discard(key) # used up: a later retry of the paper runs
|
|
878
|
+
return {**fail(key, "skipped by user", error_type="skipped", action="none"), "attempts": steps}
|
|
879
|
+
if _cancelled() and not result.get("success"):
|
|
880
|
+
return {**fail(key, "cancelled by user", error_type="cancelled", action="none"), "attempts": steps}
|
|
881
|
+
if steps and isinstance(result, dict):
|
|
882
|
+
result.setdefault("attempts", steps) # which ways were tried, how long, why they failed
|
|
883
|
+
if result.get("success"):
|
|
884
|
+
from ..publishers.strategies._core import clear_no_access
|
|
885
|
+
clear_no_access(key)
|
|
886
|
+
_SKIPPED.discard(key) # a skip that came too late must not skip the next try
|
|
887
|
+
return result
|
|
888
|
+
finally:
|
|
889
|
+
_DL_TLS.key = prev_key
|
|
890
|
+
try: # whatever this paper waited on the user for is over now
|
|
891
|
+
from ..download import progress as _pr
|
|
892
|
+
_pr.clear_attention(f"wait:{normalize_doi(key) if not is_arxiv_identifier(key) else key}")
|
|
893
|
+
except Exception:
|
|
894
|
+
pass
|
|
895
|
+
if prev_gen is None:
|
|
896
|
+
_DL_TLS.gen = None
|
|
897
|
+
with _INFLIGHT_LOCK:
|
|
898
|
+
_attempt.end(key)
|
|
899
|
+
_SETTLED.add(key)
|
|
900
|
+
# Racing returns on the first success but waived sources keep their
|
|
901
|
+
# browser windows open. Close what this download launched (its own
|
|
902
|
+
# browsers only, so a concurrent download keeps its windows).
|
|
903
|
+
try:
|
|
904
|
+
from ..browser.backend import close_browsers_since
|
|
905
|
+
n = close_browsers_since(t0, owner=key)
|
|
906
|
+
if n:
|
|
907
|
+
log.info(f" Closed {n} browser(s) left open by this download")
|
|
908
|
+
except Exception as exc:
|
|
909
|
+
log.info(f" Browser cleanup skipped: {exc}")
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def _download_impl(
|
|
913
|
+
identifier: str,
|
|
914
|
+
output_dir: str | Path | None = None,
|
|
915
|
+
*,
|
|
916
|
+
scihub_enabled: bool | None = None,
|
|
917
|
+
use_tor: bool = False,
|
|
918
|
+
use_vpnsci: bool = False,
|
|
919
|
+
bibtex: bool = False,
|
|
920
|
+
rename: bool = True,
|
|
921
|
+
_institutional: bool = True,
|
|
922
|
+
strategy: str | None = None,
|
|
923
|
+
_batch: bool = False,
|
|
924
|
+
_scihub_http_only: bool = False,
|
|
925
|
+
_school_only: bool = False,
|
|
926
|
+
) -> dict[str, Any]:
|
|
927
|
+
race_failures: list[dict[str, str]] = []
|
|
928
|
+
config = load_config()
|
|
929
|
+
if _scihub_http_only:
|
|
930
|
+
# Sci-Hub's gate solved over HTTP only: no browser, so the paper does
|
|
931
|
+
# not queue for the one browser slot (a batch's first round).
|
|
932
|
+
config["scihub_use_browser"] = False
|
|
933
|
+
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
934
|
+
|
|
935
|
+
if scihub_enabled is not None:
|
|
936
|
+
config["scihub_enabled"] = scihub_enabled
|
|
937
|
+
|
|
938
|
+
if strategy is not None:
|
|
939
|
+
config["download_strategy"] = strategy
|
|
940
|
+
|
|
941
|
+
# PMC allows automated retrieval only through its Cloud / OAI-PMH /
|
|
942
|
+
# E-utilities / BioC services (https://pmc.ncbi.nlm.nih.gov/tools/developers/),
|
|
943
|
+
# so its PDF pages are used for single downloads only, never in a batch.
|
|
944
|
+
config["_batch"] = _batch
|
|
945
|
+
|
|
946
|
+
target_dir = Path(output_dir) if output_dir else Path(config["output_dir"])
|
|
947
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
948
|
+
|
|
949
|
+
_cleanup_stale_files(target_dir)
|
|
950
|
+
|
|
951
|
+
identifier = identifier.strip()
|
|
952
|
+
output_path = target_dir / f"{safe_filename(identifier)}.pdf"
|
|
953
|
+
|
|
954
|
+
cached = cache_get(identifier, config)
|
|
955
|
+
if cached and cached.get("success"):
|
|
956
|
+
cached_file = Path(cached.get("file", ""))
|
|
957
|
+
cached_doi = str(cached.get("doi") or "")
|
|
958
|
+
# A result cached before the paper check existed may be a different paper
|
|
959
|
+
# (a JSTOR essay came back from the cache for a Ceramics International DOI, 2026-10-28).
|
|
960
|
+
if cached_file.is_file() and cached_doi and _is_wrong_paper(cached_file, cached_doi, config, "cache"):
|
|
961
|
+
log.info(" The cached file is a different paper; downloading again")
|
|
962
|
+
cached_file = Path("")
|
|
963
|
+
if cached_file.is_file() and not _same_dir(cached_file.parent, target_dir):
|
|
964
|
+
# Got before into another folder (the save location has changed since,
|
|
965
|
+
# 2026-10-19): the user expects it here. Copy it, no new download.
|
|
966
|
+
try:
|
|
967
|
+
import shutil
|
|
968
|
+
copied = target_dir / cached_file.name
|
|
969
|
+
if not copied.exists():
|
|
970
|
+
shutil.copy2(cached_file, copied)
|
|
971
|
+
log.info(f" Cached file is in another folder; copied {cached_file.name} here")
|
|
972
|
+
cached = {**cached, "file": str(copied)}
|
|
973
|
+
cached_file = copied
|
|
974
|
+
if cached.get("doi"):
|
|
975
|
+
_update_doi_index(target_dir, str(cached["doi"]), copied,
|
|
976
|
+
source=str(cached.get("source") or ""), config=config)
|
|
977
|
+
cache_set(identifier, cached, config)
|
|
978
|
+
except OSError as exc:
|
|
979
|
+
log.info(f" Cannot copy the cached file here ({exc}); downloading again")
|
|
980
|
+
cached_file = Path("")
|
|
981
|
+
if cached_file.is_file():
|
|
982
|
+
cached["cached"] = True
|
|
983
|
+
if bibtex:
|
|
984
|
+
from ..citations.bibtex_crossref import fetch_bibtex
|
|
985
|
+
cached["bibtex"] = fetch_bibtex(identifier, config)
|
|
986
|
+
return cached
|
|
987
|
+
|
|
988
|
+
# Scan output dir for existing file with same rename pattern
|
|
989
|
+
doi = normalize_doi(identifier) if not is_arxiv_identifier(identifier) else identifier
|
|
990
|
+
|
|
991
|
+
# Validate DOI before attempting download
|
|
992
|
+
if not is_arxiv_identifier(identifier):
|
|
993
|
+
from ..identifiers import validate_doi
|
|
994
|
+
valid, msg = validate_doi(doi)
|
|
995
|
+
if not valid:
|
|
996
|
+
log.info(f" DOI validation failed: {msg}")
|
|
997
|
+
return {"success": False, "identifier": identifier, "doi": doi, "error": f"Invalid DOI: {msg}"}
|
|
998
|
+
|
|
999
|
+
# Scan output dir for any PDF already downloaded for this DOI.
|
|
1000
|
+
# Uses a DOI→file index to catch renamed files reliably.
|
|
1001
|
+
doi_index = target_dir / ".doi_index.json"
|
|
1002
|
+
if doi_index.exists():
|
|
1003
|
+
try:
|
|
1004
|
+
idx = json.loads(doi_index.read_text(encoding="utf-8"))
|
|
1005
|
+
raw_entry = idx.get(doi)
|
|
1006
|
+
if raw_entry:
|
|
1007
|
+
entry = _index_entry(raw_entry)
|
|
1008
|
+
candidate = Path(entry["file"]) if entry and entry.get("file") else None
|
|
1009
|
+
if entry and candidate and _index_entry_is_fresh(entry, config) and _index_strategy_compatible(entry, config):
|
|
1010
|
+
if candidate.exists():
|
|
1011
|
+
log.info(f" Found existing file (index): {candidate.name}")
|
|
1012
|
+
result = {
|
|
1013
|
+
"success": True, "identifier": identifier,
|
|
1014
|
+
"doi": doi, "file": str(candidate),
|
|
1015
|
+
"source": "local_cache", "cached": True,
|
|
1016
|
+
}
|
|
1017
|
+
# Upgrade legacy entries to records and refresh provenance.
|
|
1018
|
+
idx[doi] = {
|
|
1019
|
+
"file": str(candidate),
|
|
1020
|
+
"source": entry["source"] or "local_cache",
|
|
1021
|
+
"strategy": entry["strategy"] or config.get("download_strategy", ""),
|
|
1022
|
+
"ts": time.time(),
|
|
1023
|
+
}
|
|
1024
|
+
doi_index.write_text(json.dumps(idx, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
1025
|
+
cache_set(identifier, result, config)
|
|
1026
|
+
return result
|
|
1027
|
+
# Stale (file deleted), TTL-expired, or strategy-mismatched entry —
|
|
1028
|
+
# drop it so the download is retried instead of serving stale state.
|
|
1029
|
+
del idx[doi]
|
|
1030
|
+
doi_index.write_text(json.dumps(idx, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
1031
|
+
except Exception:
|
|
1032
|
+
pass
|
|
1033
|
+
|
|
1034
|
+
from ..citations.formats import fetch_metadata
|
|
1035
|
+
metadata = fetch_metadata(doi, config)
|
|
1036
|
+
if metadata:
|
|
1037
|
+
from ..download.rename import rename_format
|
|
1038
|
+
expected_name = rename_pdf_generate_filename(metadata, doi=doi, fmt=rename_format(config))
|
|
1039
|
+
if expected_name:
|
|
1040
|
+
# Generated names can collide across papers, so a name match
|
|
1041
|
+
# alone is not proof: skip files the index gives to
|
|
1042
|
+
# another DOI and require this DOI inside the PDF.
|
|
1043
|
+
claimed = _files_claimed_by_other_dois(target_dir, doi)
|
|
1044
|
+
for suffix in ("", "_2", "_3", "_4", "_5"):
|
|
1045
|
+
candidate = target_dir / f"{expected_name}{suffix}.pdf"
|
|
1046
|
+
if (candidate.exists() and _norm_path(candidate) not in claimed
|
|
1047
|
+
and _pdf_mentions_doi(candidate, doi)):
|
|
1048
|
+
log.info(f" Found existing file: {candidate.name}")
|
|
1049
|
+
result = {
|
|
1050
|
+
"success": True, "identifier": identifier,
|
|
1051
|
+
"doi": doi, "file": str(candidate),
|
|
1052
|
+
"source": "local_cache", "cached": True,
|
|
1053
|
+
}
|
|
1054
|
+
cache_set(identifier, result, config)
|
|
1055
|
+
return result
|
|
1056
|
+
|
|
1057
|
+
log.info(f"LitLoop - {identifier}")
|
|
1058
|
+
if _cancelled(): # skipped or cancelled while the metadata lookup ran
|
|
1059
|
+
return fail(identifier, "skipped by user" if getattr(_DL_TLS, "key", None) in _SKIPPED else "cancelled by user")
|
|
1060
|
+
|
|
1061
|
+
if is_arxiv_identifier(identifier):
|
|
1062
|
+
log.info(" [L0] arXiv direct")
|
|
1063
|
+
result = try_arxiv(identifier, output_path, config)
|
|
1064
|
+
if result:
|
|
1065
|
+
_update_doi_index(
|
|
1066
|
+
target_dir,
|
|
1067
|
+
identifier,
|
|
1068
|
+
Path(result.get("file", "")),
|
|
1069
|
+
source=result.get("source") or "arXiv",
|
|
1070
|
+
strategy=config.get("download_strategy", ""),
|
|
1071
|
+
config=config,
|
|
1072
|
+
)
|
|
1073
|
+
if rename:
|
|
1074
|
+
_auto_rename(result, identifier, config, doi=identifier, target_dir=target_dir)
|
|
1075
|
+
cache_set(identifier, result, config)
|
|
1076
|
+
if bibtex:
|
|
1077
|
+
from ..citations.bibtex_crossref import fetch_bibtex
|
|
1078
|
+
result["bibtex"] = fetch_bibtex(identifier, config)
|
|
1079
|
+
return result
|
|
1080
|
+
return fail(identifier, "arXiv PDF not available")
|
|
1081
|
+
|
|
1082
|
+
doi = normalize_doi(identifier)
|
|
1083
|
+
|
|
1084
|
+
# A preprint is a file for this DOI but not the version of record: keep it
|
|
1085
|
+
# aside and let the later phases look for the published PDF (2026-11-03).
|
|
1086
|
+
preprint_only: dict[str, Any] = {"result": None}
|
|
1087
|
+
|
|
1088
|
+
def _stash_preprint(res: dict[str, Any]) -> bool:
|
|
1089
|
+
if not (res and res.get("preprint")):
|
|
1090
|
+
return False
|
|
1091
|
+
path = Path(res.get("file", ""))
|
|
1092
|
+
marked = path.with_name(f"{path.stem}_preprint{path.suffix}")
|
|
1093
|
+
if path.exists() and marked != path:
|
|
1094
|
+
try:
|
|
1095
|
+
path.replace(marked)
|
|
1096
|
+
res["file"] = str(marked)
|
|
1097
|
+
except OSError:
|
|
1098
|
+
pass
|
|
1099
|
+
preprint_only["result"] = res
|
|
1100
|
+
log.info(f" {res['preprint']} preprint of this paper; still looking for the published version")
|
|
1101
|
+
return True
|
|
1102
|
+
|
|
1103
|
+
def _use_preprint() -> dict[str, Any] | None:
|
|
1104
|
+
res = preprint_only["result"]
|
|
1105
|
+
if res is None:
|
|
1106
|
+
return None
|
|
1107
|
+
if rename:
|
|
1108
|
+
_auto_rename(res, identifier, config, doi=doi, target_dir=target_dir)
|
|
1109
|
+
if bibtex:
|
|
1110
|
+
from ..citations.bibtex_crossref import fetch_bibtex
|
|
1111
|
+
res["bibtex"] = fetch_bibtex(doi, config)
|
|
1112
|
+
log.info(f" No version of record available; keeping the {res.get('preprint')} preprint")
|
|
1113
|
+
return res
|
|
1114
|
+
|
|
1115
|
+
# Phase 1: Free sources (OA + grey) — parallel race
|
|
1116
|
+
t_phase = time.time()
|
|
1117
|
+
p_phase = time.perf_counter() # close_browsers_since's clock
|
|
1118
|
+
# _school_only: a batch's second pass; its first pass already tried these.
|
|
1119
|
+
free_sources = [] if _school_only else _build_free_sources(doi, config)
|
|
1120
|
+
log.info(f" [timing] phase 1 sources built: {time.time() - t_phase:.1f}s")
|
|
1121
|
+
if free_sources:
|
|
1122
|
+
result = _run_tiers_parallel(
|
|
1123
|
+
[(free_sources, "Free", 15)], doi, target_dir, output_path, config, use_tor, 15,
|
|
1124
|
+
failures=race_failures,
|
|
1125
|
+
)
|
|
1126
|
+
log.info(f" [timing] phase 1 (free sources and publisher site): {time.time() - t_phase:.1f}s, "
|
|
1127
|
+
f"{'PDF' if result else 'no PDF'}")
|
|
1128
|
+
if result and _stash_preprint(result):
|
|
1129
|
+
pass
|
|
1130
|
+
elif result:
|
|
1131
|
+
_update_doi_index(
|
|
1132
|
+
target_dir,
|
|
1133
|
+
doi,
|
|
1134
|
+
Path(result.get("file", "")),
|
|
1135
|
+
source=result.get("source") or "",
|
|
1136
|
+
strategy=config.get("download_strategy", ""),
|
|
1137
|
+
config=config,
|
|
1138
|
+
)
|
|
1139
|
+
if rename:
|
|
1140
|
+
_auto_rename(result, identifier, config, doi=doi, target_dir=target_dir)
|
|
1141
|
+
cache_set(identifier, result, config)
|
|
1142
|
+
if bibtex:
|
|
1143
|
+
from ..citations.bibtex_crossref import fetch_bibtex
|
|
1144
|
+
result["bibtex"] = fetch_bibtex(doi, config)
|
|
1145
|
+
return result
|
|
1146
|
+
|
|
1147
|
+
# Phase 2: Institutional access — only when Phase 1 failed
|
|
1148
|
+
# Skip institutional fallback for grey_only/scihub_only strategy
|
|
1149
|
+
from ..publishers.strategies._core import no_access
|
|
1150
|
+
if _institutional and no_access(doi) and not config.get("school_routes_after_publisher_login", False):
|
|
1151
|
+
log.info(" Logged into the publisher as the school and still no full text: skipping the school routes")
|
|
1152
|
+
preprint = _use_preprint()
|
|
1153
|
+
if preprint is not None:
|
|
1154
|
+
return preprint
|
|
1155
|
+
return fail(identifier, "the school does not subscribe: logged into the publisher with the school's identity, still no full text",
|
|
1156
|
+
error_type="not_entitled", action="no_subscription")
|
|
1157
|
+
if _institutional and config.get("download_strategy") not in ("scihub_only", "grey_only") and not _cancelled():
|
|
1158
|
+
# Phase 1's lanes that are still at work (waived) keep their browsers and
|
|
1159
|
+
# the browser slot: on 2026-10-18 a stuck publisher-site lane held it
|
|
1160
|
+
# for 16 minutes and the WebVPN step queued 697 s behind it. Phase 1 is
|
|
1161
|
+
# over, so close what it opened before the school routes need a browser.
|
|
1162
|
+
try:
|
|
1163
|
+
from ..browser.backend import close_browsers_since
|
|
1164
|
+
n = close_browsers_since(p_phase, owner=getattr(_DL_TLS, "key", None) or doi)
|
|
1165
|
+
if n:
|
|
1166
|
+
log.info(f" Closed {n} browser(s) phase 1 left open")
|
|
1167
|
+
except Exception as exc:
|
|
1168
|
+
log.info(f" Phase 1 browser cleanup skipped: {exc}")
|
|
1169
|
+
# Session self-healing: validate and refresh institutional sessions
|
|
1170
|
+
# before racing them (WebVPN only; CARSI self-heals inside login()).
|
|
1171
|
+
t_heal = time.time()
|
|
1172
|
+
try:
|
|
1173
|
+
from ..institutional.session_heal import ensure_institutional_sessions
|
|
1174
|
+
heal_report = ensure_institutional_sessions(
|
|
1175
|
+
config, use_vpnsci=use_vpnsci or bool(config.get("vpnsci_enabled")))
|
|
1176
|
+
for channel, state in heal_report.items():
|
|
1177
|
+
log.info(f" [session-heal] {channel}: {state.get('status')}")
|
|
1178
|
+
except Exception as exc:
|
|
1179
|
+
log.info(f" [session-heal] skipped: {exc}")
|
|
1180
|
+
log.info(f" [timing] session check: {time.time() - t_heal:.1f}s")
|
|
1181
|
+
if _cancelled(): # skipped or cancelled while the session check waited for a login
|
|
1182
|
+
return fail(identifier, "skipped by user" if getattr(_DL_TLS, "key", None) in _SKIPPED else "cancelled by user")
|
|
1183
|
+
inst_sources = _build_institutional_sources(doi, config, use_vpnsci=use_vpnsci)
|
|
1184
|
+
if inst_sources:
|
|
1185
|
+
log.info(" Phase 1 failed, trying institutional access...")
|
|
1186
|
+
result = _run_institutional_sequence(inst_sources, doi, target_dir, output_path, config,
|
|
1187
|
+
failures=race_failures)
|
|
1188
|
+
if result and _stash_preprint(result):
|
|
1189
|
+
pass
|
|
1190
|
+
elif result:
|
|
1191
|
+
_update_doi_index(
|
|
1192
|
+
target_dir,
|
|
1193
|
+
doi,
|
|
1194
|
+
Path(result.get("file", "")),
|
|
1195
|
+
source=result.get("source") or "",
|
|
1196
|
+
strategy=config.get("download_strategy", ""),
|
|
1197
|
+
config=config,
|
|
1198
|
+
)
|
|
1199
|
+
if rename:
|
|
1200
|
+
_auto_rename(result, identifier, config, doi=doi, target_dir=target_dir)
|
|
1201
|
+
cache_set(identifier, result, config)
|
|
1202
|
+
if bibtex:
|
|
1203
|
+
from ..citations.bibtex_crossref import fetch_bibtex
|
|
1204
|
+
result["bibtex"] = fetch_bibtex(doi, config)
|
|
1205
|
+
return result
|
|
1206
|
+
|
|
1207
|
+
# Late capture: wait briefly for browser downloads that complete after race timeout,
|
|
1208
|
+
# then scan for any PDFs that were saved to disk by racing threads.
|
|
1209
|
+
import time as _time
|
|
1210
|
+
_time.sleep(2) # grace period for browser threads to finish writing
|
|
1211
|
+
from ..download.pdf import is_pdf_file
|
|
1212
|
+
held = Path(preprint_only["result"]["file"]) if preprint_only["result"] else None
|
|
1213
|
+
for p in target_dir.glob(f"{safe_filename(identifier)}*.pdf"):
|
|
1214
|
+
if held is not None and p == held:
|
|
1215
|
+
continue # the kept preprint is not a late capture of the published PDF
|
|
1216
|
+
# Only a whole PDF of this very paper: a racing thread may have left a
|
|
1217
|
+
# different paper under this name (Sci-Hub mapped a Ceramics
|
|
1218
|
+
# International DOI to a JSTOR essay; it was taken here, 2026-10-17).
|
|
1219
|
+
if p.stat().st_size > 5000 and is_pdf_file(p) and not _is_wrong_paper(p, doi, config, "late capture"):
|
|
1220
|
+
result = {
|
|
1221
|
+
"success": True, "identifier": identifier, "doi": doi,
|
|
1222
|
+
"file": str(p), "source": "late_capture",
|
|
1223
|
+
}
|
|
1224
|
+
cache_set(identifier, result, config)
|
|
1225
|
+
if rename:
|
|
1226
|
+
_auto_rename(result, identifier, config, doi=doi, target_dir=target_dir)
|
|
1227
|
+
if bibtex:
|
|
1228
|
+
from ..citations.bibtex_crossref import fetch_bibtex
|
|
1229
|
+
result["bibtex"] = fetch_bibtex(identifier, config)
|
|
1230
|
+
return result
|
|
1231
|
+
|
|
1232
|
+
# Nothing delivered the published PDF: a preprint of this paper is better
|
|
1233
|
+
# than reporting failure, and its file name says what it is.
|
|
1234
|
+
preprint = _use_preprint()
|
|
1235
|
+
if preprint is not None:
|
|
1236
|
+
return preprint
|
|
1237
|
+
|
|
1238
|
+
# What this paper's own browser strategies reported (never another paper's).
|
|
1239
|
+
att = _attempt.get(getattr(_DL_TLS, "key", None))
|
|
1240
|
+
error_type, error_action = att.last_error if att is not None else ("", "")
|
|
1241
|
+
guidance = _build_failure_guidance(doi, config, error_type)
|
|
1242
|
+
|
|
1243
|
+
hint: dict[str, Any] = {"manual_url": f"https://sci-hub.ru/{doi}", "guidance": guidance}
|
|
1244
|
+
|
|
1245
|
+
reason = "no PDF found"
|
|
1246
|
+
if error_type in ("paywall", "cloudflare_blocked"):
|
|
1247
|
+
from ..desktop.error_text import raw_for
|
|
1248
|
+
reason = raw_for(error_type)
|
|
1249
|
+
|
|
1250
|
+
# Add agent hint for paywall - clear instruction on what to do next
|
|
1251
|
+
if error_type == "paywall" or error_action == "login_required":
|
|
1252
|
+
result = fail(
|
|
1253
|
+
identifier,
|
|
1254
|
+
reason,
|
|
1255
|
+
hint,
|
|
1256
|
+
error_type=error_type or "paywall",
|
|
1257
|
+
action=error_action or "login_required",
|
|
1258
|
+
)
|
|
1259
|
+
result["agent_hint"] = (
|
|
1260
|
+
f"此论文需要机构登录才能下载。请运行 litloop_login(identifier=\"{doi}\") "
|
|
1261
|
+
"打开浏览器让用户登录机构账号,登录后关闭浏览器,然后重试下载。"
|
|
1262
|
+
)
|
|
1263
|
+
else:
|
|
1264
|
+
result = fail(
|
|
1265
|
+
identifier,
|
|
1266
|
+
reason,
|
|
1267
|
+
hint,
|
|
1268
|
+
error_type=error_type or "not_found",
|
|
1269
|
+
action=error_action or "",
|
|
1270
|
+
)
|
|
1271
|
+
result["source"] = "none"
|
|
1272
|
+
|
|
1273
|
+
# Surface WHY each channel failed so the user can decide (channels are
|
|
1274
|
+
# often only temporarily unreachable; some need a user decision, e.g.
|
|
1275
|
+
# Unpaywall requires the user's real email).
|
|
1276
|
+
source_failures = race_failures
|
|
1277
|
+
if source_failures:
|
|
1278
|
+
result["source_failures"] = source_failures
|
|
1279
|
+
config_needs = [f for f in source_failures if f.get("error_type") == "config_needed"]
|
|
1280
|
+
if config_needs and "agent_hint" not in result:
|
|
1281
|
+
channels = ", ".join(sorted({f["source"] for f in config_needs}))
|
|
1282
|
+
result["agent_hint"] = (
|
|
1283
|
+
f"配置缺失({channels}):Unpaywall 等渠道需要用户真实邮箱。"
|
|
1284
|
+
'请询问用户的邮箱地址,调用 litloop_config(key="email", value="<用户邮箱>") '
|
|
1285
|
+
"后重试下载。"
|
|
1286
|
+
)
|
|
1287
|
+
return result
|
|
1288
|
+
|
|
1289
|
+
|
|
1290
|
+
# Re-export symbols split into internal modules so callers, tests, and
|
|
1291
|
+
# monkeypatches against ``litloop.sources`` keep working unchanged.
|
|
1292
|
+
from ._index import (
|
|
1293
|
+
_EXPLICIT_STRATEGIES,
|
|
1294
|
+
_auto_rename,
|
|
1295
|
+
_files_claimed_by_other_dois,
|
|
1296
|
+
_index_entry,
|
|
1297
|
+
_index_entry_is_fresh,
|
|
1298
|
+
_index_strategy_compatible,
|
|
1299
|
+
_norm_path,
|
|
1300
|
+
_pdf_mentions_doi,
|
|
1301
|
+
_same_dir,
|
|
1302
|
+
_update_doi_index,
|
|
1303
|
+
)
|
|
1304
|
+
from ._routes import (
|
|
1305
|
+
DEFAULT_INSTITUTIONAL_ORDER,
|
|
1306
|
+
_INSTITUTIONAL_STEPS,
|
|
1307
|
+
_OPEN_HOSTS,
|
|
1308
|
+
_STEP_TEXT,
|
|
1309
|
+
_build_failure_guidance,
|
|
1310
|
+
_build_free_sources,
|
|
1311
|
+
_build_institutional_sources,
|
|
1312
|
+
_run_institutional_sequence,
|
|
1313
|
+
_step_watchdog,
|
|
1314
|
+
_webvpn_browser,
|
|
1315
|
+
_webvpn_http,
|
|
1316
|
+
)
|
|
1317
|
+
from ._batch import (
|
|
1318
|
+
_apply_preprint_fallbacks,
|
|
1319
|
+
_batch_institutional_phase,
|
|
1320
|
+
_clear_progress,
|
|
1321
|
+
_close_batch_browsers,
|
|
1322
|
+
_get_progress_file,
|
|
1323
|
+
_load_progress,
|
|
1324
|
+
_progress_lock,
|
|
1325
|
+
_save_progress,
|
|
1326
|
+
_school_routes_each,
|
|
1327
|
+
_write_download_results,
|
|
1328
|
+
batch_download,
|
|
1329
|
+
institutional_download,
|
|
1330
|
+
)
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
try:
|
|
1334
|
+
from ..browser import backend as _bb
|
|
1335
|
+
_bb.LAUNCH_GUARD = _source_may_continue
|
|
1336
|
+
# A browser belongs to the paper the launching thread works for.
|
|
1337
|
+
_bb.OWNER = lambda: getattr(_SOURCE_TLS, "doi", None) or getattr(_DL_TLS, "key", None)
|
|
1338
|
+
except Exception:
|
|
1339
|
+
pass
|