litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1166 @@
1
+ """Unified job pipeline: normalize any input into a channel-annotated download queue.
2
+
3
+ Queue file format (one paper per line, TAB-separated):
4
+
5
+ identifier # plain DOI / arXiv ID
6
+ identifier<TAB>channel # with explicit channel hint
7
+ identifier<TAB>channel<TAB>oa_url # with a known open-access PDF URL
8
+
9
+ ``channel`` is one of ``oa | elsevier | grey | institution | auto``. Lines
10
+ starting with ``#`` are comments. Plain DOI lists, doi.org/arXiv URLs, and
11
+ CSV / XLSX / TSV tables are all accepted and normalized into the same shape.
12
+
13
+ The channel is a *prediction* that feeds the lane scheduler (run_lanes):
14
+ lanes decide where a paper starts, not where it is allowed to end up —
15
+ failures overflow to slower lanes exactly like the racing tiers do.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import csv
21
+ import re
22
+ from concurrent.futures import ThreadPoolExecutor
23
+ from dataclasses import dataclass
24
+ import json
25
+ import logging
26
+ import threading
27
+ import time
28
+ from pathlib import Path
29
+
30
+ logger = logging.getLogger(__name__)
31
+ from typing import Any
32
+
33
+ from ..identifiers import normalize_doi, normalize_arxiv_id
34
+ from ..identifiers import OLD_ARXIV_RE
35
+
36
+ CHANNELS = ("oa", "elsevier", "grey", "institution", "auto")
37
+
38
+ TABLE_SUFFIXES = (".csv", ".xlsx", ".tsv", ".tab")
39
+
40
+ # DOI prefix -> fast HTTP lane. Extend as new publisher API fast paths land.
41
+ CHANNEL_BY_PREFIX = {
42
+ "10.1016": "elsevier", # Elsevier / ScienceDirect / Cell / Lancet
43
+ "10.3390": "oa", # MDPI — fully OA; fast lane constructs mdpi-res CDN URLs
44
+ "10.26599": "oa", # SciOpen (Tsinghua University Press, e.g. J Adv Ceram) — OA, see sciopen_pdf_url
45
+ }
46
+
47
+
48
+ def sciopen_pdf_url(doi: str) -> str:
49
+ """SciOpen's own PDF address for a 10.26599 DOI (checked 2026-10-18: 206
50
+ application/pdf). OpenAlex knows it only for some papers; the article page's
51
+ "PDF" button is not a link the browser lanes recognise."""
52
+ d = (doi or "").strip()
53
+ if not d.lower().startswith("10.26599/"):
54
+ return ""
55
+ prefix, suffix = d.split("/", 1)
56
+ return f"https://www.sciopen.com/local/article_pdf/{prefix}/{suffix.upper()}.pdf"
57
+
58
+ DOI_RE = re.compile(r"\b(10\.\d{4,9}/[^\s\"'<>]+)", re.I)
59
+
60
+
61
+ @dataclass
62
+ class QueueEntry:
63
+ identifier: str = ""
64
+ channel: str = "auto"
65
+ oa_url: str = ""
66
+ title: str = ""
67
+ raw: str = ""
68
+ unresolved: bool = False
69
+
70
+
71
+ def institution_configured(config: dict[str, Any]) -> bool:
72
+ """A school route exists: a login (WebVPN, CARSI, EZProxy) or the campus network."""
73
+ return any(bool(config.get(k)) for k in ("vpnsci_enabled", "carsi_enabled", "ezproxy_enabled", "is_campus_network"))
74
+
75
+
76
+ def predict_channel(identifier: str) -> str:
77
+ """Zero-cost channel prediction from the DOI prefix."""
78
+ m = re.match(r"^(10\.\d{4,9})/", identifier.strip())
79
+ if m:
80
+ return CHANNEL_BY_PREFIX.get(m.group(1).lower(), "auto")
81
+ return "auto"
82
+
83
+
84
+ from ..publishers.supplementary import fetch_supplementary
85
+
86
+ _PDF_URL_HINT = re.compile(
87
+ r"\.pdf($|[?#])|/pdf(/|\?|$)|article-pdf|/epdf/|/pdfdirect", re.IGNORECASE)
88
+ _RETRYABLE_STATUS = {429, 500, 502, 503, 504}
89
+
90
+
91
+ def _looks_like_pdf_url(url: str) -> bool:
92
+ return bool(url) and bool(_PDF_URL_HINT.search(url))
93
+
94
+
95
+ def _fetch_oa_pdf(doi: str, config: dict[str, Any]) -> str:
96
+ """OpenAlex lookup: best OA direct-PDF URL for a DOI ('' when none/error).
97
+
98
+ Cached per DOI (including negative results) so repeated batches and the
99
+ retry pass don't re-query. All failures are silent — enrichment is an
100
+ optimization, never a hard dependency.
101
+ """
102
+ try:
103
+ from ..cache import cache_get, cache_set
104
+ cached = cache_get(f"oa:{doi}", config) if config.get("cache_dir") else None
105
+ if cached is not None:
106
+ return str(cached.get("pdf_url") or "")
107
+ except Exception:
108
+ pass
109
+
110
+ import requests
111
+
112
+ proxy = config.get("network_proxy", "")
113
+ proxies = {"http": proxy, "https": proxy} if proxy else None
114
+ params: dict[str, str] = {"select": "open_access,best_oa_location"}
115
+ if config.get("email"):
116
+ params["mailto"] = str(config["email"])
117
+ if config.get("openalex_api_key"):
118
+ params["api_key"] = str(config["openalex_api_key"])
119
+ pdf = ""
120
+ lookup_ok = False
121
+ try:
122
+ resp = requests.get(
123
+ f"https://api.openalex.org/works/doi:{doi}",
124
+ params=params, timeout=(15, 20), proxies=proxies,
125
+ headers={"User-Agent": _user_agent()},
126
+ )
127
+ if resp.status_code == 200:
128
+ lookup_ok = True
129
+ work = resp.json()
130
+ oa = work.get("open_access") or {}
131
+ loc = work.get("best_oa_location") or {}
132
+ candidate = str(loc.get("pdf_url") or "") or str(oa.get("oa_url") or "")
133
+ if _looks_like_pdf_url(candidate):
134
+ pdf = candidate
135
+ elif resp.status_code not in _RETRYABLE_STATUS:
136
+ lookup_ok = True # definitive negative (e.g. 404): cacheable
137
+ except Exception:
138
+ pass
139
+
140
+ if not lookup_ok:
141
+ # OpenAlex failed (timeout / 429 / 5xx) — Unpaywall mirrors the same
142
+ # OA data on a different host; a transient hiccup must not lose the
143
+ # enrichment.
144
+ try:
145
+ unpay_params = dict(params)
146
+ unpay_params.pop("api_key", None)
147
+ resp2 = requests.get(
148
+ f"https://api.unpaywall.org/v2/{doi}",
149
+ params=unpay_params, timeout=(15, 20), proxies=proxies,
150
+ headers={"User-Agent": _user_agent()},
151
+ )
152
+ if resp2.status_code == 200:
153
+ lookup_ok = True
154
+ loc = resp2.json().get("best_oa_location") or {}
155
+ candidate = str(loc.get("url_for_pdf") or "")
156
+ if _looks_like_pdf_url(candidate):
157
+ pdf = candidate
158
+ elif resp2.status_code not in _RETRYABLE_STATUS:
159
+ lookup_ok = True
160
+ except Exception:
161
+ pass
162
+
163
+ # Cache only completed lookups: a timed-out proxy handshake must not
164
+ # poison the negative cache for cache_ttl_hours.
165
+ if lookup_ok:
166
+ try:
167
+ if config.get("cache_dir"):
168
+ from ..cache import cache_set
169
+ cache_set(f"oa:{doi}", {"pdf_url": pdf}, config)
170
+ except Exception:
171
+ pass
172
+ return pdf
173
+
174
+
175
+ def _s2_batch_oa_urls(dois: list[str], config: dict[str, Any]) -> dict[str, str]:
176
+ """Semantic Scholar batch OA lookup: up to 500 ids per POST, no email/key.
177
+
178
+ Returns {doi: pdf_url}; empty when nothing was found or the endpoint is
179
+ unreachable. All errors are silent — enrichment is an optimization and
180
+ the caller falls back to per-DOI lookups. Field test: 5,645 DOIs in 12
181
+ requests (~3 min) vs 5,645 single-DOI calls.
182
+ """
183
+ import requests as _requests
184
+
185
+ proxy = config.get("network_proxy", "")
186
+ proxies = {"http": proxy, "https": proxy} if proxy else None
187
+ url = ("https://api.semanticscholar.org/graph/v1/paper/batch"
188
+ "?fields=isOpenAccess,openAccessPdf")
189
+ out: dict[str, str] = {}
190
+ for i in range(0, len(dois), 500):
191
+ chunk = [d.strip().lower() for d in dois[i:i + 500]]
192
+ for attempt in range(3):
193
+ try:
194
+ resp = _requests.post(
195
+ url, json={"ids": [f"DOI:{d}" for d in chunk]},
196
+ timeout=(10, 45), proxies=proxies,
197
+ headers={"User-Agent": _user_agent()},
198
+ )
199
+ if resp.status_code == 429: # common; clears in 10-20s
200
+ time.sleep(min(20, 5 * (attempt + 1)))
201
+ continue
202
+ if resp.status_code != 200:
203
+ return out
204
+ data = resp.json()
205
+ if not isinstance(data, list):
206
+ return out
207
+ for d, item in zip(chunk, data):
208
+ if not isinstance(item, dict):
209
+ continue
210
+ pdf = str(((item.get("openAccessPdf") or {}).get("url")) or "")
211
+ if item.get("isOpenAccess") and _looks_like_pdf_url(pdf):
212
+ out[d] = pdf
213
+ break
214
+ except Exception:
215
+ if attempt == 2:
216
+ return out
217
+ time.sleep(3)
218
+ return out
219
+
220
+
221
+ # MDPI DOI-prefix -> full journal name for mdpi-res.com CDN URLs. Mappings
222
+ # verified live (2026-09, HTTP %PDF probe) except where noted; single-letter
223
+ # and non-identity prefixes MUST be mapped here or the constructed URL misses.
224
+ # engproc (conference proceedings) uses special segment numbering not covered
225
+ # by the vol/article formula — those entries fall through to other lanes.
226
+ _MDPI_SLUGS = {
227
+ "su": "sustainability", "atmos": "atmosphere", "w": "water", "f": "forests",
228
+ "min": "minerals", "en": "energies", "polym": "polymers",
229
+ "applbiosci": "appliedbiosciences",
230
+ "app": "applsci", "educsci": "education", "tourhosp": "tourismhosp",
231
+ "d": "diversity", "a": "algorithms", "dj": "dentistry",
232
+ # 2026-10-18, HTTP probe: 10.3390/ma17071608 -> materials-17-01608.pdf,
233
+ # 10.3390/nano8110961 -> nanomaterials-08-00961.pdf (the short form 404s).
234
+ "ma": "materials", "nano": "nanomaterials",
235
+ }
236
+
237
+
238
+ def _mdpi_variants(doi: str) -> list[str]:
239
+ """Constructible mdpi-res.com CDN URLs for an MDPI DOI.
240
+
241
+ www.mdpi.com/pdf is bot-walled (Cloudflare TLS fingerprinting) but the
242
+ CDN is open. Filename = {slug}-{vol:02d}-{art:05d}[-v2|-v3|-v4].pdf;
243
+ slug is the FULL journal name (not the DOI prefix), there is no issue
244
+ number, and old journals have 1-digit volumes — try both splits.
245
+ """
246
+ m = re.match(r"^10\.3390/([a-z]+)(\d{6,})$", doi.strip().lower())
247
+ if not m:
248
+ return []
249
+ pref, digits = m.groups()
250
+ slug = _MDPI_SLUGS.get(pref, pref)
251
+ out: list[str] = []
252
+ seen: set[str] = set()
253
+ for vol_take in (2, 1):
254
+ if len(digits) <= vol_take + 2:
255
+ continue
256
+ vol = str(int(digits[:vol_take])).zfill(2)
257
+ art = str(int(digits[vol_take + 2:])).zfill(5)
258
+ base = f"{slug}-{vol}-{art}"
259
+ if base in seen:
260
+ continue
261
+ seen.add(base)
262
+ for suffix in ("", "-v2", "-v3", "-v4"):
263
+ out.append(f"https://mdpi-res.com/d_attachment/{slug}/{base}/"
264
+ f"article_deploy/{base}{suffix}.pdf")
265
+ return out
266
+
267
+
268
+ def _mdpi_cdn_fetch(doi: str, out: Path,
269
+ headers: dict[str, str],
270
+ proxies: dict[str, str] | None) -> Path | None:
271
+ """Try the constructible CDN variants until one returns a real PDF."""
272
+ import time as _time
273
+
274
+ for url in _mdpi_variants(doi):
275
+ path = _download_url(url, out, doi, headers, proxies)
276
+ if path:
277
+ return path
278
+ _time.sleep(0.15) # rapid-fire misses earn a temporary IP block
279
+ return None
280
+
281
+
282
+ def _enrich_oa_urls(entries: list[QueueEntry], config: dict[str, Any]) -> None:
283
+ """Route gold/hybrid OA papers into the fast lane before scheduling.
284
+
285
+ The prefix DB only knows Elsevier/MDPI, so plain-DOI batches send every
286
+ other publisher to the grey or institutional lanes — OA journals (NAR,
287
+ PLoS, Nat Commun, …) end up at publisher bot-walls needing manual
288
+ Turnstile clicks. For larger batches one S2 batch request per 500 DOIs
289
+ fills ``oa_url`` first; per-DOI OpenAlex/Unpaywall lookups mop up the
290
+ remainder (and serve small batches). The fast lane's own %PDF/10KB
291
+ validation keeps junk from landing. Failures and the kill switch
292
+ (``lane_oa_enrich: false``) leave entries exactly as they were.
293
+ """
294
+ if not config.get("lane_oa_enrich", True):
295
+ return
296
+ for e in entries:
297
+ if not e.oa_url and sciopen_pdf_url(e.identifier):
298
+ e.oa_url = sciopen_pdf_url(e.identifier)
299
+ targets = [
300
+ e for e in entries
301
+ if e.identifier.lower().startswith("10.")
302
+ and not e.oa_url
303
+ and predict_channel(e.identifier) not in ("elsevier", "institution")
304
+ # MDPI is covered deterministically by the CDN constructor in the
305
+ # fast lane — no enrichment requests spent on it.
306
+ and not e.identifier.lower().startswith("10.3390/")
307
+ ]
308
+ if not targets:
309
+ return
310
+
311
+ # Batch sniff first: 1 request per 500 DOIs instead of 1 per DOI.
312
+ s2_min = max(1, int(config.get("lane_s2_batch_min", 10)))
313
+ if len(targets) >= s2_min:
314
+ try:
315
+ batch = _s2_batch_oa_urls([e.identifier for e in targets], config)
316
+ except Exception:
317
+ batch = {}
318
+ for e in targets:
319
+ pdf = batch.get(e.identifier.strip().lower())
320
+ if pdf:
321
+ e.oa_url = pdf
322
+ if e.channel == "auto":
323
+ e.channel = "oa"
324
+ targets = [e for e in targets if not e.oa_url]
325
+
326
+ def lookup(entry: QueueEntry) -> None:
327
+ try:
328
+ pdf = _fetch_oa_pdf(entry.identifier, config)
329
+ except Exception:
330
+ return # enrichment must never break scheduling
331
+ if pdf:
332
+ entry.oa_url = pdf
333
+ if entry.channel == "auto":
334
+ entry.channel = "oa"
335
+
336
+ with ThreadPoolExecutor(max_workers=8) as ex:
337
+ list(ex.map(lookup, targets))
338
+
339
+
340
+ def extract_identifier(text: str) -> str | None:
341
+ """Extract a DOI or arXiv ID from a raw string, URL, or citation fragment."""
342
+ raw = text.strip().rstrip(".,;)")
343
+ if not raw:
344
+ return None
345
+ arxiv = normalize_arxiv_id(raw)
346
+ if arxiv and ("arxiv" in raw.lower()
347
+ or re.match(r"^\d{4}\.\d{4,5}(v\d+)?$", raw)
348
+ or OLD_ARXIV_RE.match(raw)):
349
+ # 旧式 arXiv(quant-ph/9901001)裸写时原文不含 "arxiv",
350
+ # 需要显式匹配旧式模式,否则会被当成无法识别的行。
351
+ return arxiv
352
+ doi_m = DOI_RE.search(raw)
353
+ if doi_m:
354
+ doi = _clean_doi(doi_m.group(1))
355
+ if doi:
356
+ return doi
357
+ return None
358
+
359
+
360
+ def _clean_doi(doi: str) -> str | None:
361
+ doi = normalize_doi(doi)
362
+ doi = doi.rstrip(".,;:)")
363
+ # Trailing author glue, e.g. "10.1002/ird.2673Hamed" -> "10.1002/ird.2673"
364
+ doi = re.sub(r"[A-Z][a-z\u00C0-\u024F]{1,}$", "", doi)
365
+ return doi if re.match(r"^10\.\d{4,9}/\S", doi) else None
366
+
367
+
368
+ def parse_queue(text: str) -> list[QueueEntry]:
369
+ """Parse queue/DOI-list text into entries; unresolvable lines are kept."""
370
+ entries: list[QueueEntry] = []
371
+ for line in text.splitlines():
372
+ line = line.strip()
373
+ if not line or line.startswith("#"):
374
+ continue
375
+ parts = [p.strip() for p in line.split("\t")]
376
+ ident = extract_identifier(parts[0])
377
+ if not ident:
378
+ entries.append(QueueEntry(raw=line, unresolved=True))
379
+ continue
380
+ channel = parts[1].lower() if len(parts) > 1 and parts[1].lower() in CHANNELS else predict_channel(ident)
381
+ oa_url = parts[2] if len(parts) > 2 and parts[2].lower().startswith("http") else ""
382
+ entries.append(QueueEntry(identifier=ident, channel=channel, oa_url=oa_url, raw=line))
383
+ return entries
384
+
385
+
386
+ def read_table(path: str | Path) -> list[dict[str, str]]:
387
+ """Read a csv / tsv / xlsx table into rows of string dicts."""
388
+ p = Path(path)
389
+ suffix = p.suffix.lower()
390
+ if suffix in (".csv", ".tsv", ".tab"):
391
+ delim = "\t" if suffix in (".tsv", ".tab") else ","
392
+ with open(p, newline="", encoding="utf-8-sig") as f:
393
+ first = f.readline()
394
+ rest = f.read()
395
+ first_cells = [c.strip() for c in first.rstrip("\r\n").split(delim)]
396
+ # 表头嗅探:首行首列能解析出 DOI/arXiv = 无表头队列契约(首行是数据行)
397
+ headerless = bool(first_cells) and extract_identifier(first_cells[0]) is not None
398
+ if headerless:
399
+ names = ["identifier", "channel", "oa_url"] + [
400
+ f"col{i}" for i in range(len(first_cells) - 3)]
401
+ out = []
402
+ for line in (first + rest).splitlines():
403
+ if not line.strip():
404
+ continue
405
+ cells = [c.strip() for c in line.rstrip("\r\n").split(delim)]
406
+ out.append({name: (cells[i] if i < len(cells) else "")
407
+ for i, name in enumerate(names)})
408
+ return out
409
+ import io as _io
410
+ return [dict(r) for r in csv.DictReader(_io.StringIO(first + rest), delimiter=delim)]
411
+ if suffix == ".xlsx":
412
+ try:
413
+ import openpyxl
414
+ except ImportError as exc: # pragma: no cover
415
+ raise RuntimeError("xlsx 支持需要 openpyxl:pip install openpyxl") from exc
416
+ wb = openpyxl.load_workbook(p, read_only=True, data_only=True)
417
+ ws = wb.active
418
+ rows = ws.iter_rows(values_only=True)
419
+ try:
420
+ header = [str(c or "").strip() for c in next(rows)]
421
+ except StopIteration:
422
+ return []
423
+ out = []
424
+ for r in rows:
425
+ if all(c is None for c in r):
426
+ continue
427
+ out.append({h: ("" if c is None else str(c)) for h, c in zip(header, r)})
428
+ return out
429
+ raise ValueError(f"Unsupported table format: {suffix}")
430
+
431
+
432
+ def entries_from_table(rows: list[dict[str, str]]) -> list[QueueEntry]:
433
+ """Map table rows to queue entries; the DOI column is sniffed if needed.
434
+
435
+ 队列契约列同样尊重:channel(∈ oa/elsevier/grey/institution/auto)与
436
+ oa_url——无表头三列队列(identifier/channel/oa_url)由此生效。
437
+ """
438
+ if not rows:
439
+ return []
440
+ cols = list(rows[0].keys())
441
+ doi_col = next((c for c in cols if re.search(r"\bdoi\b", c, re.I)), None)
442
+ if doi_col is None:
443
+ for c in cols:
444
+ if any(DOI_RE.search(str(v)) for v in (r.get(c, "") for r in rows[:20])):
445
+ doi_col = c
446
+ break
447
+ title_col = next((c for c in cols if "title" in c.lower() or "标题" in c or "题名" in c), "")
448
+ channel_col = next((c for c in cols if re.search(r"\bchannel\b", c, re.I)), None)
449
+ oa_url_col = next((c for c in cols if re.search(r"oa.?url", c, re.I)), None)
450
+ entries: list[QueueEntry] = []
451
+ for r in rows:
452
+ raw_val = str(r.get(doi_col, "")).strip() if doi_col else ""
453
+ ident = extract_identifier(raw_val) if raw_val else None
454
+ if not ident:
455
+ entries.append(QueueEntry(raw=str(r)[:200], unresolved=True))
456
+ continue
457
+ channel = (str(r.get(channel_col, "") or "").strip()
458
+ if channel_col else "") or predict_channel(ident)
459
+ oa_url = (str(r.get(oa_url_col, "") or "").strip()
460
+ if oa_url_col else "")
461
+ entries.append(QueueEntry(
462
+ identifier=ident,
463
+ channel=channel,
464
+ oa_url=oa_url,
465
+ title=(str(r.get(title_col, "") or "") if title_col else ""),
466
+ ))
467
+ return entries
468
+
469
+
470
+ def load_job(path: str | Path) -> list[QueueEntry]:
471
+ """Load any supported input file into queue entries.
472
+
473
+ Accepts queue TSV, plain DOI lists, doi.org/arXiv URLs, APA lists,
474
+ BibTeX, and csv / xlsx / tsv tables.
475
+ """
476
+ p = Path(path)
477
+ if p.suffix.lower() in TABLE_SUFFIXES:
478
+ return entries_from_table(read_table(p))
479
+
480
+ text = _read_text(p)
481
+ apa_like = bool(re.search(r"[A-Z][a-z\u00C0-\u024F]+,\s+[A-Z]\.", text)) and "doi.org" in text.lower()
482
+ if p.suffix.lower() == ".bib" or apa_like:
483
+ from ..citations.paper_list import parse_paper_list
484
+ return _from_paper_entries(parse_paper_list(p))
485
+ return parse_queue(text)
486
+
487
+
488
+ def _from_paper_entries(paper_entries) -> list[QueueEntry]:
489
+ out: list[QueueEntry] = []
490
+ for e in paper_entries:
491
+ if e.doi:
492
+ out.append(QueueEntry(identifier=e.doi, channel=predict_channel(e.doi), title=e.title, raw=e.raw))
493
+ else:
494
+ out.append(QueueEntry(raw=e.title or e.raw[:200], unresolved=True))
495
+ return out
496
+
497
+
498
+ def _read_text(p: Path) -> str:
499
+ try:
500
+ return p.read_text(encoding="utf-8")
501
+ except UnicodeDecodeError:
502
+ return p.read_text(encoding="latin-1")
503
+
504
+
505
+ def write_queue(entries: list[QueueEntry], path: str | Path) -> Path:
506
+ """Persist entries as a queue file; unresolved lines are kept as comments."""
507
+ lines = ["# identifier\tchannel\toa_url"]
508
+ for e in entries:
509
+ if e.unresolved or not e.identifier:
510
+ lines.append(f"# unresolved: {e.raw[:160]}")
511
+ continue
512
+ row = [e.identifier, e.channel or predict_channel(e.identifier)]
513
+ if e.oa_url:
514
+ row.append(e.oa_url)
515
+ lines.append("\t".join(row))
516
+ p = Path(path)
517
+ p.parent.mkdir(parents=True, exist_ok=True)
518
+ p.write_text("\n".join(lines) + "\n", encoding="utf-8")
519
+ return p
520
+
521
+
522
+ # ---------------------------------------------------------------------------
523
+ # Lane scheduler
524
+ # ---------------------------------------------------------------------------
525
+
526
+ def grey_allowed(config: dict[str, Any] | None) -> bool:
527
+ """Whether grey-source (Sci-Hub/LibGen/SciBban) authorization survives lane
528
+ scheduling. A user-explicit restriction is a veto: ``scihub_enabled=false``
529
+ or ``download_strategy=legal_only`` can never be widened by ``--lanes`` or
530
+ any other scheduling mode. Grey sources are off unless the user turned
531
+ ``scihub_enabled`` on.
532
+ """
533
+ cfg = config or {}
534
+ if not cfg.get("scihub_enabled", False):
535
+ return False
536
+ return cfg.get("download_strategy", "fastest") != "legal_only"
537
+
538
+
539
+ def run_scihub_then_single(
540
+ entries: list[QueueEntry],
541
+ output_dir: str | Path,
542
+ config: dict[str, Any] | None = None,
543
+ allow_grey: bool = True,
544
+ workers: int = 1,
545
+ rounds: str = "all",
546
+ ) -> list[dict[str, Any]]:
547
+ """A short list (up to batch_small_max papers): open-access links, then
548
+ Sci-Hub for every paper at once, then a full single download for each miss.
549
+
550
+ Round 0 is the lanes' HTTP fast lane for papers with a known open-access
551
+ PDF (or an open-access publisher, or the Elsevier API): seconds, no browser.
552
+ Round 1 asks Sci-Hub only (no browser: its ALTCHA gate is solved over
553
+ HTTP) for all papers in parallel; when the user turned gray sources off
554
+ there is no round 1. Round 2 runs each remaining paper exactly as a single
555
+ download does - every source racing, then the school's routes, WebVPN when
556
+ it is switched on - one paper at a time by default (workers): these papers
557
+ need the browser, there is one, and a paper queuing for it would run out
558
+ its race clock while it waits. (The lanes in run_lanes
559
+ serialize stage by stage, which costs minutes on a dozen papers.)
560
+
561
+ rounds: "all"; "free" stops after round 1 (the app's long lists: what is
562
+ left waits in per-publisher queues the user starts, each paper of it then
563
+ a ``{"queued": True}`` row); "single" is round 2 alone (such a queue).
564
+ """
565
+ from concurrent.futures import ThreadPoolExecutor, as_completed
566
+
567
+ from . import progress as _progress
568
+ from .. import sources as _sources
569
+ from ..sources import download
570
+
571
+ config = config or {}
572
+ gen0 = _sources._CANCEL_GEN[0] # cancel_downloads() bumps it: papers not yet started stop too
573
+ out = Path(output_dir)
574
+ out.mkdir(parents=True, exist_ok=True)
575
+ ids = [e.identifier for e in entries if e.identifier and not e.unresolved]
576
+ _progress.start_task("文献下载", total=len(ids))
577
+ _progress.set_output_dir(str(out))
578
+ results: dict[str, dict[str, Any]] = {}
579
+ t_all = t = time.time()
580
+
581
+ def _lap(what: str) -> None:
582
+ nonlocal t
583
+ logger.info(f"[timing] {what}: {time.time() - t:.1f}s (total {time.time() - t_all:.0f}s)")
584
+ t = time.time()
585
+
586
+ if rounds == "single":
587
+ entries = [] # rounds 0 and 1 were run for these papers: straight to round 2
588
+ _enrich_oa_urls(entries, config)
589
+ _lap("open-access links looked up")
590
+ springer_key = str(config.get("springer_api_key", "") or "")
591
+ from ..sources.elsevier_api import fulltext_enabled
592
+ fast_channels = ("oa", "elsevier") if fulltext_enabled(config) else ("oa",)
593
+ fast = [e for e in entries if e.identifier in ids and (
594
+ e.oa_url or (e.channel if e.channel not in ("", "auto") else predict_channel(e.identifier)) in fast_channels
595
+ or (springer_key and e.identifier.lower().startswith("10.1007/")))]
596
+ if fast:
597
+ _progress.update(phase="开放获取")
598
+ for e in fast:
599
+ _progress.item_stage(e.identifier, "正在下载开放获取全文")
600
+ from ..sources import _auto_rename, _update_doi_index
601
+ for r in _run_fast_lane(fast, out, config, workers=min(8, len(fast)), progress=None):
602
+ if r.get("success"):
603
+ # as a single download leaves it: indexed (the app sees it) and renamed
604
+ try:
605
+ _update_doi_index(out, r["doi"], Path(r["file"]), source=r.get("source") or "",
606
+ strategy=config.get("download_strategy", ""), config=config)
607
+ _auto_rename(r, r["doi"], config, doi=r["doi"], target_dir=out)
608
+ except Exception:
609
+ pass
610
+ results[r["doi"]] = r
611
+ _progress.advance(True, current=r["doi"])
612
+ _progress.item_result(r["doi"], r)
613
+ _lap(f"open-access round, {len(results)}/{len(fast)} papers")
614
+
615
+ def _one(ident: str, **kw: Any) -> dict[str, Any]:
616
+ if _sources._CANCEL_GEN[0] != gen0:
617
+ return {"success": False, "doi": ident, "identifier": ident, "error": "cancelled by user",
618
+ "error_type": "cancelled"}
619
+ try:
620
+ r = download(ident, str(out), _batch=True, **kw) or {}
621
+ except Exception as exc:
622
+ r = {"success": False, "error": str(exc)}
623
+ r.setdefault("doi", ident)
624
+ r.setdefault("identifier", ident)
625
+ return r
626
+
627
+ from ..sources.scihub import too_new
628
+ grey_ids = [] if rounds == "single" else [i for i in ids if i not in results and not too_new(i, config)]
629
+ if grey_ids and allow_grey and grey_allowed(config):
630
+ # Which mirror and route (proxy or direct) is fastest right now: the
631
+ # whole round uses it (scihub_route; a no-op for "direct").
632
+ _progress.update(phase="Sci-Hub 测速")
633
+ for i in grey_ids:
634
+ _progress.item_stage(i, "正在测试 Sci-Hub 镜像速度")
635
+ try:
636
+ from ..sources.scihub_speed import measure
637
+ measure(config)
638
+ except Exception as exc:
639
+ logger.info(f"Sci-Hub speed test skipped: {exc}")
640
+ _lap("Sci-Hub speed test")
641
+ _progress.update(phase="Sci-Hub")
642
+ with ThreadPoolExecutor(max_workers=min(int(config.get("scihub_round_workers", 10) or 10), len(grey_ids))) as ex:
643
+ for i in grey_ids:
644
+ _progress.item_stage(i, "正在查找 Sci-Hub")
645
+ # Over HTTP only: no browser, so these really run side by side; a
646
+ # paper whose gate needs the browser gets it in round 2.
647
+ futs = {ex.submit(_one, i, strategy="scihub_only", _institutional=False, _scihub_http_only=True): i
648
+ for i in grey_ids}
649
+ for f in as_completed(futs):
650
+ i, r = futs[f], f.result()
651
+ if r.get("success"):
652
+ results[i] = r
653
+ _progress.item_result(i, r)
654
+ _progress.advance(bool(r.get("success")), current=i) # a miss goes on to round 2
655
+ _lap(f"Sci-Hub round, {sum(1 for i in grey_ids if i in results)}/{len(grey_ids)} papers")
656
+
657
+ left = [i for i in ids if i not in results]
658
+ if rounds == "free":
659
+ left, queued = [], left
660
+ for i in queued:
661
+ results[i] = {"success": False, "doi": i, "identifier": i, "queued": True, "error_type": "queued"}
662
+ if left:
663
+ _progress.update(phase="逐篇下载")
664
+ use_vpnsci = bool(config.get("vpnsci_enabled") or config.get("ezproxy_enabled"))
665
+ with ThreadPoolExecutor(max_workers=max(1, min(workers, len(left)))) as ex:
666
+ searched = set(grey_ids) if grey_ids and allow_grey and grey_allowed(config) else set()
667
+ for i in left:
668
+ # Say what really happened: only papers the Sci-Hub round asked were "not found" there.
669
+ why = ("Sci-Hub 上没有" if i in searched
670
+ else "发表较新,不查 Sci-Hub" if allow_grey and grey_allowed(config) and too_new(i, config)
671
+ else "")
672
+ _progress.item_stage(i, f"等待中:{why},接下来逐篇查找" if why else "等待中,接下来逐篇查找")
673
+
674
+ def _staged(i: str) -> dict[str, Any]:
675
+ _progress.update(current=i)
676
+ _progress.item_stage(i, "正在查找开放获取版本和出版商官网")
677
+ t_one = time.time()
678
+ r = _one(i, _institutional=True, use_vpnsci=use_vpnsci)
679
+ logger.info(f"[timing] {i}: {time.time() - t_one:.1f}s, "
680
+ f"{r.get('source') if r.get('success') else 'failed'}")
681
+ return r
682
+ futs = {ex.submit(_staged, i): i for i in left}
683
+ for f in as_completed(futs):
684
+ i, r = futs[f], f.result()
685
+ results[i] = r
686
+ _progress.advance(bool(r.get("success")), current=i)
687
+ _progress.item_result(i, r) # final: nothing else will be tried for it
688
+ _lap(f"one-by-one round, {sum(1 for i in left if results[i].get('success'))}/{len(left)} papers")
689
+ logger.info(f"[timing] whole list: {time.time() - t_all:.0f}s, "
690
+ f"{sum(1 for r in results.values() if r.get('success'))}/{len(ids)} papers")
691
+ _progress.finish()
692
+ return [results[i] for i in ids if i in results]
693
+
694
+
695
+ def run_lanes(
696
+ entries: list[QueueEntry],
697
+ output_dir: str | Path,
698
+ config: dict[str, Any] | None = None,
699
+ allow_grey: bool = True,
700
+ allow_institution: bool = True,
701
+ workers_fast: int = 8,
702
+ ) -> list[dict[str, Any]]:
703
+ """Channel-lane scheduler: fast HTTP first, grey racing and institutional
704
+ cascade as slower lanes. Failures overflow to slower lanes.
705
+
706
+ Result dicts reuse the racing shape: {success, doi, file, source, error}.
707
+ """
708
+ out = Path(output_dir)
709
+ out.mkdir(parents=True, exist_ok=True)
710
+ config = config or {}
711
+
712
+ _enrich_oa_urls(entries, config)
713
+
714
+ from . import progress as _progress
715
+ _progress.start_task("文献下载", total=len(entries))
716
+ _progress.set_output_dir(str(out))
717
+
718
+ fast: list[QueueEntry] = []
719
+ grey: list[str] = []
720
+ inst: list[str] = []
721
+ springer_key = str(config.get("springer_api_key", "") or "")
722
+ for e in entries:
723
+ if e.unresolved or not e.identifier:
724
+ continue
725
+ ch = e.channel or predict_channel(e.identifier)
726
+ springer_fast = (springer_key
727
+ and e.identifier.lower().startswith("10.1007/"))
728
+ if ch == "institution":
729
+ inst.append(e.identifier)
730
+ elif ch in ("oa", "elsevier") or e.oa_url or springer_fast:
731
+ fast.append(e)
732
+ else:
733
+ grey.append(e.identifier)
734
+
735
+ results: list[dict[str, Any]] = []
736
+ fast_failures: list[str] = []
737
+
738
+ if fast:
739
+ lane_results = _run_fast_lane(fast, out, config, workers=workers_fast, progress=_progress)
740
+ results += lane_results
741
+ fast_failures = [r["doi"] for r in lane_results if not r.get("success")]
742
+
743
+ grey_ids = grey + fast_failures if allow_grey else list(grey)
744
+ # Lane scheduling must not widen source authorization: if the user has
745
+ # disabled grey sources, overflow simply does not get a grey retry.
746
+ if grey_ids and allow_grey and grey_allowed(config):
747
+ from ..sources import batch_download
748
+ _progress.update(phase="灰色源竞速")
749
+ raw = batch_download(
750
+ grey_ids, str(out), scihub_enabled=True,
751
+ progress_callback=lambda done, total, current, result: (
752
+ _progress.advance(
753
+ bool(result and result.get("success")),
754
+ current=str(current),
755
+ )
756
+ ),
757
+ )
758
+ results += _normalize_engine_results(raw)
759
+
760
+ # What the faster lanes did not get goes through the school's routes, as in
761
+ # a single download (sources.institutional_download: session refresh, then
762
+ # WebVPN / CARSI / EZProxy), one paper at a time so a login opens once.
763
+ # Without this a batch never reached the school at all.
764
+ if allow_institution:
765
+ from ..sources import institutional_download
766
+ got = {str(r.get("doi") or "").strip().lower() for r in results if r.get("success")}
767
+ left = [e.identifier for e in entries
768
+ if e.identifier and not e.unresolved and e.identifier.lower() not in got
769
+ and e.identifier not in inst and (e.channel or predict_channel(e.identifier)) != "oa"]
770
+ if left:
771
+ _progress.update(phase="学校通道")
772
+ for i, doi in enumerate(left):
773
+ try:
774
+ r = institutional_download(doi, out, heal=(i == 0))
775
+ except Exception as exc:
776
+ r = {"success": False, "doi": doi, "error": f"school route: {exc}"}
777
+ if r is None:
778
+ break # no school route configured: the rest would get the same answer
779
+ r.setdefault("doi", doi)
780
+ _progress.advance(bool(r.get("success")), current=doi)
781
+ results.append(r)
782
+
783
+ if inst and allow_institution:
784
+ from ..institutional.config_adapter import ConfigAdapter
785
+ from .fetcher import PaperFetcher
786
+
787
+ _progress.update(phase="机构级联")
788
+
789
+ def _inst_fetch_chunk(chunk: list[str]) -> list[dict[str, Any]]:
790
+ from ..browser.backend import shared
791
+ with shared(): # the fetcher's browser serves every paper of the chunk
792
+ return _inst_fetch_chunk_inner(chunk)
793
+
794
+ def _inst_fetch_chunk_inner(chunk: list[str]) -> list[dict[str, Any]]:
795
+ cfg = ConfigAdapter.load()
796
+ cfg._config["output_dir"] = str(out)
797
+ fetcher = PaperFetcher(cfg)
798
+ rows: list[dict[str, Any]] = []
799
+ try:
800
+ for doi in chunk:
801
+ print(f" [institution] {doi}")
802
+ try:
803
+ r = fetcher.fetch_with_result(doi).to_dict()
804
+ r.setdefault("doi", doi)
805
+ except Exception as exc:
806
+ r = {"doi": doi, "success": False, "error": str(exc)}
807
+ _progress.advance(bool(r.get("success")), current=doi)
808
+ rows.append(r)
809
+ finally:
810
+ fetcher.close()
811
+ return rows
812
+
813
+ workers_inst = max(1, min(int(config.get("institutional_workers", 1) or 1), 2))
814
+ if workers_inst <= 1 or len(inst) < 2:
815
+ results.extend(_inst_fetch_chunk(inst))
816
+ else:
817
+ chunks = [inst[i::workers_inst] for i in range(workers_inst)]
818
+ with ThreadPoolExecutor(max_workers=workers_inst) as ex:
819
+ for rows in ex.map(_inst_fetch_chunk, chunks):
820
+ results.extend(rows)
821
+
822
+ adopted = _adopt_late_files(results, out, config)
823
+ if adopted:
824
+ logger.info(f" {adopted} 篇的 PDF 在超时之后才落盘,按已有文件算成功")
825
+ _transient_retry(results, entries, out, config)
826
+ _adopt_late_files(results, out, config) # files that landed while the retry cooled down
827
+ _progress.finish()
828
+ _fetch_si_for_results(results, out, config)
829
+
830
+ # Result guarantee: every input entry must produce a row. Entries whose
831
+ # only lane was disabled (grey vetoed by config, institutional off, …)
832
+ # used to vanish silently — the caller saw "0/0 succeeded" with no
833
+ # explanation. Surface exactly WHY each unprocessed entry has no result.
834
+ seen_dois = {str(r.get("doi", "")).strip().lower() for r in results}
835
+ for e in entries:
836
+ if e.unresolved or not e.identifier:
837
+ continue
838
+ if e.identifier.lower() in seen_dois:
839
+ continue
840
+ ch = e.channel or predict_channel(e.identifier)
841
+ went_grey = ch not in ("oa", "elsevier", "institution") and not e.oa_url
842
+ reason = ("grey lane disabled (scihub_enabled=false or config veto)"
843
+ if went_grey and not allow_grey else
844
+ "no lane produced a result — check channel routing")
845
+ results.append({"doi": e.identifier, "success": False,
846
+ "error": reason, "source": "none"})
847
+ return results
848
+
849
+
850
+ def _adopt_late_files(results: list[dict[str, Any]], out: Path,
851
+ config: dict[str, Any]) -> int:
852
+ """Count PDFs that landed after their paper was already marked failed.
853
+
854
+ A browser lane can finish a minute after its paper's race window: the file
855
+ lands under ``"<doi>_<source>.pdf"`` while the result says failure, and the
856
+ cooling retry (or the next run) fetches the same paper again. Adopt the
857
+ file instead — same paper, no second download. Files are named after the
858
+ DOI, but a name is not proof, so the PDF has to pass ``paper_check`` too.
859
+ """
860
+ from ..download.pdf import is_pdf_file, is_suspicious_pdf, paper_check
861
+ from ..identifiers import safe_filename
862
+
863
+ adopted = 0
864
+ owned = {str(r.get("file") or "") for r in results if r.get("success")}
865
+ for r in results:
866
+ doi = str(r.get("doi") or "")
867
+ if r.get("success") or not doi:
868
+ continue
869
+ cands = [p for p in out.glob(f"{safe_filename(doi)}*.pdf")
870
+ if p.is_file() and str(p) not in owned and is_pdf_file(p)
871
+ and not is_suspicious_pdf(p, doi)]
872
+ if not cands:
873
+ continue
874
+ cand = max(cands, key=lambda p: p.stat().st_mtime)
875
+ if paper_check(cand, doi, config) == "wrong":
876
+ continue
877
+ r.update({"success": True, "identifier": r.get("identifier") or doi,
878
+ "doi": doi, "file": str(cand), "source": "late_file"})
879
+ for key in ("error", "reason"):
880
+ r.pop(key, None)
881
+ owned.add(str(cand))
882
+ adopted += 1
883
+ try:
884
+ from ..sources import _auto_rename
885
+ _auto_rename(r, r["identifier"], config, target_dir=out)
886
+ except Exception:
887
+ pass # a nicer name is optional; the file is already the paper
888
+ return adopted
889
+
890
+
891
+ def _transient_retry(results: list[dict[str, Any]], entries: list[QueueEntry],
892
+ out: Path, config: dict[str, Any]) -> None:
893
+ """瞬时失败冷却重试:限流/超时类失败等 60s 后按原车道重试一次。
894
+
895
+ 实测(2026-08-31,Elsevier 批量):64 篇失败中 56 篇为瞬时限流
896
+ (冷却后 HEAD 探测 100% 可得),高速重试仅收回 9 篇——**下载通道也
897
+ 需要节流**。本函数在车道全部跑完后执行一次带退避的重试。
898
+ 权限类失败(403/NOT_ENTITLED)不重试——那是真实边界。
899
+ """
900
+ if not config.get("fast_retry", True):
901
+ return
902
+ transient_marks = ("fast lane failed", "no PDF found", "timeout", "ERR_",
903
+ "API miss", "timed out")
904
+ failed = [r for r in results
905
+ if not r.get("success") and r.get("doi")
906
+ and any(m in (str(r.get("error", "")) + str(r.get("reason", "")))
907
+ for m in transient_marks)]
908
+ if not failed:
909
+ return
910
+ wait_sec = max(15, int(config.get("fast_retry_wait_sec", 60)))
911
+ failed_dois = {r["doi"] for r in failed}
912
+ retry_entries = [e for e in entries if e.identifier in failed_dois]
913
+ if not retry_entries:
914
+ return
915
+ logger.info(f" 瞬时失败冷却重试: {len(retry_entries)} 篇,等待 {wait_sec}s 后原车道重试")
916
+ try:
917
+ from . import progress as _progress
918
+ _progress.update(phase="冷却重试")
919
+ except Exception:
920
+ pass
921
+ time.sleep(wait_sec)
922
+ delay = max(1.0, float(config.get("fast_retry_delay_sec", 15)))
923
+ retry_results = _run_fast_lane(retry_entries, out, config, workers=1, progress=None, delay=delay)
924
+ recovered = 0
925
+ by_doi = {r["doi"]: r for r in retry_results if r.get("doi")}
926
+ for i, r in enumerate(results):
927
+ d = r.get("doi")
928
+ if d in by_doi and by_doi[d].get("success") and not r.get("success"):
929
+ results[i] = by_doi[d]
930
+ recovered += 1
931
+ logger.info(f" 冷却重试收回 {recovered}/{len(retry_entries)} 篇")
932
+
933
+
934
+ def _fetch_si_for_results(results: list[dict[str, Any]], out: Path, config: dict[str, Any]) -> None:
935
+ """抓取成功论文的附件/补充材料(config: download_si,默认关)。
936
+
937
+ 存到主 PDF 旁边,命名为 {DOI}_SI{n}.{ext},并写 si_manifest.json 便于核对。
938
+ 任何失败只减少附件数量,不影响主结果。
939
+ """
940
+ if not config.get("download_si"):
941
+ return
942
+ ok = [r for r in results if r.get("success") and r.get("doi")]
943
+ if not ok:
944
+ return
945
+ try:
946
+ from . import progress as _progress
947
+ _progress.update(phase="抓取附件")
948
+ except Exception:
949
+ pass
950
+ manifest: dict[str, list[str]] = {}
951
+ manifest_lock = threading.Lock()
952
+
953
+ def _si_one(row: dict[str, Any]) -> None:
954
+ from ..browser.backend import shared
955
+ with shared(): # the pool thread's browser serves the next papers too
956
+ _si_one_inner(row)
957
+
958
+ def _si_one_inner(row: dict[str, Any]) -> None:
959
+ doi = row["doi"]
960
+ try:
961
+ files = fetch_supplementary(doi, out, config)
962
+ except Exception:
963
+ files = []
964
+ if files:
965
+ with manifest_lock:
966
+ manifest[doi] = files
967
+ print(f" [SI] {doi}: {len(files)} 个附件")
968
+
969
+ workers_si = max(1, min(4, len(ok)))
970
+ if workers_si == 1:
971
+ for r in ok:
972
+ _si_one(r)
973
+ else:
974
+ with ThreadPoolExecutor(max_workers=workers_si) as ex:
975
+ list(ex.map(_si_one, ok))
976
+ if manifest:
977
+ try:
978
+ (out / "si_manifest.json").write_text(
979
+ json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8"
980
+ )
981
+ except Exception:
982
+ pass
983
+
984
+
985
+ _OA_BROWSER_LOCK = threading.Lock()
986
+
987
+
988
+ def _run_fast_lane(
989
+ entries: list[QueueEntry],
990
+ out: Path,
991
+ config: dict[str, Any],
992
+ workers: int = 8,
993
+ progress: Any = None,
994
+ delay: float = 0.0,
995
+ ) -> list[dict[str, Any]]:
996
+ """HTTP fast lane: known OA URLs and the Elsevier API, in parallel."""
997
+ import requests
998
+
999
+ from concurrent.futures import as_completed
1000
+ from ..network import configured_proxy
1001
+ from ..sources.elsevier_api import fetch_pdf
1002
+
1003
+ api_key = config.get("elsevier_api_key", "")
1004
+ inst_token = config.get("elsevier_insttoken", "")
1005
+ proxy = configured_proxy(config)
1006
+ proxies = {"http": proxy, "https": proxy} if proxy else None
1007
+ headers = {"User-Agent": _user_agent()}
1008
+
1009
+ def one(e: QueueEntry) -> dict[str, Any]:
1010
+ # MDPI: deterministic CDN URLs first — 94% field hit rate, no bot wall.
1011
+ if (config.get("lane_mdpi_cdn", True)
1012
+ and e.identifier.lower().startswith("10.3390/")):
1013
+ path = _mdpi_cdn_fetch(e.identifier, out, headers, proxies)
1014
+ if path:
1015
+ return {"success": True, "doi": e.identifier, "file": str(path),
1016
+ "source": "mdpi_cdn"}
1017
+ # Springer: entitled full text via the TDM API (JATS XML artifact)
1018
+ if (str(config.get("springer_api_key", "") or "")
1019
+ and e.identifier.lower().startswith("10.1007/")):
1020
+ from ..sources.springer_tdm import try_springer_tdm
1021
+
1022
+ r = try_springer_tdm(e.identifier, out / _safe_name(e.identifier), config)
1023
+ if r and r.get("success"):
1024
+ return {"success": True, "doi": e.identifier,
1025
+ "file": r["file"], "source": "springer_tdm"}
1026
+ if e.oa_url:
1027
+ path = _download_url(e.oa_url, out, e.identifier, headers, proxies)
1028
+ if not path and config.get("oa_browser_fallback", True):
1029
+ # OA 直链被出版商反爬 403(如 Wiley pdfdirect 挡无指纹请求),
1030
+ # 内容公开可得——浏览器指纹可过,回退常驻池浏览器捕获。
1031
+ try:
1032
+ from ..browser.engine import download_pdf_via_browser
1033
+
1034
+ bpath = out / _safe_name(e.identifier)
1035
+ # One at a time (each worker thread would start its own visible
1036
+ # Chrome) and short: a miss here is tried again paper by paper.
1037
+ with _OA_BROWSER_LOCK:
1038
+ got = download_pdf_via_browser(
1039
+ e.oa_url, bpath, config,
1040
+ timeout=float(config.get("oa_browser_fallback_timeout", 25)))
1041
+ if got:
1042
+ return {"success": True, "doi": e.identifier,
1043
+ "file": str(bpath), "source": "oa_browser"}
1044
+ except Exception:
1045
+ pass
1046
+ if path:
1047
+ return {"success": True, "doi": e.identifier, "file": str(path), "source": "oa_url"}
1048
+ if api_key and (e.channel == "elsevier" or e.identifier.lower().startswith("10.1016/")):
1049
+ try:
1050
+ pdf = fetch_pdf(e.identifier, api_key, inst_token, config=config)
1051
+ except Exception:
1052
+ pdf = None
1053
+ if pdf and pdf[:4] == b"%PDF":
1054
+ path = out / _safe_name(e.identifier)
1055
+ path.write_bytes(pdf)
1056
+ return {"success": True, "doi": e.identifier, "file": str(path), "source": "elsevier_api"}
1057
+ return {"success": False, "doi": e.identifier, "error": "fast lane failed (no OA url / API miss)"}
1058
+
1059
+ if progress is not None:
1060
+ progress.update(phase="快车道")
1061
+ results: list[dict[str, Any]] = []
1062
+ import time as _time
1063
+
1064
+ def _one_paced(e: QueueEntry) -> dict[str, Any]:
1065
+ if delay > 0:
1066
+ _time.sleep(delay) # 节流:冷却重试期间按间隔逐篇取,绕开限流
1067
+ t0 = _time.time()
1068
+ r = one(e)
1069
+ # per paper: the round waits for its slowest one (2026-10-17: 110 s, cause unlogged)
1070
+ logger.info(f" [timing] fast lane {e.identifier}: {_time.time() - t0:.1f}s, "
1071
+ f"{r.get('source') or 'no PDF'}")
1072
+ return r
1073
+
1074
+ with ThreadPoolExecutor(max_workers=max(1, workers)) as ex:
1075
+ futures = {ex.submit(_one_paced, e): e for e in entries}
1076
+ for fut in as_completed(futures):
1077
+ entry = futures[fut]
1078
+ try:
1079
+ r = fut.result()
1080
+ except Exception as exc:
1081
+ r = {"success": False, "doi": entry.identifier, "error": str(exc)}
1082
+ if progress is not None:
1083
+ progress.advance(bool(r.get("success")), current=entry.identifier)
1084
+ results.append(r)
1085
+ return results
1086
+
1087
+
1088
+ def _download_url(
1089
+ url: str,
1090
+ out: Path,
1091
+ identifier: str,
1092
+ headers: dict[str, str],
1093
+ proxies: dict[str, str] | None,
1094
+ ) -> Path | None:
1095
+ """Stream a URL to disk, accepting only real PDFs (magic bytes, >=10KB)."""
1096
+ import requests
1097
+
1098
+ try:
1099
+ s = requests.Session()
1100
+ s.trust_env = False
1101
+ resp = s.get(url, headers=headers, proxies=proxies, timeout=30, stream=True, allow_redirects=True)
1102
+ if resp.status_code != 200:
1103
+ return None
1104
+ first = next(resp.iter_content(chunk_size=8192), b"")
1105
+ if not first.startswith(b"%PDF"):
1106
+ return None
1107
+ path = out / _safe_name(identifier)
1108
+ with open(path, "wb") as f:
1109
+ f.write(first)
1110
+ for chunk in resp.iter_content(chunk_size=65536):
1111
+ f.write(chunk)
1112
+ if path.stat().st_size < 10_000:
1113
+ path.unlink(missing_ok=True)
1114
+ return None
1115
+ return path
1116
+ except Exception:
1117
+ return None
1118
+
1119
+
1120
+ def _safe_name(identifier: str) -> str:
1121
+ name = re.sub(r"[^A-Za-z0-9._-]+", "_", identifier.strip()).strip("_")
1122
+ return f"{name}.pdf"
1123
+
1124
+
1125
+ def _user_agent() -> str:
1126
+ try:
1127
+ from ..network import USER_AGENT
1128
+ return USER_AGENT
1129
+ except Exception: # pragma: no cover
1130
+ return "Mozilla/5.0 (compatible; litloop)"
1131
+
1132
+
1133
+ def _normalize_engine_results(raw: Any) -> list[dict[str, Any]]:
1134
+ """Engine outputs come in two shapes: a result list, or a summary dict
1135
+ ({"entries"/"results": [...], "total", "succeeded", ...}). Keep dicts only.
1136
+ """
1137
+ if isinstance(raw, dict):
1138
+ for key in ("entries", "results"):
1139
+ v = raw.get(key)
1140
+ if isinstance(v, list):
1141
+ return [r for r in v if isinstance(r, dict)]
1142
+ return []
1143
+ if isinstance(raw, list):
1144
+ return [r for r in raw if isinstance(r, dict)]
1145
+ return []
1146
+
1147
+
1148
+ def collect_failures(results: Any) -> list[str]:
1149
+ """Extract failed identifiers from any results shape (racing summary dict,
1150
+ cascade list, or plain entries)."""
1151
+ if isinstance(results, dict):
1152
+ for key in ("entries", "results"):
1153
+ v = results.get(key)
1154
+ if isinstance(v, list):
1155
+ return collect_failures(v)
1156
+ return [f for f in (results.get("failed_dois") or []) if isinstance(f, str)]
1157
+ if not isinstance(results, list):
1158
+ return []
1159
+ failed = []
1160
+ for r in results:
1161
+ if not isinstance(r, dict) or r.get("success") or r.get("status") == "success":
1162
+ continue
1163
+ ident = r.get("doi") or r.get("identifier") or ""
1164
+ if ident and ident not in failed:
1165
+ failed.append(ident)
1166
+ return failed