litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,529 @@
|
|
|
1
|
+
"""Batch download orchestration, JSONL resume state, preprint fallbacks, and standalone institutional download.
|
|
2
|
+
|
|
3
|
+
Extracted from ``litloop.sources.__init__`` on 2026-11-20; ``litloop.sources``
|
|
4
|
+
re-exports all symbols defined here so callers, tests, and monkeypatches remain unchanged.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import sys
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from ..download.pdf import fail
|
|
18
|
+
from ..identifiers import is_arxiv_identifier, normalize_doi, safe_filename
|
|
19
|
+
from ..log import get_logger
|
|
20
|
+
|
|
21
|
+
log = get_logger()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _pkg() -> Any:
|
|
25
|
+
return sys.modules[__package__]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _get_progress_file(batch_id: str) -> Path:
|
|
29
|
+
"""Get path to batch progress file."""
|
|
30
|
+
return _pkg().DATA_DIR / "batch_progress" / f"{batch_id}.jsonl"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
_progress_lock = threading.Lock()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _save_progress(batch_id: str, identifier: str, result: dict[str, Any]) -> None:
|
|
37
|
+
"""Append a single result to the progress file (thread-safe)."""
|
|
38
|
+
progress_file = _pkg()._get_progress_file(batch_id)
|
|
39
|
+
progress_file.parent.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
entry = {
|
|
41
|
+
"identifier": identifier,
|
|
42
|
+
"success": result.get("success", False),
|
|
43
|
+
"source": result.get("source", "none"),
|
|
44
|
+
"file": result.get("file", ""),
|
|
45
|
+
"doi": result.get("doi", ""),
|
|
46
|
+
}
|
|
47
|
+
line = json.dumps(entry, ensure_ascii=False) + "\n"
|
|
48
|
+
with _progress_lock:
|
|
49
|
+
with progress_file.open("a", encoding="utf-8") as f:
|
|
50
|
+
f.write(line)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _load_progress(batch_id: str) -> dict[str, dict[str, Any]]:
|
|
54
|
+
"""Load completed results from progress file. Returns {identifier: result}."""
|
|
55
|
+
progress_file = _pkg()._get_progress_file(batch_id)
|
|
56
|
+
completed: dict[str, dict[str, Any]] = {}
|
|
57
|
+
if not progress_file.exists():
|
|
58
|
+
return completed
|
|
59
|
+
try:
|
|
60
|
+
with progress_file.open("r", encoding="utf-8") as f:
|
|
61
|
+
for line in f:
|
|
62
|
+
line = line.strip()
|
|
63
|
+
if not line:
|
|
64
|
+
continue
|
|
65
|
+
entry = json.loads(line)
|
|
66
|
+
ident = entry.get("identifier", "")
|
|
67
|
+
if ident and entry.get("success"):
|
|
68
|
+
completed[ident] = entry
|
|
69
|
+
except Exception:
|
|
70
|
+
pass
|
|
71
|
+
return completed
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _clear_progress(batch_id: str) -> None:
|
|
75
|
+
"""Remove progress file after successful completion."""
|
|
76
|
+
progress_file = _pkg()._get_progress_file(batch_id)
|
|
77
|
+
if progress_file.exists():
|
|
78
|
+
try:
|
|
79
|
+
progress_file.unlink()
|
|
80
|
+
except OSError:
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _batch_institutional_phase(
|
|
85
|
+
failed_dois: list[str],
|
|
86
|
+
output_dir: Path,
|
|
87
|
+
config: dict[str, Any],
|
|
88
|
+
batch_id: str,
|
|
89
|
+
results_map: dict[str, dict[str, Any]],
|
|
90
|
+
batch_gen: int | None = None,
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Phase 2 of a batch: the school routes (WebVPN, CARSI), one paper at a time.
|
|
93
|
+
|
|
94
|
+
Through the same engine as single downloads (2026-10-20: the per-publisher
|
|
95
|
+
PublisherBatchDownloader is gone), so session self-healing, failure reasons,
|
|
96
|
+
skip and cancel work here too. Phase 1 already tried the free sources and
|
|
97
|
+
the publisher's own site, so only the school routes run. Modifies
|
|
98
|
+
results_map in place.
|
|
99
|
+
"""
|
|
100
|
+
prev_gen = getattr(_pkg()._DL_TLS, "gen", None)
|
|
101
|
+
if batch_gen is not None:
|
|
102
|
+
_pkg()._DL_TLS.gen = batch_gen # the downloads below belong to the batch: its cancel stops them
|
|
103
|
+
try:
|
|
104
|
+
_pkg()._school_routes_each(failed_dois, output_dir, config, batch_id, results_map, batch_gen)
|
|
105
|
+
finally:
|
|
106
|
+
_pkg()._DL_TLS.gen = prev_gen
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _school_routes_each(failed_dois, output_dir, config, batch_id, results_map, batch_gen) -> None:
|
|
110
|
+
for doi in failed_dois:
|
|
111
|
+
if doi in results_map and results_map[doi].get("success"):
|
|
112
|
+
continue
|
|
113
|
+
if batch_gen is not None and _pkg()._CANCEL_GEN[0] != batch_gen:
|
|
114
|
+
log.info(f"Batch {batch_id}: cancelled — stopping phase 2")
|
|
115
|
+
break
|
|
116
|
+
# A PDF already on disk (an earlier run, a partial download)
|
|
117
|
+
existing = next((p for p in output_dir.glob(f"{safe_filename(doi)}*.pdf") if p.stat().st_size > 5000), None)
|
|
118
|
+
if existing:
|
|
119
|
+
result = {"success": True, "doi": doi, "identifier": doi, "file": str(existing), "source": "local_cache"}
|
|
120
|
+
else:
|
|
121
|
+
log.info(f" [Batch] Phase 2 (school routes): {doi}")
|
|
122
|
+
result = _pkg().download(
|
|
123
|
+
doi,
|
|
124
|
+
output_dir,
|
|
125
|
+
scihub_enabled=config.get("scihub_enabled", False),
|
|
126
|
+
use_vpnsci=True,
|
|
127
|
+
_institutional=True,
|
|
128
|
+
_batch=True,
|
|
129
|
+
_school_only=True,
|
|
130
|
+
)
|
|
131
|
+
results_map[doi] = result
|
|
132
|
+
_pkg()._save_progress(batch_id, doi, result)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _write_download_results(results: list[dict[str, Any]], output_dir: str | Path | None, **extra: Any) -> None:
|
|
136
|
+
"""Persist batch outcomes as download_results.json for scansci-find reconcile."""
|
|
137
|
+
try:
|
|
138
|
+
out = Path(output_dir) if output_dir else Path(_pkg().load_config().get("output_dir", "."))
|
|
139
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
140
|
+
report: dict[str, Any] = {
|
|
141
|
+
"service": "litloop",
|
|
142
|
+
"total": len(results),
|
|
143
|
+
"succeeded": sum(1 for r in results if r and r.get("success")),
|
|
144
|
+
"entries": [
|
|
145
|
+
{
|
|
146
|
+
"identifier": r.get("identifier") or r.get("doi") or "",
|
|
147
|
+
"doi": r.get("doi") or "",
|
|
148
|
+
"success": bool(r.get("success")),
|
|
149
|
+
"source": r.get("source") or "",
|
|
150
|
+
"file": r.get("file") or "",
|
|
151
|
+
"cached": bool(r.get("cached")),
|
|
152
|
+
"preprint_fallback": r.get("preprint_fallback") or "",
|
|
153
|
+
"error_type": r.get("error_type") or "",
|
|
154
|
+
"reason": r.get("reason") or r.get("error") or "",
|
|
155
|
+
}
|
|
156
|
+
for r in results
|
|
157
|
+
if r
|
|
158
|
+
],
|
|
159
|
+
}
|
|
160
|
+
report.update(extra)
|
|
161
|
+
target = out / "download_results.json"
|
|
162
|
+
tmp = out / "download_results.json.tmp"
|
|
163
|
+
tmp.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
164
|
+
tmp.replace(target)
|
|
165
|
+
except Exception:
|
|
166
|
+
pass
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _apply_preprint_fallbacks(
|
|
170
|
+
fallbacks: dict[str, list[str]],
|
|
171
|
+
identifiers: list[str],
|
|
172
|
+
final_map: dict[str, dict[str, Any]],
|
|
173
|
+
batch_id: str,
|
|
174
|
+
*,
|
|
175
|
+
output_dir: str | Path | None,
|
|
176
|
+
scihub_enabled: bool | None,
|
|
177
|
+
use_tor: bool,
|
|
178
|
+
use_vpnsci: bool,
|
|
179
|
+
) -> None:
|
|
180
|
+
"""Retry failed DOIs with their matched preprint arXiv IDs, in place.
|
|
181
|
+
|
|
182
|
+
Updates ``final_map`` (and the batch progress file) when a fallback
|
|
183
|
+
download succeeds so downstream result aggregation picks it up.
|
|
184
|
+
"""
|
|
185
|
+
for ident in [i for i in identifiers if not (final_map.get(i) or {}).get("success")]:
|
|
186
|
+
for fallback_id in fallbacks.get(ident, []):
|
|
187
|
+
log.info(f" [Fallback] {ident} -> preprint {fallback_id}")
|
|
188
|
+
fb_result = _pkg().download(
|
|
189
|
+
fallback_id,
|
|
190
|
+
output_dir,
|
|
191
|
+
scihub_enabled=scihub_enabled,
|
|
192
|
+
use_tor=use_tor,
|
|
193
|
+
use_vpnsci=False,
|
|
194
|
+
_institutional=False,
|
|
195
|
+
_batch=True,
|
|
196
|
+
)
|
|
197
|
+
if fb_result and fb_result.get("success"):
|
|
198
|
+
fb_result["identifier"] = ident
|
|
199
|
+
fb_result["doi"] = ident
|
|
200
|
+
fb_result["preprint_fallback"] = fallback_id
|
|
201
|
+
_pkg()._save_progress(batch_id, ident, fb_result)
|
|
202
|
+
final_map[ident] = fb_result
|
|
203
|
+
break
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def institutional_download(
|
|
207
|
+
identifier: str,
|
|
208
|
+
output_dir: str | Path | None = None,
|
|
209
|
+
*,
|
|
210
|
+
use_vpnsci: bool = False,
|
|
211
|
+
heal: bool = True,
|
|
212
|
+
) -> dict[str, Any] | None:
|
|
213
|
+
"""Phase 2 of a single download on its own: the school's routes (WebVPN,
|
|
214
|
+
CARSI, EZProxy, one after another), after refreshing their sessions.
|
|
215
|
+
|
|
216
|
+
Batches run it, one paper at a time, for what the faster lanes missed: a
|
|
217
|
+
single download always had this phase, a batch had none (its grey lane
|
|
218
|
+
turns it off so ten workers do not open ten login windows). heal=False
|
|
219
|
+
skips the session refresh for the papers after the first.
|
|
220
|
+
"""
|
|
221
|
+
config = _pkg().load_config()
|
|
222
|
+
if config.get("download_strategy") in ("scihub_only", "grey_only"):
|
|
223
|
+
return None
|
|
224
|
+
target_dir = Path(output_dir) if output_dir else Path(config["output_dir"])
|
|
225
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
226
|
+
identifier = identifier.strip()
|
|
227
|
+
doi = normalize_doi(identifier) if not is_arxiv_identifier(identifier) else identifier
|
|
228
|
+
output_path = target_dir / f"{safe_filename(identifier)}.pdf"
|
|
229
|
+
if heal:
|
|
230
|
+
try:
|
|
231
|
+
from ..institutional.session_heal import ensure_institutional_sessions
|
|
232
|
+
for channel, state in ensure_institutional_sessions(
|
|
233
|
+
config, use_vpnsci=use_vpnsci or bool(config.get("vpnsci_enabled"))).items():
|
|
234
|
+
log.info(f" [session-heal] {channel}: {state.get('status')}")
|
|
235
|
+
except Exception as exc:
|
|
236
|
+
log.info(f" [session-heal] skipped: {exc}")
|
|
237
|
+
inst_sources = _pkg()._build_institutional_sources(doi, config, use_vpnsci=use_vpnsci)
|
|
238
|
+
if not inst_sources:
|
|
239
|
+
return None
|
|
240
|
+
log.info(f" [school route] {doi}")
|
|
241
|
+
result = _pkg()._run_institutional_sequence(inst_sources, doi, target_dir, output_path, config)
|
|
242
|
+
if not result:
|
|
243
|
+
return None
|
|
244
|
+
from ..publishers.strategies._core import clear_no_access
|
|
245
|
+
clear_no_access(doi) # the school routes got it: "not subscribed" was wrong
|
|
246
|
+
_pkg()._update_doi_index(target_dir, doi, Path(result.get("file", "")), source=result.get("source") or "",
|
|
247
|
+
strategy=config.get("download_strategy", ""), config=config)
|
|
248
|
+
_pkg()._auto_rename(result, identifier, config, doi=doi, target_dir=target_dir)
|
|
249
|
+
_pkg().cache_set(identifier, result, config)
|
|
250
|
+
return result
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def batch_download(
|
|
254
|
+
identifiers: list[str],
|
|
255
|
+
output_dir: str | Path | None = None,
|
|
256
|
+
*,
|
|
257
|
+
scihub_enabled: bool | None = None,
|
|
258
|
+
use_tor: bool = False,
|
|
259
|
+
use_vpnsci: bool = False,
|
|
260
|
+
progress_callback: Any = None,
|
|
261
|
+
batch_id: str | None = None,
|
|
262
|
+
resume: bool = True,
|
|
263
|
+
fallbacks: dict[str, list[str]] | None = None,
|
|
264
|
+
) -> dict[str, Any]:
|
|
265
|
+
config = _pkg().load_config()
|
|
266
|
+
workers = config.get("batch_workers", 5)
|
|
267
|
+
batch_t0 = time.perf_counter()
|
|
268
|
+
|
|
269
|
+
# Duplicate detection via DOI normalization
|
|
270
|
+
seen_dois: set[str] = set()
|
|
271
|
+
unique_identifiers: list[str] = []
|
|
272
|
+
skipped_duplicates = 0
|
|
273
|
+
for ident in identifiers:
|
|
274
|
+
normalized = normalize_doi(ident.strip()) if not is_arxiv_identifier(ident) else ident.strip()
|
|
275
|
+
if normalized.lower() in seen_dois:
|
|
276
|
+
skipped_duplicates += 1
|
|
277
|
+
continue
|
|
278
|
+
seen_dois.add(normalized.lower())
|
|
279
|
+
unique_identifiers.append(ident)
|
|
280
|
+
|
|
281
|
+
if skipped_duplicates > 0:
|
|
282
|
+
log.info(f"Skipped {skipped_duplicates} duplicate identifiers")
|
|
283
|
+
|
|
284
|
+
# Auto-generate batch_id from identifiers if not provided
|
|
285
|
+
if not batch_id:
|
|
286
|
+
import hashlib as _hashlib
|
|
287
|
+
batch_id = _hashlib.md5("|".join(sorted(unique_identifiers)).encode()).hexdigest()[:12]
|
|
288
|
+
|
|
289
|
+
# Load previous progress for resume
|
|
290
|
+
completed_map: dict[str, dict[str, Any]] = {}
|
|
291
|
+
if resume:
|
|
292
|
+
completed_map = _pkg()._load_progress(batch_id)
|
|
293
|
+
if completed_map:
|
|
294
|
+
log.info(f"Resuming batch {batch_id}: {len(completed_map)} already completed")
|
|
295
|
+
|
|
296
|
+
# Filter out already-completed identifiers
|
|
297
|
+
pending_identifiers = [i for i in unique_identifiers if i not in completed_map]
|
|
298
|
+
skipped_completed = len(unique_identifiers) - len(pending_identifiers)
|
|
299
|
+
|
|
300
|
+
if not pending_identifiers:
|
|
301
|
+
log.info("All items already completed")
|
|
302
|
+
all_results = [completed_map[i] for i in unique_identifiers]
|
|
303
|
+
succeeded = sum(1 for r in all_results if r.get("success"))
|
|
304
|
+
summary = {
|
|
305
|
+
"total": len(identifiers),
|
|
306
|
+
"unique": len(unique_identifiers),
|
|
307
|
+
"skipped_duplicates": skipped_duplicates,
|
|
308
|
+
"skipped_completed": skipped_completed,
|
|
309
|
+
"succeeded": succeeded,
|
|
310
|
+
"failed": len(unique_identifiers) - succeeded,
|
|
311
|
+
"results": all_results,
|
|
312
|
+
"failed_dois": [i for i in unique_identifiers if not completed_map.get(i, {}).get("success")],
|
|
313
|
+
"batch_id": batch_id,
|
|
314
|
+
}
|
|
315
|
+
_pkg()._write_download_results(all_results, output_dir)
|
|
316
|
+
return summary
|
|
317
|
+
|
|
318
|
+
# Pre-validate DOIs concurrently
|
|
319
|
+
from ..identifiers import validate_doi
|
|
320
|
+
valid_identifiers: list[str] = []
|
|
321
|
+
invalid_results: list[dict[str, Any]] = []
|
|
322
|
+
arxiv_ids = [i for i in pending_identifiers if is_arxiv_identifier(i)]
|
|
323
|
+
doi_ids = [i for i in pending_identifiers if not is_arxiv_identifier(i)]
|
|
324
|
+
|
|
325
|
+
if doi_ids:
|
|
326
|
+
log.info(f"Batch {batch_id}: validating {len(doi_ids)} DOIs...")
|
|
327
|
+
with ThreadPoolExecutor(max_workers=min(10, len(doi_ids))) as pool:
|
|
328
|
+
futures = {pool.submit(validate_doi, normalize_doi(i)): i for i in doi_ids}
|
|
329
|
+
for future in as_completed(futures, timeout=60):
|
|
330
|
+
ident = futures[future]
|
|
331
|
+
try:
|
|
332
|
+
valid, msg = future.result()
|
|
333
|
+
except Exception:
|
|
334
|
+
valid, msg = True, "validation error"
|
|
335
|
+
if valid:
|
|
336
|
+
valid_identifiers.append(ident)
|
|
337
|
+
else:
|
|
338
|
+
log.info(f" SKIP {ident}: {msg}")
|
|
339
|
+
r = fail(ident, f"Invalid DOI: {msg}")
|
|
340
|
+
invalid_results.append(r)
|
|
341
|
+
_pkg()._save_progress(batch_id, ident, r)
|
|
342
|
+
|
|
343
|
+
valid_identifiers.extend(arxiv_ids)
|
|
344
|
+
pending_identifiers = valid_identifiers
|
|
345
|
+
|
|
346
|
+
if invalid_results:
|
|
347
|
+
log.info(f"Batch {batch_id}: {len(invalid_results)} invalid DOIs skipped")
|
|
348
|
+
|
|
349
|
+
if not pending_identifiers:
|
|
350
|
+
log.info("No valid identifiers to download")
|
|
351
|
+
all_results = [completed_map.get(i) or fail(i, "invalid") for i in unique_identifiers]
|
|
352
|
+
succeeded = sum(1 for r in all_results if r and r.get("success"))
|
|
353
|
+
return {
|
|
354
|
+
"total": len(identifiers),
|
|
355
|
+
"unique": len(unique_identifiers),
|
|
356
|
+
"skipped_duplicates": skipped_duplicates,
|
|
357
|
+
"skipped_completed": skipped_completed,
|
|
358
|
+
"skipped_invalid": len(invalid_results),
|
|
359
|
+
"succeeded": succeeded,
|
|
360
|
+
"failed": len(unique_identifiers) - succeeded,
|
|
361
|
+
"results": all_results,
|
|
362
|
+
"batch_id": batch_id,
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
log.info(f"Batch {batch_id}: downloading {len(pending_identifiers)} items ({skipped_completed} skipped, {len(invalid_results)} invalid)")
|
|
366
|
+
|
|
367
|
+
delay_lock = threading.Lock()
|
|
368
|
+
last_download_time = [0.0]
|
|
369
|
+
delay_between = float(config.get("batch_stagger_seconds", 0.3))
|
|
370
|
+
total = len(pending_identifiers)
|
|
371
|
+
completed_count = [0]
|
|
372
|
+
num_invalid = len(invalid_results)
|
|
373
|
+
|
|
374
|
+
batch_gen = _pkg()._CANCEL_GEN[0]
|
|
375
|
+
|
|
376
|
+
def _staggered_download(ident: str) -> dict[str, Any]:
|
|
377
|
+
if _pkg()._CANCEL_GEN[0] != batch_gen:
|
|
378
|
+
return fail(ident, "cancelled by user", error_type="cancelled", action="none")
|
|
379
|
+
with delay_lock:
|
|
380
|
+
elapsed = time.time() - last_download_time[0]
|
|
381
|
+
if elapsed < delay_between:
|
|
382
|
+
time.sleep(delay_between - elapsed)
|
|
383
|
+
last_download_time[0] = time.time()
|
|
384
|
+
return _pkg().download(
|
|
385
|
+
ident,
|
|
386
|
+
output_dir,
|
|
387
|
+
scihub_enabled=scihub_enabled,
|
|
388
|
+
use_tor=use_tor,
|
|
389
|
+
use_vpnsci=use_vpnsci,
|
|
390
|
+
_institutional=False,
|
|
391
|
+
_batch=True,
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
results: list[dict[str, Any] | None] = [None] * total
|
|
395
|
+
total_budget = float(config.get("batch_total_timeout", 600))
|
|
396
|
+
pool = ThreadPoolExecutor(max_workers=workers)
|
|
397
|
+
future_to_idx = {pool.submit(_staggered_download, ident): i for i, ident in enumerate(pending_identifiers)}
|
|
398
|
+
try:
|
|
399
|
+
for future in as_completed(future_to_idx, timeout=total_budget):
|
|
400
|
+
idx = future_to_idx[future]
|
|
401
|
+
try:
|
|
402
|
+
result = future.result()
|
|
403
|
+
except Exception:
|
|
404
|
+
result = fail(pending_identifiers[idx], "download exception")
|
|
405
|
+
results[idx] = result
|
|
406
|
+
|
|
407
|
+
# Save progress immediately
|
|
408
|
+
_pkg()._save_progress(batch_id, pending_identifiers[idx], result)
|
|
409
|
+
|
|
410
|
+
completed_count[0] += 1
|
|
411
|
+
if progress_callback:
|
|
412
|
+
try:
|
|
413
|
+
progress_callback(
|
|
414
|
+
completed_count[0] + skipped_completed + num_invalid,
|
|
415
|
+
len(unique_identifiers),
|
|
416
|
+
pending_identifiers[idx],
|
|
417
|
+
result,
|
|
418
|
+
)
|
|
419
|
+
except Exception:
|
|
420
|
+
pass
|
|
421
|
+
except TimeoutError:
|
|
422
|
+
log.info(f"Batch {batch_id}: timeout after {int(total_budget)}s")
|
|
423
|
+
finally:
|
|
424
|
+
# Waive stragglers so the batch report always lands within budget;
|
|
425
|
+
# per-source I/O deadlines bound the lifetime of abandoned workers.
|
|
426
|
+
alive = [f for f in future_to_idx if not f.done()]
|
|
427
|
+
if alive:
|
|
428
|
+
log.info(f"Batch {batch_id}: {len(alive)} source(s) still running (waived)")
|
|
429
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
430
|
+
|
|
431
|
+
for i, r in enumerate(results):
|
|
432
|
+
if r is None:
|
|
433
|
+
results[i] = fail(pending_identifiers[i], "timeout or incomplete")
|
|
434
|
+
|
|
435
|
+
# Phase 2: Publisher-grouped institutional download for failed DOIs
|
|
436
|
+
phase1_results = dict(zip(pending_identifiers, results))
|
|
437
|
+
failed_phase1 = [doi for doi, r in phase1_results.items() if r and not r.get("success")]
|
|
438
|
+
if failed_phase1 and _pkg()._CANCEL_GEN[0] != batch_gen:
|
|
439
|
+
log.info(f"Batch {batch_id}: cancelled — skipping Phase 2")
|
|
440
|
+
failed_phase1 = []
|
|
441
|
+
if failed_phase1:
|
|
442
|
+
log.info(f"Batch {batch_id}: Phase 1 failed for {len(failed_phase1)} DOIs, starting Phase 2...")
|
|
443
|
+
_pkg()._batch_institutional_phase(
|
|
444
|
+
failed_phase1,
|
|
445
|
+
Path(config["output_dir"]) if not output_dir else Path(output_dir),
|
|
446
|
+
config,
|
|
447
|
+
batch_id,
|
|
448
|
+
phase1_results,
|
|
449
|
+
batch_gen,
|
|
450
|
+
)
|
|
451
|
+
# Update results list with Phase 2 outcomes from progress file
|
|
452
|
+
phase2_progress = _pkg()._load_progress(batch_id)
|
|
453
|
+
for i, ident in enumerate(pending_identifiers):
|
|
454
|
+
if ident in phase2_progress:
|
|
455
|
+
results[i] = phase2_progress[ident]
|
|
456
|
+
|
|
457
|
+
# Reload progress to include newly-saved invalid results
|
|
458
|
+
final_map = _pkg()._load_progress(batch_id)
|
|
459
|
+
|
|
460
|
+
# Preprint fallback: DOIs that failed every phase retry with the arXiv
|
|
461
|
+
# preprint IDs recorded in the ScanSci Find download queue (fallbacks map).
|
|
462
|
+
if fallbacks:
|
|
463
|
+
_pkg()._apply_preprint_fallbacks(
|
|
464
|
+
fallbacks, pending_identifiers, final_map, batch_id,
|
|
465
|
+
output_dir=output_dir, scihub_enabled=scihub_enabled, use_tor=use_tor, use_vpnsci=use_vpnsci,
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
# Build a lookup from pending_identifiers → download results
|
|
469
|
+
pending_results = dict(zip(pending_identifiers, results))
|
|
470
|
+
|
|
471
|
+
# Merge completed_map with new results (dict lookup, no fragile index counter)
|
|
472
|
+
all_results = []
|
|
473
|
+
for ident in unique_identifiers:
|
|
474
|
+
if ident in final_map:
|
|
475
|
+
all_results.append(final_map[ident])
|
|
476
|
+
elif ident in completed_map:
|
|
477
|
+
all_results.append(completed_map[ident])
|
|
478
|
+
elif ident in pending_results:
|
|
479
|
+
all_results.append(pending_results[ident])
|
|
480
|
+
else:
|
|
481
|
+
all_results.append(fail(ident, "missing result"))
|
|
482
|
+
|
|
483
|
+
succeeded = sum(1 for r in all_results if r and r.get("success"))
|
|
484
|
+
failed_dois = [r["identifier"] for r in all_results if r and not r.get("success")]
|
|
485
|
+
|
|
486
|
+
# Clean up progress file if all succeeded
|
|
487
|
+
if not failed_dois:
|
|
488
|
+
_pkg()._clear_progress(batch_id)
|
|
489
|
+
|
|
490
|
+
summary = {
|
|
491
|
+
"total": len(identifiers),
|
|
492
|
+
"unique": len(unique_identifiers),
|
|
493
|
+
"skipped_duplicates": skipped_duplicates,
|
|
494
|
+
"skipped_completed": skipped_completed,
|
|
495
|
+
"skipped_invalid": len(invalid_results),
|
|
496
|
+
"succeeded": succeeded,
|
|
497
|
+
"failed": len(failed_dois),
|
|
498
|
+
"results": all_results,
|
|
499
|
+
"failed_dois": failed_dois,
|
|
500
|
+
"batch_id": batch_id,
|
|
501
|
+
}
|
|
502
|
+
_pkg()._close_batch_browsers(batch_t0, unique_identifiers)
|
|
503
|
+
_pkg()._write_download_results(all_results, output_dir)
|
|
504
|
+
return summary
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _close_batch_browsers(t0: float, identifiers: list[str]) -> None:
|
|
508
|
+
"""End-of-batch cleanup: waived downloads stop launching, their windows close.
|
|
509
|
+
|
|
510
|
+
Per-paper cleanup in download() is skipped while other papers of the batch
|
|
511
|
+
are still in flight, so windows of waived stragglers would otherwise stay
|
|
512
|
+
open after the batch reports. Only closes when every download still in
|
|
513
|
+
flight belongs to this batch, never a concurrent unrelated download.
|
|
514
|
+
"""
|
|
515
|
+
keys = {normalize_doi(i) if not is_arxiv_identifier(i) else i for i in identifiers}
|
|
516
|
+
pkg = _pkg()
|
|
517
|
+
with pkg._INFLIGHT_LOCK:
|
|
518
|
+
pkg._SETTLED.update(keys)
|
|
519
|
+
foreign = set(pkg._attempt.active_keys()) - keys
|
|
520
|
+
if foreign:
|
|
521
|
+
log.info(f"Batch cleanup skipped: {len(foreign)} unrelated download(s) in flight")
|
|
522
|
+
return
|
|
523
|
+
try:
|
|
524
|
+
from ..browser.backend import close_browsers_since
|
|
525
|
+
n = close_browsers_since(t0)
|
|
526
|
+
if n:
|
|
527
|
+
log.info(f"Batch cleanup: closed {n} browser(s) left open by this batch")
|
|
528
|
+
except Exception as exc:
|
|
529
|
+
log.info(f"Batch browser cleanup skipped: {exc}")
|