litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Local DOI index, cache validation, and PDF auto-rename helpers.
|
|
2
|
+
|
|
3
|
+
Extracted on 2026-11-20 from ``sources/__init__.py``; all exported names are
|
|
4
|
+
re-exported by ``sources/__init__.py``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from ..identifiers import safe_filename
|
|
16
|
+
from ..log import get_logger
|
|
17
|
+
|
|
18
|
+
log = get_logger()
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"_EXPLICIT_STRATEGIES",
|
|
22
|
+
"_index_entry",
|
|
23
|
+
"_index_entry_is_fresh",
|
|
24
|
+
"_index_strategy_compatible",
|
|
25
|
+
"_norm_path",
|
|
26
|
+
"_files_claimed_by_other_dois",
|
|
27
|
+
"_pdf_mentions_doi",
|
|
28
|
+
"_same_dir",
|
|
29
|
+
"_update_doi_index",
|
|
30
|
+
"_auto_rename",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _pkg() -> Any:
|
|
35
|
+
return sys.modules[__package__]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
#: Strategies that encode an explicit user source preference. For these, a
|
|
39
|
+
#: cached file obtained through a different strategy is treated as a miss so
|
|
40
|
+
#: the requested channel is actually used; the default strategies reuse cache.
|
|
41
|
+
_EXPLICIT_STRATEGIES = ("scihub_only", "legal_only", "scihub_first")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _index_entry(entry: Any) -> dict[str, Any] | None:
|
|
45
|
+
"""Normalize a .doi_index entry to a record dict.
|
|
46
|
+
|
|
47
|
+
Legacy ``str -> path`` entries have no provenance and no timestamp; they
|
|
48
|
+
are treated as fresh (ts=None) and upgraded on the next cache hit.
|
|
49
|
+
"""
|
|
50
|
+
if isinstance(entry, str):
|
|
51
|
+
return {"file": entry, "source": "", "strategy": "", "ts": None}
|
|
52
|
+
if isinstance(entry, dict):
|
|
53
|
+
ts = entry.get("ts")
|
|
54
|
+
try:
|
|
55
|
+
ts_value = float(ts) if ts is not None else None
|
|
56
|
+
except (TypeError, ValueError):
|
|
57
|
+
ts_value = None
|
|
58
|
+
return {
|
|
59
|
+
"file": str(entry.get("file") or ""),
|
|
60
|
+
"source": str(entry.get("source") or ""),
|
|
61
|
+
"strategy": str(entry.get("strategy") or ""),
|
|
62
|
+
"ts": ts_value,
|
|
63
|
+
}
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _index_entry_is_fresh(entry: dict[str, Any], config: dict[str, Any]) -> bool:
|
|
68
|
+
"""Apply the cache TTL (cache_ttl_hours) to an index entry."""
|
|
69
|
+
try:
|
|
70
|
+
ttl_hours = float(config.get("cache_ttl_hours", 168))
|
|
71
|
+
except (TypeError, ValueError):
|
|
72
|
+
ttl_hours = 168.0
|
|
73
|
+
if ttl_hours <= 0 or entry.get("ts") is None:
|
|
74
|
+
return True
|
|
75
|
+
return (time.time() - entry["ts"]) <= ttl_hours * 3600
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _index_strategy_compatible(entry: dict[str, Any], config: dict[str, Any]) -> bool:
|
|
79
|
+
"""Honor an explicitly requested download strategy over cached files."""
|
|
80
|
+
current = str(config.get("download_strategy") or "fastest").strip()
|
|
81
|
+
if current not in _pkg()._EXPLICIT_STRATEGIES:
|
|
82
|
+
return True
|
|
83
|
+
recorded = str(entry.get("strategy") or "").strip()
|
|
84
|
+
return recorded == current
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _norm_path(p: Path | str) -> str:
|
|
88
|
+
return str(Path(p).resolve()).lower()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _files_claimed_by_other_dois(target_dir: Path, doi: str) -> set[str]:
|
|
92
|
+
"""Files the DOI index already assigns to a different DOI."""
|
|
93
|
+
try:
|
|
94
|
+
idx = json.loads((target_dir / ".doi_index.json").read_text(encoding="utf-8"))
|
|
95
|
+
except Exception:
|
|
96
|
+
return set()
|
|
97
|
+
claimed = set()
|
|
98
|
+
for key, raw in idx.items():
|
|
99
|
+
entry = _index_entry(raw)
|
|
100
|
+
if key != doi and entry and entry.get("file"):
|
|
101
|
+
claimed.add(_norm_path(entry["file"]))
|
|
102
|
+
return claimed
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _pdf_mentions_doi(path: Path, doi: str, pages: int = 3) -> bool:
|
|
106
|
+
"""True when the DOI appears in the first pages' text (publisher PDFs print it)."""
|
|
107
|
+
from ..download.pdf import doi_printed
|
|
108
|
+
return doi_printed(path, doi, pages)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _same_dir(a: Path, b: Path) -> bool:
|
|
112
|
+
try:
|
|
113
|
+
return a.resolve() == b.resolve()
|
|
114
|
+
except OSError:
|
|
115
|
+
return str(a) == str(b)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _update_doi_index(
|
|
119
|
+
target_dir: Path,
|
|
120
|
+
doi: str,
|
|
121
|
+
file_path: Path,
|
|
122
|
+
*,
|
|
123
|
+
source: str = "",
|
|
124
|
+
strategy: str = "",
|
|
125
|
+
config: dict[str, Any] | None = None,
|
|
126
|
+
) -> None:
|
|
127
|
+
"""Update the DOI→file index for dedup.
|
|
128
|
+
|
|
129
|
+
Records carry provenance (``source``/``strategy``) and a timestamp so
|
|
130
|
+
stale or channel-mismatched entries can be invalidated by TTL and by
|
|
131
|
+
explicit strategy changes instead of manual ``rm``. Existing entries keep
|
|
132
|
+
their recorded source/strategy when the caller does not supply new ones
|
|
133
|
+
(e.g. a rename that just moves the file).
|
|
134
|
+
"""
|
|
135
|
+
doi_index = target_dir / ".doi_index.json"
|
|
136
|
+
try:
|
|
137
|
+
idx: dict[str, Any] = {}
|
|
138
|
+
if doi_index.exists():
|
|
139
|
+
idx = json.loads(doi_index.read_text(encoding="utf-8"))
|
|
140
|
+
existing = _index_entry(idx.get(doi)) if doi in idx else None
|
|
141
|
+
if existing:
|
|
142
|
+
source = source or existing["source"]
|
|
143
|
+
strategy = strategy or existing["strategy"]
|
|
144
|
+
idx[doi] = {
|
|
145
|
+
"file": str(file_path),
|
|
146
|
+
"source": source or "",
|
|
147
|
+
"strategy": strategy or (config or {}).get("download_strategy", "") or "",
|
|
148
|
+
"ts": time.time(),
|
|
149
|
+
}
|
|
150
|
+
doi_index.write_text(json.dumps(idx, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
151
|
+
except Exception:
|
|
152
|
+
pass
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _auto_rename(result: dict[str, Any], identifier: str, config: dict[str, Any], doi: str = "", target_dir: Path | None = None) -> None:
|
|
156
|
+
"""Auto-rename downloaded PDF based on metadata."""
|
|
157
|
+
from ..download.rename import rename_format
|
|
158
|
+
if rename_format(config) == "off":
|
|
159
|
+
return
|
|
160
|
+
file_path = Path(result.get("file", ""))
|
|
161
|
+
if not file_path.exists():
|
|
162
|
+
return
|
|
163
|
+
if file_path.suffix.lower() != ".pdf":
|
|
164
|
+
# Non-PDF artifacts (e.g. the ElsevierAPI text/plain full text) keep
|
|
165
|
+
# their own extension — rename_pdf would force a .pdf name onto them.
|
|
166
|
+
log.info(f" Non-PDF artifact ({file_path.suffix or 'no ext'}) — skipping auto-rename")
|
|
167
|
+
return
|
|
168
|
+
# Use cached metadata or fetch from Crossref
|
|
169
|
+
from ..citations.formats import fetch_metadata
|
|
170
|
+
_doi = doi or result.get("doi", identifier)
|
|
171
|
+
metadata = fetch_metadata(_doi, config)
|
|
172
|
+
if metadata:
|
|
173
|
+
new_path = _pkg().rename_pdf(file_path, metadata, doi=_doi, fmt=rename_format(config))
|
|
174
|
+
if new_path and result.get("whole_book") and "整本书" not in new_path.stem:
|
|
175
|
+
# A chapter delivered as its whole book says so in its name.
|
|
176
|
+
tagged = new_path.with_name(f"{new_path.stem}(整本书){new_path.suffix}")
|
|
177
|
+
try:
|
|
178
|
+
new_path.rename(tagged)
|
|
179
|
+
new_path = tagged
|
|
180
|
+
except OSError:
|
|
181
|
+
pass
|
|
182
|
+
if new_path and result.get("preprint") and "_preprint" not in new_path.stem:
|
|
183
|
+
# A preprint of the paper is not the published PDF: say so in the
|
|
184
|
+
# name, so it is not mistaken for the version of record later.
|
|
185
|
+
tagged = new_path.with_name(f"{new_path.stem}_preprint{new_path.suffix}")
|
|
186
|
+
try:
|
|
187
|
+
new_path.rename(tagged)
|
|
188
|
+
new_path = tagged
|
|
189
|
+
except OSError:
|
|
190
|
+
pass
|
|
191
|
+
if new_path and new_path != file_path:
|
|
192
|
+
result["file"] = str(new_path)
|
|
193
|
+
result["renamed"] = True
|
|
194
|
+
log.info(f" Renamed: {file_path.name} -> {new_path.name}")
|
|
195
|
+
# Racing leaves one "<doi>_<Source>.pdf" copy per source that also
|
|
196
|
+
# succeeded; the winner is now saved under its final name.
|
|
197
|
+
if _doi:
|
|
198
|
+
removed = 0
|
|
199
|
+
for dup in new_path.parent.glob(f"{safe_filename(_doi)}_*.pdf"):
|
|
200
|
+
if dup != new_path:
|
|
201
|
+
try:
|
|
202
|
+
dup.unlink()
|
|
203
|
+
removed += 1
|
|
204
|
+
except OSError:
|
|
205
|
+
pass
|
|
206
|
+
if removed:
|
|
207
|
+
log.info(f" Removed {removed} duplicate source cop{'y' if removed == 1 else 'ies'}")
|
|
208
|
+
# Update DOI→file index for dedup
|
|
209
|
+
if target_dir and _doi:
|
|
210
|
+
_pkg()._update_doi_index(
|
|
211
|
+
target_dir,
|
|
212
|
+
_doi,
|
|
213
|
+
new_path,
|
|
214
|
+
source=result.get("source") or "",
|
|
215
|
+
strategy=config.get("download_strategy", ""),
|
|
216
|
+
config=config,
|
|
217
|
+
)
|
|
218
|
+
else:
|
|
219
|
+
log.info(f" Kept original name: {file_path.name}")
|
|
220
|
+
else:
|
|
221
|
+
log.info(f" No metadata for rename, keeping: {file_path.name}")
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Phase 1/2 source assembly, institutional routing sequence, watchdog, and failure guidance.
|
|
2
|
+
|
|
3
|
+
Split out of `sources/__init__.py` (2026-11-20); `sources/__init__.py` re-exports
|
|
4
|
+
all symbols below so callers and tests keep importing from there.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import sys
|
|
10
|
+
import threading
|
|
11
|
+
import time
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from ..identifiers import safe_filename
|
|
16
|
+
from ..log import get_logger
|
|
17
|
+
from .core_api import try_core
|
|
18
|
+
from .crossref import try_crossref_page_scrape
|
|
19
|
+
from .europepmc import try_europepmc, try_pmc
|
|
20
|
+
from .libgen import try_libgen
|
|
21
|
+
from .oa_discovery import try_doaj
|
|
22
|
+
from .openaire import try_openaire
|
|
23
|
+
from .openalex import try_openalex_content_api, try_openalex_oa
|
|
24
|
+
from .publishers import get_publisher_fast_sources
|
|
25
|
+
from .scibban import try_scibban
|
|
26
|
+
from .scihub import try_scihub
|
|
27
|
+
from .semantic_scholar import try_semanticscholar
|
|
28
|
+
from .unpaywall import try_unpaywall
|
|
29
|
+
|
|
30
|
+
log = get_logger()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _pkg() -> Any:
|
|
34
|
+
return sys.modules[__package__]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _build_free_sources(doi: str, config: dict[str, Any]) -> list[tuple[Any, str]]:
|
|
38
|
+
"""Build Phase 1 sources: all free/OA/grey sources, sorted by adaptive score.
|
|
39
|
+
|
|
40
|
+
Respects config['download_strategy']:
|
|
41
|
+
- fastest: all sources, EMA-sorted (default)
|
|
42
|
+
- scihub_first: all sources, but grey sources sorted first
|
|
43
|
+
- scihub_only: only SciBban + Sci-Hub + LibGen
|
|
44
|
+
- oa_first: all sources, but OA sources sorted before grey
|
|
45
|
+
- legal_only: exclude grey sources (SciBban, Sci-Hub, LibGen)
|
|
46
|
+
"""
|
|
47
|
+
from .scoring import sort_sources
|
|
48
|
+
|
|
49
|
+
strategy = config.get("download_strategy", "fastest")
|
|
50
|
+
|
|
51
|
+
publisher_fast = get_publisher_fast_sources(doi)
|
|
52
|
+
_fast_names = {label for _, label in publisher_fast}
|
|
53
|
+
|
|
54
|
+
extra_fast = []
|
|
55
|
+
for fn, label in [
|
|
56
|
+
(try_unpaywall, "Unpaywall"),
|
|
57
|
+
(try_openalex_oa, "OpenAlexOA"),
|
|
58
|
+
(try_semanticscholar, "SemanticScholar"),
|
|
59
|
+
(try_openaire, "OpenAIRE"),
|
|
60
|
+
]:
|
|
61
|
+
if label not in _fast_names:
|
|
62
|
+
extra_fast.append((fn, label))
|
|
63
|
+
|
|
64
|
+
from .elsevier_api import fulltext_enabled
|
|
65
|
+
if not fulltext_enabled(config): # off in settings, or the key has no full-text entitlement
|
|
66
|
+
publisher_fast = [(fn, label) for fn, label in publisher_fast if label != "ElsevierAPI"]
|
|
67
|
+
legal_sources = publisher_fast + extra_fast
|
|
68
|
+
# Every publisher's own site is tried here, in phase 1, from this exit:
|
|
69
|
+
# those without a browser lane of their own (PNAS, Optica, World Scientific,
|
|
70
|
+
# unlisted ones) get the generic one. Open-access hosts need none.
|
|
71
|
+
if not any("Browser" in label for label in _fast_names):
|
|
72
|
+
from .publishers import get_publisher
|
|
73
|
+
if get_publisher(doi) not in _pkg()._OPEN_HOSTS:
|
|
74
|
+
from .publishers import _FN_MAP
|
|
75
|
+
legal_sources.append((_FN_MAP["GenericBrowser"], "GenericBrowser"))
|
|
76
|
+
# Publishers whose site is not worth phase 1 (publisher_site_skip_prefixes):
|
|
77
|
+
# the school routes reach the same content faster.
|
|
78
|
+
skip = tuple(str(p).strip().lower().rstrip("/") + "/" for p in config.get("publisher_site_skip_prefixes") or [] if p)
|
|
79
|
+
if skip and doi.lower().startswith(skip):
|
|
80
|
+
legal_sources = [(fn, label) for fn, label in legal_sources if "Browser" not in label]
|
|
81
|
+
log.info(" Publisher site skipped in phase 1 (publisher_site_skip_prefixes)")
|
|
82
|
+
legal_sources += [(try_doaj, "DOAJ"), (try_crossref_page_scrape, "CrossrefPage")]
|
|
83
|
+
legal_sources += [(try_europepmc, "EuropePMC"), (try_core, "CORE")]
|
|
84
|
+
if not config.get("_batch"):
|
|
85
|
+
legal_sources.append((try_pmc, "PMC"))
|
|
86
|
+
|
|
87
|
+
if config.get("openalex_api_key"):
|
|
88
|
+
legal_sources.append((try_openalex_content_api, "OpenAlexContent"))
|
|
89
|
+
|
|
90
|
+
# Grey / shadow-library sources (Sci-Hub, SciBban, LibGen) are all gated
|
|
91
|
+
# by scihub_enabled and are off unless the user turned it on (CHARTER: the
|
|
92
|
+
# user must choose them explicitly; config.DEFAULT_CONFIG has False). A
|
|
93
|
+
# partial config dict without the key therefore means off, never on.
|
|
94
|
+
grey_sources: list[tuple[Any, str]] = []
|
|
95
|
+
if config.get("scihub_enabled", False):
|
|
96
|
+
from .scihub import recently_absent, too_new
|
|
97
|
+
new = too_new(doi, config) # published in or after scihub_skip_from_year
|
|
98
|
+
if not new:
|
|
99
|
+
grey_sources.append((try_scibban, "SciBban")) # a Sci-Hub mirror
|
|
100
|
+
grey_sources.append((try_libgen, "LibGen"))
|
|
101
|
+
if not new and not recently_absent(doi): # Sci-Hub said within the hour it does not have it
|
|
102
|
+
grey_sources.append((try_scihub, "Sci-Hub"))
|
|
103
|
+
|
|
104
|
+
if strategy == "scihub_only":
|
|
105
|
+
# Only Sci-Hub (not SciBban, not LibGen)
|
|
106
|
+
return sort_sources([(try_scihub, "Sci-Hub")]) if config.get("scihub_enabled", False) else []
|
|
107
|
+
elif strategy == "grey_only":
|
|
108
|
+
return sort_sources(grey_sources)
|
|
109
|
+
elif strategy == "legal_only":
|
|
110
|
+
return sort_sources(legal_sources)
|
|
111
|
+
elif strategy == "scihub_first":
|
|
112
|
+
# Grey sources first, then legal
|
|
113
|
+
return sort_sources(grey_sources) + sort_sources(legal_sources)
|
|
114
|
+
elif strategy == "oa_first":
|
|
115
|
+
# Legal sources first, then grey
|
|
116
|
+
return sort_sources(legal_sources) + grey_sources
|
|
117
|
+
else:
|
|
118
|
+
# fastest: all sources EMA-sorted
|
|
119
|
+
return sort_sources(legal_sources + grey_sources)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# Publishers whose papers are open: no site visit from phase 1 is needed.
|
|
123
|
+
_OPEN_HOSTS = frozenset({"MDPI", "arXiv", "bioRxiv", "PLOS", "Frontiers", "BMC", "Beilstein", "EMBO",
|
|
124
|
+
"Research Square", "chemRxiv", "Preprints.org"})
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _webvpn_http(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
128
|
+
"""WebVPN without a browser: the saved session over HTTP (and a campus
|
|
129
|
+
connector's SOCKS5 when one is configured). Asks the user nothing."""
|
|
130
|
+
from . import instsci
|
|
131
|
+
if instsci._is_campus_connector_mode(config):
|
|
132
|
+
result = instsci._try_instsci_socks5(doi, output_path, config)
|
|
133
|
+
if result:
|
|
134
|
+
return result
|
|
135
|
+
if not instsci.instsci_cookie_path(config).exists():
|
|
136
|
+
return None
|
|
137
|
+
return instsci._try_instsci_http(doi, output_path, config)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _webvpn_browser(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
141
|
+
from . import instsci
|
|
142
|
+
return instsci._try_instsci_browser(doi, output_path, config)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# The school's routes, tried one after another (institutional_order): the ones
|
|
146
|
+
# that ask the user nothing first, then those that may need a login, so at most
|
|
147
|
+
# one window waits for the user at a time. Each step: (enabled?, function, label);
|
|
148
|
+
# the label decides whether it takes the browser slot (_BROWSER_SOURCE_LABELS).
|
|
149
|
+
_INSTITUTIONAL_STEPS: dict[str, tuple[Any, Any, str]] = {
|
|
150
|
+
"webvpn_http": (lambda c: bool(c.get("vpnsci_enabled")), _webvpn_http, "WebVPN-HTTP"),
|
|
151
|
+
"webvpn_browser": (lambda c: bool(c.get("vpnsci_enabled")), _webvpn_browser, "WebVPN"),
|
|
152
|
+
"carsi": (lambda c: bool(c.get("carsi_enabled") and str(c.get("carsi_idp_name") or "").strip()),
|
|
153
|
+
lambda d, o, c: _pkg().try_carsi(d, o, c), "CARSI"),
|
|
154
|
+
"ezproxy": (lambda c: bool(c.get("ezproxy_enabled")), lambda d, o, c: _pkg().try_ezproxy(d, o, c), "EZProxy"),
|
|
155
|
+
}
|
|
156
|
+
DEFAULT_INSTITUTIONAL_ORDER = ["webvpn_http", "webvpn_browser", "carsi", "ezproxy"]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _build_institutional_sources(doi: str, config: dict[str, Any], *, use_vpnsci: bool = False) -> list[tuple[Any, str]]:
|
|
160
|
+
"""Phase 2: the school's routes that are switched on, in institutional_order.
|
|
161
|
+
|
|
162
|
+
WebVPN follows its setting (use_vpnsci is kept for callers; a single
|
|
163
|
+
download used to leave WebVPN out unless asked). The publisher's own site
|
|
164
|
+
is not here: phase 1 opened it from this same exit.
|
|
165
|
+
"""
|
|
166
|
+
order = config.get("institutional_order") or _pkg().DEFAULT_INSTITUTIONAL_ORDER
|
|
167
|
+
out = []
|
|
168
|
+
for name in order:
|
|
169
|
+
step = _pkg()._INSTITUTIONAL_STEPS.get(str(name))
|
|
170
|
+
if step and step[0](config):
|
|
171
|
+
out.append((step[1], step[2]))
|
|
172
|
+
return out
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _run_institutional_sequence(steps: list[tuple[Any, str]], doi: str, target_dir: Path, output_path: Path,
|
|
176
|
+
config: dict[str, Any], failures: list[dict[str, str]] | None = None,
|
|
177
|
+
) -> dict[str, Any] | None:
|
|
178
|
+
"""Try the school's routes one at a time; the first PDF wins."""
|
|
179
|
+
from ..download import progress as progress_reporter
|
|
180
|
+
log.info(f" School routes in turn: {', '.join(label for _, label in steps)}")
|
|
181
|
+
t_seq = time.time()
|
|
182
|
+
for fn, label in steps:
|
|
183
|
+
if _pkg()._cancelled():
|
|
184
|
+
return None
|
|
185
|
+
progress_reporter.item_stage(doi, f"正在通过学校认证下载:{_pkg()._STEP_TEXT.get(label, label)}")
|
|
186
|
+
src_output = target_dir / f"{safe_filename(doi)}_{label}.pdf"
|
|
187
|
+
done = threading.Event()
|
|
188
|
+
threading.Thread(target=_pkg()._step_watchdog, args=(doi, label, done, config), daemon=True).start()
|
|
189
|
+
try:
|
|
190
|
+
result = _pkg()._try_source(fn, doi, src_output, config, label)
|
|
191
|
+
finally:
|
|
192
|
+
done.set()
|
|
193
|
+
if result and result.get("success"):
|
|
194
|
+
final_path = Path(result.get("file", ""))
|
|
195
|
+
if final_path != output_path and final_path.exists():
|
|
196
|
+
if output_path.exists():
|
|
197
|
+
output_path.unlink()
|
|
198
|
+
final_path.rename(output_path)
|
|
199
|
+
result["file"] = str(output_path)
|
|
200
|
+
log.info(f" OK {label}")
|
|
201
|
+
log.info(f" [timing] school routes: {time.time() - t_seq:.1f}s")
|
|
202
|
+
return result
|
|
203
|
+
if failures is not None:
|
|
204
|
+
_pkg()._record_source_failure(failures, label, result or {"success": False, "error": "no PDF"})
|
|
205
|
+
if src_output.exists():
|
|
206
|
+
src_output.unlink(missing_ok=True)
|
|
207
|
+
log.info(f" [timing] school routes: {time.time() - t_seq:.1f}s, none had it")
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _step_watchdog(doi: str, label: str, done: threading.Event, config: dict[str, Any]) -> None:
|
|
212
|
+
"""End a school route that has gone quiet: after institutional_step_seconds
|
|
213
|
+
without a result (a login the user is doing keeps it alive), kill the
|
|
214
|
+
browsers it opened, so its blocked browser calls fail and the next route
|
|
215
|
+
runs. 2026-10-17: a WebVPN browser step hung over 10 minutes on a page
|
|
216
|
+
that never finished loading, with no timeout of its own firing."""
|
|
217
|
+
from ..browser.backend import close_browsers_since
|
|
218
|
+
limit = float(config.get("institutional_step_seconds", 240) or 240)
|
|
219
|
+
since = time.perf_counter() # close_browsers_since's clock
|
|
220
|
+
deadline = time.time() + limit
|
|
221
|
+
while not done.wait(5):
|
|
222
|
+
if _pkg()._user_waiting_for(doi):
|
|
223
|
+
deadline = max(deadline, time.time() + 60) # the user is at it: give them time
|
|
224
|
+
continue
|
|
225
|
+
if time.time() >= deadline:
|
|
226
|
+
n = close_browsers_since(since, owner=doi)
|
|
227
|
+
log.info(f" [timing] {label}: nothing after {limit:.0f}s; closed {n} browser(s), next route")
|
|
228
|
+
return
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
_STEP_TEXT = {"WebVPN-HTTP": "WebVPN 直接下载", "WebVPN": "WebVPN 浏览器", "CARSI": "CARSI", "EZProxy": "EZProxy"}
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _build_failure_guidance(doi: str, config: dict[str, Any], error_type: str = "") -> list[str]:
|
|
235
|
+
"""Build actionable guidance when all download sources fail.
|
|
236
|
+
|
|
237
|
+
error_type: what this paper's browser strategies reported (its DownloadAttempt).
|
|
238
|
+
"""
|
|
239
|
+
import os
|
|
240
|
+
tips = []
|
|
241
|
+
|
|
242
|
+
if error_type == "paywall":
|
|
243
|
+
tips.append("此论文需要机构订阅才能下载")
|
|
244
|
+
tips.append(f"→ 运行 litloop_login(identifier=\"{doi}\") 打开浏览器登录")
|
|
245
|
+
tips.append("→ 在浏览器中点击 'Access through your institution' 选择你的机构")
|
|
246
|
+
tips.append("→ 登录后关闭浏览器,cookies 自动保存")
|
|
247
|
+
tips.append("→ 重新运行下载命令即可")
|
|
248
|
+
elif error_type == "cloudflare_blocked":
|
|
249
|
+
tips.append("Cloudflare 防护阻止了访问")
|
|
250
|
+
tips.append("→ 安装 CloakBrowser (pip install cloakbrowser) 绕过反爬")
|
|
251
|
+
tips.append("→ 或配置代理: litloop config_set network_proxy \"socks5://127.0.0.1:1080\"")
|
|
252
|
+
elif error_type == "browser_unavailable":
|
|
253
|
+
tips.append("CloakBrowser 不可用,无法使用浏览器下载策略")
|
|
254
|
+
tips.append("→ 运行: pip install cloakbrowser")
|
|
255
|
+
|
|
256
|
+
# Check litloop proxy config
|
|
257
|
+
cfg_proxy = config.get("network_proxy", "")
|
|
258
|
+
env_proxy = os.environ.get("LITLOOP_PROXY", "")
|
|
259
|
+
|
|
260
|
+
if cfg_proxy or env_proxy:
|
|
261
|
+
active = env_proxy or cfg_proxy
|
|
262
|
+
tips.append(f"当前代理: {active} — 如果 Sci-Hub 仍不通,尝试更换代理地址")
|
|
263
|
+
else:
|
|
264
|
+
# Check if system has proxy that litloop ignores
|
|
265
|
+
sys_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or ""
|
|
266
|
+
if sys_proxy:
|
|
267
|
+
tips.append(f"检测到系统代理 {sys_proxy},但 litloop 未使用")
|
|
268
|
+
tips.append(f"→ 运行: litloop config_set network_proxy \"{sys_proxy}\"")
|
|
269
|
+
else:
|
|
270
|
+
tips.append("未配置代理 — 如果网络受限,运行: litloop config_set network_proxy \"socks5://127.0.0.1:1080\"")
|
|
271
|
+
|
|
272
|
+
# Check Tor
|
|
273
|
+
try:
|
|
274
|
+
from ..tor.proxy import check_tor_circuit
|
|
275
|
+
if not check_tor_circuit(config):
|
|
276
|
+
tips.append("Tor 未运行 → 运行: litloop tor_start(匿名访问 Sci-Hub/LibGen)")
|
|
277
|
+
except Exception:
|
|
278
|
+
pass
|
|
279
|
+
|
|
280
|
+
# Check WebVPN
|
|
281
|
+
if not config.get("vpnsci_enabled"):
|
|
282
|
+
tips.append("有高校账号?运行: litloop config_set vpnsci_enabled true")
|
|
283
|
+
|
|
284
|
+
# Manual fallback
|
|
285
|
+
tips.append(f"手动下载: https://sci-hub.ru/{doi}")
|
|
286
|
+
|
|
287
|
+
# Network diagnostic
|
|
288
|
+
tips.append("运行网络诊断: litloop network_diagnose")
|
|
289
|
+
|
|
290
|
+
return tips
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""ALTCHA proof-of-work solved over plain HTTP (no browser).
|
|
2
|
+
|
|
3
|
+
ALTCHA (used by sci-hub.ru) is a proof-of-work gate, not a human test: the
|
|
4
|
+
page's widget fetches a challenge (algorithm, salt, target hash, maxnumber) and
|
|
5
|
+
the client searches for the number n with hash(salt + n) == target, then posts
|
|
6
|
+
it back. Browsers do this in JS; doing the same in Python skips the browser
|
|
7
|
+
window entirely (~1-3 s). Ported from the user's academic-paper-downloader
|
|
8
|
+
skill (scripts/scihub_solver.py).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import base64
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import time
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import requests
|
|
21
|
+
|
|
22
|
+
from ..log import get_logger
|
|
23
|
+
|
|
24
|
+
log = get_logger()
|
|
25
|
+
|
|
26
|
+
_CHALLENGE_URL_RE = re.compile(r'challengeurl\s*=\s*["\']([^"\']+)["\']', re.I)
|
|
27
|
+
# sci-hub.ru posts the answer to its own solution URL, whose id differs from
|
|
28
|
+
# the challenge id: read it from the page script, don't derive it.
|
|
29
|
+
_SOLUTION_URL_RE = re.compile(r"""fetch\(\s*['"]([^'"]*/solution/[^'"]*)['"]""", re.I)
|
|
30
|
+
_HASHES = {"SHA-256": hashlib.sha256, "SHA-1": hashlib.sha1, "SHA-512": hashlib.sha512}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def is_altcha_page(html: str) -> bool:
|
|
34
|
+
low = html.lower()
|
|
35
|
+
return "altcha" in low and bool(_CHALLENGE_URL_RE.search(html))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def solve(challenge: dict[str, Any], max_seconds: float = 20.0) -> str | None:
|
|
39
|
+
"""Return the base64 ALTCHA payload for a challenge, or None."""
|
|
40
|
+
algorithm = str(challenge.get("algorithm") or "SHA-256").upper()
|
|
41
|
+
target = str(challenge.get("challenge") or "")
|
|
42
|
+
salt = str(challenge.get("salt") or "")
|
|
43
|
+
hasher = _HASHES.get(algorithm)
|
|
44
|
+
if not target or not salt or hasher is None:
|
|
45
|
+
return None
|
|
46
|
+
max_number = int(challenge.get("maxNumber") or challenge.get("maxnumber") or 1_000_000)
|
|
47
|
+
deadline = time.monotonic() + max_seconds
|
|
48
|
+
prefix = salt.encode("utf-8")
|
|
49
|
+
for n in range(max_number + 1):
|
|
50
|
+
if hasher(prefix + str(n).encode()).hexdigest() == target:
|
|
51
|
+
payload = {"algorithm": algorithm, "challenge": target, "number": n,
|
|
52
|
+
"salt": salt, "signature": challenge.get("signature", "")}
|
|
53
|
+
return base64.b64encode(json.dumps(payload).encode("utf-8")).decode("ascii")
|
|
54
|
+
if n % 50_000 == 0 and time.monotonic() > deadline:
|
|
55
|
+
return None
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def pass_gate(session: requests.Session, html: str, page_url: str,
|
|
60
|
+
timeout: tuple[int, int] = (10, 25)) -> str | None:
|
|
61
|
+
"""Solve the ALTCHA gate on ``page_url`` and return the page HTML behind it."""
|
|
62
|
+
m = _CHALLENGE_URL_RE.search(html)
|
|
63
|
+
if not m:
|
|
64
|
+
return None
|
|
65
|
+
origin = "{0.scheme}://{0.netloc}".format(requests.utils.urlparse(page_url))
|
|
66
|
+
challenge_url = requests.compat.urljoin(page_url, m.group(1))
|
|
67
|
+
try:
|
|
68
|
+
t0 = time.monotonic()
|
|
69
|
+
challenge = session.get(challenge_url, headers={"Referer": page_url}, timeout=timeout).json()
|
|
70
|
+
payload = solve(challenge)
|
|
71
|
+
if not payload:
|
|
72
|
+
log.info(" [altcha-http] no solution within the time budget")
|
|
73
|
+
return None
|
|
74
|
+
sm = _SOLUTION_URL_RE.search(html)
|
|
75
|
+
solution_url = (requests.compat.urljoin(page_url, sm.group(1)) if sm
|
|
76
|
+
else challenge_url.replace("/challenge/", "/solution/"))
|
|
77
|
+
answer = session.post(
|
|
78
|
+
solution_url,
|
|
79
|
+
json={"captcha": payload},
|
|
80
|
+
headers={"Origin": origin, "Referer": page_url},
|
|
81
|
+
timeout=timeout,
|
|
82
|
+
).json()
|
|
83
|
+
if answer.get("success") is not True:
|
|
84
|
+
log.info(f" [altcha-http] solution rejected: {str(answer)[:80]}")
|
|
85
|
+
return None
|
|
86
|
+
log.info(f" [altcha-http] solved in {time.monotonic() - t0:.1f}s")
|
|
87
|
+
return session.get(page_url, headers={"Referer": page_url}, timeout=timeout).text
|
|
88
|
+
except Exception as exc:
|
|
89
|
+
log.info(f" [altcha-http] failed: {type(exc).__name__}: {exc}")
|
|
90
|
+
return None
|
litloop/sources/arxiv.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""arXiv direct PDF download."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from ..identifiers import normalize_arxiv_id
|
|
9
|
+
from ..network import fetch
|
|
10
|
+
from ..download.pdf import _response_looks_pdf, is_pdf_file, success
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def try_arxiv(identifier: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
14
|
+
arxiv_id = normalize_arxiv_id(identifier)
|
|
15
|
+
if not arxiv_id:
|
|
16
|
+
return None
|
|
17
|
+
url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
|
|
18
|
+
return download_arxiv_pdf(url, output_path, config)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def download_arxiv_pdf(url: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
22
|
+
try:
|
|
23
|
+
resp = fetch(url, config, stream=True)
|
|
24
|
+
if resp.status_code >= 400:
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
28
|
+
first_chunk = next(iterator, b"")
|
|
29
|
+
if not _response_looks_pdf(resp, first_chunk):
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
from .publishers import _write_pdf_atomic
|
|
33
|
+
if not _write_pdf_atomic(output_path, first_chunk, iterator):
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
if is_pdf_file(output_path):
|
|
37
|
+
return success(output_path.stem, output_path, "arXiv")
|
|
38
|
+
else:
|
|
39
|
+
try:
|
|
40
|
+
output_path.unlink(missing_ok=True)
|
|
41
|
+
except OSError:
|
|
42
|
+
pass
|
|
43
|
+
except Exception:
|
|
44
|
+
return None
|
|
45
|
+
return None
|