litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,684 @@
|
|
|
1
|
+
"""Publisher routing table and direct download strategies.
|
|
2
|
+
|
|
3
|
+
Supports Nature direct download and MDPI open access.
|
|
4
|
+
Other publishers fall through to Crossref/Unpaywall/OpenAlex.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import requests
|
|
14
|
+
|
|
15
|
+
from .crossref import try_crossref
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _load_publisher_cookies(session: requests.Session, config: dict[str, Any]) -> None:
|
|
19
|
+
"""Load saved publisher cookies into a session (and its per-host proxy routes)."""
|
|
20
|
+
from ..network import apply_host_proxy
|
|
21
|
+
apply_host_proxy(session, config)
|
|
22
|
+
try:
|
|
23
|
+
from ..browser.cookies import inject_cookies
|
|
24
|
+
inject_cookies(session, config)
|
|
25
|
+
except Exception:
|
|
26
|
+
pass
|
|
27
|
+
# Also load CARSI/institutional login cookies
|
|
28
|
+
try:
|
|
29
|
+
from ..config import DATA_DIR
|
|
30
|
+
import json as _json
|
|
31
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
32
|
+
carsi_dir = cache_dir / "carsi_cookies"
|
|
33
|
+
if carsi_dir.is_dir():
|
|
34
|
+
for cf in carsi_dir.glob("*.json"):
|
|
35
|
+
try:
|
|
36
|
+
for c in _json.loads(cf.read_text(encoding="utf-8")):
|
|
37
|
+
session.cookies.set(
|
|
38
|
+
c.get("name", ""), c.get("value", ""),
|
|
39
|
+
domain=c.get("domain", ""), path=c.get("path", "/"),
|
|
40
|
+
)
|
|
41
|
+
except Exception:
|
|
42
|
+
pass
|
|
43
|
+
except Exception:
|
|
44
|
+
pass
|
|
45
|
+
from .nature import try_nature_direct
|
|
46
|
+
from .openalex import try_openalex_oa
|
|
47
|
+
from .semantic_scholar import try_semanticscholar
|
|
48
|
+
from .springer_tdm import try_springer_tdm
|
|
49
|
+
from .unpaywall import try_unpaywall
|
|
50
|
+
|
|
51
|
+
# ============================================================
|
|
52
|
+
# Publisher URL templates: (name, pdf_template, match_patterns, gdpr, bibtex_template)
|
|
53
|
+
# {doi} = full DOI, {doi_suffix} = part after prefix, {base} = resolved URL base
|
|
54
|
+
# ============================================================
|
|
55
|
+
# Only publishers with working direct download (no subscription required)
|
|
56
|
+
PUBLISHERS: list[tuple[str, str | None, list[str], bool, str | None]] = [
|
|
57
|
+
("Nature", "https://www.nature.com/articles/{doi_suffix}.pdf",
|
|
58
|
+
["nature.com", "10.1038/"], False, None),
|
|
59
|
+
("Frontiers", None,
|
|
60
|
+
["frontiersin.org", "10.3389/"], False, None),
|
|
61
|
+
("BMC", None,
|
|
62
|
+
["biomedcentral.com", "springeropen.com", "10.1186/"], False, None),
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
DOI_PREFIX_TO_PUBLISHER: dict[str, str] = {
|
|
66
|
+
"10.1038/": "Nature",
|
|
67
|
+
"10.1016/": "Elsevier",
|
|
68
|
+
"10.1002/": "Wiley",
|
|
69
|
+
"10.1126/": "Science",
|
|
70
|
+
"10.1073/": "PNAS",
|
|
71
|
+
"10.1093/": "Oxford",
|
|
72
|
+
"10.1021/": "ACS",
|
|
73
|
+
"10.1007/": "Springer",
|
|
74
|
+
"10.1103/": "APS",
|
|
75
|
+
"10.1088/": "IOP",
|
|
76
|
+
"10.1101/": "bioRxiv",
|
|
77
|
+
"10.3390/": "MDPI",
|
|
78
|
+
"10.1371/": "PLOS",
|
|
79
|
+
"10.3389/": "Frontiers",
|
|
80
|
+
"10.1186/": "BMC",
|
|
81
|
+
"10.15252/": "EMBO",
|
|
82
|
+
"10.1039/": "RSC",
|
|
83
|
+
"10.1063/": "AIP",
|
|
84
|
+
"10.1111/": "Wiley",
|
|
85
|
+
"10.1145/": "ACM",
|
|
86
|
+
"10.1109/": "IEEE",
|
|
87
|
+
"10.1364/": "OSA",
|
|
88
|
+
"10.1080/": "Tandfonline",
|
|
89
|
+
"10.1116/": "AIP", # AVS (pubs.aip.org)
|
|
90
|
+
"10.1177/": "SAGE",
|
|
91
|
+
"10.1061/": "ASCE",
|
|
92
|
+
"10.1146/": "AnnualReviews",
|
|
93
|
+
"10.31219/": "Research Square",
|
|
94
|
+
"10.21203/": "Research Square",
|
|
95
|
+
"10.20944/": "Preprints.org",
|
|
96
|
+
"10.26434/": "chemRxiv",
|
|
97
|
+
"10.48550/": "arXiv",
|
|
98
|
+
"10.1143/": "IOP", # JJAP (IOP-hosted)
|
|
99
|
+
"10.3938/": "KPS",
|
|
100
|
+
"10.3762/": "Beilstein",
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
# Fast sources per publisher (used by tiered racing)
|
|
104
|
+
# Browser strategies (ElsevierBrowser, etc.) use camofox for anti-bot bypass
|
|
105
|
+
PUBLISHER_TOOL_MAP: dict[str, list[str]] = {
|
|
106
|
+
"Nature": ["NatureDirect", "PublisherDirect", "NatureBrowser", "Crossref", "Unpaywall"],
|
|
107
|
+
"MDPI": ["MDPIDirect", "Crossref", "Unpaywall"],
|
|
108
|
+
"arXiv": ["arXiv"],
|
|
109
|
+
"bioRxiv": ["arXiv", "Unpaywall"],
|
|
110
|
+
"Elsevier": ["Crossref", "Unpaywall", "ElsevierAPI", "ElsevierBrowser"],
|
|
111
|
+
"Wiley": ["WileyBrowser", "Crossref", "Unpaywall"],
|
|
112
|
+
"Science": ["ScienceDirect", "ScienceBrowser", "Crossref", "Unpaywall"],
|
|
113
|
+
"PNAS": ["PNASDirect", "Crossref", "Unpaywall"],
|
|
114
|
+
"Oxford": ["OxfordBrowser", "Crossref", "Unpaywall"],
|
|
115
|
+
"ACS": ["ACSBrowser", "Crossref", "Unpaywall"],
|
|
116
|
+
"Springer": ["SpringerTDM", "SpringerBrowser", "Crossref", "Unpaywall"],
|
|
117
|
+
"APS": ["APSBrowser", "Crossref", "Unpaywall"],
|
|
118
|
+
"IOP": ["IOPBrowser", "Crossref", "Unpaywall"],
|
|
119
|
+
"PLOS": ["PLOSDirect", "Crossref", "Unpaywall"],
|
|
120
|
+
"Frontiers": ["FrontiersDirect", "Crossref", "Unpaywall"],
|
|
121
|
+
"BMC": ["BMCDirect", "Crossref", "Unpaywall"],
|
|
122
|
+
"RSC": ["RSCBrowser", "Crossref", "Unpaywall"],
|
|
123
|
+
"AIP": ["AIPBrowser", "Crossref", "Unpaywall"],
|
|
124
|
+
"ACM": ["ACMBrowser", "Crossref", "Unpaywall"],
|
|
125
|
+
"IEEE": ["IEEEBrowser", "Crossref", "Unpaywall"],
|
|
126
|
+
"OSA": ["Crossref", "Unpaywall"],
|
|
127
|
+
"Tandfonline": ["TandFBrowser", "Crossref", "Unpaywall"],
|
|
128
|
+
"SAGE": ["SAGEBrowser", "Crossref", "Unpaywall"],
|
|
129
|
+
"ASCE": ["ASCEBrowser", "Crossref", "Unpaywall"],
|
|
130
|
+
"AnnualReviews": ["Crossref", "Unpaywall"],
|
|
131
|
+
"KPS": ["Crossref", "Unpaywall"],
|
|
132
|
+
"Beilstein": ["Crossref", "Unpaywall"],
|
|
133
|
+
"EMBO": ["Crossref", "Unpaywall"],
|
|
134
|
+
"Research Square": ["Crossref", "Unpaywall"],
|
|
135
|
+
"chemRxiv": ["Crossref", "Unpaywall"],
|
|
136
|
+
"Preprints.org": ["Crossref", "Unpaywall"],
|
|
137
|
+
"WorldScientific": ["Crossref", "Unpaywall"],
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
_FN_MAP: dict[str, Any] = {}
|
|
141
|
+
|
|
142
|
+
from ..log import get_logger
|
|
143
|
+
log = get_logger()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _write_pdf_atomic(output_path: Path, first: bytes, iterator: Any) -> bool:
|
|
147
|
+
"""Write PDF to .part file then atomically rename. Cleans up on failure."""
|
|
148
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
149
|
+
tmp_path = output_path.with_suffix(output_path.suffix + ".part")
|
|
150
|
+
try:
|
|
151
|
+
with tmp_path.open("wb") as fh:
|
|
152
|
+
fh.write(first)
|
|
153
|
+
for chunk in iterator:
|
|
154
|
+
if chunk:
|
|
155
|
+
fh.write(chunk)
|
|
156
|
+
tmp_path.replace(output_path)
|
|
157
|
+
return True
|
|
158
|
+
except Exception:
|
|
159
|
+
tmp_path.unlink(missing_ok=True)
|
|
160
|
+
return False
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# ============================================================
|
|
164
|
+
# Publisher matching
|
|
165
|
+
# ============================================================
|
|
166
|
+
|
|
167
|
+
def _match_publisher(doi_or_url: str, patterns: list[str]) -> bool:
|
|
168
|
+
lower = doi_or_url.lower()
|
|
169
|
+
return any(p.lower() in lower for p in patterns)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _format_pdf_url(template: str, doi: str, url: str) -> str | None:
|
|
173
|
+
if not template:
|
|
174
|
+
return None
|
|
175
|
+
doi_suffix = doi.split("/", 1)[1] if "/" in doi else doi
|
|
176
|
+
base = ""
|
|
177
|
+
if url:
|
|
178
|
+
m = re.match(r"(https?://[^/]+)", url)
|
|
179
|
+
if m:
|
|
180
|
+
base = m.group(1)
|
|
181
|
+
return (template
|
|
182
|
+
.replace("{doi}", doi)
|
|
183
|
+
.replace("{doi_suffix}", doi_suffix)
|
|
184
|
+
.replace("{base}", base))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# ============================================================
|
|
188
|
+
# DOI resolution
|
|
189
|
+
# ============================================================
|
|
190
|
+
|
|
191
|
+
def resolve_doi(doi: str, config: dict[str, Any]) -> str | None:
|
|
192
|
+
"""Resolve DOI to publisher URL via doi.org redirect."""
|
|
193
|
+
from ..network import USER_AGENT
|
|
194
|
+
try:
|
|
195
|
+
s = requests.Session()
|
|
196
|
+
s.trust_env = False
|
|
197
|
+
# doi.org's own redirect already names the publisher URL; following it
|
|
198
|
+
# to the publisher (6+ s on the slow route to Nature's CDN) adds nothing.
|
|
199
|
+
from ..network import apply_host_proxy
|
|
200
|
+
apply_host_proxy(s, config) # doi.org may be on the proxy exception list
|
|
201
|
+
resp = s.head(f"https://doi.org/{doi}", allow_redirects=False,
|
|
202
|
+
timeout=10, headers={"User-Agent": USER_AGENT})
|
|
203
|
+
url = resp.headers.get("location", "") if resp.is_redirect else resp.url
|
|
204
|
+
known_patterns = []
|
|
205
|
+
for pub in PUBLISHERS:
|
|
206
|
+
known_patterns.extend(pub[2])
|
|
207
|
+
if url and any(p.lower() in url.lower() for p in known_patterns):
|
|
208
|
+
return url.replace("http://", "https://", 1)
|
|
209
|
+
except Exception:
|
|
210
|
+
pass
|
|
211
|
+
return None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# ============================================================
|
|
215
|
+
# Publisher direct download (with GDPR cookie bypass)
|
|
216
|
+
# ============================================================
|
|
217
|
+
|
|
218
|
+
def try_publisher_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
219
|
+
"""Try publisher direct download with HTTP first, then browser fallback."""
|
|
220
|
+
from ..network import USER_AGENT, polite_delay
|
|
221
|
+
from ..download.pdf import is_pdf_file, success, _response_looks_pdf
|
|
222
|
+
|
|
223
|
+
log.info(f" [PublisherDirect] Starting for {doi}")
|
|
224
|
+
resolved_url = resolve_doi(doi, config)
|
|
225
|
+
log.info(f" [PublisherDirect] Resolved URL: {resolved_url}")
|
|
226
|
+
|
|
227
|
+
# Phase 1: HTTP-based direct download (fast path)
|
|
228
|
+
for pub in PUBLISHERS:
|
|
229
|
+
name, template, matches, gdpr, _ = pub
|
|
230
|
+
|
|
231
|
+
if not _match_publisher(doi, matches) and not _match_publisher(resolved_url or "", matches):
|
|
232
|
+
continue
|
|
233
|
+
|
|
234
|
+
pdf_url = _format_pdf_url(template or "", doi, resolved_url or "")
|
|
235
|
+
if not pdf_url:
|
|
236
|
+
continue
|
|
237
|
+
|
|
238
|
+
log.info(f" [Publisher] {name}: {pdf_url[:80]}...")
|
|
239
|
+
polite_delay(config)
|
|
240
|
+
|
|
241
|
+
try:
|
|
242
|
+
s = requests.Session()
|
|
243
|
+
s.trust_env = False
|
|
244
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
245
|
+
_load_publisher_cookies(s, config)
|
|
246
|
+
resp = s.get(pdf_url, timeout=15, stream=True,
|
|
247
|
+
headers={"Accept": "application/pdf,*/*"},
|
|
248
|
+
allow_redirects=True)
|
|
249
|
+
|
|
250
|
+
if resp.status_code >= 400:
|
|
251
|
+
log.info(f" [Publisher] {name}: HTTP {resp.status_code}")
|
|
252
|
+
continue
|
|
253
|
+
|
|
254
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
255
|
+
first = next(iterator, b"")
|
|
256
|
+
if not _response_looks_pdf(resp, first):
|
|
257
|
+
continue
|
|
258
|
+
|
|
259
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
260
|
+
continue
|
|
261
|
+
if is_pdf_file(output_path):
|
|
262
|
+
return success(doi, output_path, f"Publisher({name})")
|
|
263
|
+
|
|
264
|
+
except Exception as e:
|
|
265
|
+
log.info(f" [Publisher] {name}: {e}")
|
|
266
|
+
|
|
267
|
+
# Phase 2: Browser-based download via camofox (handles anti-bot)
|
|
268
|
+
publisher = get_publisher(doi)
|
|
269
|
+
if publisher and config.get("camofox_enabled", True):
|
|
270
|
+
from ..browser.camofox_api import is_available as _camofox_avail
|
|
271
|
+
if _camofox_avail(config):
|
|
272
|
+
log.info(f" [PublisherDirect] Trying browser strategy for {publisher}")
|
|
273
|
+
fn = _FN_MAP.get(f"{publisher}Browser") or _FN_MAP.get("GenericBrowser")
|
|
274
|
+
if fn:
|
|
275
|
+
try:
|
|
276
|
+
result = fn(doi, output_path, config)
|
|
277
|
+
if result:
|
|
278
|
+
return result
|
|
279
|
+
except Exception as e:
|
|
280
|
+
log.info(f" [PublisherDirect] Browser strategy failed: {e}")
|
|
281
|
+
|
|
282
|
+
return None
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
# ============================================================
|
|
286
|
+
# MDPI direct download
|
|
287
|
+
# ============================================================
|
|
288
|
+
|
|
289
|
+
def try_mdpi_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
290
|
+
"""Download MDPI open access papers directly."""
|
|
291
|
+
if not doi.startswith("10.3390/"):
|
|
292
|
+
return None
|
|
293
|
+
|
|
294
|
+
from ..download.pdf import is_pdf_file, success, _response_looks_pdf
|
|
295
|
+
from ..network import USER_AGENT, polite_delay
|
|
296
|
+
|
|
297
|
+
article_id = doi.split("10.3390/")[-1]
|
|
298
|
+
|
|
299
|
+
urls = [
|
|
300
|
+
f"https://www.mdpi.com/{article_id}/pdf",
|
|
301
|
+
]
|
|
302
|
+
|
|
303
|
+
try:
|
|
304
|
+
session = requests.Session()
|
|
305
|
+
session.trust_env = False
|
|
306
|
+
session.headers.update({"User-Agent": USER_AGENT})
|
|
307
|
+
polite_delay(config)
|
|
308
|
+
|
|
309
|
+
# First try landing page to get real PDF URL
|
|
310
|
+
try:
|
|
311
|
+
resp = session.get(f"https://www.mdpi.com/{article_id}", timeout=10, allow_redirects=True)
|
|
312
|
+
if resp.status_code == 200:
|
|
313
|
+
pdf_match = re.search(r'citation_pdf_url["\s]+content="([^"]+)"', resp.text[:5000], re.I)
|
|
314
|
+
if not pdf_match:
|
|
315
|
+
pdf_match = re.search(r'href="(/[^"]*?/pdf[^"]*)"', resp.text[:10000], re.I)
|
|
316
|
+
if pdf_match:
|
|
317
|
+
pdf_url = pdf_match.group(1)
|
|
318
|
+
if pdf_url.startswith("/"):
|
|
319
|
+
pdf_url = f"https://www.mdpi.com{pdf_url}"
|
|
320
|
+
urls.insert(0, pdf_url)
|
|
321
|
+
except Exception:
|
|
322
|
+
pass
|
|
323
|
+
|
|
324
|
+
for pdf_url in urls:
|
|
325
|
+
try:
|
|
326
|
+
resp2 = session.get(pdf_url, timeout=15, stream=True,
|
|
327
|
+
headers={"Accept": "application/pdf,*/*"})
|
|
328
|
+
if resp2.status_code >= 400:
|
|
329
|
+
continue
|
|
330
|
+
|
|
331
|
+
iterator = resp2.iter_content(chunk_size=8192)
|
|
332
|
+
first = next(iterator, b"")
|
|
333
|
+
if not _response_looks_pdf(resp2, first):
|
|
334
|
+
continue
|
|
335
|
+
|
|
336
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
337
|
+
continue
|
|
338
|
+
if is_pdf_file(output_path):
|
|
339
|
+
return success(doi, output_path, "MDPIDirect")
|
|
340
|
+
except Exception:
|
|
341
|
+
continue
|
|
342
|
+
except Exception:
|
|
343
|
+
pass
|
|
344
|
+
return None
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
# ============================================================
|
|
348
|
+
# Science/AAAS direct download (Science Advances, Science, etc.)
|
|
349
|
+
# ============================================================
|
|
350
|
+
|
|
351
|
+
def try_science_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
352
|
+
"""Download Science/AAAS papers directly. Science Advances is OA."""
|
|
353
|
+
if not doi.startswith("10.1126/"):
|
|
354
|
+
return None
|
|
355
|
+
|
|
356
|
+
from ..network import polite_delay
|
|
357
|
+
from ..download.pdf import is_pdf_file, success, _response_looks_pdf
|
|
358
|
+
|
|
359
|
+
def _fetch_pdf_camofox(pdf_url: str) -> "requests.Response | None": # type: ignore[name-defined]
|
|
360
|
+
from ..browser.camofox_api import is_available as _camofox_avail, solve_url as _camofox_solve
|
|
361
|
+
if not _camofox_avail(config):
|
|
362
|
+
return None
|
|
363
|
+
result = _camofox_solve(pdf_url, config)
|
|
364
|
+
if not result:
|
|
365
|
+
return None
|
|
366
|
+
solution = result.get("solution", {})
|
|
367
|
+
if solution.get("status", 0) >= 400:
|
|
368
|
+
return None
|
|
369
|
+
import requests as _req
|
|
370
|
+
resp = _req.Response()
|
|
371
|
+
resp.status_code = solution.get("status", 200)
|
|
372
|
+
resp._content = solution.get("response", "").encode("utf-8")
|
|
373
|
+
resp.url = solution.get("url", pdf_url)
|
|
374
|
+
return resp
|
|
375
|
+
|
|
376
|
+
# Extract article ID from DOI (e.g., sciadv.1600983 -> 1600983)
|
|
377
|
+
doi_suffix = doi.split("10.1126/")[-1]
|
|
378
|
+
|
|
379
|
+
# Try Science Advances PDF URL patterns
|
|
380
|
+
pdf_urls = []
|
|
381
|
+
if "sciadv" in doi_suffix:
|
|
382
|
+
# Science Advances: https://advances.sciencemag.org/content/3/11/e1600983.full.pdf
|
|
383
|
+
article_id = doi_suffix.split(".")[-1] if "." in doi_suffix else doi_suffix
|
|
384
|
+
pdf_urls.append(f"https://advances.sciencemag.org/content/3/11/{article_id}.full.pdf")
|
|
385
|
+
pdf_urls.append(f"https://advances.sciencemag.org/content/early/{article_id}.full.pdf")
|
|
386
|
+
elif "science" in doi_suffix.lower():
|
|
387
|
+
# Science magazine
|
|
388
|
+
pdf_urls.append(f"https://www.science.org/doi/pdf/{doi}")
|
|
389
|
+
|
|
390
|
+
for pdf_url in pdf_urls:
|
|
391
|
+
try:
|
|
392
|
+
polite_delay(config)
|
|
393
|
+
resp = _fetch_pdf_camofox(pdf_url)
|
|
394
|
+
if resp is None:
|
|
395
|
+
continue
|
|
396
|
+
|
|
397
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
398
|
+
first = next(iterator, b"")
|
|
399
|
+
if not _response_looks_pdf(resp, first):
|
|
400
|
+
continue
|
|
401
|
+
|
|
402
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
403
|
+
continue
|
|
404
|
+
if is_pdf_file(output_path):
|
|
405
|
+
return success(doi, output_path, "ScienceDirect")
|
|
406
|
+
except Exception:
|
|
407
|
+
continue
|
|
408
|
+
return None
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
# ============================================================
|
|
412
|
+
# PNAS direct download
|
|
413
|
+
# ============================================================
|
|
414
|
+
|
|
415
|
+
def try_pnas_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
416
|
+
"""Download PNAS papers directly. PNAS is OA after 6 months."""
|
|
417
|
+
if not doi.startswith("10.1073/"):
|
|
418
|
+
return None
|
|
419
|
+
|
|
420
|
+
from ..network import USER_AGENT, polite_delay
|
|
421
|
+
from ..download.pdf import is_pdf_file, success, _response_looks_pdf
|
|
422
|
+
|
|
423
|
+
doi_suffix = doi.split("10.1073/")[-1]
|
|
424
|
+
pdf_url = f"https://www.pnas.org/doi/epdf/{doi}"
|
|
425
|
+
|
|
426
|
+
try:
|
|
427
|
+
polite_delay(config)
|
|
428
|
+
s = requests.Session()
|
|
429
|
+
s.trust_env = False
|
|
430
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
431
|
+
resp = s.get(pdf_url, timeout=15, stream=True,
|
|
432
|
+
headers={"Accept": "application/pdf,*/*"},
|
|
433
|
+
allow_redirects=True)
|
|
434
|
+
|
|
435
|
+
if resp.status_code >= 400:
|
|
436
|
+
return None
|
|
437
|
+
|
|
438
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
439
|
+
first = next(iterator, b"")
|
|
440
|
+
if not _response_looks_pdf(resp, first):
|
|
441
|
+
return None
|
|
442
|
+
|
|
443
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
444
|
+
return None
|
|
445
|
+
if is_pdf_file(output_path):
|
|
446
|
+
return success(doi, output_path, "PNASDirect")
|
|
447
|
+
except Exception:
|
|
448
|
+
pass
|
|
449
|
+
return None
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
# ============================================================
|
|
453
|
+
# PLOS direct download
|
|
454
|
+
# ============================================================
|
|
455
|
+
|
|
456
|
+
def try_plos_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
457
|
+
"""Download PLOS papers directly. All PLOS journals are OA."""
|
|
458
|
+
if not doi.startswith("10.1371/"):
|
|
459
|
+
return None
|
|
460
|
+
|
|
461
|
+
from ..network import USER_AGENT, polite_delay
|
|
462
|
+
from ..download.pdf import is_pdf_file, success, _response_looks_pdf
|
|
463
|
+
|
|
464
|
+
# PLOS PDF URL: https://journals.plos.org/{journal}/article/file?id={doi}&type=printable
|
|
465
|
+
# Determine journal from DOI: 10.1371/journal.pone.0123456 -> plosone
|
|
466
|
+
doi_suffix = doi.split("10.1371/")[-1] if "10.1371/" in doi else doi
|
|
467
|
+
journal_map = {
|
|
468
|
+
"journal.pone": "plosone",
|
|
469
|
+
"journal.pbio": "plosbiology",
|
|
470
|
+
"journal.pmed": "plosmedicine",
|
|
471
|
+
"journal.pgen": "plosgenetics",
|
|
472
|
+
"journal.ppat": "plospathogens",
|
|
473
|
+
"journal.pcbi": "ploscompbiol",
|
|
474
|
+
"journal.pntd": "plosntds",
|
|
475
|
+
"journal.pone": "plosone",
|
|
476
|
+
"journal.pclm": "plosclimate",
|
|
477
|
+
"journal.pdig": "plosdigitalhealth",
|
|
478
|
+
}
|
|
479
|
+
journal = "plosone" # default
|
|
480
|
+
for key, val in journal_map.items():
|
|
481
|
+
if doi_suffix.startswith(key):
|
|
482
|
+
journal = val
|
|
483
|
+
break
|
|
484
|
+
pdf_url = f"https://journals.plos.org/{journal}/article/file?id={doi}&type=printable"
|
|
485
|
+
|
|
486
|
+
try:
|
|
487
|
+
polite_delay(config)
|
|
488
|
+
s = requests.Session()
|
|
489
|
+
s.trust_env = False
|
|
490
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
491
|
+
resp = s.get(pdf_url, timeout=15, stream=True,
|
|
492
|
+
headers={"Accept": "application/pdf,*/*"},
|
|
493
|
+
allow_redirects=True)
|
|
494
|
+
|
|
495
|
+
if resp.status_code >= 400:
|
|
496
|
+
return None
|
|
497
|
+
|
|
498
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
499
|
+
first = next(iterator, b"")
|
|
500
|
+
if not _response_looks_pdf(resp, first):
|
|
501
|
+
return None
|
|
502
|
+
|
|
503
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
504
|
+
return None
|
|
505
|
+
if is_pdf_file(output_path):
|
|
506
|
+
return success(doi, output_path, "PLOSDirect")
|
|
507
|
+
except Exception:
|
|
508
|
+
pass
|
|
509
|
+
return None
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
# ============================================================
|
|
513
|
+
# Frontiers direct download
|
|
514
|
+
# ============================================================
|
|
515
|
+
|
|
516
|
+
def try_frontiers_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
517
|
+
"""Download Frontiers papers directly. All Frontiers journals are OA."""
|
|
518
|
+
if not doi.startswith("10.3389/"):
|
|
519
|
+
return None
|
|
520
|
+
|
|
521
|
+
from ..network import USER_AGENT, polite_delay
|
|
522
|
+
from ..download.pdf import is_pdf_file, success, _response_looks_pdf
|
|
523
|
+
|
|
524
|
+
# Frontiers PDF URL: https://www.frontiersin.org/articles/{doi}/pdf
|
|
525
|
+
pdf_url = f"https://www.frontiersin.org/articles/{doi}/pdf"
|
|
526
|
+
|
|
527
|
+
try:
|
|
528
|
+
polite_delay(config)
|
|
529
|
+
s = requests.Session()
|
|
530
|
+
s.trust_env = False
|
|
531
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
532
|
+
resp = s.get(pdf_url, timeout=15, stream=True,
|
|
533
|
+
headers={"Accept": "application/pdf,*/*"},
|
|
534
|
+
allow_redirects=True)
|
|
535
|
+
|
|
536
|
+
if resp.status_code >= 400:
|
|
537
|
+
return None
|
|
538
|
+
|
|
539
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
540
|
+
first = next(iterator, b"")
|
|
541
|
+
if not _response_looks_pdf(resp, first):
|
|
542
|
+
return None
|
|
543
|
+
|
|
544
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
545
|
+
return None
|
|
546
|
+
if is_pdf_file(output_path):
|
|
547
|
+
return success(doi, output_path, "FrontiersDirect")
|
|
548
|
+
except Exception:
|
|
549
|
+
pass
|
|
550
|
+
return None
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
# ============================================================
|
|
554
|
+
# BMC/SpringerOpen direct download
|
|
555
|
+
# ============================================================
|
|
556
|
+
|
|
557
|
+
def try_bmc_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
558
|
+
"""Download BMC/SpringerOpen papers directly. All BMC journals are OA."""
|
|
559
|
+
if not doi.startswith("10.1186/"):
|
|
560
|
+
return None
|
|
561
|
+
|
|
562
|
+
from ..network import USER_AGENT, polite_delay
|
|
563
|
+
from ..download.pdf import is_pdf_file, success, _response_looks_pdf
|
|
564
|
+
|
|
565
|
+
# BMC/SpringerOpen PDF: try Springer link first, then direct biomedcentral
|
|
566
|
+
pdf_urls = [
|
|
567
|
+
f"https://link.springer.com/content/pdf/{doi}.pdf",
|
|
568
|
+
f"https://link.springer.com/content/pdf/{doi}.pdf?pdf=button%20sticky",
|
|
569
|
+
]
|
|
570
|
+
|
|
571
|
+
for pdf_url in pdf_urls:
|
|
572
|
+
try:
|
|
573
|
+
polite_delay(config)
|
|
574
|
+
s = requests.Session()
|
|
575
|
+
s.trust_env = False
|
|
576
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
577
|
+
resp = s.get(pdf_url, timeout=15, stream=True,
|
|
578
|
+
headers={"Accept": "application/pdf,*/*"},
|
|
579
|
+
allow_redirects=True)
|
|
580
|
+
|
|
581
|
+
if resp.status_code >= 400:
|
|
582
|
+
continue
|
|
583
|
+
|
|
584
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
585
|
+
first = next(iterator, b"")
|
|
586
|
+
if not _response_looks_pdf(resp, first):
|
|
587
|
+
continue
|
|
588
|
+
|
|
589
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
590
|
+
continue
|
|
591
|
+
if is_pdf_file(output_path):
|
|
592
|
+
return success(doi, output_path, "BMCDirect")
|
|
593
|
+
except Exception:
|
|
594
|
+
continue
|
|
595
|
+
return None
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
# ============================================================
|
|
599
|
+
# Helpers
|
|
600
|
+
# ============================================================
|
|
601
|
+
|
|
602
|
+
def get_publisher(doi: str) -> str:
|
|
603
|
+
for prefix, publisher in DOI_PREFIX_TO_PUBLISHER.items():
|
|
604
|
+
if doi.startswith(prefix):
|
|
605
|
+
return publisher
|
|
606
|
+
return ""
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _elsevier_api_fn() -> Any:
|
|
610
|
+
"""Lazy import of the Elsevier API download function."""
|
|
611
|
+
from ..publishers.strategies import try_elsevier_api
|
|
612
|
+
return try_elsevier_api
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _browser_strategy(publisher: str) -> Any:
|
|
616
|
+
"""Create a browser download function for a publisher.
|
|
617
|
+
|
|
618
|
+
Uses the new StrategyRegistry to validate the publisher, then returns
|
|
619
|
+
the legacy ``try_*_browser`` function via the compatibility bridge.
|
|
620
|
+
"""
|
|
621
|
+
from ..publishers.strategies import StrategyRegistry
|
|
622
|
+
from ..publishers.strategies._core import try_generic_browser
|
|
623
|
+
|
|
624
|
+
strategy = StrategyRegistry.get_by_name(publisher)
|
|
625
|
+
if not strategy:
|
|
626
|
+
return try_generic_browser
|
|
627
|
+
|
|
628
|
+
# Resolve the legacy function name from the strategy's data
|
|
629
|
+
# Standard naming: try_{publisher.lower()}_browser
|
|
630
|
+
# Special cases handled by the strategy's aliases
|
|
631
|
+
fn_name = f"try_{strategy.name.lower().replace(' ', '_')}_browser"
|
|
632
|
+
from ..publishers.strategies import _core
|
|
633
|
+
return getattr(_core, fn_name, try_generic_browser)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def get_publisher_fast_sources(doi: str) -> list[tuple[Any, str]]:
|
|
637
|
+
publisher = get_publisher(doi)
|
|
638
|
+
if not publisher:
|
|
639
|
+
return []
|
|
640
|
+
tools = PUBLISHER_TOOL_MAP.get(publisher, ["Crossref", "Unpaywall"])
|
|
641
|
+
sources = []
|
|
642
|
+
for tool in tools:
|
|
643
|
+
fn = _FN_MAP.get(tool)
|
|
644
|
+
if fn:
|
|
645
|
+
sources.append((fn, tool))
|
|
646
|
+
return sources
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
# Populate function map
|
|
650
|
+
_FN_MAP.update({
|
|
651
|
+
"NatureDirect": try_nature_direct,
|
|
652
|
+
"PublisherDirect": try_publisher_direct,
|
|
653
|
+
"Crossref": try_crossref,
|
|
654
|
+
"Unpaywall": try_unpaywall,
|
|
655
|
+
"OpenAlexOA": try_openalex_oa,
|
|
656
|
+
"SemanticScholar": try_semanticscholar,
|
|
657
|
+
"MDPIDirect": try_mdpi_direct,
|
|
658
|
+
"SpringerTDM": try_springer_tdm,
|
|
659
|
+
"ScienceDirect": try_science_direct,
|
|
660
|
+
"PNASDirect": try_pnas_direct,
|
|
661
|
+
"PLOSDirect": try_plos_direct,
|
|
662
|
+
"FrontiersDirect": try_frontiers_direct,
|
|
663
|
+
"BMCDirect": try_bmc_direct,
|
|
664
|
+
# API strategies
|
|
665
|
+
"ElsevierAPI": _elsevier_api_fn(),
|
|
666
|
+
# Browser strategies via camofox
|
|
667
|
+
"ElsevierBrowser": _browser_strategy("Elsevier"),
|
|
668
|
+
"WileyBrowser": _browser_strategy("Wiley"),
|
|
669
|
+
"IEEEBrowser": _browser_strategy("IEEE"),
|
|
670
|
+
"ACSBrowser": _browser_strategy("ACS"),
|
|
671
|
+
"RSCBrowser": _browser_strategy("RSC"),
|
|
672
|
+
"AIPBrowser": _browser_strategy("AIP"),
|
|
673
|
+
"SpringerBrowser": _browser_strategy("Springer"),
|
|
674
|
+
"APSBrowser": _browser_strategy("APS"),
|
|
675
|
+
"TandFBrowser": _browser_strategy("Tandfonline"),
|
|
676
|
+
"IOPBrowser": _browser_strategy("IOP"),
|
|
677
|
+
"OxfordBrowser": _browser_strategy("Oxford"),
|
|
678
|
+
"ACMBrowser": _browser_strategy("ACM"),
|
|
679
|
+
"NatureBrowser": _browser_strategy("Nature"),
|
|
680
|
+
"ScienceBrowser": _browser_strategy("Science"),
|
|
681
|
+
"SAGEBrowser": _browser_strategy("SAGE"),
|
|
682
|
+
"ASCEBrowser": _browser_strategy("ASCE"),
|
|
683
|
+
"GenericBrowser": _browser_strategy(""),
|
|
684
|
+
})
|