litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Semantic Scholar Open Access API source."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import time
|
|
7
|
+
import urllib.parse
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
from ..network import fetch_json, polite_delay
|
|
15
|
+
from ..download.pdf import download_pdf, is_plausible_pdf_url
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
S2_API = "https://api.semanticscholar.org/graph/v1"
|
|
20
|
+
S2_FIELDS = "title,authors,year,abstract,externalIds,journal,citationCount,url"
|
|
21
|
+
_MAX_RETRIES = 3
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class SearchResult:
|
|
26
|
+
"""A single search result from Semantic Scholar."""
|
|
27
|
+
|
|
28
|
+
title: str = ""
|
|
29
|
+
authors: list[str] = field(default_factory=list)
|
|
30
|
+
year: int | None = None
|
|
31
|
+
abstract: str = ""
|
|
32
|
+
doi: str = ""
|
|
33
|
+
arxiv_id: str = ""
|
|
34
|
+
journal: str = ""
|
|
35
|
+
citation_count: int = 0
|
|
36
|
+
s2_url: str = ""
|
|
37
|
+
paper_id: str = ""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _s2_request(url: str, params: dict) -> dict | None:
|
|
41
|
+
"""Make a GET request to Semantic Scholar with retry on 429."""
|
|
42
|
+
for attempt in range(_MAX_RETRIES):
|
|
43
|
+
try:
|
|
44
|
+
resp = requests.get(url, params=params, timeout=15)
|
|
45
|
+
if resp.status_code == 404:
|
|
46
|
+
return None
|
|
47
|
+
if resp.status_code == 429:
|
|
48
|
+
wait = 2 ** (attempt + 1)
|
|
49
|
+
logger.warning("Rate limited by Semantic Scholar, retrying in %ds...", wait)
|
|
50
|
+
time.sleep(wait)
|
|
51
|
+
continue
|
|
52
|
+
resp.raise_for_status()
|
|
53
|
+
return resp.json()
|
|
54
|
+
except requests.RequestException as e:
|
|
55
|
+
logger.warning("Semantic Scholar request failed: %s", e)
|
|
56
|
+
if attempt < _MAX_RETRIES - 1:
|
|
57
|
+
time.sleep(2)
|
|
58
|
+
else:
|
|
59
|
+
return None
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _parse_paper(item: dict) -> SearchResult:
|
|
64
|
+
"""Parse a Semantic Scholar API response item into a SearchResult."""
|
|
65
|
+
ext_ids = item.get("externalIds") or {}
|
|
66
|
+
authors_data = item.get("authors") or []
|
|
67
|
+
journal_data = item.get("journal") or {}
|
|
68
|
+
return SearchResult(
|
|
69
|
+
title=item.get("title", ""),
|
|
70
|
+
authors=[a.get("name", "") for a in authors_data if a.get("name")],
|
|
71
|
+
year=item.get("year"),
|
|
72
|
+
abstract=item.get("abstract") or "",
|
|
73
|
+
doi=ext_ids.get("DOI", ""),
|
|
74
|
+
arxiv_id=ext_ids.get("ArXiv", ""),
|
|
75
|
+
journal=journal_data.get("name", "") if isinstance(journal_data, dict) else str(journal_data),
|
|
76
|
+
citation_count=item.get("citationCount", 0),
|
|
77
|
+
s2_url=item.get("url", ""),
|
|
78
|
+
paper_id=item.get("paperId", ""),
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def search(
|
|
83
|
+
query: str,
|
|
84
|
+
limit: int = 10,
|
|
85
|
+
year_range: str | None = None,
|
|
86
|
+
fields_of_study: list[str] | None = None,
|
|
87
|
+
) -> list[SearchResult]:
|
|
88
|
+
"""Search for papers on Semantic Scholar."""
|
|
89
|
+
params = {
|
|
90
|
+
"query": query,
|
|
91
|
+
"limit": min(limit, 100),
|
|
92
|
+
"fields": S2_FIELDS,
|
|
93
|
+
}
|
|
94
|
+
if year_range:
|
|
95
|
+
params["year"] = year_range
|
|
96
|
+
if fields_of_study:
|
|
97
|
+
params["fieldsOfStudy"] = ",".join(fields_of_study)
|
|
98
|
+
|
|
99
|
+
data = _s2_request(f"{S2_API}/paper/search", params)
|
|
100
|
+
if data is None:
|
|
101
|
+
return []
|
|
102
|
+
return [_parse_paper(item) for item in data.get("data", [])]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def get_paper(paper_id: str) -> SearchResult | None:
|
|
106
|
+
"""Get details for a specific paper by Semantic Scholar ID or DOI.
|
|
107
|
+
|
|
108
|
+
paper_id: S2 paper ID, DOI (prefix with "DOI:"), or arXiv ID (prefix with "ARXIV:").
|
|
109
|
+
"""
|
|
110
|
+
item = _s2_request(f"{S2_API}/paper/{paper_id}", {"fields": S2_FIELDS})
|
|
111
|
+
if item is None:
|
|
112
|
+
return None
|
|
113
|
+
return _parse_paper(item)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def try_semanticscholar(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
117
|
+
q = urllib.parse.quote(doi, safe="")
|
|
118
|
+
url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{q}?fields=openAccessPdf,externalIds"
|
|
119
|
+
payload = fetch_json(url, config)
|
|
120
|
+
if not payload:
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
oa_pdf = payload.get("openAccessPdf")
|
|
124
|
+
if isinstance(oa_pdf, dict):
|
|
125
|
+
pdf_url = oa_pdf.get("url", "")
|
|
126
|
+
if pdf_url and is_plausible_pdf_url(pdf_url):
|
|
127
|
+
polite_delay(config)
|
|
128
|
+
result = download_pdf(pdf_url, output_path, config, "SemanticScholar")
|
|
129
|
+
if result:
|
|
130
|
+
result["doi"] = doi
|
|
131
|
+
result["identifier"] = doi
|
|
132
|
+
return result
|
|
133
|
+
|
|
134
|
+
ext_ids = payload.get("externalIds", {})
|
|
135
|
+
if isinstance(ext_ids, dict):
|
|
136
|
+
arxiv_id = ext_ids.get("ArXiv")
|
|
137
|
+
if arxiv_id:
|
|
138
|
+
arxiv_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
|
|
139
|
+
polite_delay(config)
|
|
140
|
+
result = download_pdf(arxiv_url, output_path, config, "SemanticScholar(arXiv)", require_pdf_like_url=False)
|
|
141
|
+
if result:
|
|
142
|
+
result["doi"] = doi
|
|
143
|
+
result["identifier"] = doi
|
|
144
|
+
return result
|
|
145
|
+
|
|
146
|
+
return None
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Springer Nature Full Text API (TDM): subscription full text for 10.1007.
|
|
2
|
+
|
|
3
|
+
Institutional lane for Springer papers, mirroring the Elsevier API lane.
|
|
4
|
+
There is no PDF endpoint — the TDM API returns the JATS XML full text —
|
|
5
|
+
so the artifact is a .xml sibling (non-PDF artifacts keep their extension;
|
|
6
|
+
_auto_rename skips them). Entitlement follows the INSTITUTION: the account
|
|
7
|
+
must hold a Springer subscription with TDM rights (free key from
|
|
8
|
+
dev.springernature.com, affiliated via ORCID). Without entitlement the API
|
|
9
|
+
returns no full text and this source degrades to a silent miss.
|
|
10
|
+
|
|
11
|
+
Endpoint: https://api.springernature.com/xmldata/jats?q=doi:"..."&api_key=...
|
|
12
|
+
(the official springernature_api_client sends the key as a query param;
|
|
13
|
+
premium keys have the form "key/metric" and work the same way).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
SPRINGER_TDM_URL = "https://api.springernature.com/xmldata/jats"
|
|
22
|
+
# Free portal keys are scoped per API: an Open Access API key gets 401 from
|
|
23
|
+
# the TDM endpoint but is a valid key. Probe the OA endpoint to tell the two
|
|
24
|
+
# apart, so an OA-only key is a silent miss instead of "recreate your key".
|
|
25
|
+
SPRINGER_OA_URL = "https://api.springernature.com/openaccess/jats"
|
|
26
|
+
_OA_ONLY_KEYS: dict[str, bool] = {}
|
|
27
|
+
|
|
28
|
+
# Known paywalled article (Math. Ann., verified closed via OpenAlex) used to
|
|
29
|
+
# probe whether a key carries subscription full-text entitlement.
|
|
30
|
+
_ENTITLEMENT_PROBE_DOI = "10.1007/s00208-025-03286-4"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _session(config: dict[str, Any]):
|
|
34
|
+
import requests
|
|
35
|
+
|
|
36
|
+
s = requests.Session()
|
|
37
|
+
s.trust_env = False
|
|
38
|
+
proxy = config.get("network_proxy", "")
|
|
39
|
+
if proxy:
|
|
40
|
+
s.proxies = {"http": proxy, "https": proxy}
|
|
41
|
+
from ..network import USER_AGENT
|
|
42
|
+
|
|
43
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
44
|
+
from ..network import apply_host_proxy
|
|
45
|
+
apply_host_proxy(s, config)
|
|
46
|
+
return s
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def fetch_fulltext_xml(doi: str, api_key: str,
|
|
50
|
+
config: dict[str, Any]) -> tuple[str | None, str]:
|
|
51
|
+
"""Fetch the JATS XML full text for one DOI.
|
|
52
|
+
|
|
53
|
+
Returns (xml, status); status is one of
|
|
54
|
+
ok / not_found / invalid_key / not_entitled / rate_limited / error.
|
|
55
|
+
"""
|
|
56
|
+
try:
|
|
57
|
+
resp = _session(config).get(
|
|
58
|
+
SPRINGER_TDM_URL,
|
|
59
|
+
params={"q": f'doi:"{doi}"', "p": 1, "s": 1, "api_key": api_key},
|
|
60
|
+
timeout=(10, 45),
|
|
61
|
+
)
|
|
62
|
+
except Exception as e:
|
|
63
|
+
return None, f"error:{type(e).__name__}"
|
|
64
|
+
|
|
65
|
+
if resp.status_code == 401 and _is_oa_only_key(api_key, config):
|
|
66
|
+
return None, "oa_only"
|
|
67
|
+
if resp.status_code in (401, 403):
|
|
68
|
+
# 401 = bad key; 403 = key valid but no entitlement for this content
|
|
69
|
+
return None, "invalid_key" if resp.status_code == 401 else "not_entitled"
|
|
70
|
+
if resp.status_code == 429:
|
|
71
|
+
return None, "rate_limited"
|
|
72
|
+
if resp.status_code != 200:
|
|
73
|
+
return None, f"error:http_{resp.status_code}"
|
|
74
|
+
|
|
75
|
+
xml = resp.text or ""
|
|
76
|
+
if "<body" in xml:
|
|
77
|
+
return xml, "ok"
|
|
78
|
+
# 200 without a body: either the DOI is unknown to the API or the key
|
|
79
|
+
# lacks full-text rights (both surface identically) — not a hard failure.
|
|
80
|
+
return None, "not_entitled" if "<response" in xml else "not_found"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _is_oa_only_key(api_key: str, config: dict[str, Any]) -> bool:
|
|
84
|
+
"""True when the OA API accepts a key that the TDM endpoint rejected."""
|
|
85
|
+
if api_key not in _OA_ONLY_KEYS:
|
|
86
|
+
try:
|
|
87
|
+
resp = _session(config).get(
|
|
88
|
+
SPRINGER_OA_URL,
|
|
89
|
+
params={"q": f'doi:"{_ENTITLEMENT_PROBE_DOI}"', "p": 1, "s": 1,
|
|
90
|
+
"api_key": api_key},
|
|
91
|
+
timeout=(10, 30),
|
|
92
|
+
)
|
|
93
|
+
except Exception:
|
|
94
|
+
return False # undecided: don't cache, fall back to invalid_key
|
|
95
|
+
_OA_ONLY_KEYS[api_key] = resp.status_code not in (401, 403)
|
|
96
|
+
return _OA_ONLY_KEYS[api_key]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def try_springer_tdm(doi: str, output_path: Path,
|
|
100
|
+
config: dict[str, Any]) -> dict[str, Any] | None:
|
|
101
|
+
"""Racing source: Springer full text via the TDM API (10.1007 only).
|
|
102
|
+
|
|
103
|
+
Writes {doi}.xml next to the expected PDF path. PDF lanes (OA direct,
|
|
104
|
+
browser) race in parallel and a PDF win is the better artifact, so this
|
|
105
|
+
source never blocks or shadows them.
|
|
106
|
+
"""
|
|
107
|
+
api_key = str(config.get("springer_api_key", "") or "")
|
|
108
|
+
if not api_key:
|
|
109
|
+
return None
|
|
110
|
+
if not doi.lower().startswith("10.1007/"):
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
xml, status = fetch_fulltext_xml(doi, api_key, config)
|
|
114
|
+
if status == "invalid_key":
|
|
115
|
+
from ..download.pdf import fail
|
|
116
|
+
|
|
117
|
+
return fail(
|
|
118
|
+
doi,
|
|
119
|
+
"Springer TDM API key rejected (HTTP 401)",
|
|
120
|
+
error_type="config_needed",
|
|
121
|
+
action="recreate_api_key",
|
|
122
|
+
)
|
|
123
|
+
if status != "ok" or not xml:
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
xml_path = output_path.with_suffix(".xml")
|
|
127
|
+
try:
|
|
128
|
+
xml_path.parent.mkdir(parents=True, exist_ok=True)
|
|
129
|
+
xml_path.write_text(xml, encoding="utf-8")
|
|
130
|
+
except OSError:
|
|
131
|
+
return None
|
|
132
|
+
|
|
133
|
+
from ..download.pdf import success
|
|
134
|
+
|
|
135
|
+
return success(doi, xml_path, "SpringerTDM")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def validate_springer_key(api_key: str,
|
|
139
|
+
config: dict[str, Any]) -> dict[str, str]:
|
|
140
|
+
"""Probe a key with a known paywalled article: does it carry entitlement?
|
|
141
|
+
|
|
142
|
+
Used by the setup tool; distinguishes invalid key / valid key without
|
|
143
|
+
subscription full-text rights / entitled.
|
|
144
|
+
"""
|
|
145
|
+
if not api_key:
|
|
146
|
+
return {"status": "no_key", "detail": "springer_api_key is empty"}
|
|
147
|
+
xml, status = fetch_fulltext_xml(_ENTITLEMENT_PROBE_DOI, api_key, config)
|
|
148
|
+
if status == "ok":
|
|
149
|
+
return {
|
|
150
|
+
"status": "entitled",
|
|
151
|
+
"detail": "subscription full text available (probe DOI returned JATS body)",
|
|
152
|
+
}
|
|
153
|
+
if status == "invalid_key":
|
|
154
|
+
return {"status": "invalid_key", "detail": "API rejected the key (HTTP 401)"}
|
|
155
|
+
if status == "oa_only":
|
|
156
|
+
return {
|
|
157
|
+
"status": "oa_only",
|
|
158
|
+
"detail": (
|
|
159
|
+
"key is valid for the Open Access API only — no TDM full-text "
|
|
160
|
+
"rights; Springer papers keep using the PDF lanes"
|
|
161
|
+
),
|
|
162
|
+
}
|
|
163
|
+
if status == "not_entitled":
|
|
164
|
+
return {
|
|
165
|
+
"status": "not_entitled",
|
|
166
|
+
"detail": (
|
|
167
|
+
"key accepted but no full text for a paywalled article — the "
|
|
168
|
+
"account needs an institutional Springer subscription with TDM "
|
|
169
|
+
"rights (affiliated via ORCID); OA-only keys work only for OA content"
|
|
170
|
+
),
|
|
171
|
+
}
|
|
172
|
+
return {"status": status, "detail": f"probe returned status={status}"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Unpaywall source for open-access papers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import urllib.parse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..config import DEFAULT_CONFIG
|
|
10
|
+
from ..network import fetch, polite_delay
|
|
11
|
+
from ..download.pdf import download_pdf, is_plausible_pdf_url, dedupe, fail
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _is_placeholder_email(email: str) -> bool:
|
|
15
|
+
"""Unpaywall 422s placeholder/anonymous addresses; they are never usable."""
|
|
16
|
+
return (
|
|
17
|
+
not email
|
|
18
|
+
or "@example.invalid" in email
|
|
19
|
+
or "@example.com" in email
|
|
20
|
+
or email.startswith("mailto:")
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def extract_unpaywall_pdf_candidates(payload: dict[str, Any]) -> list[str]:
|
|
25
|
+
"""Extract PDF URLs prioritized: best_oa > repository > publisher.
|
|
26
|
+
|
|
27
|
+
Repository URLs (EuropePMC, arXiv, PMC) are more likely to be freely
|
|
28
|
+
accessible than publisher URLs which often require subscription.
|
|
29
|
+
"""
|
|
30
|
+
repo_urls: list[str] = []
|
|
31
|
+
publisher_urls: list[str] = []
|
|
32
|
+
|
|
33
|
+
# best_oa_location first
|
|
34
|
+
best = payload.get("best_oa_location")
|
|
35
|
+
if isinstance(best, dict):
|
|
36
|
+
url = best.get("url_for_pdf") or ""
|
|
37
|
+
if is_plausible_pdf_url(url):
|
|
38
|
+
if best.get("host_type") == "publisher":
|
|
39
|
+
publisher_urls.append(url)
|
|
40
|
+
else:
|
|
41
|
+
repo_urls.append(url)
|
|
42
|
+
|
|
43
|
+
# Separate by host_type
|
|
44
|
+
all_locations = payload.get("oa_locations", [])
|
|
45
|
+
if isinstance(all_locations, list):
|
|
46
|
+
for loc in all_locations:
|
|
47
|
+
if not isinstance(loc, dict):
|
|
48
|
+
continue
|
|
49
|
+
url = loc.get("url_for_pdf") or ""
|
|
50
|
+
if not is_plausible_pdf_url(url):
|
|
51
|
+
continue
|
|
52
|
+
is_publisher = loc.get("host_type") == "publisher"
|
|
53
|
+
if is_publisher:
|
|
54
|
+
if url not in publisher_urls:
|
|
55
|
+
publisher_urls.append(url)
|
|
56
|
+
else:
|
|
57
|
+
if url not in repo_urls:
|
|
58
|
+
repo_urls.append(url)
|
|
59
|
+
|
|
60
|
+
# Repository URLs first (more likely free), then publisher URLs
|
|
61
|
+
return dedupe(repo_urls + publisher_urls)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def try_unpaywall(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
65
|
+
raw_email = str(config.get("email") or DEFAULT_CONFIG["email"]).strip()
|
|
66
|
+
if _is_placeholder_email(raw_email):
|
|
67
|
+
# Unpaywall ALWAYS requires a real email and rejects placeholders with
|
|
68
|
+
# 422. Ask the user for theirs instead of silently losing the channel.
|
|
69
|
+
return fail(
|
|
70
|
+
doi,
|
|
71
|
+
"Unpaywall needs the user's real email address — only a placeholder is configured",
|
|
72
|
+
error_type="config_needed",
|
|
73
|
+
action="ask_user_email",
|
|
74
|
+
)
|
|
75
|
+
email = urllib.parse.quote(raw_email)
|
|
76
|
+
q = urllib.parse.quote(doi, safe="")
|
|
77
|
+
url = f"https://api.unpaywall.org/v2/{q}?email={email}"
|
|
78
|
+
try:
|
|
79
|
+
resp = fetch(url, config, headers={"Accept": "application/json"})
|
|
80
|
+
status = int(getattr(resp, "status_code", 0) or 0)
|
|
81
|
+
except Exception:
|
|
82
|
+
status = 0
|
|
83
|
+
if status == 422:
|
|
84
|
+
return fail(
|
|
85
|
+
doi,
|
|
86
|
+
"Unpaywall rejected the configured email address (HTTP 422)",
|
|
87
|
+
error_type="config_needed",
|
|
88
|
+
action="ask_user_email",
|
|
89
|
+
)
|
|
90
|
+
if status == 429:
|
|
91
|
+
# Temporary, per-IP limit — retry later rather than dropping the channel.
|
|
92
|
+
return fail(
|
|
93
|
+
doi,
|
|
94
|
+
"Unpaywall rate limit (HTTP 429) — temporary, retry later",
|
|
95
|
+
error_type="rate_limited",
|
|
96
|
+
action="retry_later",
|
|
97
|
+
extra={"status_code": 429},
|
|
98
|
+
)
|
|
99
|
+
if status != 200:
|
|
100
|
+
# 404 = DOI genuinely not in Unpaywall (silent skip is correct);
|
|
101
|
+
# other transient errors just let the race continue elsewhere.
|
|
102
|
+
return None
|
|
103
|
+
try:
|
|
104
|
+
payload = resp.json()
|
|
105
|
+
except Exception:
|
|
106
|
+
return None
|
|
107
|
+
|
|
108
|
+
candidates = extract_unpaywall_pdf_candidates(payload)
|
|
109
|
+
for pdf_url in candidates[: int(config.get("max_unpaywall_candidates", 2))]:
|
|
110
|
+
polite_delay(config)
|
|
111
|
+
result = download_pdf(pdf_url, output_path, config, "Unpaywall")
|
|
112
|
+
if result:
|
|
113
|
+
result["doi"] = doi
|
|
114
|
+
result["identifier"] = doi
|
|
115
|
+
return result
|
|
116
|
+
return None
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""WebVPN institutional source — ``vpnsci`` naming aliases for ``instsci``.
|
|
2
|
+
|
|
3
|
+
v1.5.1 consolidated the original ``vpnsci.py`` into ``instsci.py``. v1.7.0
|
|
4
|
+
renamed the user-facing surface (config keys, MCP tools, ``download()`` kwarg)
|
|
5
|
+
back to ``vpnsci`` while the implementation stayed as ``instsci.py``. This
|
|
6
|
+
module restores the ``vpnsci`` import path by re-exporting the ``instsci``
|
|
7
|
+
implementation under both naming conventions, so callers using either name
|
|
8
|
+
keep working.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from .instsci import (
|
|
14
|
+
convert_url,
|
|
15
|
+
_get_webvpn_base,
|
|
16
|
+
_validate_session,
|
|
17
|
+
instsci_cookie_path,
|
|
18
|
+
instsci_is_configured,
|
|
19
|
+
instsci_login,
|
|
20
|
+
try_instsci,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# vpnsci-named aliases — the public surface expected by sources/__init__.py,
|
|
24
|
+
# server.py and config.py on v1.7.0+.
|
|
25
|
+
try_vpnsci = try_instsci
|
|
26
|
+
vpnsci_login = instsci_login
|
|
27
|
+
vpnsci_is_configured = instsci_is_configured
|
|
28
|
+
vpnsci_cookie_path = instsci_cookie_path
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"try_vpnsci",
|
|
32
|
+
"vpnsci_login",
|
|
33
|
+
"vpnsci_is_configured",
|
|
34
|
+
"vpnsci_cookie_path",
|
|
35
|
+
"convert_url",
|
|
36
|
+
"_validate_session",
|
|
37
|
+
"_get_webvpn_base",
|
|
38
|
+
]
|