litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,146 @@
1
+ """Semantic Scholar Open Access API source."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import time
7
+ import urllib.parse
8
+ from dataclasses import dataclass, field
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import requests
13
+
14
+ from ..network import fetch_json, polite_delay
15
+ from ..download.pdf import download_pdf, is_plausible_pdf_url
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ S2_API = "https://api.semanticscholar.org/graph/v1"
20
+ S2_FIELDS = "title,authors,year,abstract,externalIds,journal,citationCount,url"
21
+ _MAX_RETRIES = 3
22
+
23
+
24
+ @dataclass
25
+ class SearchResult:
26
+ """A single search result from Semantic Scholar."""
27
+
28
+ title: str = ""
29
+ authors: list[str] = field(default_factory=list)
30
+ year: int | None = None
31
+ abstract: str = ""
32
+ doi: str = ""
33
+ arxiv_id: str = ""
34
+ journal: str = ""
35
+ citation_count: int = 0
36
+ s2_url: str = ""
37
+ paper_id: str = ""
38
+
39
+
40
+ def _s2_request(url: str, params: dict) -> dict | None:
41
+ """Make a GET request to Semantic Scholar with retry on 429."""
42
+ for attempt in range(_MAX_RETRIES):
43
+ try:
44
+ resp = requests.get(url, params=params, timeout=15)
45
+ if resp.status_code == 404:
46
+ return None
47
+ if resp.status_code == 429:
48
+ wait = 2 ** (attempt + 1)
49
+ logger.warning("Rate limited by Semantic Scholar, retrying in %ds...", wait)
50
+ time.sleep(wait)
51
+ continue
52
+ resp.raise_for_status()
53
+ return resp.json()
54
+ except requests.RequestException as e:
55
+ logger.warning("Semantic Scholar request failed: %s", e)
56
+ if attempt < _MAX_RETRIES - 1:
57
+ time.sleep(2)
58
+ else:
59
+ return None
60
+ return None
61
+
62
+
63
+ def _parse_paper(item: dict) -> SearchResult:
64
+ """Parse a Semantic Scholar API response item into a SearchResult."""
65
+ ext_ids = item.get("externalIds") or {}
66
+ authors_data = item.get("authors") or []
67
+ journal_data = item.get("journal") or {}
68
+ return SearchResult(
69
+ title=item.get("title", ""),
70
+ authors=[a.get("name", "") for a in authors_data if a.get("name")],
71
+ year=item.get("year"),
72
+ abstract=item.get("abstract") or "",
73
+ doi=ext_ids.get("DOI", ""),
74
+ arxiv_id=ext_ids.get("ArXiv", ""),
75
+ journal=journal_data.get("name", "") if isinstance(journal_data, dict) else str(journal_data),
76
+ citation_count=item.get("citationCount", 0),
77
+ s2_url=item.get("url", ""),
78
+ paper_id=item.get("paperId", ""),
79
+ )
80
+
81
+
82
+ def search(
83
+ query: str,
84
+ limit: int = 10,
85
+ year_range: str | None = None,
86
+ fields_of_study: list[str] | None = None,
87
+ ) -> list[SearchResult]:
88
+ """Search for papers on Semantic Scholar."""
89
+ params = {
90
+ "query": query,
91
+ "limit": min(limit, 100),
92
+ "fields": S2_FIELDS,
93
+ }
94
+ if year_range:
95
+ params["year"] = year_range
96
+ if fields_of_study:
97
+ params["fieldsOfStudy"] = ",".join(fields_of_study)
98
+
99
+ data = _s2_request(f"{S2_API}/paper/search", params)
100
+ if data is None:
101
+ return []
102
+ return [_parse_paper(item) for item in data.get("data", [])]
103
+
104
+
105
+ def get_paper(paper_id: str) -> SearchResult | None:
106
+ """Get details for a specific paper by Semantic Scholar ID or DOI.
107
+
108
+ paper_id: S2 paper ID, DOI (prefix with "DOI:"), or arXiv ID (prefix with "ARXIV:").
109
+ """
110
+ item = _s2_request(f"{S2_API}/paper/{paper_id}", {"fields": S2_FIELDS})
111
+ if item is None:
112
+ return None
113
+ return _parse_paper(item)
114
+
115
+
116
+ def try_semanticscholar(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
117
+ q = urllib.parse.quote(doi, safe="")
118
+ url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{q}?fields=openAccessPdf,externalIds"
119
+ payload = fetch_json(url, config)
120
+ if not payload:
121
+ return None
122
+
123
+ oa_pdf = payload.get("openAccessPdf")
124
+ if isinstance(oa_pdf, dict):
125
+ pdf_url = oa_pdf.get("url", "")
126
+ if pdf_url and is_plausible_pdf_url(pdf_url):
127
+ polite_delay(config)
128
+ result = download_pdf(pdf_url, output_path, config, "SemanticScholar")
129
+ if result:
130
+ result["doi"] = doi
131
+ result["identifier"] = doi
132
+ return result
133
+
134
+ ext_ids = payload.get("externalIds", {})
135
+ if isinstance(ext_ids, dict):
136
+ arxiv_id = ext_ids.get("ArXiv")
137
+ if arxiv_id:
138
+ arxiv_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
139
+ polite_delay(config)
140
+ result = download_pdf(arxiv_url, output_path, config, "SemanticScholar(arXiv)", require_pdf_like_url=False)
141
+ if result:
142
+ result["doi"] = doi
143
+ result["identifier"] = doi
144
+ return result
145
+
146
+ return None
@@ -0,0 +1,172 @@
1
+ """Springer Nature Full Text API (TDM): subscription full text for 10.1007.
2
+
3
+ Institutional lane for Springer papers, mirroring the Elsevier API lane.
4
+ There is no PDF endpoint — the TDM API returns the JATS XML full text —
5
+ so the artifact is a .xml sibling (non-PDF artifacts keep their extension;
6
+ _auto_rename skips them). Entitlement follows the INSTITUTION: the account
7
+ must hold a Springer subscription with TDM rights (free key from
8
+ dev.springernature.com, affiliated via ORCID). Without entitlement the API
9
+ returns no full text and this source degrades to a silent miss.
10
+
11
+ Endpoint: https://api.springernature.com/xmldata/jats?q=doi:"..."&api_key=...
12
+ (the official springernature_api_client sends the key as a query param;
13
+ premium keys have the form "key/metric" and work the same way).
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ SPRINGER_TDM_URL = "https://api.springernature.com/xmldata/jats"
22
+ # Free portal keys are scoped per API: an Open Access API key gets 401 from
23
+ # the TDM endpoint but is a valid key. Probe the OA endpoint to tell the two
24
+ # apart, so an OA-only key is a silent miss instead of "recreate your key".
25
+ SPRINGER_OA_URL = "https://api.springernature.com/openaccess/jats"
26
+ _OA_ONLY_KEYS: dict[str, bool] = {}
27
+
28
+ # Known paywalled article (Math. Ann., verified closed via OpenAlex) used to
29
+ # probe whether a key carries subscription full-text entitlement.
30
+ _ENTITLEMENT_PROBE_DOI = "10.1007/s00208-025-03286-4"
31
+
32
+
33
+ def _session(config: dict[str, Any]):
34
+ import requests
35
+
36
+ s = requests.Session()
37
+ s.trust_env = False
38
+ proxy = config.get("network_proxy", "")
39
+ if proxy:
40
+ s.proxies = {"http": proxy, "https": proxy}
41
+ from ..network import USER_AGENT
42
+
43
+ s.headers.update({"User-Agent": USER_AGENT})
44
+ from ..network import apply_host_proxy
45
+ apply_host_proxy(s, config)
46
+ return s
47
+
48
+
49
+ def fetch_fulltext_xml(doi: str, api_key: str,
50
+ config: dict[str, Any]) -> tuple[str | None, str]:
51
+ """Fetch the JATS XML full text for one DOI.
52
+
53
+ Returns (xml, status); status is one of
54
+ ok / not_found / invalid_key / not_entitled / rate_limited / error.
55
+ """
56
+ try:
57
+ resp = _session(config).get(
58
+ SPRINGER_TDM_URL,
59
+ params={"q": f'doi:"{doi}"', "p": 1, "s": 1, "api_key": api_key},
60
+ timeout=(10, 45),
61
+ )
62
+ except Exception as e:
63
+ return None, f"error:{type(e).__name__}"
64
+
65
+ if resp.status_code == 401 and _is_oa_only_key(api_key, config):
66
+ return None, "oa_only"
67
+ if resp.status_code in (401, 403):
68
+ # 401 = bad key; 403 = key valid but no entitlement for this content
69
+ return None, "invalid_key" if resp.status_code == 401 else "not_entitled"
70
+ if resp.status_code == 429:
71
+ return None, "rate_limited"
72
+ if resp.status_code != 200:
73
+ return None, f"error:http_{resp.status_code}"
74
+
75
+ xml = resp.text or ""
76
+ if "<body" in xml:
77
+ return xml, "ok"
78
+ # 200 without a body: either the DOI is unknown to the API or the key
79
+ # lacks full-text rights (both surface identically) — not a hard failure.
80
+ return None, "not_entitled" if "<response" in xml else "not_found"
81
+
82
+
83
+ def _is_oa_only_key(api_key: str, config: dict[str, Any]) -> bool:
84
+ """True when the OA API accepts a key that the TDM endpoint rejected."""
85
+ if api_key not in _OA_ONLY_KEYS:
86
+ try:
87
+ resp = _session(config).get(
88
+ SPRINGER_OA_URL,
89
+ params={"q": f'doi:"{_ENTITLEMENT_PROBE_DOI}"', "p": 1, "s": 1,
90
+ "api_key": api_key},
91
+ timeout=(10, 30),
92
+ )
93
+ except Exception:
94
+ return False # undecided: don't cache, fall back to invalid_key
95
+ _OA_ONLY_KEYS[api_key] = resp.status_code not in (401, 403)
96
+ return _OA_ONLY_KEYS[api_key]
97
+
98
+
99
+ def try_springer_tdm(doi: str, output_path: Path,
100
+ config: dict[str, Any]) -> dict[str, Any] | None:
101
+ """Racing source: Springer full text via the TDM API (10.1007 only).
102
+
103
+ Writes {doi}.xml next to the expected PDF path. PDF lanes (OA direct,
104
+ browser) race in parallel and a PDF win is the better artifact, so this
105
+ source never blocks or shadows them.
106
+ """
107
+ api_key = str(config.get("springer_api_key", "") or "")
108
+ if not api_key:
109
+ return None
110
+ if not doi.lower().startswith("10.1007/"):
111
+ return None
112
+
113
+ xml, status = fetch_fulltext_xml(doi, api_key, config)
114
+ if status == "invalid_key":
115
+ from ..download.pdf import fail
116
+
117
+ return fail(
118
+ doi,
119
+ "Springer TDM API key rejected (HTTP 401)",
120
+ error_type="config_needed",
121
+ action="recreate_api_key",
122
+ )
123
+ if status != "ok" or not xml:
124
+ return None
125
+
126
+ xml_path = output_path.with_suffix(".xml")
127
+ try:
128
+ xml_path.parent.mkdir(parents=True, exist_ok=True)
129
+ xml_path.write_text(xml, encoding="utf-8")
130
+ except OSError:
131
+ return None
132
+
133
+ from ..download.pdf import success
134
+
135
+ return success(doi, xml_path, "SpringerTDM")
136
+
137
+
138
+ def validate_springer_key(api_key: str,
139
+ config: dict[str, Any]) -> dict[str, str]:
140
+ """Probe a key with a known paywalled article: does it carry entitlement?
141
+
142
+ Used by the setup tool; distinguishes invalid key / valid key without
143
+ subscription full-text rights / entitled.
144
+ """
145
+ if not api_key:
146
+ return {"status": "no_key", "detail": "springer_api_key is empty"}
147
+ xml, status = fetch_fulltext_xml(_ENTITLEMENT_PROBE_DOI, api_key, config)
148
+ if status == "ok":
149
+ return {
150
+ "status": "entitled",
151
+ "detail": "subscription full text available (probe DOI returned JATS body)",
152
+ }
153
+ if status == "invalid_key":
154
+ return {"status": "invalid_key", "detail": "API rejected the key (HTTP 401)"}
155
+ if status == "oa_only":
156
+ return {
157
+ "status": "oa_only",
158
+ "detail": (
159
+ "key is valid for the Open Access API only — no TDM full-text "
160
+ "rights; Springer papers keep using the PDF lanes"
161
+ ),
162
+ }
163
+ if status == "not_entitled":
164
+ return {
165
+ "status": "not_entitled",
166
+ "detail": (
167
+ "key accepted but no full text for a paywalled article — the "
168
+ "account needs an institutional Springer subscription with TDM "
169
+ "rights (affiliated via ORCID); OA-only keys work only for OA content"
170
+ ),
171
+ }
172
+ return {"status": status, "detail": f"probe returned status={status}"}
@@ -0,0 +1,116 @@
1
+ """Unpaywall source for open-access papers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import urllib.parse
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from ..config import DEFAULT_CONFIG
10
+ from ..network import fetch, polite_delay
11
+ from ..download.pdf import download_pdf, is_plausible_pdf_url, dedupe, fail
12
+
13
+
14
+ def _is_placeholder_email(email: str) -> bool:
15
+ """Unpaywall 422s placeholder/anonymous addresses; they are never usable."""
16
+ return (
17
+ not email
18
+ or "@example.invalid" in email
19
+ or "@example.com" in email
20
+ or email.startswith("mailto:")
21
+ )
22
+
23
+
24
+ def extract_unpaywall_pdf_candidates(payload: dict[str, Any]) -> list[str]:
25
+ """Extract PDF URLs prioritized: best_oa > repository > publisher.
26
+
27
+ Repository URLs (EuropePMC, arXiv, PMC) are more likely to be freely
28
+ accessible than publisher URLs which often require subscription.
29
+ """
30
+ repo_urls: list[str] = []
31
+ publisher_urls: list[str] = []
32
+
33
+ # best_oa_location first
34
+ best = payload.get("best_oa_location")
35
+ if isinstance(best, dict):
36
+ url = best.get("url_for_pdf") or ""
37
+ if is_plausible_pdf_url(url):
38
+ if best.get("host_type") == "publisher":
39
+ publisher_urls.append(url)
40
+ else:
41
+ repo_urls.append(url)
42
+
43
+ # Separate by host_type
44
+ all_locations = payload.get("oa_locations", [])
45
+ if isinstance(all_locations, list):
46
+ for loc in all_locations:
47
+ if not isinstance(loc, dict):
48
+ continue
49
+ url = loc.get("url_for_pdf") or ""
50
+ if not is_plausible_pdf_url(url):
51
+ continue
52
+ is_publisher = loc.get("host_type") == "publisher"
53
+ if is_publisher:
54
+ if url not in publisher_urls:
55
+ publisher_urls.append(url)
56
+ else:
57
+ if url not in repo_urls:
58
+ repo_urls.append(url)
59
+
60
+ # Repository URLs first (more likely free), then publisher URLs
61
+ return dedupe(repo_urls + publisher_urls)
62
+
63
+
64
+ def try_unpaywall(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
65
+ raw_email = str(config.get("email") or DEFAULT_CONFIG["email"]).strip()
66
+ if _is_placeholder_email(raw_email):
67
+ # Unpaywall ALWAYS requires a real email and rejects placeholders with
68
+ # 422. Ask the user for theirs instead of silently losing the channel.
69
+ return fail(
70
+ doi,
71
+ "Unpaywall needs the user's real email address — only a placeholder is configured",
72
+ error_type="config_needed",
73
+ action="ask_user_email",
74
+ )
75
+ email = urllib.parse.quote(raw_email)
76
+ q = urllib.parse.quote(doi, safe="")
77
+ url = f"https://api.unpaywall.org/v2/{q}?email={email}"
78
+ try:
79
+ resp = fetch(url, config, headers={"Accept": "application/json"})
80
+ status = int(getattr(resp, "status_code", 0) or 0)
81
+ except Exception:
82
+ status = 0
83
+ if status == 422:
84
+ return fail(
85
+ doi,
86
+ "Unpaywall rejected the configured email address (HTTP 422)",
87
+ error_type="config_needed",
88
+ action="ask_user_email",
89
+ )
90
+ if status == 429:
91
+ # Temporary, per-IP limit — retry later rather than dropping the channel.
92
+ return fail(
93
+ doi,
94
+ "Unpaywall rate limit (HTTP 429) — temporary, retry later",
95
+ error_type="rate_limited",
96
+ action="retry_later",
97
+ extra={"status_code": 429},
98
+ )
99
+ if status != 200:
100
+ # 404 = DOI genuinely not in Unpaywall (silent skip is correct);
101
+ # other transient errors just let the race continue elsewhere.
102
+ return None
103
+ try:
104
+ payload = resp.json()
105
+ except Exception:
106
+ return None
107
+
108
+ candidates = extract_unpaywall_pdf_candidates(payload)
109
+ for pdf_url in candidates[: int(config.get("max_unpaywall_candidates", 2))]:
110
+ polite_delay(config)
111
+ result = download_pdf(pdf_url, output_path, config, "Unpaywall")
112
+ if result:
113
+ result["doi"] = doi
114
+ result["identifier"] = doi
115
+ return result
116
+ return None
@@ -0,0 +1,38 @@
1
+ """WebVPN institutional source — ``vpnsci`` naming aliases for ``instsci``.
2
+
3
+ v1.5.1 consolidated the original ``vpnsci.py`` into ``instsci.py``. v1.7.0
4
+ renamed the user-facing surface (config keys, MCP tools, ``download()`` kwarg)
5
+ back to ``vpnsci`` while the implementation stayed as ``instsci.py``. This
6
+ module restores the ``vpnsci`` import path by re-exporting the ``instsci``
7
+ implementation under both naming conventions, so callers using either name
8
+ keep working.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from .instsci import (
14
+ convert_url,
15
+ _get_webvpn_base,
16
+ _validate_session,
17
+ instsci_cookie_path,
18
+ instsci_is_configured,
19
+ instsci_login,
20
+ try_instsci,
21
+ )
22
+
23
+ # vpnsci-named aliases — the public surface expected by sources/__init__.py,
24
+ # server.py and config.py on v1.7.0+.
25
+ try_vpnsci = try_instsci
26
+ vpnsci_login = instsci_login
27
+ vpnsci_is_configured = instsci_is_configured
28
+ vpnsci_cookie_path = instsci_cookie_path
29
+
30
+ __all__ = [
31
+ "try_vpnsci",
32
+ "vpnsci_login",
33
+ "vpnsci_is_configured",
34
+ "vpnsci_cookie_path",
35
+ "convert_url",
36
+ "_validate_session",
37
+ "_get_webvpn_base",
38
+ ]