litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Taylor & Francis (Tandfonline) publisher strategy."""
|
|
2
|
+
|
|
3
|
+
from .base import BasePublisherStrategy
|
|
4
|
+
from .registry import StrategyRegistry
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@StrategyRegistry.register
|
|
8
|
+
class TandfonlineStrategy(BasePublisherStrategy):
|
|
9
|
+
name = "Tandfonline"
|
|
10
|
+
aliases = ("tandfonline", "taylorandfrancis", "tandf")
|
|
11
|
+
doi_prefixes = ("10.1080/",)
|
|
12
|
+
base_domains = ("tandfonline.com",)
|
|
13
|
+
sample_dois = ("10.1080/00268976.2019.1660819",)
|
|
14
|
+
article_url_template = "https://www.tandfonline.com/doi/{doi}"
|
|
15
|
+
pdf_url_templates = (
|
|
16
|
+
"https://www.tandfonline.com/doi/pdf/{doi}",
|
|
17
|
+
)
|
|
18
|
+
success_url_markers = ("tandfonline.com/doi/",)
|
|
19
|
+
auth_url_markers = (
|
|
20
|
+
"id.tsinghua.edu.cn",
|
|
21
|
+
"idp.tsinghua.edu.cn",
|
|
22
|
+
"login.openathens.net",
|
|
23
|
+
)
|
|
24
|
+
sso_text_markers = ("Access through your institution",)
|
|
25
|
+
sso_url_patterns = ("/ssostart", "/shibboleth", "/institutional")
|
|
26
|
+
institution_input_selectors = (
|
|
27
|
+
'input[placeholder*="institution"]',
|
|
28
|
+
'input[placeholder*="Type the name"]',
|
|
29
|
+
"#searchInstitution",
|
|
30
|
+
"input[name='query']",
|
|
31
|
+
)
|
|
32
|
+
institution_result_selectors = ("#searchInstitution",)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Wiley publisher strategy."""
|
|
2
|
+
|
|
3
|
+
from .base import BasePublisherStrategy
|
|
4
|
+
from .registry import StrategyRegistry
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@StrategyRegistry.register
|
|
8
|
+
class WileyStrategy(BasePublisherStrategy):
|
|
9
|
+
name = "Wiley"
|
|
10
|
+
aliases = ("wiley",)
|
|
11
|
+
doi_prefixes = ("10.1002/", "10.1111/")
|
|
12
|
+
base_domains = ("onlinelibrary.wiley.com", "wiley.com")
|
|
13
|
+
sample_dois = ("10.1002/adma.201706259",)
|
|
14
|
+
article_url_template = "https://onlinelibrary.wiley.com/doi/{doi}"
|
|
15
|
+
pdf_url_templates = (
|
|
16
|
+
"https://onlinelibrary.wiley.com/doi/pdfdirect/{doi}",
|
|
17
|
+
"https://onlinelibrary.wiley.com/doi/pdf/{doi}",
|
|
18
|
+
)
|
|
19
|
+
success_url_markers = ("onlinelibrary.wiley.com/doi/",)
|
|
20
|
+
auth_url_markers = (
|
|
21
|
+
"id.tsinghua.edu.cn",
|
|
22
|
+
"idp.tsinghua.edu.cn",
|
|
23
|
+
"login.openathens.net",
|
|
24
|
+
)
|
|
25
|
+
sso_text_markers = (
|
|
26
|
+
"Access through your institution",
|
|
27
|
+
"Institutional Login",
|
|
28
|
+
"Institution Login",
|
|
29
|
+
)
|
|
30
|
+
sso_url_patterns = ("/ssostart",)
|
|
31
|
+
institution_input_selectors = ("#searchInstitution",)
|
|
32
|
+
institution_result_selectors = ("#searchInstitution",)
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Supplementary Information (SI) discovery and download.
|
|
2
|
+
|
|
3
|
+
v2: lenient publishers over plain HTTP; Cloudflare-protected ones (Elsevier's
|
|
4
|
+
linkinghub bounce) fall back to the stealth-browser backend, reusing its
|
|
5
|
+
cookies for the attachment downloads. SI failure never fails the main download.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import time
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from urllib.parse import urljoin
|
|
14
|
+
|
|
15
|
+
_LINK_RE = re.compile(r"""href=["']([^"']+)["']""", re.I)
|
|
16
|
+
_EXT_RE = re.compile(r"\.(pdf|docx?|xlsx?|pptx?|zip|gz|csv|txt)(?:[?#]|$)", re.I)
|
|
17
|
+
_MARKER_RE = re.compile(r"(suppl|supp_|supporting|_si\d|/cms/|attachment|mmc\d|moesm)", re.I)
|
|
18
|
+
# URLs that scream "attachment" even without a file extension (PLOS-style
|
|
19
|
+
# .../file?id=<doi>.s001&type=supplementary-file) — the download step validates
|
|
20
|
+
# by content-type instead of trusting the URL. MOESM files carry extensions,
|
|
21
|
+
# so they ride the weak rule; page anchors (#MOESM1) stay excluded.
|
|
22
|
+
_STRONG_RE = re.compile(r"(supplementary[-_]?file|/cms/|attachment|mmc\d)", re.I)
|
|
23
|
+
_MAIN_PDF_RE = re.compile(r"(?:^|/)(?:main|mainext|article)\.pdf(?:[?#]|$)", re.I)
|
|
24
|
+
|
|
25
|
+
_CT_EXT = (
|
|
26
|
+
("application/pdf", ".pdf"),
|
|
27
|
+
("wordprocessingml.document", ".docx"),
|
|
28
|
+
("msword", ".doc"),
|
|
29
|
+
("spreadsheetml.sheet", ".xlsx"),
|
|
30
|
+
("vnd.ms-excel", ".xls"),
|
|
31
|
+
("presentationml.presentation", ".pptx"),
|
|
32
|
+
("text/csv", ".csv"),
|
|
33
|
+
("application/zip", ".zip"),
|
|
34
|
+
("application/gzip", ".gz"),
|
|
35
|
+
("text/plain", ".txt"),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def extract_supplementary_links(html: str, base_url: str, max_files: int = 10) -> list[str]:
|
|
40
|
+
"""Return absolute candidate SI URLs found on an article landing page."""
|
|
41
|
+
seen: set[str] = set()
|
|
42
|
+
out: list[str] = []
|
|
43
|
+
for m in _LINK_RE.finditer(html or ""):
|
|
44
|
+
href = m.group(1)
|
|
45
|
+
if not _MARKER_RE.search(href):
|
|
46
|
+
continue
|
|
47
|
+
if _MAIN_PDF_RE.search(href):
|
|
48
|
+
continue # the article's own main PDF, not an attachment
|
|
49
|
+
# Weak marker + known extension is enough; strong markers (PLOS-style
|
|
50
|
+
# supplementary-file URLs) qualify even without an extension.
|
|
51
|
+
if not _EXT_RE.search(href) and not _STRONG_RE.search(href):
|
|
52
|
+
continue
|
|
53
|
+
url = urljoin(base_url or "", href)
|
|
54
|
+
if url in seen:
|
|
55
|
+
continue
|
|
56
|
+
seen.add(url)
|
|
57
|
+
out.append(url)
|
|
58
|
+
if len(out) >= max_files:
|
|
59
|
+
break
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def fetch_supplementary(
|
|
64
|
+
doi: str,
|
|
65
|
+
output_dir: str | Path,
|
|
66
|
+
config: dict | None = None,
|
|
67
|
+
max_files: int = 10,
|
|
68
|
+
timeout: int = 30,
|
|
69
|
+
) -> list[str]:
|
|
70
|
+
"""Download SI attachments for ``doi``; returns saved file paths.
|
|
71
|
+
|
|
72
|
+
Never raises: problems are reported by returning fewer/zero files.
|
|
73
|
+
"""
|
|
74
|
+
config = config or {}
|
|
75
|
+
out = Path(output_dir)
|
|
76
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
77
|
+
|
|
78
|
+
article_url = _article_url(doi)
|
|
79
|
+
if not article_url:
|
|
80
|
+
return []
|
|
81
|
+
|
|
82
|
+
html, fetch_bytes = _fetch_page(article_url, config, timeout)
|
|
83
|
+
if not html:
|
|
84
|
+
return []
|
|
85
|
+
links = extract_supplementary_links(html, article_url, max_files=max_files)
|
|
86
|
+
|
|
87
|
+
saved: list[str] = []
|
|
88
|
+
stem = _safe_stem(doi)
|
|
89
|
+
for i, url in enumerate(links, 1):
|
|
90
|
+
body, ctype = fetch_bytes(url)
|
|
91
|
+
if not body:
|
|
92
|
+
continue
|
|
93
|
+
ctype = (ctype or "").lower()
|
|
94
|
+
if "text/html" in ctype:
|
|
95
|
+
continue # another landing page, not an attachment
|
|
96
|
+
ext = _ext_of(url) or _ext_from_content_type(ctype) or ".bin"
|
|
97
|
+
path = out / f"{stem}_SI{i}{ext}"
|
|
98
|
+
path.write_bytes(body)
|
|
99
|
+
saved.append(str(path))
|
|
100
|
+
return saved
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _fetch_page(article_url: str, config: dict, timeout: int):
|
|
104
|
+
"""Get (html, fetch_bytes) for the article page.
|
|
105
|
+
|
|
106
|
+
Plain HTTP first. When the publisher bounces us (Elsevier's linkinghub) or
|
|
107
|
+
returns a non-200, retry through the stealth-browser backend so Cloudflare
|
|
108
|
+
cookies are shared with the attachment downloads. fetch_bytes(url) returns
|
|
109
|
+
(body | None, content-type | None).
|
|
110
|
+
"""
|
|
111
|
+
import requests
|
|
112
|
+
|
|
113
|
+
proxy = (config or {}).get("network_proxy", "")
|
|
114
|
+
proxies = {"http": proxy, "https": proxy} if proxy else None
|
|
115
|
+
headers = {"User-Agent": _user_agent()}
|
|
116
|
+
|
|
117
|
+
session = requests.Session()
|
|
118
|
+
session.trust_env = False
|
|
119
|
+
blocked = False
|
|
120
|
+
html = ""
|
|
121
|
+
try:
|
|
122
|
+
resp = session.get(article_url, headers=headers, proxies=proxies, timeout=timeout, allow_redirects=True)
|
|
123
|
+
blocked = resp.status_code != 200 or "linkinghub." in str(resp.url)
|
|
124
|
+
if resp.status_code == 200:
|
|
125
|
+
html = resp.text
|
|
126
|
+
except Exception:
|
|
127
|
+
blocked = True
|
|
128
|
+
|
|
129
|
+
if not blocked:
|
|
130
|
+
def plain_fetch(url: str):
|
|
131
|
+
try:
|
|
132
|
+
r = session.get(url, headers=headers, proxies=proxies, timeout=timeout, allow_redirects=True)
|
|
133
|
+
return (r.content, r.headers.get("content-type")) if r.status_code == 200 else (None, None)
|
|
134
|
+
except Exception:
|
|
135
|
+
return None, None
|
|
136
|
+
return html, plain_fetch
|
|
137
|
+
|
|
138
|
+
fetched = _browser_page(article_url, config, timeout)
|
|
139
|
+
if fetched:
|
|
140
|
+
return fetched
|
|
141
|
+
return (html, None) if html else (None, None)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _browser_page(article_url: str, config: dict, timeout: int):
|
|
145
|
+
"""Render the article page through the shared stealth-browser backend.
|
|
146
|
+
|
|
147
|
+
Uses the browser-worker pool (browser_engine) instead of a cold start per
|
|
148
|
+
call, and downloads attachments through the browser context's request API
|
|
149
|
+
so Cloudflare cookies are shared. Returns (page_html, fetch_bytes); only
|
|
150
|
+
the PAGE is closed - the pooled browser stays alive for the next paper.
|
|
151
|
+
"""
|
|
152
|
+
try:
|
|
153
|
+
from ..browser.engine import get_browser_page
|
|
154
|
+
|
|
155
|
+
page = get_browser_page(config)
|
|
156
|
+
except Exception:
|
|
157
|
+
return None
|
|
158
|
+
if page is None:
|
|
159
|
+
return None
|
|
160
|
+
|
|
161
|
+
try:
|
|
162
|
+
page.goto(article_url, wait_until="domcontentloaded", timeout=max(timeout, 45) * 1000)
|
|
163
|
+
time.sleep(2.5) # not page.wait_for_timeout: it hangs on a page stuck loading
|
|
164
|
+
html = page.content()
|
|
165
|
+
|
|
166
|
+
def fetch_bytes(url: str):
|
|
167
|
+
try:
|
|
168
|
+
r = page.context.request.get(url, timeout=max(timeout, 45) * 1000)
|
|
169
|
+
if r.status != 200:
|
|
170
|
+
return None, None
|
|
171
|
+
return r.body(), r.headers.get("content-type")
|
|
172
|
+
except Exception:
|
|
173
|
+
return None, None
|
|
174
|
+
|
|
175
|
+
return html, fetch_bytes
|
|
176
|
+
except Exception:
|
|
177
|
+
try:
|
|
178
|
+
ctx.close()
|
|
179
|
+
except Exception:
|
|
180
|
+
pass
|
|
181
|
+
return None
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _ext_from_content_type(ctype: str) -> str | None:
|
|
185
|
+
for frag, ext in _CT_EXT:
|
|
186
|
+
if frag in ctype:
|
|
187
|
+
return ext
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _article_url(doi: str) -> str | None:
|
|
192
|
+
try:
|
|
193
|
+
from .strategies import StrategyRegistry
|
|
194
|
+
strategy = StrategyRegistry.get_for_doi(doi)
|
|
195
|
+
except Exception:
|
|
196
|
+
return None
|
|
197
|
+
if strategy is None:
|
|
198
|
+
return None
|
|
199
|
+
try:
|
|
200
|
+
return strategy.article_url(doi)
|
|
201
|
+
except Exception:
|
|
202
|
+
return None
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _ext_of(url: str) -> str | None:
|
|
206
|
+
m = _EXT_RE.search(url)
|
|
207
|
+
return f".{m.group(1).lower()}" if m else None
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _safe_stem(identifier: str) -> str:
|
|
211
|
+
return re.sub(r"[^A-Za-z0-9._-]+", "_", identifier.strip()).strip("_")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _user_agent() -> str:
|
|
215
|
+
try:
|
|
216
|
+
from ..network import USER_AGENT
|
|
217
|
+
return USER_AGENT
|
|
218
|
+
except Exception: # pragma: no cover
|
|
219
|
+
return "Mozilla/5.0 (compatible; litloop)"
|
litloop/quota.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Usage and limits of the rate-limited APIs LitLoop calls.
|
|
2
|
+
|
|
3
|
+
Every request to OpenAlex, Scopus, Semantic Scholar or Crossref goes through
|
|
4
|
+
``slot(provider)`` (a per-provider cap on parallel requests, so bursts do not
|
|
5
|
+
trip per-second throttles) and ``record(provider, response)``, which keeps:
|
|
6
|
+
|
|
7
|
+
* requests made today, and how many were throttled (HTTP 429);
|
|
8
|
+
* what the provider last said about its budget (remaining, limit, reset),
|
|
9
|
+
read from the X-RateLimit-* headers when it sends them.
|
|
10
|
+
|
|
11
|
+
The numbers live in ~/.litloop/api_usage.json, shared by the MCP server, the
|
|
12
|
+
command line and scripts. ``summary()`` is what diagnostics, deep search and
|
|
13
|
+
(later) the app show; ``warnings()`` names the providers close to their limit.
|
|
14
|
+
The caps only coordinate threads within one process; separate processes
|
|
15
|
+
still see each other's usage through the file.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from contextlib import contextmanager
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from . import config as _config
|
|
27
|
+
|
|
28
|
+
# Official limits (checked 2026-10-16; full list with sources in docs/API_LIMITS.md).
|
|
29
|
+
# PER_SECOND stays a little below each published per-second figure. CONCURRENCY
|
|
30
|
+
# is official only for Crossref (1 public / 3 polite); the providers below that
|
|
31
|
+
# publish no concurrency limit get LitLoop's own conservative choice.
|
|
32
|
+
# * OpenAlex: 30 requests/s with a key, but 5/s for searches with more than five
|
|
33
|
+
# boolean operators; budget 10,000 credits/day with a free key (list 1 credit,
|
|
34
|
+
# keyword search 10), reset at midnight UTC.
|
|
35
|
+
# https://help.openalex.org/how-to-use-the-api/rate-limits-and-authentication
|
|
36
|
+
# * Scopus Abstract Retrieval: 9 requests/s, 10,000 per week.
|
|
37
|
+
# https://dev.elsevier.com/api_key_settings.html
|
|
38
|
+
# * Crossref: public pool 5 single-record requests/s, 1 at a time; polite pool
|
|
39
|
+
# (with a contact email) 10/s, 3 at a time.
|
|
40
|
+
# https://crossref.org/blog/announcing-changes-to-rest-api-rate-limits/
|
|
41
|
+
# * Semantic Scholar: keyless requests share one pool and are often throttled;
|
|
42
|
+
# a key gives 1 request/s. https://www.semanticscholar.org/product/api
|
|
43
|
+
# * Springer Nature Meta API, free plan: 500 requests/day, 100/min.
|
|
44
|
+
# https://dev.springernature.com/subscription/
|
|
45
|
+
# Lanes: "openalex" assumes a key (30/s); without one OpenAlex allows 10/s and
|
|
46
|
+
# 1/s for boolean-heavy searches, and only $0.10 (1,000 credits) a day.
|
|
47
|
+
CONCURRENCY = {"openalex": 4, "openalex_boolean": 2, "openalex_nokey": 2, "openalex_boolean_nokey": 1,
|
|
48
|
+
"scopus": 5, "semantic_scholar": 1, "crossref": 1, "crossref_polite": 3, "springer": 2}
|
|
49
|
+
PER_SECOND = {"openalex": 25.0, "openalex_boolean": 4.0, "openalex_nokey": 8.0, "openalex_boolean_nokey": 0.9,
|
|
50
|
+
"scopus": 8.0, "semantic_scholar": 1.0, "crossref": 4.0, "crossref_polite": 8.0, "springer": 1.5}
|
|
51
|
+
WARN_FRACTION = 0.1 # warn when less than this share of the budget is left
|
|
52
|
+
|
|
53
|
+
_lock = threading.Lock()
|
|
54
|
+
_slots = {name: threading.Semaphore(n) for name, n in CONCURRENCY.items()}
|
|
55
|
+
_next_free = {name: 0.0 for name in PER_SECOND}
|
|
56
|
+
_spent: dict[str, int] = {} # budget units used by this process (OpenAlex credits, others requests)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _path():
|
|
60
|
+
return _config.DATA_DIR / "api_usage.json"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _load() -> dict[str, Any]:
|
|
64
|
+
try:
|
|
65
|
+
return json.loads(_path().read_text(encoding="utf-8"))
|
|
66
|
+
except (OSError, ValueError):
|
|
67
|
+
return {}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _save(data: dict[str, Any]) -> None:
|
|
71
|
+
try:
|
|
72
|
+
_path().parent.mkdir(parents=True, exist_ok=True)
|
|
73
|
+
tmp = _path().with_suffix(".tmp")
|
|
74
|
+
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
75
|
+
tmp.replace(_path())
|
|
76
|
+
except OSError:
|
|
77
|
+
pass # usage tracking must never break a search
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@contextmanager
|
|
81
|
+
def slot(provider: str):
|
|
82
|
+
"""Hold one of ``provider``'s parallel-request slots, spaced to its per-second limit."""
|
|
83
|
+
sem = _slots.get(provider)
|
|
84
|
+
if sem is None:
|
|
85
|
+
yield
|
|
86
|
+
return
|
|
87
|
+
with sem:
|
|
88
|
+
rate = PER_SECOND.get(provider)
|
|
89
|
+
if rate:
|
|
90
|
+
with _lock:
|
|
91
|
+
now = time.monotonic()
|
|
92
|
+
start = max(now, _next_free[provider])
|
|
93
|
+
_next_free[provider] = start + 1.0 / rate
|
|
94
|
+
if start > now:
|
|
95
|
+
time.sleep(start - now)
|
|
96
|
+
yield
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def spent(provider: str) -> int:
|
|
100
|
+
"""Budget units ``provider`` has used in this process (OpenAlex: credits)."""
|
|
101
|
+
return _spent.get(provider, 0)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def record(provider: str, response: Any = None, *, throttled: bool | None = None) -> None:
|
|
105
|
+
"""Count one request to ``provider`` and keep the budget headers it returned."""
|
|
106
|
+
today = time.strftime("%Y-%m-%d")
|
|
107
|
+
headers = getattr(response, "headers", None) or {}
|
|
108
|
+
status = getattr(response, "status_code", None)
|
|
109
|
+
with _lock:
|
|
110
|
+
data = _load()
|
|
111
|
+
p = data.setdefault(provider, {})
|
|
112
|
+
if p.get("day") != today:
|
|
113
|
+
p.update(day=today, requests=0, throttled=0, credits=0)
|
|
114
|
+
p["requests"] = p.get("requests", 0) + 1
|
|
115
|
+
cost = _int(headers.get("X-RateLimit-Credits-Used"))
|
|
116
|
+
cost = 1 if cost is None else cost
|
|
117
|
+
if status is not None and status < 400:
|
|
118
|
+
p["credits"] = p.get("credits", 0) + cost
|
|
119
|
+
_spent[provider] = _spent.get(provider, 0) + cost
|
|
120
|
+
if throttled or status == 429:
|
|
121
|
+
p["throttled"] = p.get("throttled", 0) + 1
|
|
122
|
+
p["last_throttled"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
123
|
+
remaining = headers.get("X-RateLimit-Remaining")
|
|
124
|
+
if "QUOTA_EXCEEDED" in str(headers.get("X-ELS-Status") or ""):
|
|
125
|
+
p["remaining"] = 0 # Elsevier drops the Remaining header once the quota is spent
|
|
126
|
+
if remaining is not None:
|
|
127
|
+
p["remaining"] = _int(remaining)
|
|
128
|
+
p["limit"] = _int(headers.get("X-RateLimit-Limit"))
|
|
129
|
+
reset = _int(headers.get("X-RateLimit-Reset"))
|
|
130
|
+
# OpenAlex sends seconds until reset, Scopus an epoch time.
|
|
131
|
+
if reset is not None:
|
|
132
|
+
p["resets_at"] = time.strftime("%Y-%m-%d %H:%M", time.localtime(
|
|
133
|
+
reset if reset > 10**9 else time.time() + reset))
|
|
134
|
+
p["checked_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
135
|
+
_save(data)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _int(value: Any) -> int | None:
|
|
139
|
+
try:
|
|
140
|
+
return int(float(value))
|
|
141
|
+
except (TypeError, ValueError):
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def summary() -> dict[str, Any]:
|
|
146
|
+
"""Per provider: requests today, throttled today, remaining / limit, reset time."""
|
|
147
|
+
today = time.strftime("%Y-%m-%d")
|
|
148
|
+
out = {}
|
|
149
|
+
for name, p in _load().items():
|
|
150
|
+
out[name] = {
|
|
151
|
+
"requests_today": p.get("requests", 0) if p.get("day") == today else 0,
|
|
152
|
+
"credits_today": p.get("credits", 0) if p.get("day") == today else 0,
|
|
153
|
+
"throttled_today": p.get("throttled", 0) if p.get("day") == today else 0,
|
|
154
|
+
"remaining": p.get("remaining"), "limit": p.get("limit"),
|
|
155
|
+
"resets_at": p.get("resets_at"), "checked_at": p.get("checked_at"),
|
|
156
|
+
"concurrency": CONCURRENCY.get(name),
|
|
157
|
+
}
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
DISPLAY = {"openalex": "OpenAlex", "scopus": "Scopus", "springer": "Springer Nature", "crossref": "Crossref",
|
|
162
|
+
"semantic_scholar": "Semantic Scholar"}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def warnings(lang: str = "en") -> list[str]:
|
|
166
|
+
"""Plain-language notes for providers that are nearly out of budget or being throttled.
|
|
167
|
+
|
|
168
|
+
``lang="zh"`` gives the app's wording (docs/UI_COPY.md); agents get English.
|
|
169
|
+
"""
|
|
170
|
+
notes = []
|
|
171
|
+
for name, s in summary().items():
|
|
172
|
+
shown = DISPLAY.get(name, name)
|
|
173
|
+
if s["remaining"] is not None and s["limit"] and s["remaining"] < s["limit"] * WARN_FRACTION:
|
|
174
|
+
credits = name == "openalex" # OpenAlex's headers count credits
|
|
175
|
+
if lang == "zh":
|
|
176
|
+
notes.append(f"{shown} 剩余 {s['remaining']:,} / {s['limit']:,} {'积分' if credits else '次'}"
|
|
177
|
+
+ (f",重置 {s['resets_at']}" if s["resets_at"] else ""))
|
|
178
|
+
else:
|
|
179
|
+
notes.append(f"{name}: only {s['remaining']} of {s['limit']} {'credits' if credits else 'requests'} left"
|
|
180
|
+
+ (f", resets at {s['resets_at']}" if s["resets_at"] else ""))
|
|
181
|
+
if s["throttled_today"] >= 5:
|
|
182
|
+
notes.append(f"{shown} 今日被限流 {s['throttled_today']} 次,已放慢请求" if lang == "zh"
|
|
183
|
+
else f"{name}: throttled {s['throttled_today']} times today; requests are slowed down")
|
|
184
|
+
return notes
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def openalex_live(timeout: float = 20.0) -> dict[str, Any] | None:
|
|
188
|
+
"""OpenAlex's own account of today's budget, from its /rate-limit endpoint
|
|
189
|
+
(no credits charged): used, remaining, limit, reset time, price per call type."""
|
|
190
|
+
import requests
|
|
191
|
+
key = str(_config.load_config().get("openalex_api_key") or "").strip()
|
|
192
|
+
try:
|
|
193
|
+
r = requests.get("https://api.openalex.org/rate-limit", params={"api_key": key} if key else None,
|
|
194
|
+
timeout=timeout, proxies={"http": None, "https": None})
|
|
195
|
+
if r.status_code != 200:
|
|
196
|
+
return {"error": f"HTTP {r.status_code}"}
|
|
197
|
+
info = r.json().get("rate_limit") or {}
|
|
198
|
+
except Exception as exc: # diagnostics only
|
|
199
|
+
return {"error": str(exc)}
|
|
200
|
+
return {"credits_used": info.get("credits_used"), "credits_remaining": info.get("credits_remaining"),
|
|
201
|
+
"credits_limit": info.get("credits_limit"), "resets_at": info.get("resets_at"),
|
|
202
|
+
"credit_costs": info.get("credit_costs"), "has_key": bool(key)}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Literature search: quick search, deep search, screening, and title resolution."""
|