litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,32 @@
1
+ """Taylor & Francis (Tandfonline) publisher strategy."""
2
+
3
+ from .base import BasePublisherStrategy
4
+ from .registry import StrategyRegistry
5
+
6
+
7
+ @StrategyRegistry.register
8
+ class TandfonlineStrategy(BasePublisherStrategy):
9
+ name = "Tandfonline"
10
+ aliases = ("tandfonline", "taylorandfrancis", "tandf")
11
+ doi_prefixes = ("10.1080/",)
12
+ base_domains = ("tandfonline.com",)
13
+ sample_dois = ("10.1080/00268976.2019.1660819",)
14
+ article_url_template = "https://www.tandfonline.com/doi/{doi}"
15
+ pdf_url_templates = (
16
+ "https://www.tandfonline.com/doi/pdf/{doi}",
17
+ )
18
+ success_url_markers = ("tandfonline.com/doi/",)
19
+ auth_url_markers = (
20
+ "id.tsinghua.edu.cn",
21
+ "idp.tsinghua.edu.cn",
22
+ "login.openathens.net",
23
+ )
24
+ sso_text_markers = ("Access through your institution",)
25
+ sso_url_patterns = ("/ssostart", "/shibboleth", "/institutional")
26
+ institution_input_selectors = (
27
+ 'input[placeholder*="institution"]',
28
+ 'input[placeholder*="Type the name"]',
29
+ "#searchInstitution",
30
+ "input[name='query']",
31
+ )
32
+ institution_result_selectors = ("#searchInstitution",)
@@ -0,0 +1,32 @@
1
+ """Wiley publisher strategy."""
2
+
3
+ from .base import BasePublisherStrategy
4
+ from .registry import StrategyRegistry
5
+
6
+
7
+ @StrategyRegistry.register
8
+ class WileyStrategy(BasePublisherStrategy):
9
+ name = "Wiley"
10
+ aliases = ("wiley",)
11
+ doi_prefixes = ("10.1002/", "10.1111/")
12
+ base_domains = ("onlinelibrary.wiley.com", "wiley.com")
13
+ sample_dois = ("10.1002/adma.201706259",)
14
+ article_url_template = "https://onlinelibrary.wiley.com/doi/{doi}"
15
+ pdf_url_templates = (
16
+ "https://onlinelibrary.wiley.com/doi/pdfdirect/{doi}",
17
+ "https://onlinelibrary.wiley.com/doi/pdf/{doi}",
18
+ )
19
+ success_url_markers = ("onlinelibrary.wiley.com/doi/",)
20
+ auth_url_markers = (
21
+ "id.tsinghua.edu.cn",
22
+ "idp.tsinghua.edu.cn",
23
+ "login.openathens.net",
24
+ )
25
+ sso_text_markers = (
26
+ "Access through your institution",
27
+ "Institutional Login",
28
+ "Institution Login",
29
+ )
30
+ sso_url_patterns = ("/ssostart",)
31
+ institution_input_selectors = ("#searchInstitution",)
32
+ institution_result_selectors = ("#searchInstitution",)
@@ -0,0 +1,219 @@
1
+ """Supplementary Information (SI) discovery and download.
2
+
3
+ v2: lenient publishers over plain HTTP; Cloudflare-protected ones (Elsevier's
4
+ linkinghub bounce) fall back to the stealth-browser backend, reusing its
5
+ cookies for the attachment downloads. SI failure never fails the main download.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ import time
12
+ from pathlib import Path
13
+ from urllib.parse import urljoin
14
+
15
+ _LINK_RE = re.compile(r"""href=["']([^"']+)["']""", re.I)
16
+ _EXT_RE = re.compile(r"\.(pdf|docx?|xlsx?|pptx?|zip|gz|csv|txt)(?:[?#]|$)", re.I)
17
+ _MARKER_RE = re.compile(r"(suppl|supp_|supporting|_si\d|/cms/|attachment|mmc\d|moesm)", re.I)
18
+ # URLs that scream "attachment" even without a file extension (PLOS-style
19
+ # .../file?id=<doi>.s001&type=supplementary-file) — the download step validates
20
+ # by content-type instead of trusting the URL. MOESM files carry extensions,
21
+ # so they ride the weak rule; page anchors (#MOESM1) stay excluded.
22
+ _STRONG_RE = re.compile(r"(supplementary[-_]?file|/cms/|attachment|mmc\d)", re.I)
23
+ _MAIN_PDF_RE = re.compile(r"(?:^|/)(?:main|mainext|article)\.pdf(?:[?#]|$)", re.I)
24
+
25
+ _CT_EXT = (
26
+ ("application/pdf", ".pdf"),
27
+ ("wordprocessingml.document", ".docx"),
28
+ ("msword", ".doc"),
29
+ ("spreadsheetml.sheet", ".xlsx"),
30
+ ("vnd.ms-excel", ".xls"),
31
+ ("presentationml.presentation", ".pptx"),
32
+ ("text/csv", ".csv"),
33
+ ("application/zip", ".zip"),
34
+ ("application/gzip", ".gz"),
35
+ ("text/plain", ".txt"),
36
+ )
37
+
38
+
39
+ def extract_supplementary_links(html: str, base_url: str, max_files: int = 10) -> list[str]:
40
+ """Return absolute candidate SI URLs found on an article landing page."""
41
+ seen: set[str] = set()
42
+ out: list[str] = []
43
+ for m in _LINK_RE.finditer(html or ""):
44
+ href = m.group(1)
45
+ if not _MARKER_RE.search(href):
46
+ continue
47
+ if _MAIN_PDF_RE.search(href):
48
+ continue # the article's own main PDF, not an attachment
49
+ # Weak marker + known extension is enough; strong markers (PLOS-style
50
+ # supplementary-file URLs) qualify even without an extension.
51
+ if not _EXT_RE.search(href) and not _STRONG_RE.search(href):
52
+ continue
53
+ url = urljoin(base_url or "", href)
54
+ if url in seen:
55
+ continue
56
+ seen.add(url)
57
+ out.append(url)
58
+ if len(out) >= max_files:
59
+ break
60
+ return out
61
+
62
+
63
+ def fetch_supplementary(
64
+ doi: str,
65
+ output_dir: str | Path,
66
+ config: dict | None = None,
67
+ max_files: int = 10,
68
+ timeout: int = 30,
69
+ ) -> list[str]:
70
+ """Download SI attachments for ``doi``; returns saved file paths.
71
+
72
+ Never raises: problems are reported by returning fewer/zero files.
73
+ """
74
+ config = config or {}
75
+ out = Path(output_dir)
76
+ out.mkdir(parents=True, exist_ok=True)
77
+
78
+ article_url = _article_url(doi)
79
+ if not article_url:
80
+ return []
81
+
82
+ html, fetch_bytes = _fetch_page(article_url, config, timeout)
83
+ if not html:
84
+ return []
85
+ links = extract_supplementary_links(html, article_url, max_files=max_files)
86
+
87
+ saved: list[str] = []
88
+ stem = _safe_stem(doi)
89
+ for i, url in enumerate(links, 1):
90
+ body, ctype = fetch_bytes(url)
91
+ if not body:
92
+ continue
93
+ ctype = (ctype or "").lower()
94
+ if "text/html" in ctype:
95
+ continue # another landing page, not an attachment
96
+ ext = _ext_of(url) or _ext_from_content_type(ctype) or ".bin"
97
+ path = out / f"{stem}_SI{i}{ext}"
98
+ path.write_bytes(body)
99
+ saved.append(str(path))
100
+ return saved
101
+
102
+
103
+ def _fetch_page(article_url: str, config: dict, timeout: int):
104
+ """Get (html, fetch_bytes) for the article page.
105
+
106
+ Plain HTTP first. When the publisher bounces us (Elsevier's linkinghub) or
107
+ returns a non-200, retry through the stealth-browser backend so Cloudflare
108
+ cookies are shared with the attachment downloads. fetch_bytes(url) returns
109
+ (body | None, content-type | None).
110
+ """
111
+ import requests
112
+
113
+ proxy = (config or {}).get("network_proxy", "")
114
+ proxies = {"http": proxy, "https": proxy} if proxy else None
115
+ headers = {"User-Agent": _user_agent()}
116
+
117
+ session = requests.Session()
118
+ session.trust_env = False
119
+ blocked = False
120
+ html = ""
121
+ try:
122
+ resp = session.get(article_url, headers=headers, proxies=proxies, timeout=timeout, allow_redirects=True)
123
+ blocked = resp.status_code != 200 or "linkinghub." in str(resp.url)
124
+ if resp.status_code == 200:
125
+ html = resp.text
126
+ except Exception:
127
+ blocked = True
128
+
129
+ if not blocked:
130
+ def plain_fetch(url: str):
131
+ try:
132
+ r = session.get(url, headers=headers, proxies=proxies, timeout=timeout, allow_redirects=True)
133
+ return (r.content, r.headers.get("content-type")) if r.status_code == 200 else (None, None)
134
+ except Exception:
135
+ return None, None
136
+ return html, plain_fetch
137
+
138
+ fetched = _browser_page(article_url, config, timeout)
139
+ if fetched:
140
+ return fetched
141
+ return (html, None) if html else (None, None)
142
+
143
+
144
+ def _browser_page(article_url: str, config: dict, timeout: int):
145
+ """Render the article page through the shared stealth-browser backend.
146
+
147
+ Uses the browser-worker pool (browser_engine) instead of a cold start per
148
+ call, and downloads attachments through the browser context's request API
149
+ so Cloudflare cookies are shared. Returns (page_html, fetch_bytes); only
150
+ the PAGE is closed - the pooled browser stays alive for the next paper.
151
+ """
152
+ try:
153
+ from ..browser.engine import get_browser_page
154
+
155
+ page = get_browser_page(config)
156
+ except Exception:
157
+ return None
158
+ if page is None:
159
+ return None
160
+
161
+ try:
162
+ page.goto(article_url, wait_until="domcontentloaded", timeout=max(timeout, 45) * 1000)
163
+ time.sleep(2.5) # not page.wait_for_timeout: it hangs on a page stuck loading
164
+ html = page.content()
165
+
166
+ def fetch_bytes(url: str):
167
+ try:
168
+ r = page.context.request.get(url, timeout=max(timeout, 45) * 1000)
169
+ if r.status != 200:
170
+ return None, None
171
+ return r.body(), r.headers.get("content-type")
172
+ except Exception:
173
+ return None, None
174
+
175
+ return html, fetch_bytes
176
+ except Exception:
177
+ try:
178
+ ctx.close()
179
+ except Exception:
180
+ pass
181
+ return None
182
+
183
+
184
+ def _ext_from_content_type(ctype: str) -> str | None:
185
+ for frag, ext in _CT_EXT:
186
+ if frag in ctype:
187
+ return ext
188
+ return None
189
+
190
+
191
+ def _article_url(doi: str) -> str | None:
192
+ try:
193
+ from .strategies import StrategyRegistry
194
+ strategy = StrategyRegistry.get_for_doi(doi)
195
+ except Exception:
196
+ return None
197
+ if strategy is None:
198
+ return None
199
+ try:
200
+ return strategy.article_url(doi)
201
+ except Exception:
202
+ return None
203
+
204
+
205
+ def _ext_of(url: str) -> str | None:
206
+ m = _EXT_RE.search(url)
207
+ return f".{m.group(1).lower()}" if m else None
208
+
209
+
210
+ def _safe_stem(identifier: str) -> str:
211
+ return re.sub(r"[^A-Za-z0-9._-]+", "_", identifier.strip()).strip("_")
212
+
213
+
214
+ def _user_agent() -> str:
215
+ try:
216
+ from ..network import USER_AGENT
217
+ return USER_AGENT
218
+ except Exception: # pragma: no cover
219
+ return "Mozilla/5.0 (compatible; litloop)"
litloop/quota.py ADDED
@@ -0,0 +1,202 @@
1
+ """Usage and limits of the rate-limited APIs LitLoop calls.
2
+
3
+ Every request to OpenAlex, Scopus, Semantic Scholar or Crossref goes through
4
+ ``slot(provider)`` (a per-provider cap on parallel requests, so bursts do not
5
+ trip per-second throttles) and ``record(provider, response)``, which keeps:
6
+
7
+ * requests made today, and how many were throttled (HTTP 429);
8
+ * what the provider last said about its budget (remaining, limit, reset),
9
+ read from the X-RateLimit-* headers when it sends them.
10
+
11
+ The numbers live in ~/.litloop/api_usage.json, shared by the MCP server, the
12
+ command line and scripts. ``summary()`` is what diagnostics, deep search and
13
+ (later) the app show; ``warnings()`` names the providers close to their limit.
14
+ The caps only coordinate threads within one process; separate processes
15
+ still see each other's usage through the file.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ import threading
22
+ import time
23
+ from contextlib import contextmanager
24
+ from typing import Any
25
+
26
+ from . import config as _config
27
+
28
+ # Official limits (checked 2026-10-16; full list with sources in docs/API_LIMITS.md).
29
+ # PER_SECOND stays a little below each published per-second figure. CONCURRENCY
30
+ # is official only for Crossref (1 public / 3 polite); the providers below that
31
+ # publish no concurrency limit get LitLoop's own conservative choice.
32
+ # * OpenAlex: 30 requests/s with a key, but 5/s for searches with more than five
33
+ # boolean operators; budget 10,000 credits/day with a free key (list 1 credit,
34
+ # keyword search 10), reset at midnight UTC.
35
+ # https://help.openalex.org/how-to-use-the-api/rate-limits-and-authentication
36
+ # * Scopus Abstract Retrieval: 9 requests/s, 10,000 per week.
37
+ # https://dev.elsevier.com/api_key_settings.html
38
+ # * Crossref: public pool 5 single-record requests/s, 1 at a time; polite pool
39
+ # (with a contact email) 10/s, 3 at a time.
40
+ # https://crossref.org/blog/announcing-changes-to-rest-api-rate-limits/
41
+ # * Semantic Scholar: keyless requests share one pool and are often throttled;
42
+ # a key gives 1 request/s. https://www.semanticscholar.org/product/api
43
+ # * Springer Nature Meta API, free plan: 500 requests/day, 100/min.
44
+ # https://dev.springernature.com/subscription/
45
+ # Lanes: "openalex" assumes a key (30/s); without one OpenAlex allows 10/s and
46
+ # 1/s for boolean-heavy searches, and only $0.10 (1,000 credits) a day.
47
+ CONCURRENCY = {"openalex": 4, "openalex_boolean": 2, "openalex_nokey": 2, "openalex_boolean_nokey": 1,
48
+ "scopus": 5, "semantic_scholar": 1, "crossref": 1, "crossref_polite": 3, "springer": 2}
49
+ PER_SECOND = {"openalex": 25.0, "openalex_boolean": 4.0, "openalex_nokey": 8.0, "openalex_boolean_nokey": 0.9,
50
+ "scopus": 8.0, "semantic_scholar": 1.0, "crossref": 4.0, "crossref_polite": 8.0, "springer": 1.5}
51
+ WARN_FRACTION = 0.1 # warn when less than this share of the budget is left
52
+
53
+ _lock = threading.Lock()
54
+ _slots = {name: threading.Semaphore(n) for name, n in CONCURRENCY.items()}
55
+ _next_free = {name: 0.0 for name in PER_SECOND}
56
+ _spent: dict[str, int] = {} # budget units used by this process (OpenAlex credits, others requests)
57
+
58
+
59
+ def _path():
60
+ return _config.DATA_DIR / "api_usage.json"
61
+
62
+
63
+ def _load() -> dict[str, Any]:
64
+ try:
65
+ return json.loads(_path().read_text(encoding="utf-8"))
66
+ except (OSError, ValueError):
67
+ return {}
68
+
69
+
70
+ def _save(data: dict[str, Any]) -> None:
71
+ try:
72
+ _path().parent.mkdir(parents=True, exist_ok=True)
73
+ tmp = _path().with_suffix(".tmp")
74
+ tmp.write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
75
+ tmp.replace(_path())
76
+ except OSError:
77
+ pass # usage tracking must never break a search
78
+
79
+
80
+ @contextmanager
81
+ def slot(provider: str):
82
+ """Hold one of ``provider``'s parallel-request slots, spaced to its per-second limit."""
83
+ sem = _slots.get(provider)
84
+ if sem is None:
85
+ yield
86
+ return
87
+ with sem:
88
+ rate = PER_SECOND.get(provider)
89
+ if rate:
90
+ with _lock:
91
+ now = time.monotonic()
92
+ start = max(now, _next_free[provider])
93
+ _next_free[provider] = start + 1.0 / rate
94
+ if start > now:
95
+ time.sleep(start - now)
96
+ yield
97
+
98
+
99
+ def spent(provider: str) -> int:
100
+ """Budget units ``provider`` has used in this process (OpenAlex: credits)."""
101
+ return _spent.get(provider, 0)
102
+
103
+
104
+ def record(provider: str, response: Any = None, *, throttled: bool | None = None) -> None:
105
+ """Count one request to ``provider`` and keep the budget headers it returned."""
106
+ today = time.strftime("%Y-%m-%d")
107
+ headers = getattr(response, "headers", None) or {}
108
+ status = getattr(response, "status_code", None)
109
+ with _lock:
110
+ data = _load()
111
+ p = data.setdefault(provider, {})
112
+ if p.get("day") != today:
113
+ p.update(day=today, requests=0, throttled=0, credits=0)
114
+ p["requests"] = p.get("requests", 0) + 1
115
+ cost = _int(headers.get("X-RateLimit-Credits-Used"))
116
+ cost = 1 if cost is None else cost
117
+ if status is not None and status < 400:
118
+ p["credits"] = p.get("credits", 0) + cost
119
+ _spent[provider] = _spent.get(provider, 0) + cost
120
+ if throttled or status == 429:
121
+ p["throttled"] = p.get("throttled", 0) + 1
122
+ p["last_throttled"] = time.strftime("%Y-%m-%d %H:%M:%S")
123
+ remaining = headers.get("X-RateLimit-Remaining")
124
+ if "QUOTA_EXCEEDED" in str(headers.get("X-ELS-Status") or ""):
125
+ p["remaining"] = 0 # Elsevier drops the Remaining header once the quota is spent
126
+ if remaining is not None:
127
+ p["remaining"] = _int(remaining)
128
+ p["limit"] = _int(headers.get("X-RateLimit-Limit"))
129
+ reset = _int(headers.get("X-RateLimit-Reset"))
130
+ # OpenAlex sends seconds until reset, Scopus an epoch time.
131
+ if reset is not None:
132
+ p["resets_at"] = time.strftime("%Y-%m-%d %H:%M", time.localtime(
133
+ reset if reset > 10**9 else time.time() + reset))
134
+ p["checked_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
135
+ _save(data)
136
+
137
+
138
+ def _int(value: Any) -> int | None:
139
+ try:
140
+ return int(float(value))
141
+ except (TypeError, ValueError):
142
+ return None
143
+
144
+
145
+ def summary() -> dict[str, Any]:
146
+ """Per provider: requests today, throttled today, remaining / limit, reset time."""
147
+ today = time.strftime("%Y-%m-%d")
148
+ out = {}
149
+ for name, p in _load().items():
150
+ out[name] = {
151
+ "requests_today": p.get("requests", 0) if p.get("day") == today else 0,
152
+ "credits_today": p.get("credits", 0) if p.get("day") == today else 0,
153
+ "throttled_today": p.get("throttled", 0) if p.get("day") == today else 0,
154
+ "remaining": p.get("remaining"), "limit": p.get("limit"),
155
+ "resets_at": p.get("resets_at"), "checked_at": p.get("checked_at"),
156
+ "concurrency": CONCURRENCY.get(name),
157
+ }
158
+ return out
159
+
160
+
161
+ DISPLAY = {"openalex": "OpenAlex", "scopus": "Scopus", "springer": "Springer Nature", "crossref": "Crossref",
162
+ "semantic_scholar": "Semantic Scholar"}
163
+
164
+
165
+ def warnings(lang: str = "en") -> list[str]:
166
+ """Plain-language notes for providers that are nearly out of budget or being throttled.
167
+
168
+ ``lang="zh"`` gives the app's wording (docs/UI_COPY.md); agents get English.
169
+ """
170
+ notes = []
171
+ for name, s in summary().items():
172
+ shown = DISPLAY.get(name, name)
173
+ if s["remaining"] is not None and s["limit"] and s["remaining"] < s["limit"] * WARN_FRACTION:
174
+ credits = name == "openalex" # OpenAlex's headers count credits
175
+ if lang == "zh":
176
+ notes.append(f"{shown} 剩余 {s['remaining']:,} / {s['limit']:,} {'积分' if credits else '次'}"
177
+ + (f",重置 {s['resets_at']}" if s["resets_at"] else ""))
178
+ else:
179
+ notes.append(f"{name}: only {s['remaining']} of {s['limit']} {'credits' if credits else 'requests'} left"
180
+ + (f", resets at {s['resets_at']}" if s["resets_at"] else ""))
181
+ if s["throttled_today"] >= 5:
182
+ notes.append(f"{shown} 今日被限流 {s['throttled_today']} 次,已放慢请求" if lang == "zh"
183
+ else f"{name}: throttled {s['throttled_today']} times today; requests are slowed down")
184
+ return notes
185
+
186
+
187
+ def openalex_live(timeout: float = 20.0) -> dict[str, Any] | None:
188
+ """OpenAlex's own account of today's budget, from its /rate-limit endpoint
189
+ (no credits charged): used, remaining, limit, reset time, price per call type."""
190
+ import requests
191
+ key = str(_config.load_config().get("openalex_api_key") or "").strip()
192
+ try:
193
+ r = requests.get("https://api.openalex.org/rate-limit", params={"api_key": key} if key else None,
194
+ timeout=timeout, proxies={"http": None, "https": None})
195
+ if r.status_code != 200:
196
+ return {"error": f"HTTP {r.status_code}"}
197
+ info = r.json().get("rate_limit") or {}
198
+ except Exception as exc: # diagnostics only
199
+ return {"error": str(exc)}
200
+ return {"credits_used": info.get("credits_used"), "credits_remaining": info.get("credits_remaining"),
201
+ "credits_limit": info.get("credits_limit"), "resets_at": info.get("resets_at"),
202
+ "credit_costs": info.get("credit_costs"), "has_key": bool(key)}
@@ -0,0 +1 @@
1
+ """Literature search: quick search, deep search, screening, and title resolution."""