litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/network.py ADDED
@@ -0,0 +1,417 @@
1
+ """Network session management and HTTP helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import random
7
+ import threading
8
+ import time
9
+ from typing import Any
10
+
11
+ import requests
12
+ from requests.adapters import HTTPAdapter
13
+
14
+ from .config import load_config
15
+ from .log import get_logger
16
+
17
+ log = get_logger()
18
+
19
+ # Check SOCKS proxy support
20
+ try:
21
+ import socks # noqa: F401
22
+ HAS_SOCKS = True
23
+ except ImportError:
24
+ HAS_SOCKS = False
25
+
26
+ def user_agent(extra: str = "") -> str:
27
+ """Return the LitLoop User-Agent header value."""
28
+ try:
29
+ from . import __version__
30
+ except (ImportError, AttributeError):
31
+ try:
32
+ import litloop
33
+ __version__ = getattr(litloop, "__version__", "1.0")
34
+ except Exception:
35
+ __version__ = "1.0"
36
+ base = f"LitLoop/{__version__}"
37
+ suffix = extra.strip()
38
+ if suffix:
39
+ return f"{base} {suffix}"
40
+ return base
41
+
42
+
43
+ USER_AGENT = (
44
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
45
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
46
+ f"Chrome/124.0 Safari/537.36 {user_agent()}"
47
+ )
48
+
49
+ # Browser UA without the LitLoop suffix: some CDNs (sci.bban.top PDF edge,
50
+ # observed 2026-09) 403 any UA that names a non-browser client, even with a
51
+ # valid Referer. Use only where impersonation is required to read at all.
52
+ CHROME_UA = (
53
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
54
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
55
+ "Chrome/126.0.0.0 Safari/537.36"
56
+ )
57
+
58
+ CLOUDFLARE_CHALLENGE_SIGNALS = (
59
+ "just a moment", "attention required", "verify", "security check",
60
+ "请稍候", "正在验证", "checking", "cloudflare",
61
+ )
62
+
63
+
64
+ def is_cloudflare_challenge(title: str) -> bool:
65
+ """Check if a page title indicates a Cloudflare/anti-bot challenge."""
66
+ lower = title.lower()
67
+ return any(sig in lower for sig in CLOUDFLARE_CHALLENGE_SIGNALS)
68
+
69
+ _session_pool: dict[str, requests.Session] = {}
70
+
71
+
72
+ def configured_proxy(config: dict[str, Any] | None = None) -> str:
73
+ """The one explicit proxy setting: LITLOOP_PROXY env > network_proxy config.
74
+
75
+ HTTP_PROXY/HTTPS_PROXY are ignored project-wide (NO_PROXY='*' in the
76
+ package __init__, sessions run trust_env=False — the ZCode proxy CA
77
+ workaround). Campus-bound traffic (Elsevier/Springer TDM entitlement is
78
+ IP-based) must go through THIS setting or it egresses off-campus.
79
+ """
80
+ return os.environ.get("LITLOOP_PROXY") or (config or {}).get("network_proxy") or ""
81
+
82
+
83
+ _PROXY_UP: dict[str, tuple[float, bool]] = {}
84
+
85
+
86
+ def _proxy_reachable(proxy_url: str) -> bool:
87
+ """Is the proxy's port accepting connections? Cached for 30 s."""
88
+ import socket
89
+ from urllib.parse import urlparse
90
+ now = time.monotonic()
91
+ cached = _PROXY_UP.get(proxy_url)
92
+ if cached and now - cached[0] < 30:
93
+ return cached[1]
94
+ u = urlparse(proxy_url)
95
+ try:
96
+ with socket.create_connection((u.hostname or "127.0.0.1", u.port or 80), timeout=1):
97
+ up = True
98
+ except OSError:
99
+ up = False
100
+ _PROXY_UP[proxy_url] = (now, up)
101
+ return up
102
+
103
+
104
+ def host_proxy(config: dict[str, Any] | None) -> tuple[str, tuple[str, ...]]:
105
+ """(proxy_url, domains) of the per-host proxy exception, or ("", ()) when off/down."""
106
+ config = config or {}
107
+ url = str(config.get("host_proxy_url") or "").strip()
108
+ hosts = tuple(str(h).strip().lstrip(".").lower() for h in config.get("host_proxy_hosts") or [] if str(h).strip())
109
+ if not url or not hosts or not _proxy_reachable(url):
110
+ return "", ()
111
+ return url, hosts
112
+
113
+
114
+ def grey_hosts(config: dict[str, Any] | None) -> tuple[str, ...]:
115
+ """Sci-Hub and LibGen mirrors (from the settings' domain lists)."""
116
+ from urllib.parse import urlparse
117
+ config = config or {}
118
+ from .sources.libgen import LIBGEN_MIRRORS
119
+ urls = list(config.get("scihub_domains") or []) + list(LIBGEN_MIRRORS) + ["https://bban.top"]
120
+ return tuple(sorted({(urlparse(str(u)).hostname or "").lower() for u in urls} - {""}))
121
+
122
+
123
+ def grey_proxy(config: dict[str, Any] | None) -> str:
124
+ """The proxy Sci-Hub and LibGen go through: host_proxy_url while its port
125
+ answers, when scihub_route says proxy (or auto measured the proxy faster,
126
+ sources/scihub_speed.py); else "" (direct)."""
127
+ config = config or {}
128
+ url = str(config.get("host_proxy_url") or "").strip()
129
+ from .sources.scihub_speed import route
130
+ if route(config) != "proxy" or not url or not _proxy_reachable(url):
131
+ return ""
132
+ return url
133
+
134
+
135
+ def host_uses_proxy(host: str, hosts: tuple[str, ...]) -> bool:
136
+ host = (host or "").lower()
137
+ return any(host == h or host.endswith("." + h) for h in hosts)
138
+
139
+
140
+ class _HostProxyAdapter(HTTPAdapter):
141
+ """Sends requests for the configured domains through the exception proxy."""
142
+
143
+ def __init__(self, proxy_url: str, hosts: tuple[str, ...]) -> None:
144
+ super().__init__()
145
+ self._proxy = proxy_url
146
+ self._hosts = hosts
147
+
148
+ def send(self, request, **kwargs): # type: ignore[override]
149
+ from urllib.parse import urlparse
150
+ if host_uses_proxy(urlparse(request.url).hostname or "", self._hosts):
151
+ kwargs["proxies"] = {"http": self._proxy, "https": self._proxy}
152
+ return super().send(request, **kwargs)
153
+
154
+
155
+ def apply_host_proxy(session: requests.Session, config: dict[str, Any] | None) -> None:
156
+ """Route the per-host proxy exception's domains through its proxy (no-op when off)."""
157
+ url, hosts = host_proxy(config)
158
+ if url:
159
+ adapter = _HostProxyAdapter(url, hosts)
160
+ session.mount("https://", adapter)
161
+ session.mount("http://", adapter)
162
+
163
+
164
+ def _get_session(config: dict[str, Any]) -> requests.Session:
165
+ proxy = configured_proxy(config)
166
+ key = proxy or "__none__"
167
+ if key not in _session_pool:
168
+ s = requests.Session()
169
+ s.trust_env = False
170
+ adapter = HTTPAdapter(pool_connections=20, pool_maxsize=20, max_retries=0)
171
+ s.mount("https://", adapter)
172
+ s.mount("http://", adapter)
173
+ s.headers.update({"User-Agent": USER_AGENT})
174
+ if proxy:
175
+ if proxy.startswith("socks") and not HAS_SOCKS:
176
+ log.warning("SOCKS proxy configured but PySocks not installed. Install: pip install requests[socks]")
177
+ s.proxies = {"http": proxy, "https": proxy}
178
+ _session_pool[key] = s
179
+ return _session_pool[key]
180
+
181
+
182
+ def request_timeout(config: dict[str, Any]) -> tuple[int, int]:
183
+ return (int(config.get("connect_timeout", 3)), int(config.get("read_timeout", 7)))
184
+
185
+
186
+ def proxy_dict(proxy: str | None) -> dict[str, str] | None:
187
+ if not proxy:
188
+ return None
189
+ return {"http": proxy, "https": proxy}
190
+
191
+
192
+ def select_proxy_for_url(url: str, config: dict[str, Any], use_tor: bool = False) -> str | None:
193
+ if use_tor:
194
+ from .tor.proxy import ensure_tor
195
+ tor_proxy = ensure_tor(config)
196
+ if tor_proxy:
197
+ return tor_proxy
198
+ log.warning("Tor requested but unavailable — falling back to direct connection")
199
+
200
+ # Some hosts rate-limit by egress IP, and every proxy user shares ONE exit
201
+ # — so proxied requests get walled quickly (measured live on sci-hub.ru,
202
+ # 2026-08-30: proxied = standing verification wall, direct = clean). These
203
+ # hosts route DIRECT by default; ``direct_domains`` extends the set,
204
+ # ``scihub_direct: false`` disables the behavior.
205
+ from urllib.parse import urlparse
206
+
207
+ try:
208
+ host = (urlparse(url).hostname or "").lower()
209
+ except Exception:
210
+ host = ""
211
+ grey = grey_proxy(config)
212
+ if grey and host_uses_proxy(host, grey_hosts(config)):
213
+ return grey # the user's choice wins over the direct default below
214
+ direct_hosts = {"sci-hub.ru"}
215
+ direct_hosts |= {str(d).lower().lstrip(".") for d in (config.get("direct_domains") or [])}
216
+ if config.get("scihub_direct", True) and any(
217
+ host == h or host.endswith("." + h) for h in direct_hosts if h
218
+ ):
219
+ return None
220
+
221
+ explicit = configured_proxy(config)
222
+ if explicit:
223
+ return explicit
224
+ return None
225
+
226
+
227
+ def fetch(
228
+ url: str,
229
+ config: dict[str, Any],
230
+ *,
231
+ headers: dict[str, str] | None = None,
232
+ stream: bool = False,
233
+ method: str = "GET",
234
+ use_tor: bool = False,
235
+ ) -> requests.Response:
236
+ domain_rate_limit(url)
237
+ host_concurrency_acquire(url, config)
238
+ try:
239
+ merged_headers = {"User-Agent": USER_AGENT}
240
+ if headers:
241
+ merged_headers.update(headers)
242
+ session = _get_session(config)
243
+ proxies = proxy_dict(select_proxy_for_url(url, config, use_tor=use_tor))
244
+ return session.request(
245
+ method,
246
+ url,
247
+ headers=merged_headers,
248
+ timeout=request_timeout(config),
249
+ proxies=proxies,
250
+ allow_redirects=True,
251
+ stream=stream,
252
+ )
253
+ finally:
254
+ host_concurrency_release(url, config)
255
+
256
+
257
+ def fetch_json(
258
+ url: str,
259
+ config: dict[str, Any],
260
+ *,
261
+ headers: dict[str, str] | None = None,
262
+ use_tor: bool = False,
263
+ ) -> dict[str, Any] | None:
264
+ # In-memory probe cache: avoid hammering metadata endpoints during racing.
265
+ ttl = float(config.get("json_probe_cache_seconds", 0) or 0)
266
+ now = time.time()
267
+ if ttl > 0:
268
+ cached = _json_cache.get(url)
269
+ if cached is not None and _json_cache_expires.get(url, 0) > now:
270
+ return cached
271
+ try:
272
+ resp = fetch(url, config, headers={"Accept": "application/json", **(headers or {})}, use_tor=use_tor)
273
+ if resp.status_code >= 400:
274
+ return None
275
+ data = resp.json()
276
+ except Exception:
277
+ return None
278
+ if ttl > 0:
279
+ _json_cache[url] = data
280
+ _json_cache_expires[url] = now + ttl
281
+ return data
282
+
283
+
284
+ # Module-level JSON probe caches (see fetch_json). Mutable for test reset.
285
+ _json_cache: dict[str, dict[str, Any]] = {}
286
+ _json_cache_expires: dict[str, float] = {}
287
+
288
+
289
+ def _is_cloudflare_block(resp: requests.Response) -> bool:
290
+ """Check if response is a Cloudflare/anti-bot challenge."""
291
+ if resp.status_code in (403, 503):
292
+ server = resp.headers.get("server", "").lower()
293
+ if "cloudflare" in server:
294
+ return True
295
+ try:
296
+ body = resp.text[:2000].lower()
297
+ if "cf-browser-verification" in body or "challenge-platform" in body:
298
+ return True
299
+ except Exception:
300
+ pass
301
+ return False
302
+
303
+
304
+ def polite_delay(config: dict[str, Any]) -> None:
305
+ # Polite delay is opt-in: only sleep when fixed_request_delay_enabled is True.
306
+ # Without it, racing/parallel sources would be needlessly slowed.
307
+ if not config.get("fixed_request_delay_enabled", False):
308
+ return
309
+ lo = float(config.get("request_delay_min", 0))
310
+ hi = float(config.get("request_delay_max", 0))
311
+ if hi > 0:
312
+ time.sleep(random.uniform(lo, max(lo, hi)))
313
+
314
+
315
+ # ============================================================
316
+ # Per-domain rate limiting: max 1 request per domain per interval
317
+ # ============================================================
318
+ _domain_locks: dict[str, threading.Lock] = {}
319
+ _domain_last_request: dict[str, float] = {}
320
+ _DOMAIN_RATE_LIMIT = 0.5 # seconds between requests to same domain
321
+ # Longer gaps where the provider's terms ask for them (docs/API_LIMITS.md):
322
+ # arXiv API "no more than one request every three seconds"; CORE "five single
323
+ # requests per 10 seconds".
324
+ _DOMAIN_MIN_INTERVAL = {"export.arxiv.org": 3.0, "arxiv.org": 3.0, "api.core.ac.uk": 2.0}
325
+ # Hourly allowances: OpenAIRE allows 60 requests/hour without a token. Once
326
+ # used up the host is skipped (an OA probe, other sources cover it) instead of
327
+ # stalling downloads for up to an hour.
328
+ _DOMAIN_HOURLY_CAP = {"api.openaire.eu": 55}
329
+ _domain_hour_log: dict[str, list[float]] = {}
330
+
331
+
332
+ def _extract_domain(url: str) -> str:
333
+ from urllib.parse import urlparse
334
+ try:
335
+ return urlparse(url).netloc.lower()
336
+ except Exception:
337
+ return url
338
+
339
+
340
+ def domain_rate_limit(url: str) -> None:
341
+ """Wait if needed to respect per-domain rate limit."""
342
+ domain = _extract_domain(url)
343
+ if not domain:
344
+ return
345
+ if domain not in _domain_locks:
346
+ _domain_locks[domain] = threading.Lock()
347
+ with _domain_locks[domain]:
348
+ cap = _DOMAIN_HOURLY_CAP.get(domain)
349
+ if cap:
350
+ recent = [s for s in _domain_hour_log.get(domain, []) if time.time() - s < 3600]
351
+ if len(recent) >= cap:
352
+ raise requests.ConnectionError(f"{domain}: hourly allowance of {cap} requests used; skipped")
353
+ _domain_hour_log[domain] = recent + [time.time()]
354
+ gap = _DOMAIN_MIN_INTERVAL.get(domain, _DOMAIN_RATE_LIMIT)
355
+ now = time.time()
356
+ last = _domain_last_request.get(domain, 0)
357
+ elapsed = now - last
358
+ if elapsed < gap:
359
+ time.sleep(gap - elapsed)
360
+ _domain_last_request[domain] = time.time()
361
+
362
+
363
+ # ============================================================
364
+ # Per-host concurrency control (scimesh-style)
365
+ # ============================================================
366
+ # Inherited from the original project; not published by the providers, except
367
+ # arXiv (single connection) and CORE (see _DOMAIN_MIN_INTERVAL). docs/API_LIMITS.md.
368
+ _DEFAULT_HOST_CONCURRENCY: dict[str, int] = {
369
+ "unpaywall.org": 3,
370
+ "api.openalex.org": 5,
371
+ "content.openalex.org": 3,
372
+ "api.crossref.org": 5,
373
+ "api.openaire.eu": 3,
374
+ "doaj.org": 3,
375
+ "api.semanticscholar.org": 3,
376
+ "api.core.ac.uk": 1, # CORE: five single requests per 10 seconds
377
+ "europepmc.org": 3,
378
+ "www.ebi.ac.uk": 3,
379
+ "arxiv.org": 1, # arXiv terms: a single connection at a time
380
+ "export.arxiv.org": 1,
381
+ }
382
+
383
+ _host_semaphores: dict[str, threading.Semaphore] = {}
384
+
385
+
386
+ def _get_host_concurrency(host: str, config: dict[str, Any]) -> int:
387
+ """Get max concurrent requests for a host."""
388
+ custom = config.get("host_concurrency", {})
389
+ if isinstance(custom, dict) and host in custom:
390
+ return int(custom[host])
391
+ # Match by suffix (e.g., "sci-hub.ru" matches default)
392
+ for pattern, limit in _DEFAULT_HOST_CONCURRENCY.items():
393
+ if host == pattern or host.endswith("." + pattern):
394
+ return limit
395
+ return 0
396
+
397
+
398
+ def host_concurrency_acquire(url: str, config: dict[str, Any]) -> None:
399
+ """Acquire a per-host concurrency slot. Blocks if at limit."""
400
+ host = _extract_domain(url)
401
+ if not host:
402
+ return
403
+ limit = _get_host_concurrency(host, config)
404
+ if limit <= 0:
405
+ return
406
+ if host not in _host_semaphores:
407
+ _host_semaphores[host] = threading.Semaphore(limit)
408
+ _host_semaphores[host].acquire()
409
+
410
+
411
+ def host_concurrency_release(url: str, config: dict[str, Any]) -> None:
412
+ """Release a per-host concurrency slot."""
413
+ host = _extract_domain(url)
414
+ if not host:
415
+ return
416
+ if host in _host_semaphores:
417
+ _host_semaphores[host].release()