litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/network.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
"""Network session management and HTTP helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
from requests.adapters import HTTPAdapter
|
|
13
|
+
|
|
14
|
+
from .config import load_config
|
|
15
|
+
from .log import get_logger
|
|
16
|
+
|
|
17
|
+
log = get_logger()
|
|
18
|
+
|
|
19
|
+
# Check SOCKS proxy support
|
|
20
|
+
try:
|
|
21
|
+
import socks # noqa: F401
|
|
22
|
+
HAS_SOCKS = True
|
|
23
|
+
except ImportError:
|
|
24
|
+
HAS_SOCKS = False
|
|
25
|
+
|
|
26
|
+
def user_agent(extra: str = "") -> str:
|
|
27
|
+
"""Return the LitLoop User-Agent header value."""
|
|
28
|
+
try:
|
|
29
|
+
from . import __version__
|
|
30
|
+
except (ImportError, AttributeError):
|
|
31
|
+
try:
|
|
32
|
+
import litloop
|
|
33
|
+
__version__ = getattr(litloop, "__version__", "1.0")
|
|
34
|
+
except Exception:
|
|
35
|
+
__version__ = "1.0"
|
|
36
|
+
base = f"LitLoop/{__version__}"
|
|
37
|
+
suffix = extra.strip()
|
|
38
|
+
if suffix:
|
|
39
|
+
return f"{base} {suffix}"
|
|
40
|
+
return base
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
USER_AGENT = (
|
|
44
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
45
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
46
|
+
f"Chrome/124.0 Safari/537.36 {user_agent()}"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# Browser UA without the LitLoop suffix: some CDNs (sci.bban.top PDF edge,
|
|
50
|
+
# observed 2026-09) 403 any UA that names a non-browser client, even with a
|
|
51
|
+
# valid Referer. Use only where impersonation is required to read at all.
|
|
52
|
+
CHROME_UA = (
|
|
53
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
54
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
55
|
+
"Chrome/126.0.0.0 Safari/537.36"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
CLOUDFLARE_CHALLENGE_SIGNALS = (
|
|
59
|
+
"just a moment", "attention required", "verify", "security check",
|
|
60
|
+
"请稍候", "正在验证", "checking", "cloudflare",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def is_cloudflare_challenge(title: str) -> bool:
|
|
65
|
+
"""Check if a page title indicates a Cloudflare/anti-bot challenge."""
|
|
66
|
+
lower = title.lower()
|
|
67
|
+
return any(sig in lower for sig in CLOUDFLARE_CHALLENGE_SIGNALS)
|
|
68
|
+
|
|
69
|
+
_session_pool: dict[str, requests.Session] = {}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def configured_proxy(config: dict[str, Any] | None = None) -> str:
|
|
73
|
+
"""The one explicit proxy setting: LITLOOP_PROXY env > network_proxy config.
|
|
74
|
+
|
|
75
|
+
HTTP_PROXY/HTTPS_PROXY are ignored project-wide (NO_PROXY='*' in the
|
|
76
|
+
package __init__, sessions run trust_env=False — the ZCode proxy CA
|
|
77
|
+
workaround). Campus-bound traffic (Elsevier/Springer TDM entitlement is
|
|
78
|
+
IP-based) must go through THIS setting or it egresses off-campus.
|
|
79
|
+
"""
|
|
80
|
+
return os.environ.get("LITLOOP_PROXY") or (config or {}).get("network_proxy") or ""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
_PROXY_UP: dict[str, tuple[float, bool]] = {}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _proxy_reachable(proxy_url: str) -> bool:
|
|
87
|
+
"""Is the proxy's port accepting connections? Cached for 30 s."""
|
|
88
|
+
import socket
|
|
89
|
+
from urllib.parse import urlparse
|
|
90
|
+
now = time.monotonic()
|
|
91
|
+
cached = _PROXY_UP.get(proxy_url)
|
|
92
|
+
if cached and now - cached[0] < 30:
|
|
93
|
+
return cached[1]
|
|
94
|
+
u = urlparse(proxy_url)
|
|
95
|
+
try:
|
|
96
|
+
with socket.create_connection((u.hostname or "127.0.0.1", u.port or 80), timeout=1):
|
|
97
|
+
up = True
|
|
98
|
+
except OSError:
|
|
99
|
+
up = False
|
|
100
|
+
_PROXY_UP[proxy_url] = (now, up)
|
|
101
|
+
return up
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def host_proxy(config: dict[str, Any] | None) -> tuple[str, tuple[str, ...]]:
|
|
105
|
+
"""(proxy_url, domains) of the per-host proxy exception, or ("", ()) when off/down."""
|
|
106
|
+
config = config or {}
|
|
107
|
+
url = str(config.get("host_proxy_url") or "").strip()
|
|
108
|
+
hosts = tuple(str(h).strip().lstrip(".").lower() for h in config.get("host_proxy_hosts") or [] if str(h).strip())
|
|
109
|
+
if not url or not hosts or not _proxy_reachable(url):
|
|
110
|
+
return "", ()
|
|
111
|
+
return url, hosts
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def grey_hosts(config: dict[str, Any] | None) -> tuple[str, ...]:
|
|
115
|
+
"""Sci-Hub and LibGen mirrors (from the settings' domain lists)."""
|
|
116
|
+
from urllib.parse import urlparse
|
|
117
|
+
config = config or {}
|
|
118
|
+
from .sources.libgen import LIBGEN_MIRRORS
|
|
119
|
+
urls = list(config.get("scihub_domains") or []) + list(LIBGEN_MIRRORS) + ["https://bban.top"]
|
|
120
|
+
return tuple(sorted({(urlparse(str(u)).hostname or "").lower() for u in urls} - {""}))
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def grey_proxy(config: dict[str, Any] | None) -> str:
|
|
124
|
+
"""The proxy Sci-Hub and LibGen go through: host_proxy_url while its port
|
|
125
|
+
answers, when scihub_route says proxy (or auto measured the proxy faster,
|
|
126
|
+
sources/scihub_speed.py); else "" (direct)."""
|
|
127
|
+
config = config or {}
|
|
128
|
+
url = str(config.get("host_proxy_url") or "").strip()
|
|
129
|
+
from .sources.scihub_speed import route
|
|
130
|
+
if route(config) != "proxy" or not url or not _proxy_reachable(url):
|
|
131
|
+
return ""
|
|
132
|
+
return url
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def host_uses_proxy(host: str, hosts: tuple[str, ...]) -> bool:
|
|
136
|
+
host = (host or "").lower()
|
|
137
|
+
return any(host == h or host.endswith("." + h) for h in hosts)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class _HostProxyAdapter(HTTPAdapter):
|
|
141
|
+
"""Sends requests for the configured domains through the exception proxy."""
|
|
142
|
+
|
|
143
|
+
def __init__(self, proxy_url: str, hosts: tuple[str, ...]) -> None:
|
|
144
|
+
super().__init__()
|
|
145
|
+
self._proxy = proxy_url
|
|
146
|
+
self._hosts = hosts
|
|
147
|
+
|
|
148
|
+
def send(self, request, **kwargs): # type: ignore[override]
|
|
149
|
+
from urllib.parse import urlparse
|
|
150
|
+
if host_uses_proxy(urlparse(request.url).hostname or "", self._hosts):
|
|
151
|
+
kwargs["proxies"] = {"http": self._proxy, "https": self._proxy}
|
|
152
|
+
return super().send(request, **kwargs)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def apply_host_proxy(session: requests.Session, config: dict[str, Any] | None) -> None:
|
|
156
|
+
"""Route the per-host proxy exception's domains through its proxy (no-op when off)."""
|
|
157
|
+
url, hosts = host_proxy(config)
|
|
158
|
+
if url:
|
|
159
|
+
adapter = _HostProxyAdapter(url, hosts)
|
|
160
|
+
session.mount("https://", adapter)
|
|
161
|
+
session.mount("http://", adapter)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _get_session(config: dict[str, Any]) -> requests.Session:
|
|
165
|
+
proxy = configured_proxy(config)
|
|
166
|
+
key = proxy or "__none__"
|
|
167
|
+
if key not in _session_pool:
|
|
168
|
+
s = requests.Session()
|
|
169
|
+
s.trust_env = False
|
|
170
|
+
adapter = HTTPAdapter(pool_connections=20, pool_maxsize=20, max_retries=0)
|
|
171
|
+
s.mount("https://", adapter)
|
|
172
|
+
s.mount("http://", adapter)
|
|
173
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
174
|
+
if proxy:
|
|
175
|
+
if proxy.startswith("socks") and not HAS_SOCKS:
|
|
176
|
+
log.warning("SOCKS proxy configured but PySocks not installed. Install: pip install requests[socks]")
|
|
177
|
+
s.proxies = {"http": proxy, "https": proxy}
|
|
178
|
+
_session_pool[key] = s
|
|
179
|
+
return _session_pool[key]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def request_timeout(config: dict[str, Any]) -> tuple[int, int]:
|
|
183
|
+
return (int(config.get("connect_timeout", 3)), int(config.get("read_timeout", 7)))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def proxy_dict(proxy: str | None) -> dict[str, str] | None:
|
|
187
|
+
if not proxy:
|
|
188
|
+
return None
|
|
189
|
+
return {"http": proxy, "https": proxy}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def select_proxy_for_url(url: str, config: dict[str, Any], use_tor: bool = False) -> str | None:
|
|
193
|
+
if use_tor:
|
|
194
|
+
from .tor.proxy import ensure_tor
|
|
195
|
+
tor_proxy = ensure_tor(config)
|
|
196
|
+
if tor_proxy:
|
|
197
|
+
return tor_proxy
|
|
198
|
+
log.warning("Tor requested but unavailable — falling back to direct connection")
|
|
199
|
+
|
|
200
|
+
# Some hosts rate-limit by egress IP, and every proxy user shares ONE exit
|
|
201
|
+
# — so proxied requests get walled quickly (measured live on sci-hub.ru,
|
|
202
|
+
# 2026-08-30: proxied = standing verification wall, direct = clean). These
|
|
203
|
+
# hosts route DIRECT by default; ``direct_domains`` extends the set,
|
|
204
|
+
# ``scihub_direct: false`` disables the behavior.
|
|
205
|
+
from urllib.parse import urlparse
|
|
206
|
+
|
|
207
|
+
try:
|
|
208
|
+
host = (urlparse(url).hostname or "").lower()
|
|
209
|
+
except Exception:
|
|
210
|
+
host = ""
|
|
211
|
+
grey = grey_proxy(config)
|
|
212
|
+
if grey and host_uses_proxy(host, grey_hosts(config)):
|
|
213
|
+
return grey # the user's choice wins over the direct default below
|
|
214
|
+
direct_hosts = {"sci-hub.ru"}
|
|
215
|
+
direct_hosts |= {str(d).lower().lstrip(".") for d in (config.get("direct_domains") or [])}
|
|
216
|
+
if config.get("scihub_direct", True) and any(
|
|
217
|
+
host == h or host.endswith("." + h) for h in direct_hosts if h
|
|
218
|
+
):
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
explicit = configured_proxy(config)
|
|
222
|
+
if explicit:
|
|
223
|
+
return explicit
|
|
224
|
+
return None
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def fetch(
|
|
228
|
+
url: str,
|
|
229
|
+
config: dict[str, Any],
|
|
230
|
+
*,
|
|
231
|
+
headers: dict[str, str] | None = None,
|
|
232
|
+
stream: bool = False,
|
|
233
|
+
method: str = "GET",
|
|
234
|
+
use_tor: bool = False,
|
|
235
|
+
) -> requests.Response:
|
|
236
|
+
domain_rate_limit(url)
|
|
237
|
+
host_concurrency_acquire(url, config)
|
|
238
|
+
try:
|
|
239
|
+
merged_headers = {"User-Agent": USER_AGENT}
|
|
240
|
+
if headers:
|
|
241
|
+
merged_headers.update(headers)
|
|
242
|
+
session = _get_session(config)
|
|
243
|
+
proxies = proxy_dict(select_proxy_for_url(url, config, use_tor=use_tor))
|
|
244
|
+
return session.request(
|
|
245
|
+
method,
|
|
246
|
+
url,
|
|
247
|
+
headers=merged_headers,
|
|
248
|
+
timeout=request_timeout(config),
|
|
249
|
+
proxies=proxies,
|
|
250
|
+
allow_redirects=True,
|
|
251
|
+
stream=stream,
|
|
252
|
+
)
|
|
253
|
+
finally:
|
|
254
|
+
host_concurrency_release(url, config)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def fetch_json(
|
|
258
|
+
url: str,
|
|
259
|
+
config: dict[str, Any],
|
|
260
|
+
*,
|
|
261
|
+
headers: dict[str, str] | None = None,
|
|
262
|
+
use_tor: bool = False,
|
|
263
|
+
) -> dict[str, Any] | None:
|
|
264
|
+
# In-memory probe cache: avoid hammering metadata endpoints during racing.
|
|
265
|
+
ttl = float(config.get("json_probe_cache_seconds", 0) or 0)
|
|
266
|
+
now = time.time()
|
|
267
|
+
if ttl > 0:
|
|
268
|
+
cached = _json_cache.get(url)
|
|
269
|
+
if cached is not None and _json_cache_expires.get(url, 0) > now:
|
|
270
|
+
return cached
|
|
271
|
+
try:
|
|
272
|
+
resp = fetch(url, config, headers={"Accept": "application/json", **(headers or {})}, use_tor=use_tor)
|
|
273
|
+
if resp.status_code >= 400:
|
|
274
|
+
return None
|
|
275
|
+
data = resp.json()
|
|
276
|
+
except Exception:
|
|
277
|
+
return None
|
|
278
|
+
if ttl > 0:
|
|
279
|
+
_json_cache[url] = data
|
|
280
|
+
_json_cache_expires[url] = now + ttl
|
|
281
|
+
return data
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
# Module-level JSON probe caches (see fetch_json). Mutable for test reset.
|
|
285
|
+
_json_cache: dict[str, dict[str, Any]] = {}
|
|
286
|
+
_json_cache_expires: dict[str, float] = {}
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _is_cloudflare_block(resp: requests.Response) -> bool:
|
|
290
|
+
"""Check if response is a Cloudflare/anti-bot challenge."""
|
|
291
|
+
if resp.status_code in (403, 503):
|
|
292
|
+
server = resp.headers.get("server", "").lower()
|
|
293
|
+
if "cloudflare" in server:
|
|
294
|
+
return True
|
|
295
|
+
try:
|
|
296
|
+
body = resp.text[:2000].lower()
|
|
297
|
+
if "cf-browser-verification" in body or "challenge-platform" in body:
|
|
298
|
+
return True
|
|
299
|
+
except Exception:
|
|
300
|
+
pass
|
|
301
|
+
return False
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def polite_delay(config: dict[str, Any]) -> None:
|
|
305
|
+
# Polite delay is opt-in: only sleep when fixed_request_delay_enabled is True.
|
|
306
|
+
# Without it, racing/parallel sources would be needlessly slowed.
|
|
307
|
+
if not config.get("fixed_request_delay_enabled", False):
|
|
308
|
+
return
|
|
309
|
+
lo = float(config.get("request_delay_min", 0))
|
|
310
|
+
hi = float(config.get("request_delay_max", 0))
|
|
311
|
+
if hi > 0:
|
|
312
|
+
time.sleep(random.uniform(lo, max(lo, hi)))
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
# ============================================================
|
|
316
|
+
# Per-domain rate limiting: max 1 request per domain per interval
|
|
317
|
+
# ============================================================
|
|
318
|
+
_domain_locks: dict[str, threading.Lock] = {}
|
|
319
|
+
_domain_last_request: dict[str, float] = {}
|
|
320
|
+
_DOMAIN_RATE_LIMIT = 0.5 # seconds between requests to same domain
|
|
321
|
+
# Longer gaps where the provider's terms ask for them (docs/API_LIMITS.md):
|
|
322
|
+
# arXiv API "no more than one request every three seconds"; CORE "five single
|
|
323
|
+
# requests per 10 seconds".
|
|
324
|
+
_DOMAIN_MIN_INTERVAL = {"export.arxiv.org": 3.0, "arxiv.org": 3.0, "api.core.ac.uk": 2.0}
|
|
325
|
+
# Hourly allowances: OpenAIRE allows 60 requests/hour without a token. Once
|
|
326
|
+
# used up the host is skipped (an OA probe, other sources cover it) instead of
|
|
327
|
+
# stalling downloads for up to an hour.
|
|
328
|
+
_DOMAIN_HOURLY_CAP = {"api.openaire.eu": 55}
|
|
329
|
+
_domain_hour_log: dict[str, list[float]] = {}
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _extract_domain(url: str) -> str:
|
|
333
|
+
from urllib.parse import urlparse
|
|
334
|
+
try:
|
|
335
|
+
return urlparse(url).netloc.lower()
|
|
336
|
+
except Exception:
|
|
337
|
+
return url
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def domain_rate_limit(url: str) -> None:
|
|
341
|
+
"""Wait if needed to respect per-domain rate limit."""
|
|
342
|
+
domain = _extract_domain(url)
|
|
343
|
+
if not domain:
|
|
344
|
+
return
|
|
345
|
+
if domain not in _domain_locks:
|
|
346
|
+
_domain_locks[domain] = threading.Lock()
|
|
347
|
+
with _domain_locks[domain]:
|
|
348
|
+
cap = _DOMAIN_HOURLY_CAP.get(domain)
|
|
349
|
+
if cap:
|
|
350
|
+
recent = [s for s in _domain_hour_log.get(domain, []) if time.time() - s < 3600]
|
|
351
|
+
if len(recent) >= cap:
|
|
352
|
+
raise requests.ConnectionError(f"{domain}: hourly allowance of {cap} requests used; skipped")
|
|
353
|
+
_domain_hour_log[domain] = recent + [time.time()]
|
|
354
|
+
gap = _DOMAIN_MIN_INTERVAL.get(domain, _DOMAIN_RATE_LIMIT)
|
|
355
|
+
now = time.time()
|
|
356
|
+
last = _domain_last_request.get(domain, 0)
|
|
357
|
+
elapsed = now - last
|
|
358
|
+
if elapsed < gap:
|
|
359
|
+
time.sleep(gap - elapsed)
|
|
360
|
+
_domain_last_request[domain] = time.time()
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
# ============================================================
|
|
364
|
+
# Per-host concurrency control (scimesh-style)
|
|
365
|
+
# ============================================================
|
|
366
|
+
# Inherited from the original project; not published by the providers, except
|
|
367
|
+
# arXiv (single connection) and CORE (see _DOMAIN_MIN_INTERVAL). docs/API_LIMITS.md.
|
|
368
|
+
_DEFAULT_HOST_CONCURRENCY: dict[str, int] = {
|
|
369
|
+
"unpaywall.org": 3,
|
|
370
|
+
"api.openalex.org": 5,
|
|
371
|
+
"content.openalex.org": 3,
|
|
372
|
+
"api.crossref.org": 5,
|
|
373
|
+
"api.openaire.eu": 3,
|
|
374
|
+
"doaj.org": 3,
|
|
375
|
+
"api.semanticscholar.org": 3,
|
|
376
|
+
"api.core.ac.uk": 1, # CORE: five single requests per 10 seconds
|
|
377
|
+
"europepmc.org": 3,
|
|
378
|
+
"www.ebi.ac.uk": 3,
|
|
379
|
+
"arxiv.org": 1, # arXiv terms: a single connection at a time
|
|
380
|
+
"export.arxiv.org": 1,
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
_host_semaphores: dict[str, threading.Semaphore] = {}
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _get_host_concurrency(host: str, config: dict[str, Any]) -> int:
|
|
387
|
+
"""Get max concurrent requests for a host."""
|
|
388
|
+
custom = config.get("host_concurrency", {})
|
|
389
|
+
if isinstance(custom, dict) and host in custom:
|
|
390
|
+
return int(custom[host])
|
|
391
|
+
# Match by suffix (e.g., "sci-hub.ru" matches default)
|
|
392
|
+
for pattern, limit in _DEFAULT_HOST_CONCURRENCY.items():
|
|
393
|
+
if host == pattern or host.endswith("." + pattern):
|
|
394
|
+
return limit
|
|
395
|
+
return 0
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def host_concurrency_acquire(url: str, config: dict[str, Any]) -> None:
|
|
399
|
+
"""Acquire a per-host concurrency slot. Blocks if at limit."""
|
|
400
|
+
host = _extract_domain(url)
|
|
401
|
+
if not host:
|
|
402
|
+
return
|
|
403
|
+
limit = _get_host_concurrency(host, config)
|
|
404
|
+
if limit <= 0:
|
|
405
|
+
return
|
|
406
|
+
if host not in _host_semaphores:
|
|
407
|
+
_host_semaphores[host] = threading.Semaphore(limit)
|
|
408
|
+
_host_semaphores[host].acquire()
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def host_concurrency_release(url: str, config: dict[str, Any]) -> None:
|
|
412
|
+
"""Release a per-host concurrency slot."""
|
|
413
|
+
host = _extract_domain(url)
|
|
414
|
+
if not host:
|
|
415
|
+
return
|
|
416
|
+
if host in _host_semaphores:
|
|
417
|
+
_host_semaphores[host].release()
|