litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/tor/embedded.py
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"""Embedded Tor: auto-download and manage Tor binary as subprocess.
|
|
2
|
+
|
|
3
|
+
Downloads Tor Expert Bundle to ~/.litloop/tor/ on first use,
|
|
4
|
+
starts it as a SOCKS5 proxy subprocess, and cleans up on exit.
|
|
5
|
+
Supports obfs4 bridges for restricted networks.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import io
|
|
11
|
+
import os
|
|
12
|
+
import platform
|
|
13
|
+
import shutil
|
|
14
|
+
import signal
|
|
15
|
+
import stat
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
import tarfile
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from ..log import get_logger
|
|
25
|
+
|
|
26
|
+
log = get_logger()
|
|
27
|
+
|
|
28
|
+
# Default bridges for restricted networks (obfs4)
|
|
29
|
+
DEFAULT_OBFS4_BRIDGES = [
|
|
30
|
+
"obfs4 154.35.22.13:16844 8FB268DC3037E3F5A5E64C9031B2F92384321B93 cert=bjNGcJBJqUTRqtA4q0z2CU8wmsGdDJnVBPpnrZqK+z8kfH3ZNxFbcFhsBdz2t317W77YOA iat-mode=0",
|
|
31
|
+
"obfs4 192.95.36.142:443 3424DB3F0F1F5A7935872FDE7573F9FB912AAEF1 cert=nmPlqulSC4W5X+9mrrq1xLObRTFM81QjPcJm687Dsr0PQRw4UaI8OCkuNfYklA3/aWF3sA iat-mode=0",
|
|
32
|
+
"obfs4 51.222.13.177:50000 1C46E0A5B76D624B99260522F818D1F0287FD4B4 cert=uxPGjNskSCqHGoODKx8j3wcllOJBuJnlkGEXXf9geUZG4YEmyrPK1sc0ePsSVuLD8P9Kg iat-mode=0",
|
|
33
|
+
"obfs4 51.81.223.139:50000 B3A09F6EE345B6B246D3C462FF6643BB30D639FF cert=+yoRAwJXN3Ge1mH2kDjDRTXWbJa1yANPN4kvP1H4ozuFr3S9NbfPQ6YMclexjGqIIAFJcA iat-mode=0",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
# Mirror URLs for downloading Tor Expert Bundle.
|
|
37
|
+
# Files live under {mirror}/torbrowser/{version}/ on both mirrors.
|
|
38
|
+
TOR_DOWNLOAD_MIRRORS = [
|
|
39
|
+
"https://dist.torproject.org/torbrowser",
|
|
40
|
+
"https://archive.torproject.org/tor-package-archive/torbrowser",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
# Tor Browser version shipping the Expert Bundle. Bump when torproject.org
|
|
44
|
+
# cuts a new stable release. Verified 2026-07-04 (15.0.17, tor 0.4.9.11).
|
|
45
|
+
TOR_VERSION = "15.0.17"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _download_url() -> tuple[str, str]:
|
|
49
|
+
"""Get the download URL path and filename for the Tor Expert Bundle.
|
|
50
|
+
|
|
51
|
+
Returns ``(url_path, filename)`` where ``url_path`` is relative to a
|
|
52
|
+
mirror root (e.g. ``"15.0.17"``) and ``filename`` is the archive name.
|
|
53
|
+
All Expert Bundle archives are .tar.gz on every platform since 0.4.7.x.
|
|
54
|
+
"""
|
|
55
|
+
system = platform.system()
|
|
56
|
+
machine = platform.machine().lower()
|
|
57
|
+
|
|
58
|
+
# Map (system, machine) → the os-arch slug used in the filename.
|
|
59
|
+
if system == "Windows":
|
|
60
|
+
os_arch = "windows-x86_64" # i686 build retired; x86_64 covers all supported Windows
|
|
61
|
+
elif system == "Darwin":
|
|
62
|
+
os_arch = "macos-aarch64" if machine in ("arm64", "aarch64") else "macos-x86_64"
|
|
63
|
+
else:
|
|
64
|
+
os_arch = "linux-aarch64" if machine in ("aarch64", "arm64") else "linux-x86_64"
|
|
65
|
+
|
|
66
|
+
filename = f"tor-expert-bundle-{os_arch}-{TOR_VERSION}.tar.gz"
|
|
67
|
+
return TOR_VERSION, filename
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _tor_dir(config: dict[str, Any]) -> Path:
|
|
71
|
+
from ..config import DATA_DIR
|
|
72
|
+
return Path(config.get("cache_dir", str(DATA_DIR / "cache"))).parent / "tor"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _tor_binary(config: dict[str, Any]) -> Path | None:
|
|
76
|
+
"""Find the tor binary path, downloading if needed."""
|
|
77
|
+
tor_dir = _tor_dir(config)
|
|
78
|
+
|
|
79
|
+
# Check for existing binary
|
|
80
|
+
if platform.system() == "Windows":
|
|
81
|
+
tor_exe = tor_dir / "tor" / "tor.exe"
|
|
82
|
+
else:
|
|
83
|
+
tor_exe = tor_dir / "tor" / "tor"
|
|
84
|
+
|
|
85
|
+
if tor_exe.exists():
|
|
86
|
+
return tor_exe
|
|
87
|
+
|
|
88
|
+
# Check system PATH
|
|
89
|
+
system_tor = shutil.which("tor")
|
|
90
|
+
if system_tor:
|
|
91
|
+
return Path(system_tor)
|
|
92
|
+
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _download_url() -> tuple[str, str]:
|
|
97
|
+
"""Get the download URL path and filename for the Tor Expert Bundle.
|
|
98
|
+
|
|
99
|
+
Returns ``(version_path, filename)`` where ``version_path`` is the
|
|
100
|
+
relative path under a mirror root (the Tor Browser version) and
|
|
101
|
+
``filename`` is the archive name. All Expert Bundle archives are
|
|
102
|
+
.tar.gz on every platform (the legacy .zip was retired with 0.4.7.x).
|
|
103
|
+
"""
|
|
104
|
+
system = platform.system()
|
|
105
|
+
machine = platform.machine().lower()
|
|
106
|
+
|
|
107
|
+
if system == "Windows":
|
|
108
|
+
os_arch = "windows-x86_64"
|
|
109
|
+
elif system == "Darwin":
|
|
110
|
+
os_arch = "macos-aarch64" if machine in ("arm64", "aarch64") else "macos-x86_64"
|
|
111
|
+
else:
|
|
112
|
+
os_arch = "linux-aarch64" if machine in ("aarch64", "arm64") else "linux-x86_64"
|
|
113
|
+
|
|
114
|
+
filename = f"tor-expert-bundle-{os_arch}-{TOR_VERSION}.tar.gz"
|
|
115
|
+
return TOR_VERSION, filename
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def download_tor(config: dict[str, Any]) -> Path | None:
|
|
119
|
+
"""Download and extract Tor Expert Bundle. Returns path to tor binary."""
|
|
120
|
+
tor_dir = _tor_dir(config)
|
|
121
|
+
tor_dir.mkdir(parents=True, exist_ok=True)
|
|
122
|
+
|
|
123
|
+
version_path, filename = _download_url()
|
|
124
|
+
# Try mirrors, using proxy if configured
|
|
125
|
+
import requests as req
|
|
126
|
+
proxies = {}
|
|
127
|
+
proxy = os.environ.get("LITLOOP_PROXY") or os.environ.get("HTTPS_PROXY") or config.get("network_proxy")
|
|
128
|
+
if proxy:
|
|
129
|
+
proxies = {"http": proxy, "https": proxy}
|
|
130
|
+
# Expert Bundle archives are .tar.gz on every platform now.
|
|
131
|
+
for mirror in TOR_DOWNLOAD_MIRRORS:
|
|
132
|
+
url = f"{mirror}/{version_path}/{filename}"
|
|
133
|
+
try:
|
|
134
|
+
log.info(f"Downloading Tor from {url}")
|
|
135
|
+
resp = req.get(url, timeout=300, stream=True, proxies=proxies)
|
|
136
|
+
if resp.status_code != 200:
|
|
137
|
+
continue
|
|
138
|
+
|
|
139
|
+
total = int(resp.headers.get("content-length", 0))
|
|
140
|
+
data = io.BytesIO()
|
|
141
|
+
downloaded = 0
|
|
142
|
+
for chunk in resp.iter_content(8192):
|
|
143
|
+
data.write(chunk)
|
|
144
|
+
downloaded += len(chunk)
|
|
145
|
+
if total and downloaded % (1024 * 1024) < 8192:
|
|
146
|
+
log.info(f" Downloaded {downloaded / 1024 / 1024:.1f}/{total / 1024 / 1024:.1f} MB")
|
|
147
|
+
|
|
148
|
+
data.seek(0)
|
|
149
|
+
log.info(f"Extracting Tor to {tor_dir}")
|
|
150
|
+
|
|
151
|
+
with tarfile.open(fileobj=data, mode="r:gz") as tf:
|
|
152
|
+
tf.extractall(tor_dir)
|
|
153
|
+
|
|
154
|
+
# Find and set executable permission on non-Windows
|
|
155
|
+
binary = _tor_binary(config)
|
|
156
|
+
if binary and binary.exists():
|
|
157
|
+
if platform.system() != "Windows":
|
|
158
|
+
binary.chmod(binary.stat().st_mode | stat.S_IEXEC)
|
|
159
|
+
log.info(f"Tor installed: {binary}")
|
|
160
|
+
return binary
|
|
161
|
+
|
|
162
|
+
except Exception as e:
|
|
163
|
+
log.warning(f"Failed to download from {mirror}: {e}")
|
|
164
|
+
continue
|
|
165
|
+
|
|
166
|
+
log.error("Failed to download Tor from all mirrors")
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _write_torrc(tor_dir: Path, socks_port: int, use_bridges: bool = False) -> Path:
|
|
171
|
+
"""Generate a minimal torrc configuration file."""
|
|
172
|
+
torrc_path = tor_dir / "torrc"
|
|
173
|
+
lines = [
|
|
174
|
+
f"SocksPort {socks_port}",
|
|
175
|
+
"SocksListenAddress 127.0.0.1",
|
|
176
|
+
"AvoidDiskWrites 1",
|
|
177
|
+
"Log notice stdout",
|
|
178
|
+
"GeoIPFile unreachable",
|
|
179
|
+
"GeoIPv6File unreachable",
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
if use_bridges:
|
|
183
|
+
lines.append("UseBridges 1")
|
|
184
|
+
lines.append("ClientTransportPlugin obfs4 exec obfs4proxy")
|
|
185
|
+
for bridge in DEFAULT_OBFS4_BRIDGES:
|
|
186
|
+
lines.append(f"Bridge {bridge}")
|
|
187
|
+
|
|
188
|
+
torrc_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
189
|
+
return torrc_path
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class EmbeddedTor:
|
|
193
|
+
"""Manages an embedded Tor subprocess as a SOCKS5 proxy."""
|
|
194
|
+
|
|
195
|
+
def __init__(self, config: dict[str, Any], socks_port: int = 0, use_bridges: bool = False):
|
|
196
|
+
self.config = config
|
|
197
|
+
self.socks_port = socks_port or self._find_free_port()
|
|
198
|
+
self.use_bridges = use_bridges
|
|
199
|
+
self._process: subprocess.Popen | None = None
|
|
200
|
+
self._binary: Path | None = None
|
|
201
|
+
|
|
202
|
+
@staticmethod
|
|
203
|
+
def _find_free_port() -> int:
|
|
204
|
+
import socket
|
|
205
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
206
|
+
s.bind(("127.0.0.1", 0))
|
|
207
|
+
return s.getsockname()[1]
|
|
208
|
+
|
|
209
|
+
@property
|
|
210
|
+
def proxy_url(self) -> str:
|
|
211
|
+
return f"socks5h://127.0.0.1:{self.socks_port}"
|
|
212
|
+
|
|
213
|
+
def start(self, timeout: int = 60) -> bool:
|
|
214
|
+
"""Start Tor subprocess and wait for it to be ready."""
|
|
215
|
+
self._binary = _tor_binary(self.config)
|
|
216
|
+
|
|
217
|
+
# Download if not found
|
|
218
|
+
if not self._binary:
|
|
219
|
+
log.info("Tor binary not found, downloading...")
|
|
220
|
+
self._binary = download_tor(self.config)
|
|
221
|
+
if not self._binary:
|
|
222
|
+
return False
|
|
223
|
+
|
|
224
|
+
# Warn when something else already holds our port — typically a
|
|
225
|
+
# foreign/orphaned tor from a recycled MCP server process. Our own
|
|
226
|
+
# tor would then die on bind and start() reports failure while the
|
|
227
|
+
# proxy actually still works on that port (#61).
|
|
228
|
+
import socket as _socket
|
|
229
|
+
try:
|
|
230
|
+
with _socket.create_connection(("127.0.0.1", self.socks_port), timeout=1):
|
|
231
|
+
log.warning(
|
|
232
|
+
f"Tor: port {self.socks_port} is already occupied by another process — "
|
|
233
|
+
"if this is an old embedded Tor, either reuse it via tor_proxy "
|
|
234
|
+
f"({self.proxy_url}) or stop it before starting a new instance"
|
|
235
|
+
)
|
|
236
|
+
except OSError:
|
|
237
|
+
pass
|
|
238
|
+
|
|
239
|
+
tor_dir = self._binary.parent
|
|
240
|
+
torrc = _write_torrc(tor_dir.parent, self.socks_port, self.use_bridges)
|
|
241
|
+
|
|
242
|
+
cmd = [str(self._binary), "-f", str(torrc)]
|
|
243
|
+
log.info(f"Starting Tor: {' '.join(cmd[:2])}")
|
|
244
|
+
|
|
245
|
+
try:
|
|
246
|
+
self._process = subprocess.Popen(
|
|
247
|
+
cmd,
|
|
248
|
+
stdout=subprocess.PIPE,
|
|
249
|
+
stderr=subprocess.PIPE,
|
|
250
|
+
creationflags=subprocess.CREATE_NO_WINDOW if platform.system() == "Windows" else 0,
|
|
251
|
+
)
|
|
252
|
+
except Exception as e:
|
|
253
|
+
log.error(f"Failed to start Tor: {e}")
|
|
254
|
+
return False
|
|
255
|
+
|
|
256
|
+
# Wait for SOCKS port to be ready
|
|
257
|
+
import socket
|
|
258
|
+
deadline = time.time() + timeout
|
|
259
|
+
while time.time() < deadline:
|
|
260
|
+
try:
|
|
261
|
+
with socket.create_connection(("127.0.0.1", self.socks_port), timeout=2):
|
|
262
|
+
log.info(f"Tor ready on {self.proxy_url}")
|
|
263
|
+
return True
|
|
264
|
+
except (ConnectionRefusedError, OSError):
|
|
265
|
+
# Check if process died
|
|
266
|
+
if self._process.poll() is not None:
|
|
267
|
+
log.error(f"Tor exited with code {self._process.returncode}")
|
|
268
|
+
stderr = self._process.stderr.read().decode("utf-8", errors="replace")[:500]
|
|
269
|
+
log.error(f"Tor stderr: {stderr}")
|
|
270
|
+
self._process = None
|
|
271
|
+
return False
|
|
272
|
+
time.sleep(1)
|
|
273
|
+
|
|
274
|
+
log.warning("Tor startup timed out")
|
|
275
|
+
self.stop()
|
|
276
|
+
return False
|
|
277
|
+
|
|
278
|
+
def stop(self) -> None:
|
|
279
|
+
"""Stop the Tor subprocess."""
|
|
280
|
+
if self._process and self._process.poll() is None:
|
|
281
|
+
log.info("Stopping embedded Tor")
|
|
282
|
+
try:
|
|
283
|
+
if platform.system() == "Windows":
|
|
284
|
+
self._process.terminate()
|
|
285
|
+
else:
|
|
286
|
+
self._process.send_signal(signal.SIGTERM)
|
|
287
|
+
self._process.wait(timeout=10)
|
|
288
|
+
except Exception:
|
|
289
|
+
try:
|
|
290
|
+
self._process.kill()
|
|
291
|
+
except Exception:
|
|
292
|
+
pass
|
|
293
|
+
self._process = None
|
|
294
|
+
|
|
295
|
+
def is_running(self) -> bool:
|
|
296
|
+
return self._process is not None and self._process.poll() is None
|
|
297
|
+
|
|
298
|
+
def __enter__(self):
|
|
299
|
+
return self
|
|
300
|
+
|
|
301
|
+
def __exit__(self, *args):
|
|
302
|
+
self.stop()
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
# Global singleton for embedded Tor process
|
|
306
|
+
_embedded_tor: EmbeddedTor | None = None
|
|
307
|
+
_tor_lock = threading.Lock()
|
|
308
|
+
_tor_unavailable_since: float = 0.0 # timestamp when Tor last failed
|
|
309
|
+
_TOR_UNAVAILABLE_TTL = 3600 # 1 hour before retrying Tor download
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def get_embedded_tor(config: dict[str, Any]) -> EmbeddedTor | None:
|
|
313
|
+
"""Get or create a global embedded Tor instance.
|
|
314
|
+
|
|
315
|
+
If Tor download previously failed, skips retry for 1 hour to avoid
|
|
316
|
+
blocking on unreachable torproject.org mirrors.
|
|
317
|
+
"""
|
|
318
|
+
global _embedded_tor, _tor_unavailable_since
|
|
319
|
+
with _tor_lock:
|
|
320
|
+
if _embedded_tor and _embedded_tor.is_running():
|
|
321
|
+
return _embedded_tor
|
|
322
|
+
# Fast skip: Tor download previously failed within the TTL
|
|
323
|
+
if _tor_unavailable_since and (time.time() - _tor_unavailable_since) < _TOR_UNAVAILABLE_TTL:
|
|
324
|
+
return None
|
|
325
|
+
|
|
326
|
+
use_bridges = config.get("tor_use_bridges", False)
|
|
327
|
+
tor = EmbeddedTor(config, use_bridges=use_bridges)
|
|
328
|
+
if tor.start():
|
|
329
|
+
_embedded_tor = tor
|
|
330
|
+
_tor_unavailable_since = 0.0 # reset on success
|
|
331
|
+
return tor
|
|
332
|
+
_tor_unavailable_since = time.time() # cache failure timestamp
|
|
333
|
+
return None
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def running_embedded_tor() -> "EmbeddedTor | None":
|
|
337
|
+
"""Return the embedded instance only if it is ALREADY running.
|
|
338
|
+
|
|
339
|
+
Unlike get_embedded_tor this never starts Tor or downloads a binary —
|
|
340
|
+
safe to call from a health probe that must stay side-effect free (#61).
|
|
341
|
+
"""
|
|
342
|
+
with _tor_lock:
|
|
343
|
+
if _embedded_tor and _embedded_tor.is_running():
|
|
344
|
+
return _embedded_tor
|
|
345
|
+
return None
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def stop_embedded_tor() -> None:
|
|
349
|
+
"""Stop the global embedded Tor instance."""
|
|
350
|
+
global _embedded_tor
|
|
351
|
+
with _tor_lock:
|
|
352
|
+
if _embedded_tor:
|
|
353
|
+
_embedded_tor.stop()
|
|
354
|
+
_embedded_tor = None
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def is_tor_installed(config: dict[str, Any]) -> bool:
|
|
358
|
+
"""Check if Tor binary is available (embedded or system)."""
|
|
359
|
+
return _tor_binary(config) is not None
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def install_tor(config: dict[str, Any]) -> dict[str, Any]:
|
|
363
|
+
"""Download and install Tor. Returns status dict."""
|
|
364
|
+
binary = _tor_binary(config)
|
|
365
|
+
if binary:
|
|
366
|
+
return {"installed": True, "path": str(binary), "message": "Tor already installed"}
|
|
367
|
+
|
|
368
|
+
binary = download_tor(config)
|
|
369
|
+
if binary:
|
|
370
|
+
return {"installed": True, "path": str(binary), "message": f"Tor installed to {binary}"}
|
|
371
|
+
return {"installed": False, "error": "Failed to download Tor. Check network connectivity."}
|
litloop/tor/proxy.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Tor SOCKS5 proxy integration with embedded Tor support."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from ..log import get_logger
|
|
9
|
+
|
|
10
|
+
log = get_logger()
|
|
11
|
+
|
|
12
|
+
DEFAULT_TOR_PROXY = "socks5h://127.0.0.1:1080"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_tor_proxy(config: dict[str, Any] | None = None) -> str | None:
|
|
16
|
+
"""Get Tor proxy URL.
|
|
17
|
+
|
|
18
|
+
Priority:
|
|
19
|
+
1. TOR_PROXY env var
|
|
20
|
+
2. Config tor_proxy
|
|
21
|
+
3. Embedded instance, if one is already running (never started here)
|
|
22
|
+
4. Default 1080 port
|
|
23
|
+
|
|
24
|
+
Consulting the embedded instance keeps the health probe in sync with the
|
|
25
|
+
download path (ensure_tor), which falls back to embedded Tor when the
|
|
26
|
+
configured proxy is absent (#61).
|
|
27
|
+
"""
|
|
28
|
+
proxy = os.environ.get("TOR_PROXY", "")
|
|
29
|
+
if proxy:
|
|
30
|
+
return proxy
|
|
31
|
+
if config and config.get("tor_proxy"):
|
|
32
|
+
return config["tor_proxy"]
|
|
33
|
+
try:
|
|
34
|
+
from .embedded import running_embedded_tor
|
|
35
|
+
tor = running_embedded_tor()
|
|
36
|
+
if tor:
|
|
37
|
+
return tor.proxy_url
|
|
38
|
+
except Exception:
|
|
39
|
+
pass
|
|
40
|
+
return DEFAULT_TOR_PROXY
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def check_tor_circuit(config: dict[str, Any] | None = None) -> bool:
|
|
44
|
+
"""Check if Tor SOCKS5 proxy is reachable."""
|
|
45
|
+
import socket
|
|
46
|
+
try:
|
|
47
|
+
proxy = get_tor_proxy(config)
|
|
48
|
+
if not proxy:
|
|
49
|
+
return False
|
|
50
|
+
host_port = proxy.split("://")[-1]
|
|
51
|
+
host, port = host_port.rsplit(":", 1)
|
|
52
|
+
with socket.create_connection((host, int(port)), timeout=3):
|
|
53
|
+
return True
|
|
54
|
+
except Exception:
|
|
55
|
+
return False
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def ensure_tor(config: dict[str, Any]) -> str | None:
|
|
59
|
+
"""Ensure Tor is available, starting embedded Tor if needed.
|
|
60
|
+
|
|
61
|
+
Returns the SOCKS5 proxy URL or None if Tor is unavailable.
|
|
62
|
+
"""
|
|
63
|
+
# 1. Check if external Tor is already running
|
|
64
|
+
if check_tor_circuit(config):
|
|
65
|
+
return get_tor_proxy(config)
|
|
66
|
+
|
|
67
|
+
# 2. Try starting embedded Tor (auto-install if needed)
|
|
68
|
+
try:
|
|
69
|
+
from .embedded import get_embedded_tor
|
|
70
|
+
log.info("Tor: starting embedded Tor...")
|
|
71
|
+
tor = get_embedded_tor(config)
|
|
72
|
+
if tor and tor.is_running():
|
|
73
|
+
log.info(f"Tor: embedded Tor started at {tor.proxy_url}")
|
|
74
|
+
return tor.proxy_url
|
|
75
|
+
log.warning("Tor: embedded Tor failed to start")
|
|
76
|
+
except Exception as e:
|
|
77
|
+
log.warning(f"Tor: embedded Tor error: {e}")
|
|
78
|
+
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def start_embedded_tor(config: dict[str, Any]) -> dict[str, Any]:
|
|
83
|
+
"""Start embedded Tor proxy. Returns status dict."""
|
|
84
|
+
try:
|
|
85
|
+
from .embedded import get_embedded_tor
|
|
86
|
+
tor = get_embedded_tor(config)
|
|
87
|
+
if tor:
|
|
88
|
+
return {"running": True, "proxy": tor.proxy_url}
|
|
89
|
+
return {"running": False, "error": "Failed to start embedded Tor"}
|
|
90
|
+
except Exception as e:
|
|
91
|
+
return {"running": False, "error": str(e)}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def stop_embedded_tor() -> dict[str, Any]:
|
|
95
|
+
"""Stop embedded Tor proxy."""
|
|
96
|
+
try:
|
|
97
|
+
from .embedded import stop_embedded_tor as _stop
|
|
98
|
+
_stop()
|
|
99
|
+
return {"stopped": True}
|
|
100
|
+
except Exception as e:
|
|
101
|
+
return {"stopped": False, "error": str(e)}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def install_tor(config: dict[str, Any]) -> dict[str, Any]:
|
|
105
|
+
"""Download and install Tor Expert Bundle."""
|
|
106
|
+
try:
|
|
107
|
+
from .embedded import install_tor as _install
|
|
108
|
+
return _install(config)
|
|
109
|
+
except Exception as e:
|
|
110
|
+
return {"installed": False, "error": str(e)}
|
litloop/updates.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""New-version reminder for LitLoop, easyseek (scansci-find) and zotero-mcp.
|
|
2
|
+
|
|
3
|
+
Checked at most every ``update_check_days`` days (config; 0 = never), in a
|
|
4
|
+
background thread, so no search or download waits for it. The result is kept
|
|
5
|
+
in ``~/.litloop/update_check.json``; ``notice()`` reads that file and, when it
|
|
6
|
+
is stale, starts the next check. Nothing is installed: the notice says how to
|
|
7
|
+
update.
|
|
8
|
+
|
|
9
|
+
Where the versions come from:
|
|
10
|
+
- LitLoop, easyseek: ``version`` in ``pyproject.toml`` on the main branch
|
|
11
|
+
(git tags stopped at v1.9.0, so they are not used).
|
|
12
|
+
- zotero-mcp: PyPI (``zotero-mcp-server``), against the version installed in
|
|
13
|
+
zotero-mcp's own Python.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import importlib.metadata
|
|
19
|
+
import re
|
|
20
|
+
import subprocess
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
import requests
|
|
26
|
+
|
|
27
|
+
from . import __version__, config, store
|
|
28
|
+
|
|
29
|
+
RAW = "https://raw.githubusercontent.com/dysprosium231/{repo}/main/pyproject.toml"
|
|
30
|
+
PYPI = "https://pypi.org/pypi/{name}/json"
|
|
31
|
+
_running = threading.Lock()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _path():
|
|
35
|
+
return config.DATA_DIR / "update_check.json"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _parse(v: str | None) -> tuple[int, ...]:
|
|
39
|
+
return tuple(int(x) for x in re.findall(r"\d+", v or "")[:3])
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _remote_pyproject(repo: str) -> str | None:
|
|
43
|
+
r = requests.get(RAW.format(repo=repo), timeout=15)
|
|
44
|
+
r.raise_for_status()
|
|
45
|
+
m = re.search(r'^version\s*=\s*"([^"]+)"', r.text, re.M)
|
|
46
|
+
return m.group(1) if m else None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _pypi(name: str) -> str | None:
|
|
50
|
+
r = requests.get(PYPI.format(name=name), timeout=15)
|
|
51
|
+
r.raise_for_status()
|
|
52
|
+
return r.json()["info"]["version"]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _installed(dist: str) -> str | None:
|
|
56
|
+
try:
|
|
57
|
+
return importlib.metadata.version(dist)
|
|
58
|
+
except importlib.metadata.PackageNotFoundError:
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _zotero_mcp_installed(cfg: dict[str, Any]) -> str | None:
|
|
63
|
+
from .zotero.client import discover_zotero_mcp
|
|
64
|
+
found = discover_zotero_mcp(cfg)
|
|
65
|
+
if not found:
|
|
66
|
+
return None
|
|
67
|
+
r = subprocess.run([str(found["python"]), "-c",
|
|
68
|
+
"import importlib.metadata as m; print(m.version('zotero-mcp-server'))"],
|
|
69
|
+
capture_output=True, text=True, timeout=30,
|
|
70
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
71
|
+
return r.stdout.strip() or None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# name, how to read the installed version, how to read the latest one, how to update
|
|
75
|
+
COMPONENTS = [
|
|
76
|
+
("LitLoop", lambda cfg: __version__, lambda: _remote_pyproject("litloop-paper-agent"),
|
|
77
|
+
"运行 python scripts/deploy.py 更新"),
|
|
78
|
+
("easyseek", lambda cfg: _installed("scansci-find"), lambda: _remote_pyproject("easyseek"),
|
|
79
|
+
"运行 python scripts/deploy.py 更新"),
|
|
80
|
+
("zotero-mcp", _zotero_mcp_installed, lambda: _pypi("zotero-mcp-server"),
|
|
81
|
+
"在 zotero-mcp 的 Python 里运行 pip install -U zotero-mcp-server"),
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def check(cfg: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
86
|
+
"""Ask now; store and return {"checked": ts, "components": [...]}. A component that fails is skipped."""
|
|
87
|
+
cfg = cfg or config.load_config()
|
|
88
|
+
out = []
|
|
89
|
+
for name, installed, latest, how in COMPONENTS:
|
|
90
|
+
try:
|
|
91
|
+
have, new = installed(cfg), latest()
|
|
92
|
+
except Exception: # offline, rate-limited, not installed: try again next time
|
|
93
|
+
continue
|
|
94
|
+
if have and new:
|
|
95
|
+
out.append({"name": name, "installed": have, "latest": new,
|
|
96
|
+
"update": _parse(new) > _parse(have), "how": how})
|
|
97
|
+
data = {"checked": time.time(), "components": out}
|
|
98
|
+
store.write_json(_path(), data)
|
|
99
|
+
return data
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _due(cfg: dict[str, Any]) -> bool:
|
|
103
|
+
days = float(cfg.get("update_check_days", 7) or 0)
|
|
104
|
+
if days <= 0:
|
|
105
|
+
return False
|
|
106
|
+
try:
|
|
107
|
+
last = float(store.read_json(_path()).get("checked") or 0)
|
|
108
|
+
except (OSError, ValueError):
|
|
109
|
+
last = 0
|
|
110
|
+
return time.time() - last > days * 86400
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _refresh_in_background(cfg: dict[str, Any]) -> None:
|
|
114
|
+
if not _running.acquire(blocking=False):
|
|
115
|
+
return
|
|
116
|
+
|
|
117
|
+
def run() -> None:
|
|
118
|
+
try:
|
|
119
|
+
check(cfg)
|
|
120
|
+
except Exception:
|
|
121
|
+
pass
|
|
122
|
+
finally:
|
|
123
|
+
_running.release()
|
|
124
|
+
threading.Thread(target=run, name="litloop-update-check", daemon=True).start()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def available(cfg: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
128
|
+
"""Components with a newer version, from the last check; starts a check when one is due."""
|
|
129
|
+
cfg = cfg or config.load_config()
|
|
130
|
+
if float(cfg.get("update_check_days", 7) or 0) <= 0:
|
|
131
|
+
return []
|
|
132
|
+
if _due(cfg):
|
|
133
|
+
_refresh_in_background(cfg)
|
|
134
|
+
try:
|
|
135
|
+
data = store.read_json(_path())
|
|
136
|
+
except (OSError, ValueError):
|
|
137
|
+
return []
|
|
138
|
+
# LitLoop itself may have been updated since: compare with the running version.
|
|
139
|
+
return [c for c in data.get("components", [])
|
|
140
|
+
if c.get("update") and not (c["name"] == "LitLoop" and _parse(c["latest"]) <= _parse(__version__))]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def notice(cfg: dict[str, Any] | None = None) -> str | None:
|
|
144
|
+
"""One line for tool results, or None."""
|
|
145
|
+
comps = available(cfg)
|
|
146
|
+
if not comps:
|
|
147
|
+
return None
|
|
148
|
+
return "有新版本:" + ";".join(f"{c['name']} {c['latest']}(当前 {c['installed']}),{c['how']}" for c in comps)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Zotero MCP client integration and local data bridge."""
|