mcp-modelmanager 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcp_modelmanager/__init__.py +1 -0
- mcp_modelmanager/connection.py +501 -0
- mcp_modelmanager/estimation.py +161 -0
- mcp_modelmanager/jobs.py +170 -0
- mcp_modelmanager/operations.py +821 -0
- mcp_modelmanager/protocol.py +88 -0
- mcp_modelmanager/server.py +1984 -0
- mcp_modelmanager/services.py +206 -0
- mcp_modelmanager/validation.py +768 -0
- mcp_modelmanager-0.1.0.dist-info/METADATA +544 -0
- mcp_modelmanager-0.1.0.dist-info/RECORD +15 -0
- mcp_modelmanager-0.1.0.dist-info/WHEEL +5 -0
- mcp_modelmanager-0.1.0.dist-info/entry_points.txt +2 -0
- mcp_modelmanager-0.1.0.dist-info/licenses/LICENSE +21 -0
- mcp_modelmanager-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""MCP server to manage a local Ollama/vLLM model machine."""
|
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
"""The two channels to the model machine.
|
|
2
|
+
|
|
3
|
+
Data channel (key A): an SSH tunnel that brings the VM's two service ports to
|
|
4
|
+
the own machine. The key must get no shell on the VM and forward only these two
|
|
5
|
+
targets.
|
|
6
|
+
|
|
7
|
+
Control channel (key B): runs exactly one operation from the catalog. On the VM
|
|
8
|
+
the key is bound firmly to the wrapper; here the same catalog is validated
|
|
9
|
+
before sending.
|
|
10
|
+
|
|
11
|
+
Both channels talk only to the local network, see
|
|
12
|
+
validation.check_local_target.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
import shutil
|
|
19
|
+
import socket
|
|
20
|
+
import subprocess
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
|
|
25
|
+
from mcp_modelmanager import operations
|
|
26
|
+
from mcp_modelmanager import protocol
|
|
27
|
+
from mcp_modelmanager.validation import (
|
|
28
|
+
ValidationError,
|
|
29
|
+
check_local_target,
|
|
30
|
+
_configured_int,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
# Configuration, exclusively from environment variables
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
#
|
|
37
|
+
# There are NO default values for the target machine here. Every user has their
|
|
38
|
+
# own machine; a built-in default would be either wrong or would point at
|
|
39
|
+
# someone else's machine. If a required field is missing, the server exits at
|
|
40
|
+
# startup with a clear message instead of silently falling back on something.
|
|
41
|
+
# How to set the fields is described in SETUP.md.
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _required(name: str, hint: str) -> str:
|
|
45
|
+
value = os.environ.get(name, "").strip()
|
|
46
|
+
if not value:
|
|
47
|
+
raise SystemExit(
|
|
48
|
+
f"{name} is not set. {hint} "
|
|
49
|
+
f"Without this value the server does not start. See SETUP.md."
|
|
50
|
+
)
|
|
51
|
+
return value
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Address of the own model machine. Three cases are foreseen and all valid, as
|
|
55
|
+
# long as the address is on the own machine or the local network (see
|
|
56
|
+
# check_local_target below):
|
|
57
|
+
# - the same machine as this server: 127.0.0.1
|
|
58
|
+
# - a machine on the same network: the LAN IP (e.g. 192.168.1.10)
|
|
59
|
+
# - reached over a tunnel/VPN: the address at the tunnel/VPN end
|
|
60
|
+
VM_HOST = _required(
|
|
61
|
+
"MM_VM_HOST",
|
|
62
|
+
"Enter the address of your model machine (127.0.0.1 for the same machine, "
|
|
63
|
+
"otherwise your LAN or tunnel/VPN address).",
|
|
64
|
+
)
|
|
65
|
+
# SSH user on the model machine. Only the control channel (management: pulling
|
|
66
|
+
# models, switching the service, training) needs it; plain queries run over HTTP
|
|
67
|
+
# without SSH.
|
|
68
|
+
VM_USER = _required(
|
|
69
|
+
"MM_VM_USER",
|
|
70
|
+
"Enter the SSH user name on your model machine.",
|
|
71
|
+
)
|
|
72
|
+
# Optional: a SECOND machine that may currently hold the shared GPU. Only there
|
|
73
|
+
# to interpret an unreachability of the first machine ("GPU probably on the
|
|
74
|
+
# second machine right now"). Whoever has only one machine leaves this empty;
|
|
75
|
+
# the interpretation then drops entirely.
|
|
76
|
+
VM2_HOST = os.environ.get("MM_VM2_HOST", "").strip()
|
|
77
|
+
|
|
78
|
+
_KEY_DIR = os.environ.get(
|
|
79
|
+
"MM_KEY_DIR", os.path.expanduser("~/.ssh/modelmanager")
|
|
80
|
+
)
|
|
81
|
+
KEY_DATA = os.environ.get(
|
|
82
|
+
"MM_KEY_DATA", os.path.join(_KEY_DIR, "datachannel")
|
|
83
|
+
)
|
|
84
|
+
KEY_CONTROL = os.environ.get(
|
|
85
|
+
"MM_KEY_CONTROL", os.path.join(_KEY_DIR, "controlchannel")
|
|
86
|
+
)
|
|
87
|
+
# Pinned host key of the model machine. Written once by setup/install.sh (via
|
|
88
|
+
# ssh-keyscan plus explicit operator confirmation of the fingerprint), never by
|
|
89
|
+
# this program. Checking against a fixed file instead of the trust-on-first-use
|
|
90
|
+
# default (accept-new) means a machine-in-the-middle at connect time is
|
|
91
|
+
# rejected instead of silently pinned.
|
|
92
|
+
KNOWN_HOSTS = os.environ.get(
|
|
93
|
+
"MM_KNOWN_HOSTS", os.path.join(_KEY_DIR, "known_hosts")
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# Ports of the services on the VM (in future only on 127.0.0.1 there). Reuses
|
|
97
|
+
# validation.py's _configured_int so a non-numeric value gets a clear
|
|
98
|
+
# SystemExit naming the variable instead of a raw ValueError; the 1-65535
|
|
99
|
+
# bound is the natural range of a port, not an added restriction.
|
|
100
|
+
REMOTE_VLLM = _configured_int("MM_REMOTE_PORT_VLLM", 8000, 1, 65535)
|
|
101
|
+
REMOTE_OLLAMA = _configured_int("MM_REMOTE_PORT_OLLAMA", 11434, 1, 65535)
|
|
102
|
+
# Ports under which the tunnel exposes them here. Default to the same
|
|
103
|
+
# well-known ports as the remote side; override if those are already taken
|
|
104
|
+
# locally.
|
|
105
|
+
LOCAL_VLLM = _configured_int("MM_LOCAL_PORT_VLLM", 8000, 1, 65535)
|
|
106
|
+
LOCAL_OLLAMA = _configured_int("MM_LOCAL_PORT_OLLAMA", 11434, 1, 65535)
|
|
107
|
+
|
|
108
|
+
# "tunnel" is the target state. "direct" is only the transition, while the
|
|
109
|
+
# services still listen on the LAN.
|
|
110
|
+
ACCESS = os.environ.get("MM_ACCESS", "tunnel").strip().lower()
|
|
111
|
+
if ACCESS not in ("tunnel", "direct"):
|
|
112
|
+
raise SystemExit(f"MM_ACCESS must be 'tunnel' or 'direct', not '{ACCESS}'.")
|
|
113
|
+
|
|
114
|
+
check_local_target(VM_HOST)
|
|
115
|
+
# The second machine, if configured, is also addressed by this tool, so it must
|
|
116
|
+
# clear the same local-network gate at start-up. An empty MM_VM2_HOST means no
|
|
117
|
+
# second machine and is checked nowhere.
|
|
118
|
+
if VM2_HOST:
|
|
119
|
+
check_local_target(VM2_HOST)
|
|
120
|
+
|
|
121
|
+
# Forced key selection. "ssh -i" is only a SUGGESTION: without the following
|
|
122
|
+
# bolts the client additionally offers all agent keys, all default files and
|
|
123
|
+
# everything from ~/.ssh/config, and the far side takes the FIRST that matches.
|
|
124
|
+
# If an unrestricted old key sits there too, the control channel slips past its
|
|
125
|
+
# command= binding without anything noticing anywhere: the allowlist in the
|
|
126
|
+
# wrapper is then never reached. Exactly that happened in a real setup run.
|
|
127
|
+
# IdentitiesOnly=yes only the identity given here is offered
|
|
128
|
+
# IdentityAgent=none the agent is not even asked
|
|
129
|
+
# -F /dev/null ~/.ssh/config cannot contribute another key
|
|
130
|
+
# PreferredAuthentications=publickey no fallback to other methods
|
|
131
|
+
# Note: if the file behind -i is missing, ssh silently drops it and falls back
|
|
132
|
+
# on the default files. So the existence is checked before every build
|
|
133
|
+
# (control_channel_ready, Tunnel.ensure); without that check the forcing would
|
|
134
|
+
# not be watertight.
|
|
135
|
+
SSH_FORCE_KEY = [
|
|
136
|
+
"-F", "/dev/null",
|
|
137
|
+
"-o", "IdentitiesOnly=yes",
|
|
138
|
+
"-o", "IdentityAgent=none",
|
|
139
|
+
"-o", "PreferredAuthentications=publickey",
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
# Absolute path to the ssh program, resolved once from PATH at import. The
|
|
143
|
+
# control and data channels are security boundaries; a bare "ssh" would let a
|
|
144
|
+
# PATH manipulated at call time substitute a different binary for the one that
|
|
145
|
+
# carries the bound keys. Pinning the resolved path here closes that. If ssh is
|
|
146
|
+
# not on PATH at import, fall back to the usual location; a wrong path fails
|
|
147
|
+
# loudly at connect time rather than running an unexpected program.
|
|
148
|
+
SSH_BIN = shutil.which("ssh") or "/usr/bin/ssh"
|
|
149
|
+
|
|
150
|
+
# StrictHostKeyChecking=yes against a fixed UserKnownHostsFile (set up once by
|
|
151
|
+
# setup/install.sh, see KNOWN_HOSTS above) instead of accept-new: the host key
|
|
152
|
+
# is pinned deliberately with operator confirmation, not trusted on first use.
|
|
153
|
+
_SSH_BASE_OPTS = [
|
|
154
|
+
*SSH_FORCE_KEY,
|
|
155
|
+
"-o", "BatchMode=yes",
|
|
156
|
+
"-o", "StrictHostKeyChecking=yes",
|
|
157
|
+
"-o", f"UserKnownHostsFile={KNOWN_HOSTS}",
|
|
158
|
+
"-o", "ConnectTimeout=8",
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class AccessMissing(RuntimeError):
|
|
163
|
+
"""The key is not present or the VM does not accept it."""
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class MachineUnreachable(RuntimeError):
|
|
167
|
+
"""The model machine does not answer. That is a state of its own, not an
|
|
168
|
+
error.
|
|
169
|
+
|
|
170
|
+
One possible reason: if a second, optional host shares the GPU with this
|
|
171
|
+
machine, that second host may currently hold the card while this one is
|
|
172
|
+
off. This interpretation only applies if a second machine has been made
|
|
173
|
+
known via MM_VM2_HOST.
|
|
174
|
+
"""
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# ---------------------------------------------------------------------------
|
|
178
|
+
# Reachability
|
|
179
|
+
# ---------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def port_open(host: str, port: int, timeout: float = 2.0) -> bool:
|
|
183
|
+
try:
|
|
184
|
+
with socket.create_connection((host, port), timeout=timeout):
|
|
185
|
+
return True
|
|
186
|
+
except OSError:
|
|
187
|
+
return False
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@dataclass
|
|
191
|
+
class Reachability:
|
|
192
|
+
machine_reachable: bool
|
|
193
|
+
vm2_reachable: bool
|
|
194
|
+
state: str
|
|
195
|
+
explanation: str
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def reachability() -> Reachability:
|
|
199
|
+
# SSH port 22 is assumed hard: the tool exposes no port parameter and the
|
|
200
|
+
# channels force -F /dev/null, so a non-standard Port from ~/.ssh/config is
|
|
201
|
+
# ignored too. A model machine that only listens on a different SSH port
|
|
202
|
+
# would read here as unreachable. Standard-port setups are unaffected.
|
|
203
|
+
first_up = port_open(VM_HOST, 22)
|
|
204
|
+
if first_up:
|
|
205
|
+
return Reachability(
|
|
206
|
+
True, False, "machine_running",
|
|
207
|
+
"The model machine is reachable.",
|
|
208
|
+
)
|
|
209
|
+
if VM2_HOST:
|
|
210
|
+
vm2 = port_open(VM2_HOST, 22, 1.5)
|
|
211
|
+
if vm2:
|
|
212
|
+
return Reachability(
|
|
213
|
+
False, True, "gpu_probably_on_second_machine",
|
|
214
|
+
"The model machine does not answer, but the second machine does. "
|
|
215
|
+
"Very likely the second machine currently holds the GPU while "
|
|
216
|
+
"this one is off. That is not an error; switch back on the "
|
|
217
|
+
"GPU-holding machine.",
|
|
218
|
+
)
|
|
219
|
+
return Reachability(
|
|
220
|
+
False, False, "machine_unreachable",
|
|
221
|
+
"Neither the model machine nor the second machine answer. Possible: a "
|
|
222
|
+
"switchover is in progress, both machines are off, or a network "
|
|
223
|
+
"problem. Not a fault of this tool.",
|
|
224
|
+
)
|
|
225
|
+
return Reachability(
|
|
226
|
+
False, False, "machine_unreachable",
|
|
227
|
+
"The model machine does not answer. Possible: the machine is off, or a "
|
|
228
|
+
"network problem. Not a fault of this tool.",
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
# ---------------------------------------------------------------------------
|
|
233
|
+
# Data channel: the tunnel
|
|
234
|
+
# ---------------------------------------------------------------------------
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class Tunnel:
|
|
238
|
+
"""Holds the SSH tunnel and rebuilds it on demand.
|
|
239
|
+
|
|
240
|
+
The build happens on demand, not in advance: before every service access it
|
|
241
|
+
is checked whether the local port answers, and otherwise reconnected. A tool
|
|
242
|
+
call should not fail just because the tunnel dropped in between.
|
|
243
|
+
"""
|
|
244
|
+
|
|
245
|
+
def __init__(self) -> None:
|
|
246
|
+
self._process: subprocess.Popen[bytes] | None = None
|
|
247
|
+
self._lock = threading.Lock()
|
|
248
|
+
self.last_error: str | None = None
|
|
249
|
+
self.builds = 0
|
|
250
|
+
|
|
251
|
+
def _running(self) -> bool:
|
|
252
|
+
return self._process is not None and self._process.poll() is None
|
|
253
|
+
|
|
254
|
+
def _ready(self) -> bool:
|
|
255
|
+
# Both ports, not one: a half-standing tunnel is no tunnel.
|
|
256
|
+
return port_open("127.0.0.1", LOCAL_VLLM, 1.0) and port_open(
|
|
257
|
+
"127.0.0.1", LOCAL_OLLAMA, 1.0
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
def command(self) -> list[str]:
|
|
261
|
+
return [
|
|
262
|
+
SSH_BIN,
|
|
263
|
+
*_SSH_BASE_OPTS,
|
|
264
|
+
"-i", KEY_DATA,
|
|
265
|
+
"-N", "-T",
|
|
266
|
+
"-o", "ExitOnForwardFailure=yes",
|
|
267
|
+
"-o", "ServerAliveInterval=5",
|
|
268
|
+
"-o", "ServerAliveCountMax=2",
|
|
269
|
+
"-L", f"127.0.0.1:{LOCAL_VLLM}:127.0.0.1:{REMOTE_VLLM}",
|
|
270
|
+
"-L", f"127.0.0.1:{LOCAL_OLLAMA}:127.0.0.1:{REMOTE_OLLAMA}",
|
|
271
|
+
f"{VM_USER}@{VM_HOST}",
|
|
272
|
+
]
|
|
273
|
+
|
|
274
|
+
def _tear_down_locked(self) -> None:
|
|
275
|
+
# Caller MUST already hold self._lock. Split out so ensure() can reuse it
|
|
276
|
+
# from inside the lock it already holds: self._lock is a plain (non
|
|
277
|
+
# reentrant) Lock, so calling tear_down() there would deadlock and the
|
|
278
|
+
# lock would stay held forever.
|
|
279
|
+
if self._process is not None:
|
|
280
|
+
self._process.terminate()
|
|
281
|
+
try:
|
|
282
|
+
self._process.wait(timeout=5)
|
|
283
|
+
except subprocess.TimeoutExpired:
|
|
284
|
+
self._process.kill()
|
|
285
|
+
self._process = None
|
|
286
|
+
|
|
287
|
+
def tear_down(self) -> None:
|
|
288
|
+
with self._lock:
|
|
289
|
+
self._tear_down_locked()
|
|
290
|
+
|
|
291
|
+
def ensure(self, wait_time: float = 12.0) -> None:
|
|
292
|
+
"""Ensures the tunnel is up. Otherwise raises with a clear cause."""
|
|
293
|
+
if ACCESS == "direct":
|
|
294
|
+
return
|
|
295
|
+
# First the question whether the ports RESPOND, regardless of who holds
|
|
296
|
+
# the tunnel. If a system service holds it (see README, so a query
|
|
297
|
+
# bridge can share it), a second tunnel of our own on the same ports
|
|
298
|
+
# would not only be superfluous, it would fail on the occupied port and
|
|
299
|
+
# report an error where everything is fine.
|
|
300
|
+
# By design this only proves a port is open, not that it is OUR tunnel:
|
|
301
|
+
# an unrelated local service already listening on LOCAL_VLLM/LOCAL_OLLAMA
|
|
302
|
+
# would be taken as ready. Accepted as an edge case; the caller is
|
|
303
|
+
# expected to own these ports.
|
|
304
|
+
if self._ready():
|
|
305
|
+
return
|
|
306
|
+
|
|
307
|
+
with self._lock:
|
|
308
|
+
if self._ready():
|
|
309
|
+
return
|
|
310
|
+
if self._process is not None: # dead or half-dead, get rid of it
|
|
311
|
+
try:
|
|
312
|
+
self._process.kill()
|
|
313
|
+
except OSError:
|
|
314
|
+
pass
|
|
315
|
+
self._process = None
|
|
316
|
+
|
|
317
|
+
if not os.path.exists(KEY_DATA):
|
|
318
|
+
raise AccessMissing(
|
|
319
|
+
f"The data-channel key {KEY_DATA} is missing. Without it there "
|
|
320
|
+
f"is no tunnel and hence no access to the services. Setup: see "
|
|
321
|
+
f"README, section Isolation."
|
|
322
|
+
)
|
|
323
|
+
state = reachability()
|
|
324
|
+
if not state.machine_reachable:
|
|
325
|
+
raise MachineUnreachable(state.explanation)
|
|
326
|
+
|
|
327
|
+
# stdout/stderr are captured so a failed build can quote SSH's
|
|
328
|
+
# message (read below via .read() on the dead process). For a
|
|
329
|
+
# long-lived tunnel that stays up, nobody drains these pipes, so a
|
|
330
|
+
# very chatty ssh could in theory fill the pipe buffer and block. In
|
|
331
|
+
# practice a -N -T tunnel is silent once established; left as is
|
|
332
|
+
# rather than lose the error message on the failure path.
|
|
333
|
+
self._process = subprocess.Popen(
|
|
334
|
+
self.command(),
|
|
335
|
+
stdin=subprocess.DEVNULL,
|
|
336
|
+
stdout=subprocess.PIPE,
|
|
337
|
+
stderr=subprocess.PIPE,
|
|
338
|
+
)
|
|
339
|
+
self.builds += 1
|
|
340
|
+
|
|
341
|
+
deadline = time.monotonic() + wait_time
|
|
342
|
+
while time.monotonic() < deadline:
|
|
343
|
+
if self._process.poll() is not None:
|
|
344
|
+
error = (self._process.stderr.read() or b"").decode(
|
|
345
|
+
"utf-8", "replace"
|
|
346
|
+
).strip()
|
|
347
|
+
self.last_error = error
|
|
348
|
+
self._process = None
|
|
349
|
+
raise AccessMissing(
|
|
350
|
+
"The tunnel could not be built. Message from SSH: "
|
|
351
|
+
+ (error or "none")
|
|
352
|
+
)
|
|
353
|
+
if self._ready():
|
|
354
|
+
self.last_error = None
|
|
355
|
+
return
|
|
356
|
+
time.sleep(0.3)
|
|
357
|
+
|
|
358
|
+
self._tear_down_locked()
|
|
359
|
+
raise AccessMissing(
|
|
360
|
+
f"The tunnel was not up after {wait_time:.0f} seconds. "
|
|
361
|
+
f"Check: is key A registered on the VM and does it permitopen "
|
|
362
|
+
f"127.0.0.1:{REMOTE_VLLM} and 127.0.0.1:{REMOTE_OLLAMA}."
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
def base_vllm(self) -> str:
|
|
366
|
+
if ACCESS == "direct":
|
|
367
|
+
# Re-check at use time: the import-time check_local_target(VM_HOST)
|
|
368
|
+
# only saw one resolution. A hostname that resolved locally then
|
|
369
|
+
# could since have been rebound to an external address (DNS
|
|
370
|
+
# rebinding). Re-resolve and re-validate before egressing plain HTTP.
|
|
371
|
+
check_local_target(VM_HOST)
|
|
372
|
+
return f"http://{VM_HOST}:{REMOTE_VLLM}"
|
|
373
|
+
self.ensure()
|
|
374
|
+
return f"http://127.0.0.1:{LOCAL_VLLM}"
|
|
375
|
+
|
|
376
|
+
def base_ollama(self) -> str:
|
|
377
|
+
if ACCESS == "direct":
|
|
378
|
+
# Re-check at use time against DNS rebinding (see base_vllm).
|
|
379
|
+
check_local_target(VM_HOST)
|
|
380
|
+
return f"http://{VM_HOST}:{REMOTE_OLLAMA}"
|
|
381
|
+
self.ensure()
|
|
382
|
+
return f"http://127.0.0.1:{LOCAL_OLLAMA}"
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
TUNNEL = Tunnel()
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
# ---------------------------------------------------------------------------
|
|
389
|
+
# Control channel
|
|
390
|
+
# ---------------------------------------------------------------------------
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@dataclass
|
|
394
|
+
class Result:
|
|
395
|
+
operation: str
|
|
396
|
+
returncode: int
|
|
397
|
+
output: str
|
|
398
|
+
error: str
|
|
399
|
+
|
|
400
|
+
@property
|
|
401
|
+
def succeeded(self) -> bool:
|
|
402
|
+
return self.returncode == 0
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def control_channel_ready() -> tuple[bool, str]:
|
|
406
|
+
if not os.path.exists(KEY_CONTROL):
|
|
407
|
+
return False, (
|
|
408
|
+
f"The control-channel key {KEY_CONTROL} is missing. As long as it is "
|
|
409
|
+
f"not registered in the VM's authorized_keys, all tools that read VM "
|
|
410
|
+
f"state or steer services are non-functional."
|
|
411
|
+
)
|
|
412
|
+
return True, "Control-channel key present."
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def control_command(line: str) -> list[str]:
|
|
416
|
+
"""The SSH call of the control channel. Factored out so the selftest can
|
|
417
|
+
check it for the forced key selection without a connection."""
|
|
418
|
+
# Defense in depth: VM_USER goes into the user@host argument unvalidated. A
|
|
419
|
+
# leading "-" could in principle be read by ssh as an option. Only the
|
|
420
|
+
# operator sets MM_VM_USER (a normal SSH login name), so this is not gated
|
|
421
|
+
# here; noted so a future change does not treat it as trusted input.
|
|
422
|
+
return [
|
|
423
|
+
SSH_BIN,
|
|
424
|
+
*_SSH_BASE_OPTS,
|
|
425
|
+
"-i", KEY_CONTROL,
|
|
426
|
+
"-o", "ClearAllForwardings=yes",
|
|
427
|
+
"-n",
|
|
428
|
+
f"{VM_USER}@{VM_HOST}",
|
|
429
|
+
line,
|
|
430
|
+
]
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def call(operation_name: str, **parameters: object) -> Result:
|
|
434
|
+
"""Runs exactly one catalog operation on the VM.
|
|
435
|
+
|
|
436
|
+
The parameters are validated HERE against the catalog already, even though
|
|
437
|
+
the wrapper on the VM does the same again. Two independent checks are by
|
|
438
|
+
design: the one here catches errors early and with a better message, the one
|
|
439
|
+
on the VM holds even if this machine were compromised.
|
|
440
|
+
"""
|
|
441
|
+
operation = operations.get(operation_name)
|
|
442
|
+
operation.render(**parameters) # pre-check, result deliberately discarded
|
|
443
|
+
|
|
444
|
+
ready, message = control_channel_ready()
|
|
445
|
+
if not ready:
|
|
446
|
+
raise AccessMissing(message)
|
|
447
|
+
|
|
448
|
+
line = protocol.build(operation_name, parameters)
|
|
449
|
+
command = control_command(line)
|
|
450
|
+
try:
|
|
451
|
+
run = subprocess.run(
|
|
452
|
+
command,
|
|
453
|
+
capture_output=True,
|
|
454
|
+
text=True,
|
|
455
|
+
timeout=operation.timeout + 20.0,
|
|
456
|
+
)
|
|
457
|
+
except subprocess.TimeoutExpired:
|
|
458
|
+
raise TimeoutError(
|
|
459
|
+
f"The operation '{operation_name}' exceeded the time limit of "
|
|
460
|
+
f"{operation.timeout:.0f} seconds."
|
|
461
|
+
) from None
|
|
462
|
+
|
|
463
|
+
error_text = (run.stderr or "").strip()
|
|
464
|
+
if run.returncode == 255:
|
|
465
|
+
state = reachability()
|
|
466
|
+
if not state.machine_reachable:
|
|
467
|
+
raise MachineUnreachable(state.explanation)
|
|
468
|
+
raise AccessMissing(
|
|
469
|
+
"The VM did not accept the control-channel key. Message from SSH: "
|
|
470
|
+
+ (error_text or "none") + ". Check: is the public part of key B "
|
|
471
|
+
"registered in /home/" + VM_USER + "/.ssh/authorized_keys, with "
|
|
472
|
+
"command= on /usr/local/sbin/modelmanager."
|
|
473
|
+
)
|
|
474
|
+
return Result(operation_name, run.returncode, run.stdout or "", error_text)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def call_strict(operation_name: str, **parameters: object) -> str:
|
|
478
|
+
result = call(operation_name, **parameters)
|
|
479
|
+
if not result.succeeded:
|
|
480
|
+
raise RuntimeError(
|
|
481
|
+
f"Operation '{operation_name}' ended with return code "
|
|
482
|
+
f"{result.returncode}. Message: {result.error or result.output or 'none'}"
|
|
483
|
+
)
|
|
484
|
+
return result.output
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
__all__ = [
|
|
488
|
+
"Result",
|
|
489
|
+
"Reachability",
|
|
490
|
+
"MachineUnreachable",
|
|
491
|
+
"ValidationError",
|
|
492
|
+
"SSH_FORCE_KEY",
|
|
493
|
+
"TUNNEL",
|
|
494
|
+
"AccessMissing",
|
|
495
|
+
"reachability",
|
|
496
|
+
"port_open",
|
|
497
|
+
"call",
|
|
498
|
+
"call_strict",
|
|
499
|
+
"control_command",
|
|
500
|
+
"control_channel_ready",
|
|
501
|
+
]
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Memory estimate for a model switch.
|
|
2
|
+
|
|
3
|
+
If the card has, say, 12 GB, only one large model fits. A switch that only
|
|
4
|
+
surfaces on load costs several minutes and leaves the service broken. So it is
|
|
5
|
+
computed beforehand.
|
|
6
|
+
|
|
7
|
+
Computed, not guessed: weight size from the actual file sizes on the VM, the
|
|
8
|
+
key-value cache from the model's config.json. Only if both are missing does it
|
|
9
|
+
fall back to a rough estimate from the name, and the result then says
|
|
10
|
+
explicitly that it is uncertain.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
GB = 2 ** 30
|
|
20
|
+
|
|
21
|
+
# Fixed allowance for the CUDA context, intermediate results and the service
|
|
22
|
+
# itself. Empirical value for vLLM on a single card.
|
|
23
|
+
BASE_LOAD_GB = 1.0
|
|
24
|
+
|
|
25
|
+
_DTYPE_BYTES = {
|
|
26
|
+
"float32": 4.0, "float16": 2.0, "bfloat16": 2.0,
|
|
27
|
+
"float8_e4m3fn": 1.0, "int8": 1.0, "int4": 0.5,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def parse_specs(text: str) -> dict[str, Any]:
|
|
32
|
+
"""Reads the output of the 'hf_model_specs' operation."""
|
|
33
|
+
if "NOT_PRESENT" in text:
|
|
34
|
+
return {"present": False}
|
|
35
|
+
result: dict[str, Any] = {"present": True, "location": None,
|
|
36
|
+
"weights_bytes": None, "config": None}
|
|
37
|
+
head, _, config_part = text.partition("===CONFIG")
|
|
38
|
+
for line in head.splitlines():
|
|
39
|
+
if line.startswith("LOCATION\t"):
|
|
40
|
+
result["location"] = line.split("\t", 1)[1].strip()
|
|
41
|
+
elif line.startswith("WEIGHTS\t"):
|
|
42
|
+
raw = line.split("\t", 1)[1].strip()
|
|
43
|
+
if raw.isdigit():
|
|
44
|
+
result["weights_bytes"] = int(raw)
|
|
45
|
+
if config_part.strip():
|
|
46
|
+
try:
|
|
47
|
+
result["config"] = json.loads(config_part.strip())
|
|
48
|
+
except ValueError:
|
|
49
|
+
result["config"] = None
|
|
50
|
+
return result
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def params_from_name(name: str) -> float | None:
|
|
54
|
+
"""Rough parameter count in billions from the model name, e.g. '7B'.
|
|
55
|
+
|
|
56
|
+
For mixture-of-experts names such as 'mixtral-8x7b' this only picks up the
|
|
57
|
+
per-expert count ('7b'), not the far larger total parameter count. Treat
|
|
58
|
+
the result as a lower bound for such models.
|
|
59
|
+
"""
|
|
60
|
+
match = re.search(r"(\d+(?:[.,]\d+)?)\s*[bB](?![a-zA-Z])", name)
|
|
61
|
+
if not match:
|
|
62
|
+
return None
|
|
63
|
+
try:
|
|
64
|
+
return float(match.group(1).replace(",", "."))
|
|
65
|
+
except ValueError:
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def kv_per_token_bytes(config: dict[str, Any]) -> float | None:
|
|
70
|
+
"""Bytes per token in the key-value cache, from config.json."""
|
|
71
|
+
try:
|
|
72
|
+
layers = int(config["num_hidden_layers"])
|
|
73
|
+
except (KeyError, TypeError, ValueError):
|
|
74
|
+
return None
|
|
75
|
+
head_dim = config.get("head_dim")
|
|
76
|
+
if not head_dim:
|
|
77
|
+
try:
|
|
78
|
+
head_dim = int(config["hidden_size"]) // int(config["num_attention_heads"])
|
|
79
|
+
except (KeyError, TypeError, ValueError, ZeroDivisionError):
|
|
80
|
+
return None
|
|
81
|
+
kv_heads = config.get("num_key_value_heads") or config.get("num_attention_heads")
|
|
82
|
+
try:
|
|
83
|
+
kv_heads = int(kv_heads)
|
|
84
|
+
except (TypeError, ValueError):
|
|
85
|
+
return None
|
|
86
|
+
dbytes = _DTYPE_BYTES.get(str(config.get("torch_dtype", "bfloat16")).lower(), 2.0)
|
|
87
|
+
# Key and value, per layer, per head, per dimension.
|
|
88
|
+
return 2.0 * layers * kv_heads * float(head_dim) * dbytes
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def estimate(specs: dict[str, Any], model_name: str, context: int,
|
|
92
|
+
fraction: float, gpu_total_gb: float) -> dict[str, Any]:
|
|
93
|
+
"""Does the model with this context fit into the allotted memory."""
|
|
94
|
+
uncertain: list[str] = []
|
|
95
|
+
|
|
96
|
+
weights_gb: float | None = None
|
|
97
|
+
if specs.get("weights_bytes"):
|
|
98
|
+
weights_gb = specs["weights_bytes"] / GB
|
|
99
|
+
else:
|
|
100
|
+
billions = params_from_name(model_name)
|
|
101
|
+
if billions is not None:
|
|
102
|
+
weights_gb = billions * 2.0 # assume 16 bit
|
|
103
|
+
uncertain.append(
|
|
104
|
+
"The weight size was derived from the name and 16 bit was "
|
|
105
|
+
"assumed, because the files on the VM were not measurable."
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
kv_gb: float | None = None
|
|
109
|
+
config = specs.get("config")
|
|
110
|
+
if isinstance(config, dict):
|
|
111
|
+
per_token = kv_per_token_bytes(config)
|
|
112
|
+
if per_token:
|
|
113
|
+
kv_gb = per_token * context / GB
|
|
114
|
+
if kv_gb is None and weights_gb is not None:
|
|
115
|
+
kv_gb = 0.06 * (weights_gb / 2.0) * (context / 8192)
|
|
116
|
+
uncertain.append(
|
|
117
|
+
"The key-value cache was estimated roughly, because no config.json "
|
|
118
|
+
"could be read. The deviation can be large."
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
if weights_gb is None:
|
|
122
|
+
return {
|
|
123
|
+
"assessable": False,
|
|
124
|
+
"reason": (
|
|
125
|
+
"Neither file size nor parameter count could be determined. "
|
|
126
|
+
"Without either it cannot be said whether the model fits into "
|
|
127
|
+
"the allotted memory. The switch is therefore not attempted blindly."
|
|
128
|
+
),
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
budget_gb = gpu_total_gb * fraction
|
|
132
|
+
need_gb = weights_gb + (kv_gb or 0.0) + BASE_LOAD_GB
|
|
133
|
+
fits = need_gb <= budget_gb
|
|
134
|
+
|
|
135
|
+
reason = (
|
|
136
|
+
f"Weights {weights_gb:.2f} GB, key-value cache for {context} tokens "
|
|
137
|
+
f"{(kv_gb or 0.0):.2f} GB, base load {BASE_LOAD_GB:.2f} GB, together "
|
|
138
|
+
f"{need_gb:.2f} GB. Allotted are {fraction:.2f} of {gpu_total_gb:.1f} GB, "
|
|
139
|
+
f"so {budget_gb:.2f} GB."
|
|
140
|
+
)
|
|
141
|
+
if not fits:
|
|
142
|
+
possible = max(
|
|
143
|
+
0,
|
|
144
|
+
int((budget_gb - weights_gb - BASE_LOAD_GB) / (kv_gb / context))
|
|
145
|
+
) if kv_gb else 0
|
|
146
|
+
reason += (
|
|
147
|
+
f" That does not fit. Options: a smaller model, a quantized version, "
|
|
148
|
+
f"or lowering the context to at most about {possible} tokens."
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
"assessable": True,
|
|
153
|
+
"fits": fits,
|
|
154
|
+
"weights_gb": round(weights_gb, 2),
|
|
155
|
+
"kv_cache_gb": round(kv_gb or 0.0, 2),
|
|
156
|
+
"base_load_gb": BASE_LOAD_GB,
|
|
157
|
+
"need_gb": round(need_gb, 2),
|
|
158
|
+
"budget_gb": round(budget_gb, 2),
|
|
159
|
+
"reason": reason,
|
|
160
|
+
"uncertainties": uncertain,
|
|
161
|
+
}
|