mcp-modelmanager 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ """MCP server to manage a local Ollama/vLLM model machine."""
@@ -0,0 +1,501 @@
1
+ """The two channels to the model machine.
2
+
3
+ Data channel (key A): an SSH tunnel that brings the VM's two service ports to
4
+ the own machine. The key must get no shell on the VM and forward only these two
5
+ targets.
6
+
7
+ Control channel (key B): runs exactly one operation from the catalog. On the VM
8
+ the key is bound firmly to the wrapper; here the same catalog is validated
9
+ before sending.
10
+
11
+ Both channels talk only to the local network, see
12
+ validation.check_local_target.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import os
18
+ import shutil
19
+ import socket
20
+ import subprocess
21
+ import threading
22
+ import time
23
+ from dataclasses import dataclass
24
+
25
+ from mcp_modelmanager import operations
26
+ from mcp_modelmanager import protocol
27
+ from mcp_modelmanager.validation import (
28
+ ValidationError,
29
+ check_local_target,
30
+ _configured_int,
31
+ )
32
+
33
+ # ---------------------------------------------------------------------------
34
+ # Configuration, exclusively from environment variables
35
+ # ---------------------------------------------------------------------------
36
+ #
37
+ # There are NO default values for the target machine here. Every user has their
38
+ # own machine; a built-in default would be either wrong or would point at
39
+ # someone else's machine. If a required field is missing, the server exits at
40
+ # startup with a clear message instead of silently falling back on something.
41
+ # How to set the fields is described in SETUP.md.
42
+
43
+
44
+ def _required(name: str, hint: str) -> str:
45
+ value = os.environ.get(name, "").strip()
46
+ if not value:
47
+ raise SystemExit(
48
+ f"{name} is not set. {hint} "
49
+ f"Without this value the server does not start. See SETUP.md."
50
+ )
51
+ return value
52
+
53
+
54
+ # Address of the own model machine. Three cases are foreseen and all valid, as
55
+ # long as the address is on the own machine or the local network (see
56
+ # check_local_target below):
57
+ # - the same machine as this server: 127.0.0.1
58
+ # - a machine on the same network: the LAN IP (e.g. 192.168.1.10)
59
+ # - reached over a tunnel/VPN: the address at the tunnel/VPN end
60
+ VM_HOST = _required(
61
+ "MM_VM_HOST",
62
+ "Enter the address of your model machine (127.0.0.1 for the same machine, "
63
+ "otherwise your LAN or tunnel/VPN address).",
64
+ )
65
+ # SSH user on the model machine. Only the control channel (management: pulling
66
+ # models, switching the service, training) needs it; plain queries run over HTTP
67
+ # without SSH.
68
+ VM_USER = _required(
69
+ "MM_VM_USER",
70
+ "Enter the SSH user name on your model machine.",
71
+ )
72
+ # Optional: a SECOND machine that may currently hold the shared GPU. Only there
73
+ # to interpret an unreachability of the first machine ("GPU probably on the
74
+ # second machine right now"). Whoever has only one machine leaves this empty;
75
+ # the interpretation then drops entirely.
76
+ VM2_HOST = os.environ.get("MM_VM2_HOST", "").strip()
77
+
78
+ _KEY_DIR = os.environ.get(
79
+ "MM_KEY_DIR", os.path.expanduser("~/.ssh/modelmanager")
80
+ )
81
+ KEY_DATA = os.environ.get(
82
+ "MM_KEY_DATA", os.path.join(_KEY_DIR, "datachannel")
83
+ )
84
+ KEY_CONTROL = os.environ.get(
85
+ "MM_KEY_CONTROL", os.path.join(_KEY_DIR, "controlchannel")
86
+ )
87
+ # Pinned host key of the model machine. Written once by setup/install.sh (via
88
+ # ssh-keyscan plus explicit operator confirmation of the fingerprint), never by
89
+ # this program. Checking against a fixed file instead of the trust-on-first-use
90
+ # default (accept-new) means a machine-in-the-middle at connect time is
91
+ # rejected instead of silently pinned.
92
+ KNOWN_HOSTS = os.environ.get(
93
+ "MM_KNOWN_HOSTS", os.path.join(_KEY_DIR, "known_hosts")
94
+ )
95
+
96
+ # Ports of the services on the VM (in future only on 127.0.0.1 there). Reuses
97
+ # validation.py's _configured_int so a non-numeric value gets a clear
98
+ # SystemExit naming the variable instead of a raw ValueError; the 1-65535
99
+ # bound is the natural range of a port, not an added restriction.
100
+ REMOTE_VLLM = _configured_int("MM_REMOTE_PORT_VLLM", 8000, 1, 65535)
101
+ REMOTE_OLLAMA = _configured_int("MM_REMOTE_PORT_OLLAMA", 11434, 1, 65535)
102
+ # Ports under which the tunnel exposes them here. Default to the same
103
+ # well-known ports as the remote side; override if those are already taken
104
+ # locally.
105
+ LOCAL_VLLM = _configured_int("MM_LOCAL_PORT_VLLM", 8000, 1, 65535)
106
+ LOCAL_OLLAMA = _configured_int("MM_LOCAL_PORT_OLLAMA", 11434, 1, 65535)
107
+
108
+ # "tunnel" is the target state. "direct" is only the transition, while the
109
+ # services still listen on the LAN.
110
+ ACCESS = os.environ.get("MM_ACCESS", "tunnel").strip().lower()
111
+ if ACCESS not in ("tunnel", "direct"):
112
+ raise SystemExit(f"MM_ACCESS must be 'tunnel' or 'direct', not '{ACCESS}'.")
113
+
114
+ check_local_target(VM_HOST)
115
+ # The second machine, if configured, is also addressed by this tool, so it must
116
+ # clear the same local-network gate at start-up. An empty MM_VM2_HOST means no
117
+ # second machine and is checked nowhere.
118
+ if VM2_HOST:
119
+ check_local_target(VM2_HOST)
120
+
121
+ # Forced key selection. "ssh -i" is only a SUGGESTION: without the following
122
+ # bolts the client additionally offers all agent keys, all default files and
123
+ # everything from ~/.ssh/config, and the far side takes the FIRST that matches.
124
+ # If an unrestricted old key sits there too, the control channel slips past its
125
+ # command= binding without anything noticing anywhere: the allowlist in the
126
+ # wrapper is then never reached. Exactly that happened in a real setup run.
127
+ # IdentitiesOnly=yes only the identity given here is offered
128
+ # IdentityAgent=none the agent is not even asked
129
+ # -F /dev/null ~/.ssh/config cannot contribute another key
130
+ # PreferredAuthentications=publickey no fallback to other methods
131
+ # Note: if the file behind -i is missing, ssh silently drops it and falls back
132
+ # on the default files. So the existence is checked before every build
133
+ # (control_channel_ready, Tunnel.ensure); without that check the forcing would
134
+ # not be watertight.
135
+ SSH_FORCE_KEY = [
136
+ "-F", "/dev/null",
137
+ "-o", "IdentitiesOnly=yes",
138
+ "-o", "IdentityAgent=none",
139
+ "-o", "PreferredAuthentications=publickey",
140
+ ]
141
+
142
+ # Absolute path to the ssh program, resolved once from PATH at import. The
143
+ # control and data channels are security boundaries; a bare "ssh" would let a
144
+ # PATH manipulated at call time substitute a different binary for the one that
145
+ # carries the bound keys. Pinning the resolved path here closes that. If ssh is
146
+ # not on PATH at import, fall back to the usual location; a wrong path fails
147
+ # loudly at connect time rather than running an unexpected program.
148
+ SSH_BIN = shutil.which("ssh") or "/usr/bin/ssh"
149
+
150
+ # StrictHostKeyChecking=yes against a fixed UserKnownHostsFile (set up once by
151
+ # setup/install.sh, see KNOWN_HOSTS above) instead of accept-new: the host key
152
+ # is pinned deliberately with operator confirmation, not trusted on first use.
153
+ _SSH_BASE_OPTS = [
154
+ *SSH_FORCE_KEY,
155
+ "-o", "BatchMode=yes",
156
+ "-o", "StrictHostKeyChecking=yes",
157
+ "-o", f"UserKnownHostsFile={KNOWN_HOSTS}",
158
+ "-o", "ConnectTimeout=8",
159
+ ]
160
+
161
+
162
+ class AccessMissing(RuntimeError):
163
+ """The key is not present or the VM does not accept it."""
164
+
165
+
166
+ class MachineUnreachable(RuntimeError):
167
+ """The model machine does not answer. That is a state of its own, not an
168
+ error.
169
+
170
+ One possible reason: if a second, optional host shares the GPU with this
171
+ machine, that second host may currently hold the card while this one is
172
+ off. This interpretation only applies if a second machine has been made
173
+ known via MM_VM2_HOST.
174
+ """
175
+
176
+
177
+ # ---------------------------------------------------------------------------
178
+ # Reachability
179
+ # ---------------------------------------------------------------------------
180
+
181
+
182
+ def port_open(host: str, port: int, timeout: float = 2.0) -> bool:
183
+ try:
184
+ with socket.create_connection((host, port), timeout=timeout):
185
+ return True
186
+ except OSError:
187
+ return False
188
+
189
+
190
+ @dataclass
191
+ class Reachability:
192
+ machine_reachable: bool
193
+ vm2_reachable: bool
194
+ state: str
195
+ explanation: str
196
+
197
+
198
+ def reachability() -> Reachability:
199
+ # SSH port 22 is assumed hard: the tool exposes no port parameter and the
200
+ # channels force -F /dev/null, so a non-standard Port from ~/.ssh/config is
201
+ # ignored too. A model machine that only listens on a different SSH port
202
+ # would read here as unreachable. Standard-port setups are unaffected.
203
+ first_up = port_open(VM_HOST, 22)
204
+ if first_up:
205
+ return Reachability(
206
+ True, False, "machine_running",
207
+ "The model machine is reachable.",
208
+ )
209
+ if VM2_HOST:
210
+ vm2 = port_open(VM2_HOST, 22, 1.5)
211
+ if vm2:
212
+ return Reachability(
213
+ False, True, "gpu_probably_on_second_machine",
214
+ "The model machine does not answer, but the second machine does. "
215
+ "Very likely the second machine currently holds the GPU while "
216
+ "this one is off. That is not an error; switch back on the "
217
+ "GPU-holding machine.",
218
+ )
219
+ return Reachability(
220
+ False, False, "machine_unreachable",
221
+ "Neither the model machine nor the second machine answer. Possible: a "
222
+ "switchover is in progress, both machines are off, or a network "
223
+ "problem. Not a fault of this tool.",
224
+ )
225
+ return Reachability(
226
+ False, False, "machine_unreachable",
227
+ "The model machine does not answer. Possible: the machine is off, or a "
228
+ "network problem. Not a fault of this tool.",
229
+ )
230
+
231
+
232
+ # ---------------------------------------------------------------------------
233
+ # Data channel: the tunnel
234
+ # ---------------------------------------------------------------------------
235
+
236
+
237
+ class Tunnel:
238
+ """Holds the SSH tunnel and rebuilds it on demand.
239
+
240
+ The build happens on demand, not in advance: before every service access it
241
+ is checked whether the local port answers, and otherwise reconnected. A tool
242
+ call should not fail just because the tunnel dropped in between.
243
+ """
244
+
245
+ def __init__(self) -> None:
246
+ self._process: subprocess.Popen[bytes] | None = None
247
+ self._lock = threading.Lock()
248
+ self.last_error: str | None = None
249
+ self.builds = 0
250
+
251
+ def _running(self) -> bool:
252
+ return self._process is not None and self._process.poll() is None
253
+
254
+ def _ready(self) -> bool:
255
+ # Both ports, not one: a half-standing tunnel is no tunnel.
256
+ return port_open("127.0.0.1", LOCAL_VLLM, 1.0) and port_open(
257
+ "127.0.0.1", LOCAL_OLLAMA, 1.0
258
+ )
259
+
260
+ def command(self) -> list[str]:
261
+ return [
262
+ SSH_BIN,
263
+ *_SSH_BASE_OPTS,
264
+ "-i", KEY_DATA,
265
+ "-N", "-T",
266
+ "-o", "ExitOnForwardFailure=yes",
267
+ "-o", "ServerAliveInterval=5",
268
+ "-o", "ServerAliveCountMax=2",
269
+ "-L", f"127.0.0.1:{LOCAL_VLLM}:127.0.0.1:{REMOTE_VLLM}",
270
+ "-L", f"127.0.0.1:{LOCAL_OLLAMA}:127.0.0.1:{REMOTE_OLLAMA}",
271
+ f"{VM_USER}@{VM_HOST}",
272
+ ]
273
+
274
+ def _tear_down_locked(self) -> None:
275
+ # Caller MUST already hold self._lock. Split out so ensure() can reuse it
276
+ # from inside the lock it already holds: self._lock is a plain (non
277
+ # reentrant) Lock, so calling tear_down() there would deadlock and the
278
+ # lock would stay held forever.
279
+ if self._process is not None:
280
+ self._process.terminate()
281
+ try:
282
+ self._process.wait(timeout=5)
283
+ except subprocess.TimeoutExpired:
284
+ self._process.kill()
285
+ self._process = None
286
+
287
+ def tear_down(self) -> None:
288
+ with self._lock:
289
+ self._tear_down_locked()
290
+
291
+ def ensure(self, wait_time: float = 12.0) -> None:
292
+ """Ensures the tunnel is up. Otherwise raises with a clear cause."""
293
+ if ACCESS == "direct":
294
+ return
295
+ # First the question whether the ports RESPOND, regardless of who holds
296
+ # the tunnel. If a system service holds it (see README, so a query
297
+ # bridge can share it), a second tunnel of our own on the same ports
298
+ # would not only be superfluous, it would fail on the occupied port and
299
+ # report an error where everything is fine.
300
+ # By design this only proves a port is open, not that it is OUR tunnel:
301
+ # an unrelated local service already listening on LOCAL_VLLM/LOCAL_OLLAMA
302
+ # would be taken as ready. Accepted as an edge case; the caller is
303
+ # expected to own these ports.
304
+ if self._ready():
305
+ return
306
+
307
+ with self._lock:
308
+ if self._ready():
309
+ return
310
+ if self._process is not None: # dead or half-dead, get rid of it
311
+ try:
312
+ self._process.kill()
313
+ except OSError:
314
+ pass
315
+ self._process = None
316
+
317
+ if not os.path.exists(KEY_DATA):
318
+ raise AccessMissing(
319
+ f"The data-channel key {KEY_DATA} is missing. Without it there "
320
+ f"is no tunnel and hence no access to the services. Setup: see "
321
+ f"README, section Isolation."
322
+ )
323
+ state = reachability()
324
+ if not state.machine_reachable:
325
+ raise MachineUnreachable(state.explanation)
326
+
327
+ # stdout/stderr are captured so a failed build can quote SSH's
328
+ # message (read below via .read() on the dead process). For a
329
+ # long-lived tunnel that stays up, nobody drains these pipes, so a
330
+ # very chatty ssh could in theory fill the pipe buffer and block. In
331
+ # practice a -N -T tunnel is silent once established; left as is
332
+ # rather than lose the error message on the failure path.
333
+ self._process = subprocess.Popen(
334
+ self.command(),
335
+ stdin=subprocess.DEVNULL,
336
+ stdout=subprocess.PIPE,
337
+ stderr=subprocess.PIPE,
338
+ )
339
+ self.builds += 1
340
+
341
+ deadline = time.monotonic() + wait_time
342
+ while time.monotonic() < deadline:
343
+ if self._process.poll() is not None:
344
+ error = (self._process.stderr.read() or b"").decode(
345
+ "utf-8", "replace"
346
+ ).strip()
347
+ self.last_error = error
348
+ self._process = None
349
+ raise AccessMissing(
350
+ "The tunnel could not be built. Message from SSH: "
351
+ + (error or "none")
352
+ )
353
+ if self._ready():
354
+ self.last_error = None
355
+ return
356
+ time.sleep(0.3)
357
+
358
+ self._tear_down_locked()
359
+ raise AccessMissing(
360
+ f"The tunnel was not up after {wait_time:.0f} seconds. "
361
+ f"Check: is key A registered on the VM and does it permitopen "
362
+ f"127.0.0.1:{REMOTE_VLLM} and 127.0.0.1:{REMOTE_OLLAMA}."
363
+ )
364
+
365
+ def base_vllm(self) -> str:
366
+ if ACCESS == "direct":
367
+ # Re-check at use time: the import-time check_local_target(VM_HOST)
368
+ # only saw one resolution. A hostname that resolved locally then
369
+ # could since have been rebound to an external address (DNS
370
+ # rebinding). Re-resolve and re-validate before egressing plain HTTP.
371
+ check_local_target(VM_HOST)
372
+ return f"http://{VM_HOST}:{REMOTE_VLLM}"
373
+ self.ensure()
374
+ return f"http://127.0.0.1:{LOCAL_VLLM}"
375
+
376
+ def base_ollama(self) -> str:
377
+ if ACCESS == "direct":
378
+ # Re-check at use time against DNS rebinding (see base_vllm).
379
+ check_local_target(VM_HOST)
380
+ return f"http://{VM_HOST}:{REMOTE_OLLAMA}"
381
+ self.ensure()
382
+ return f"http://127.0.0.1:{LOCAL_OLLAMA}"
383
+
384
+
385
+ TUNNEL = Tunnel()
386
+
387
+
388
+ # ---------------------------------------------------------------------------
389
+ # Control channel
390
+ # ---------------------------------------------------------------------------
391
+
392
+
393
+ @dataclass
394
+ class Result:
395
+ operation: str
396
+ returncode: int
397
+ output: str
398
+ error: str
399
+
400
+ @property
401
+ def succeeded(self) -> bool:
402
+ return self.returncode == 0
403
+
404
+
405
+ def control_channel_ready() -> tuple[bool, str]:
406
+ if not os.path.exists(KEY_CONTROL):
407
+ return False, (
408
+ f"The control-channel key {KEY_CONTROL} is missing. As long as it is "
409
+ f"not registered in the VM's authorized_keys, all tools that read VM "
410
+ f"state or steer services are non-functional."
411
+ )
412
+ return True, "Control-channel key present."
413
+
414
+
415
+ def control_command(line: str) -> list[str]:
416
+ """The SSH call of the control channel. Factored out so the selftest can
417
+ check it for the forced key selection without a connection."""
418
+ # Defense in depth: VM_USER goes into the user@host argument unvalidated. A
419
+ # leading "-" could in principle be read by ssh as an option. Only the
420
+ # operator sets MM_VM_USER (a normal SSH login name), so this is not gated
421
+ # here; noted so a future change does not treat it as trusted input.
422
+ return [
423
+ SSH_BIN,
424
+ *_SSH_BASE_OPTS,
425
+ "-i", KEY_CONTROL,
426
+ "-o", "ClearAllForwardings=yes",
427
+ "-n",
428
+ f"{VM_USER}@{VM_HOST}",
429
+ line,
430
+ ]
431
+
432
+
433
+ def call(operation_name: str, **parameters: object) -> Result:
434
+ """Runs exactly one catalog operation on the VM.
435
+
436
+ The parameters are validated HERE against the catalog already, even though
437
+ the wrapper on the VM does the same again. Two independent checks are by
438
+ design: the one here catches errors early and with a better message, the one
439
+ on the VM holds even if this machine were compromised.
440
+ """
441
+ operation = operations.get(operation_name)
442
+ operation.render(**parameters) # pre-check, result deliberately discarded
443
+
444
+ ready, message = control_channel_ready()
445
+ if not ready:
446
+ raise AccessMissing(message)
447
+
448
+ line = protocol.build(operation_name, parameters)
449
+ command = control_command(line)
450
+ try:
451
+ run = subprocess.run(
452
+ command,
453
+ capture_output=True,
454
+ text=True,
455
+ timeout=operation.timeout + 20.0,
456
+ )
457
+ except subprocess.TimeoutExpired:
458
+ raise TimeoutError(
459
+ f"The operation '{operation_name}' exceeded the time limit of "
460
+ f"{operation.timeout:.0f} seconds."
461
+ ) from None
462
+
463
+ error_text = (run.stderr or "").strip()
464
+ if run.returncode == 255:
465
+ state = reachability()
466
+ if not state.machine_reachable:
467
+ raise MachineUnreachable(state.explanation)
468
+ raise AccessMissing(
469
+ "The VM did not accept the control-channel key. Message from SSH: "
470
+ + (error_text or "none") + ". Check: is the public part of key B "
471
+ "registered in /home/" + VM_USER + "/.ssh/authorized_keys, with "
472
+ "command= on /usr/local/sbin/modelmanager."
473
+ )
474
+ return Result(operation_name, run.returncode, run.stdout or "", error_text)
475
+
476
+
477
+ def call_strict(operation_name: str, **parameters: object) -> str:
478
+ result = call(operation_name, **parameters)
479
+ if not result.succeeded:
480
+ raise RuntimeError(
481
+ f"Operation '{operation_name}' ended with return code "
482
+ f"{result.returncode}. Message: {result.error or result.output or 'none'}"
483
+ )
484
+ return result.output
485
+
486
+
487
+ __all__ = [
488
+ "Result",
489
+ "Reachability",
490
+ "MachineUnreachable",
491
+ "ValidationError",
492
+ "SSH_FORCE_KEY",
493
+ "TUNNEL",
494
+ "AccessMissing",
495
+ "reachability",
496
+ "port_open",
497
+ "call",
498
+ "call_strict",
499
+ "control_command",
500
+ "control_channel_ready",
501
+ ]
@@ -0,0 +1,161 @@
1
+ """Memory estimate for a model switch.
2
+
3
+ If the card has, say, 12 GB, only one large model fits. A switch that only
4
+ surfaces on load costs several minutes and leaves the service broken. So it is
5
+ computed beforehand.
6
+
7
+ Computed, not guessed: weight size from the actual file sizes on the VM, the
8
+ key-value cache from the model's config.json. Only if both are missing does it
9
+ fall back to a rough estimate from the name, and the result then says
10
+ explicitly that it is uncertain.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import re
17
+ from typing import Any
18
+
19
+ GB = 2 ** 30
20
+
21
+ # Fixed allowance for the CUDA context, intermediate results and the service
22
+ # itself. Empirical value for vLLM on a single card.
23
+ BASE_LOAD_GB = 1.0
24
+
25
+ _DTYPE_BYTES = {
26
+ "float32": 4.0, "float16": 2.0, "bfloat16": 2.0,
27
+ "float8_e4m3fn": 1.0, "int8": 1.0, "int4": 0.5,
28
+ }
29
+
30
+
31
+ def parse_specs(text: str) -> dict[str, Any]:
32
+ """Reads the output of the 'hf_model_specs' operation."""
33
+ if "NOT_PRESENT" in text:
34
+ return {"present": False}
35
+ result: dict[str, Any] = {"present": True, "location": None,
36
+ "weights_bytes": None, "config": None}
37
+ head, _, config_part = text.partition("===CONFIG")
38
+ for line in head.splitlines():
39
+ if line.startswith("LOCATION\t"):
40
+ result["location"] = line.split("\t", 1)[1].strip()
41
+ elif line.startswith("WEIGHTS\t"):
42
+ raw = line.split("\t", 1)[1].strip()
43
+ if raw.isdigit():
44
+ result["weights_bytes"] = int(raw)
45
+ if config_part.strip():
46
+ try:
47
+ result["config"] = json.loads(config_part.strip())
48
+ except ValueError:
49
+ result["config"] = None
50
+ return result
51
+
52
+
53
+ def params_from_name(name: str) -> float | None:
54
+ """Rough parameter count in billions from the model name, e.g. '7B'.
55
+
56
+ For mixture-of-experts names such as 'mixtral-8x7b' this only picks up the
57
+ per-expert count ('7b'), not the far larger total parameter count. Treat
58
+ the result as a lower bound for such models.
59
+ """
60
+ match = re.search(r"(\d+(?:[.,]\d+)?)\s*[bB](?![a-zA-Z])", name)
61
+ if not match:
62
+ return None
63
+ try:
64
+ return float(match.group(1).replace(",", "."))
65
+ except ValueError:
66
+ return None
67
+
68
+
69
+ def kv_per_token_bytes(config: dict[str, Any]) -> float | None:
70
+ """Bytes per token in the key-value cache, from config.json."""
71
+ try:
72
+ layers = int(config["num_hidden_layers"])
73
+ except (KeyError, TypeError, ValueError):
74
+ return None
75
+ head_dim = config.get("head_dim")
76
+ if not head_dim:
77
+ try:
78
+ head_dim = int(config["hidden_size"]) // int(config["num_attention_heads"])
79
+ except (KeyError, TypeError, ValueError, ZeroDivisionError):
80
+ return None
81
+ kv_heads = config.get("num_key_value_heads") or config.get("num_attention_heads")
82
+ try:
83
+ kv_heads = int(kv_heads)
84
+ except (TypeError, ValueError):
85
+ return None
86
+ dbytes = _DTYPE_BYTES.get(str(config.get("torch_dtype", "bfloat16")).lower(), 2.0)
87
+ # Key and value, per layer, per head, per dimension.
88
+ return 2.0 * layers * kv_heads * float(head_dim) * dbytes
89
+
90
+
91
+ def estimate(specs: dict[str, Any], model_name: str, context: int,
92
+ fraction: float, gpu_total_gb: float) -> dict[str, Any]:
93
+ """Does the model with this context fit into the allotted memory."""
94
+ uncertain: list[str] = []
95
+
96
+ weights_gb: float | None = None
97
+ if specs.get("weights_bytes"):
98
+ weights_gb = specs["weights_bytes"] / GB
99
+ else:
100
+ billions = params_from_name(model_name)
101
+ if billions is not None:
102
+ weights_gb = billions * 2.0 # assume 16 bit
103
+ uncertain.append(
104
+ "The weight size was derived from the name and 16 bit was "
105
+ "assumed, because the files on the VM were not measurable."
106
+ )
107
+
108
+ kv_gb: float | None = None
109
+ config = specs.get("config")
110
+ if isinstance(config, dict):
111
+ per_token = kv_per_token_bytes(config)
112
+ if per_token:
113
+ kv_gb = per_token * context / GB
114
+ if kv_gb is None and weights_gb is not None:
115
+ kv_gb = 0.06 * (weights_gb / 2.0) * (context / 8192)
116
+ uncertain.append(
117
+ "The key-value cache was estimated roughly, because no config.json "
118
+ "could be read. The deviation can be large."
119
+ )
120
+
121
+ if weights_gb is None:
122
+ return {
123
+ "assessable": False,
124
+ "reason": (
125
+ "Neither file size nor parameter count could be determined. "
126
+ "Without either it cannot be said whether the model fits into "
127
+ "the allotted memory. The switch is therefore not attempted blindly."
128
+ ),
129
+ }
130
+
131
+ budget_gb = gpu_total_gb * fraction
132
+ need_gb = weights_gb + (kv_gb or 0.0) + BASE_LOAD_GB
133
+ fits = need_gb <= budget_gb
134
+
135
+ reason = (
136
+ f"Weights {weights_gb:.2f} GB, key-value cache for {context} tokens "
137
+ f"{(kv_gb or 0.0):.2f} GB, base load {BASE_LOAD_GB:.2f} GB, together "
138
+ f"{need_gb:.2f} GB. Allotted are {fraction:.2f} of {gpu_total_gb:.1f} GB, "
139
+ f"so {budget_gb:.2f} GB."
140
+ )
141
+ if not fits:
142
+ possible = max(
143
+ 0,
144
+ int((budget_gb - weights_gb - BASE_LOAD_GB) / (kv_gb / context))
145
+ ) if kv_gb else 0
146
+ reason += (
147
+ f" That does not fit. Options: a smaller model, a quantized version, "
148
+ f"or lowering the context to at most about {possible} tokens."
149
+ )
150
+
151
+ return {
152
+ "assessable": True,
153
+ "fits": fits,
154
+ "weights_gb": round(weights_gb, 2),
155
+ "kv_cache_gb": round(kv_gb or 0.0, 2),
156
+ "base_load_gb": BASE_LOAD_GB,
157
+ "need_gb": round(need_gb, 2),
158
+ "budget_gb": round(budget_gb, 2),
159
+ "reason": reason,
160
+ "uncertainties": uncertain,
161
+ }