gpu4agents 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gpu4agents/__init__.py +5 -0
- gpu4agents/__main__.py +4 -0
- gpu4agents/agent/__init__.py +1 -0
- gpu4agents/agent/install.py +173 -0
- gpu4agents/agent/jobs.py +193 -0
- gpu4agents/agent/mcp.py +646 -0
- gpu4agents/agent/skill/SKILL.md +68 -0
- gpu4agents/cli/__init__.py +0 -0
- gpu4agents/cli/_common.py +131 -0
- gpu4agents/cli/account.py +330 -0
- gpu4agents/cli/agent.py +74 -0
- gpu4agents/cli/inspect.py +207 -0
- gpu4agents/cli/lifecycle.py +127 -0
- gpu4agents/cli/main.py +163 -0
- gpu4agents/cli/providers.py +96 -0
- gpu4agents/cli/run.py +38 -0
- gpu4agents/cli/setup.py +164 -0
- gpu4agents/cli/status.py +224 -0
- gpu4agents/cli/timer.py +143 -0
- gpu4agents/cli/validate.py +57 -0
- gpu4agents/core/__init__.py +0 -0
- gpu4agents/core/beacon.py +133 -0
- gpu4agents/core/config.py +647 -0
- gpu4agents/core/credentials.py +95 -0
- gpu4agents/core/durations.py +47 -0
- gpu4agents/core/engine.py +1335 -0
- gpu4agents/core/errors.py +94 -0
- gpu4agents/core/events.py +117 -0
- gpu4agents/core/execution.py +169 -0
- gpu4agents/core/health.py +62 -0
- gpu4agents/core/hosts.py +58 -0
- gpu4agents/core/model_inspect.py +368 -0
- gpu4agents/core/models.py +313 -0
- gpu4agents/core/paths.py +42 -0
- gpu4agents/core/policy.py +264 -0
- gpu4agents/core/state.py +406 -0
- gpu4agents/core/sweep.py +478 -0
- gpu4agents/core/tool_parsers.py +63 -0
- gpu4agents/providers/__init__.py +4 -0
- gpu4agents/providers/base.py +137 -0
- gpu4agents/providers/registry.py +63 -0
- gpu4agents/providers/vast/__init__.py +3 -0
- gpu4agents/providers/vast/client.py +255 -0
- gpu4agents/providers/vast/config.py +56 -0
- gpu4agents/providers/vast/mapper.py +292 -0
- gpu4agents/providers/vast/provider.py +438 -0
- gpu4agents/runtime/__init__.py +6 -0
- gpu4agents/runtime/base.py +184 -0
- gpu4agents/runtime/vllm.py +295 -0
- gpu4agents/watchdog/__init__.py +3 -0
- gpu4agents/watchdog/agent_body.py +959 -0
- gpu4agents/watchdog/builder.py +248 -0
- gpu4agents-0.5.0.dist-info/METADATA +157 -0
- gpu4agents-0.5.0.dist-info/RECORD +59 -0
- gpu4agents-0.5.0.dist-info/WHEEL +4 -0
- gpu4agents-0.5.0.dist-info/entry_points.txt +2 -0
- gpu4agents-0.5.0.dist-info/licenses/LICENSE +21 -0
- jambu_gpu/__init__.py +9 -0
- jambu_gpu/__main__.py +4 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""Where jambu.yaml's model-specific values actually come from.
|
|
2
|
+
|
|
3
|
+
Two public, documented HuggingFace endpoints - not scraping, not guessing:
|
|
4
|
+
|
|
5
|
+
https://huggingface.co/api/models/<repo> (HF Hub API)
|
|
6
|
+
-> safetensors.total / safetensors.parameters (real on-disk param
|
|
7
|
+
count, broken down by dtype), tags, pipeline_tag, gated status
|
|
8
|
+
|
|
9
|
+
https://huggingface.co/<repo>/raw/<revision>/config.json
|
|
10
|
+
-> architectures, model_type, torch_dtype, quantization_config,
|
|
11
|
+
max_position_embeddings, rope_scaling, auto_map (its presence is
|
|
12
|
+
the actual signal HF's own `transformers`/vLLM use to decide
|
|
13
|
+
whether a repo needs --trust-remote-code)
|
|
14
|
+
|
|
15
|
+
`gpu inspect-model <repo>` fetches both, derives a VRAM estimate and a
|
|
16
|
+
tool-call-parser guess from them, and prints a ready `gpu_runtime:` snippet -
|
|
17
|
+
every number it prints traces back to one of the two documents above, and the
|
|
18
|
+
command says which field.
|
|
19
|
+
|
|
20
|
+
This is deliberately a heuristic, not an oracle: total_params * bytes_per_param
|
|
21
|
+
is the weight footprint; KV cache and activation memory depend on context
|
|
22
|
+
length and concurrency this module has no way to know, so a flat overhead
|
|
23
|
+
margin stands in for them. Hybrid/linear-attention architectures (Gated-
|
|
24
|
+
DeltaNet, Mamba, ...) additionally need a modern-enough GPU regardless of how
|
|
25
|
+
much VRAM it has - flagged separately, not folded into the VRAM number.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import math
|
|
31
|
+
from dataclasses import dataclass, field
|
|
32
|
+
from typing import Any, Optional
|
|
33
|
+
|
|
34
|
+
import httpx
|
|
35
|
+
|
|
36
|
+
from .errors import JambuError
|
|
37
|
+
from .tool_parsers import guess_tool_call_parser
|
|
38
|
+
|
|
39
|
+
HF_API_BASE = "https://huggingface.co/api/models"
|
|
40
|
+
HF_HUB_BASE = "https://huggingface.co"
|
|
41
|
+
|
|
42
|
+
# dtype/quant-method name -> bytes per parameter. Sources: the same strings
|
|
43
|
+
# HF's `safetensors` API and `config.json`'s `torch_dtype` /
|
|
44
|
+
# `quantization_config.quant_method` use.
|
|
45
|
+
_BYTES_PER_DTYPE: dict[str, float] = {
|
|
46
|
+
"f64": 8.0, "float64": 8.0,
|
|
47
|
+
"f32": 4.0, "float32": 4.0,
|
|
48
|
+
"f16": 2.0, "float16": 2.0,
|
|
49
|
+
"bf16": 2.0, "bfloat16": 2.0,
|
|
50
|
+
"f8_e4m3": 1.0, "f8_e5m2": 1.0, "fp8": 1.0, "float8": 1.0,
|
|
51
|
+
"i8": 1.0, "int8": 1.0,
|
|
52
|
+
"i4": 0.5, "int4": 0.5,
|
|
53
|
+
"awq": 0.5, "gptq": 0.5, "bitsandbytes_4bit": 0.5, "nf4": 0.5,
|
|
54
|
+
"nvfp4": 0.5,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
# model_type / architectures substrings known to use hybrid linear-attention
|
|
58
|
+
# (Gated-DeltaNet, Mamba, ...) - these need a modern GPU (Ampere or newer)
|
|
59
|
+
# for their Triton/FLA kernels regardless of how much VRAM it has. Confirmed
|
|
60
|
+
# the hard way: a Turing card (Quadro RTX 8000) failed to boot vLLM at all
|
|
61
|
+
# for a qwen3_5 model; an Ampere card (RTX A6000) worked immediately.
|
|
62
|
+
_HYBRID_ATTENTION_MARKERS = (
|
|
63
|
+
"qwen3_5", "qwen3_6", "qwen3_8", "qwen3_next",
|
|
64
|
+
"jamba", "zamba", "mamba", "bamba", "recurrent_gemma",
|
|
65
|
+
"falcon_mamba", "deltanet", "gdn", "minimax",
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ModelInspectError(JambuError):
|
|
70
|
+
exit_code = 12
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class ModelInspection:
|
|
75
|
+
repo_id: str
|
|
76
|
+
revision: str
|
|
77
|
+
|
|
78
|
+
# Raw source documents, kept so callers/tests can cite exactly where a
|
|
79
|
+
# number came from.
|
|
80
|
+
hf_api: dict[str, Any] = field(default_factory=dict)
|
|
81
|
+
hf_config: dict[str, Any] = field(default_factory=dict)
|
|
82
|
+
|
|
83
|
+
total_params: Optional[int] = None
|
|
84
|
+
param_dtype: Optional[str] = None
|
|
85
|
+
quant_method: Optional[str] = None
|
|
86
|
+
bytes_per_param: Optional[float] = None
|
|
87
|
+
|
|
88
|
+
architectures: list[str] = field(default_factory=list)
|
|
89
|
+
model_type: Optional[str] = None
|
|
90
|
+
max_position_embeddings: Optional[int] = None
|
|
91
|
+
needs_trust_remote_code: bool = False
|
|
92
|
+
is_hybrid_attention: bool = False
|
|
93
|
+
gated: bool = False
|
|
94
|
+
|
|
95
|
+
suggested_tool_call_parser: Optional[str] = None
|
|
96
|
+
|
|
97
|
+
notes: list[str] = field(default_factory=list)
|
|
98
|
+
|
|
99
|
+
# -- derived sizing -------------------------------------------------
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def weight_gb(self) -> Optional[float]:
|
|
103
|
+
if self.total_params is None or self.bytes_per_param is None:
|
|
104
|
+
return None
|
|
105
|
+
return self.total_params * self.bytes_per_param / 1e9
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def suggested_min_vram_gb(self) -> Optional[int]:
|
|
109
|
+
"""Weights + a flat headroom margin for KV cache/activations/runtime.
|
|
110
|
+
|
|
111
|
+
Not exact - context length and concurrency change real KV-cache use
|
|
112
|
+
and this function knows neither. Treat this as a floor to search
|
|
113
|
+
offers from, not a guarantee; confirm with a real health check.
|
|
114
|
+
"""
|
|
115
|
+
weight_gb = self.weight_gb
|
|
116
|
+
if weight_gb is None:
|
|
117
|
+
return None
|
|
118
|
+
return math.ceil(weight_gb * 1.3 + 2.0)
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def suggested_context_length(self) -> int:
|
|
122
|
+
cap = 32768
|
|
123
|
+
if self.max_position_embeddings and self.max_position_embeddings < cap:
|
|
124
|
+
return self.max_position_embeddings
|
|
125
|
+
return cap
|
|
126
|
+
|
|
127
|
+
def to_dict(self) -> dict[str, Any]:
|
|
128
|
+
return {
|
|
129
|
+
"repo_id": self.repo_id,
|
|
130
|
+
"revision": self.revision,
|
|
131
|
+
"total_params": self.total_params,
|
|
132
|
+
"param_dtype": self.param_dtype,
|
|
133
|
+
"quant_method": self.quant_method,
|
|
134
|
+
"bytes_per_param": self.bytes_per_param,
|
|
135
|
+
"weight_gb": self.weight_gb,
|
|
136
|
+
"suggested_min_vram_gb": self.suggested_min_vram_gb,
|
|
137
|
+
"architectures": self.architectures,
|
|
138
|
+
"model_type": self.model_type,
|
|
139
|
+
"max_position_embeddings": self.max_position_embeddings,
|
|
140
|
+
"suggested_context_length": self.suggested_context_length,
|
|
141
|
+
"needs_trust_remote_code": self.needs_trust_remote_code,
|
|
142
|
+
"is_hybrid_attention": self.is_hybrid_attention,
|
|
143
|
+
"gated": self.gated,
|
|
144
|
+
"suggested_tool_call_parser": self.suggested_tool_call_parser,
|
|
145
|
+
"notes": self.notes,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
def to_profile_fragment(self) -> dict[str, Any]:
|
|
149
|
+
"""The catalog-entry shape for ``jambu.models.yaml`` - just the
|
|
150
|
+
model-specific fields, no lifecycle/health/budget/watchdog (those are
|
|
151
|
+
project policy, not model identity, and stay in each project's own
|
|
152
|
+
jambu.yaml).
|
|
153
|
+
"""
|
|
154
|
+
model: dict[str, Any] = {"id": self.repo_id}
|
|
155
|
+
if self.gated:
|
|
156
|
+
model["hf_token_env"] = "HF_TOKEN"
|
|
157
|
+
if self.needs_trust_remote_code:
|
|
158
|
+
model["trust_remote_code"] = True
|
|
159
|
+
|
|
160
|
+
gpu: dict[str, Any] = {"count": 1}
|
|
161
|
+
if self.suggested_min_vram_gb:
|
|
162
|
+
gpu["min_vram_gb"] = self.suggested_min_vram_gb
|
|
163
|
+
if self.is_hybrid_attention and self.suggested_min_vram_gb and self.suggested_min_vram_gb <= 44:
|
|
164
|
+
gpu["name_filter"] = "A6000"
|
|
165
|
+
|
|
166
|
+
runtime: dict[str, Any] = {
|
|
167
|
+
"engine": "vllm",
|
|
168
|
+
"context_length": self.suggested_context_length,
|
|
169
|
+
# auto either way: resolves the parser automatically when
|
|
170
|
+
# recognized, stays off (never fights a model that doesn't do
|
|
171
|
+
# tool calling) when it isn't - see docs/CHOOSING_A_MODEL.md.
|
|
172
|
+
"tool_calling": "auto",
|
|
173
|
+
}
|
|
174
|
+
if self.quant_method:
|
|
175
|
+
runtime["dtype"] = "auto"
|
|
176
|
+
|
|
177
|
+
return {"model": model, "compute": {"gpu": gpu}, "runtime": runtime}
|
|
178
|
+
|
|
179
|
+
def to_yaml_snippet(self) -> str:
|
|
180
|
+
lines = ["gpu_runtime:", " model:", f" id: {self.repo_id}"]
|
|
181
|
+
if self.gated:
|
|
182
|
+
lines.append(" hf_token_env: HF_TOKEN # gated repo - needs a real HF token")
|
|
183
|
+
if self.needs_trust_remote_code:
|
|
184
|
+
lines.append(" trust_remote_code: true # config.json has auto_map (custom code)")
|
|
185
|
+
lines.append("")
|
|
186
|
+
lines.append(" compute:")
|
|
187
|
+
lines.append(" gpu:")
|
|
188
|
+
vram = self.suggested_min_vram_gb
|
|
189
|
+
if vram:
|
|
190
|
+
lines.append(f" min_vram_gb: {vram} # estimate, see notes")
|
|
191
|
+
else:
|
|
192
|
+
lines.append(" min_vram_gb: 24 # UNKNOWN - safetensors metadata unavailable, size manually")
|
|
193
|
+
if self.is_hybrid_attention:
|
|
194
|
+
# A6000 (48GB, Ampere) is only a safe default when the estimate
|
|
195
|
+
# actually fits it - naming a card too small for a 70GB+ model
|
|
196
|
+
# would just make every offer search come back empty.
|
|
197
|
+
if vram and vram <= 44:
|
|
198
|
+
lines.append(
|
|
199
|
+
" name_filter: A6000 # hybrid linear-attention: needs Ampere+, not Turing"
|
|
200
|
+
)
|
|
201
|
+
else:
|
|
202
|
+
lines.append(
|
|
203
|
+
" # hybrid linear-attention needs Ampere+ (not Turing), but this "
|
|
204
|
+
"model is too big for a single 48GB card - either search a bigger one "
|
|
205
|
+
"(A100/H100 80GB) or split across multiple Ampere+ GPUs, e.g.:"
|
|
206
|
+
)
|
|
207
|
+
lines.append(" # count: 2 # + runtime.tensor_parallel_size: 2")
|
|
208
|
+
lines.append(" count: 1")
|
|
209
|
+
lines.append("")
|
|
210
|
+
lines.append(" runtime:")
|
|
211
|
+
lines.append(" engine: vllm")
|
|
212
|
+
lines.append(f" context_length: {self.suggested_context_length}")
|
|
213
|
+
if self.quant_method:
|
|
214
|
+
lines.append(" dtype: auto # quantization declared in the checkpoint's own config.json")
|
|
215
|
+
if self.suggested_tool_call_parser:
|
|
216
|
+
lines.append(
|
|
217
|
+
f" tool_calling: auto # resolves to {self.suggested_tool_call_parser} automatically"
|
|
218
|
+
)
|
|
219
|
+
else:
|
|
220
|
+
lines.append(" tool_calling: auto # gpu couldn't guess a parser for this repo name")
|
|
221
|
+
lines.append(" # tool_call_parser: ??? # set explicitly if this model does tool calling")
|
|
222
|
+
return "\n".join(lines) + "\n"
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _extract_dtype_and_params(api_data: dict[str, Any]) -> tuple[Optional[int], Optional[str]]:
|
|
226
|
+
"""From the HF Hub API's `safetensors` field - the real, on-disk shape."""
|
|
227
|
+
safetensors = api_data.get("safetensors") or {}
|
|
228
|
+
total = safetensors.get("total")
|
|
229
|
+
per_dtype = safetensors.get("parameters") or {}
|
|
230
|
+
if not per_dtype:
|
|
231
|
+
return total, None
|
|
232
|
+
dominant = max(per_dtype.items(), key=lambda kv: kv[1])[0]
|
|
233
|
+
return total, dominant
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _bytes_per_param_for(dtype: Optional[str], quant_method: Optional[str]) -> Optional[float]:
|
|
237
|
+
for candidate in (quant_method, dtype):
|
|
238
|
+
if not candidate:
|
|
239
|
+
continue
|
|
240
|
+
key = str(candidate).lower().replace("-", "_")
|
|
241
|
+
if key in _BYTES_PER_DTYPE:
|
|
242
|
+
return _BYTES_PER_DTYPE[key]
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _is_hybrid_attention(model_type: Optional[str], architectures: list[str]) -> bool:
|
|
247
|
+
haystack = " ".join([model_type or "", *architectures]).lower()
|
|
248
|
+
return any(marker in haystack for marker in _HYBRID_ATTENTION_MARKERS)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def fetch_hf_api_metadata(
|
|
252
|
+
repo_id: str, token: Optional[str] = None, timeout: float = 20.0
|
|
253
|
+
) -> dict[str, Any]:
|
|
254
|
+
"""GET https://huggingface.co/api/models/<repo_id>"""
|
|
255
|
+
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
|
256
|
+
try:
|
|
257
|
+
response = httpx.get(f"{HF_API_BASE}/{repo_id}", headers=headers, timeout=timeout)
|
|
258
|
+
except httpx.HTTPError as exc:
|
|
259
|
+
raise ModelInspectError(f"could not reach huggingface.co: {exc}") from exc
|
|
260
|
+
if response.status_code == 404:
|
|
261
|
+
raise ModelInspectError(f"'{repo_id}' was not found on HuggingFace")
|
|
262
|
+
if response.status_code in (401, 403):
|
|
263
|
+
raise ModelInspectError(
|
|
264
|
+
f"'{repo_id}' is gated or private - set HF_TOKEN (a real token with access) and retry"
|
|
265
|
+
)
|
|
266
|
+
if response.status_code >= 400:
|
|
267
|
+
raise ModelInspectError(f"HuggingFace API returned {response.status_code} for '{repo_id}'")
|
|
268
|
+
return response.json()
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def fetch_hf_config(
|
|
272
|
+
repo_id: str, revision: str = "main", token: Optional[str] = None, timeout: float = 20.0
|
|
273
|
+
) -> dict[str, Any]:
|
|
274
|
+
"""GET https://huggingface.co/<repo_id>/raw/<revision>/config.json"""
|
|
275
|
+
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
|
276
|
+
url = f"{HF_HUB_BASE}/{repo_id}/raw/{revision}/config.json"
|
|
277
|
+
try:
|
|
278
|
+
response = httpx.get(url, headers=headers, timeout=timeout)
|
|
279
|
+
except httpx.HTTPError as exc:
|
|
280
|
+
raise ModelInspectError(f"could not reach huggingface.co: {exc}") from exc
|
|
281
|
+
if response.status_code == 404:
|
|
282
|
+
# Not every repo ships a config.json at the root (some are pure GGUF
|
|
283
|
+
# repos, for instance) - this is not fatal, just less precise.
|
|
284
|
+
return {}
|
|
285
|
+
if response.status_code in (401, 403):
|
|
286
|
+
raise ModelInspectError(
|
|
287
|
+
f"'{repo_id}' is gated or private - set HF_TOKEN (a real token with access) and retry"
|
|
288
|
+
)
|
|
289
|
+
if response.status_code >= 400:
|
|
290
|
+
raise ModelInspectError(f"HuggingFace returned {response.status_code} fetching config.json")
|
|
291
|
+
try:
|
|
292
|
+
return response.json()
|
|
293
|
+
except ValueError:
|
|
294
|
+
return {}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def inspect_model(
|
|
298
|
+
repo_id: str, revision: str = "main", token: Optional[str] = None
|
|
299
|
+
) -> ModelInspection:
|
|
300
|
+
api_data = fetch_hf_api_metadata(repo_id, token=token)
|
|
301
|
+
config = fetch_hf_config(repo_id, revision=revision, token=token)
|
|
302
|
+
|
|
303
|
+
total_params, dtype_from_weights = _extract_dtype_and_params(api_data)
|
|
304
|
+
quant_method = None
|
|
305
|
+
quant_config = config.get("quantization_config")
|
|
306
|
+
if isinstance(quant_config, dict):
|
|
307
|
+
quant_method = quant_config.get("quant_method")
|
|
308
|
+
|
|
309
|
+
dtype = dtype_from_weights or config.get("torch_dtype")
|
|
310
|
+
bytes_per_param = _bytes_per_param_for(dtype, quant_method)
|
|
311
|
+
|
|
312
|
+
architectures = list(config.get("architectures") or [])
|
|
313
|
+
model_type = config.get("model_type")
|
|
314
|
+
|
|
315
|
+
result = ModelInspection(
|
|
316
|
+
repo_id=repo_id,
|
|
317
|
+
revision=revision,
|
|
318
|
+
hf_api=api_data,
|
|
319
|
+
hf_config=config,
|
|
320
|
+
total_params=total_params,
|
|
321
|
+
param_dtype=dtype,
|
|
322
|
+
quant_method=quant_method,
|
|
323
|
+
bytes_per_param=bytes_per_param,
|
|
324
|
+
architectures=architectures,
|
|
325
|
+
model_type=model_type,
|
|
326
|
+
max_position_embeddings=config.get("max_position_embeddings"),
|
|
327
|
+
needs_trust_remote_code="auto_map" in config,
|
|
328
|
+
is_hybrid_attention=_is_hybrid_attention(model_type, architectures),
|
|
329
|
+
gated=bool(api_data.get("gated")),
|
|
330
|
+
suggested_tool_call_parser=guess_tool_call_parser(repo_id),
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
if total_params is None:
|
|
334
|
+
result.notes.append(
|
|
335
|
+
"no safetensors metadata from the HF API - this repo may be GGUF-only "
|
|
336
|
+
"or not indexed; size compute.gpu.min_vram_gb manually"
|
|
337
|
+
)
|
|
338
|
+
if bytes_per_param is None and total_params is not None:
|
|
339
|
+
result.notes.append(
|
|
340
|
+
f"could not map dtype '{dtype}' to a byte width - "
|
|
341
|
+
"suggested_min_vram_gb is unavailable, size manually"
|
|
342
|
+
)
|
|
343
|
+
if result.is_hybrid_attention:
|
|
344
|
+
result.notes.append(
|
|
345
|
+
"hybrid linear-attention architecture (Gated-DeltaNet/Mamba-style) - "
|
|
346
|
+
"needs an Ampere-or-newer GPU for its Triton/FLA kernels; a Turing card "
|
|
347
|
+
"(e.g. Quadro RTX 8000) will fail to boot regardless of VRAM"
|
|
348
|
+
)
|
|
349
|
+
if result.needs_trust_remote_code:
|
|
350
|
+
result.notes.append("config.json has auto_map - set model.trust_remote_code: true")
|
|
351
|
+
if not result.suggested_tool_call_parser:
|
|
352
|
+
result.notes.append(
|
|
353
|
+
"gpu could not guess a --tool-call-parser from this repo name "
|
|
354
|
+
"(common for fine-tunes with custom names) - if the base model does "
|
|
355
|
+
"tool calling, set runtime.tool_call_parser explicitly and verify with "
|
|
356
|
+
"a live request; if unsure, leave tool_calling: auto (off) and test later"
|
|
357
|
+
)
|
|
358
|
+
if (
|
|
359
|
+
result.max_position_embeddings
|
|
360
|
+
and result.max_position_embeddings > result.suggested_context_length
|
|
361
|
+
):
|
|
362
|
+
result.notes.append(
|
|
363
|
+
f"native context is {result.max_position_embeddings} tokens - "
|
|
364
|
+
f"suggesting {result.suggested_context_length} as a cheaper default; "
|
|
365
|
+
"raise runtime.context_length deliberately if you need more"
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
return result
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"""Provider-independent domain models (spec sections 6-8).
|
|
2
|
+
|
|
3
|
+
Nothing here may reference a concrete provider. Adapters translate their own
|
|
4
|
+
payloads into these shapes and back.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import time
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from enum import Enum
|
|
12
|
+
from typing import Any, Iterator, Optional
|
|
13
|
+
|
|
14
|
+
from .policy import (
|
|
15
|
+
BUSY,
|
|
16
|
+
DESTROYED,
|
|
17
|
+
FAILED,
|
|
18
|
+
IDLE,
|
|
19
|
+
PROVISIONING,
|
|
20
|
+
READY,
|
|
21
|
+
STARTING,
|
|
22
|
+
STOPPED,
|
|
23
|
+
UNKNOWN,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# Every instance this tool creates carries a label with this prefix. It is how
|
|
28
|
+
# account-wide commands tell our instances from everything else in the account.
|
|
29
|
+
JAMBU_LABEL_PREFIX = "jambu-"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def is_jambu_label(label: Optional[str]) -> bool:
|
|
33
|
+
return bool(label) and str(label).startswith(JAMBU_LABEL_PREFIX)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class InstanceState(str, Enum):
|
|
37
|
+
PROVISIONING = PROVISIONING
|
|
38
|
+
STARTING = STARTING
|
|
39
|
+
READY = READY
|
|
40
|
+
BUSY = BUSY
|
|
41
|
+
IDLE = IDLE
|
|
42
|
+
STOPPED = STOPPED
|
|
43
|
+
FAILED = FAILED
|
|
44
|
+
DESTROYED = DESTROYED
|
|
45
|
+
UNKNOWN = UNKNOWN
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def is_terminal(self) -> bool:
|
|
49
|
+
return self in (InstanceState.DESTROYED,)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def is_billable_compute(self) -> bool:
|
|
53
|
+
return self in (
|
|
54
|
+
InstanceState.PROVISIONING,
|
|
55
|
+
InstanceState.STARTING,
|
|
56
|
+
InstanceState.READY,
|
|
57
|
+
InstanceState.BUSY,
|
|
58
|
+
InstanceState.IDLE,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class ProviderCapabilities:
|
|
64
|
+
"""What an adapter can actually do (spec section 7).
|
|
65
|
+
|
|
66
|
+
The core branches on these flags, never on provider names.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
stop: bool = False
|
|
70
|
+
destroy: bool = True
|
|
71
|
+
persistent_disk: bool = False
|
|
72
|
+
spot_instances: bool = False
|
|
73
|
+
ssh: bool = False
|
|
74
|
+
public_ports: bool = False
|
|
75
|
+
gpu_selection: bool = False
|
|
76
|
+
pricing_query: bool = False
|
|
77
|
+
remote_exec: bool = False
|
|
78
|
+
logs: bool = False
|
|
79
|
+
# False when a stopped instance still costs money (spec section 25).
|
|
80
|
+
stop_eliminates_billing: bool = True
|
|
81
|
+
# Can enumerate this tool's instances account-wide (`gpu ls`, `gpu sweep`).
|
|
82
|
+
list_instances: bool = False
|
|
83
|
+
|
|
84
|
+
def as_row(self) -> dict[str, str]:
|
|
85
|
+
return {k: ("yes" if v else "no") for k, v in self.__dict__.items()}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class GpuSpec:
|
|
90
|
+
min_vram_gb: int = 24
|
|
91
|
+
count: int = 1
|
|
92
|
+
name_filter: Optional[str] = None
|
|
93
|
+
cuda_min: Optional[str] = None
|
|
94
|
+
# Minimum GPU compute capability (architecture), e.g. 8.0 for Ampere. The
|
|
95
|
+
# host driver's CUDA (``cuda_min``) is not the same thing: a Pascal/Volta
|
|
96
|
+
# card can sit behind a CUDA-13 driver yet still be too old for a runtime
|
|
97
|
+
# image built for newer architectures, which then dies at engine startup.
|
|
98
|
+
compute_cap_min: Optional[float] = None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass
|
|
102
|
+
class ComputeSpec:
|
|
103
|
+
"""The normalized "give me a machine like this" request."""
|
|
104
|
+
|
|
105
|
+
gpu: GpuSpec
|
|
106
|
+
disk_gb: int = 80
|
|
107
|
+
image: str = ""
|
|
108
|
+
ports: list[int] = field(default_factory=list)
|
|
109
|
+
env: dict[str, str] = field(default_factory=dict)
|
|
110
|
+
onstart: str = ""
|
|
111
|
+
label: str = "jambu-gpu"
|
|
112
|
+
spot: bool = False
|
|
113
|
+
region: Optional[str] = None
|
|
114
|
+
max_hourly_cost_usd: Optional[float] = None
|
|
115
|
+
# Machines (Offer.host_id) not to rent: hosts that already failed to boot.
|
|
116
|
+
exclude_hosts: list[str] = field(default_factory=list)
|
|
117
|
+
# Free-form provider hints; adapters ignore keys they do not understand.
|
|
118
|
+
provider_options: dict[str, Any] = field(default_factory=dict)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass
|
|
122
|
+
class Offer:
|
|
123
|
+
"""A normalized purchasable capacity offer."""
|
|
124
|
+
|
|
125
|
+
id: str
|
|
126
|
+
gpu_name: str
|
|
127
|
+
gpu_count: int
|
|
128
|
+
gpu_vram_gb: float
|
|
129
|
+
disk_gb: float
|
|
130
|
+
hourly_cost_usd: float
|
|
131
|
+
region: Optional[str] = None
|
|
132
|
+
score: Optional[float] = None
|
|
133
|
+
# Highest CUDA version the host's driver supports, when the provider says.
|
|
134
|
+
cuda_max: Optional[float] = None
|
|
135
|
+
# The GPU's compute capability (architecture), e.g. 8.6 for an RTX 3090,
|
|
136
|
+
# when the provider says. Distinct from cuda_max (the driver's CUDA).
|
|
137
|
+
compute_cap: Optional[float] = None
|
|
138
|
+
# The physical machine behind the offer, when the provider says. Offers
|
|
139
|
+
# on one machine share its network, disk and Docker daemon.
|
|
140
|
+
host_id: Optional[str] = None
|
|
141
|
+
raw: dict[str, Any] = field(default_factory=dict)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass
|
|
145
|
+
class Endpoint:
|
|
146
|
+
"""A reachable network address on the instance."""
|
|
147
|
+
|
|
148
|
+
name: str
|
|
149
|
+
url: str
|
|
150
|
+
internal_port: int
|
|
151
|
+
external_port: int
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class Instance:
|
|
156
|
+
"""Normalized handle on remote compute. Provider ids never leak semantics."""
|
|
157
|
+
|
|
158
|
+
id: str
|
|
159
|
+
provider: str
|
|
160
|
+
state: InstanceState = InstanceState.PROVISIONING
|
|
161
|
+
created_at: float = field(default_factory=time.time)
|
|
162
|
+
gpu_name: str = ""
|
|
163
|
+
gpu_count: int = 0
|
|
164
|
+
hourly_cost_usd: Optional[float] = None
|
|
165
|
+
host: Optional[str] = None
|
|
166
|
+
ssh_host: Optional[str] = None
|
|
167
|
+
ssh_port: Optional[int] = None
|
|
168
|
+
ssh_user: str = "root"
|
|
169
|
+
endpoints: dict[str, Endpoint] = field(default_factory=dict)
|
|
170
|
+
label: str = ""
|
|
171
|
+
# The machine it runs on (see Offer.host_id).
|
|
172
|
+
host_id: Optional[str] = None
|
|
173
|
+
raw: dict[str, Any] = field(default_factory=dict)
|
|
174
|
+
|
|
175
|
+
def endpoint_url(self, name: str) -> Optional[str]:
|
|
176
|
+
ep = self.endpoints.get(name)
|
|
177
|
+
return ep.url if ep else None
|
|
178
|
+
|
|
179
|
+
def to_dict(self) -> dict[str, Any]:
|
|
180
|
+
return {
|
|
181
|
+
"id": self.id,
|
|
182
|
+
"provider": self.provider,
|
|
183
|
+
"state": self.state.value,
|
|
184
|
+
"created_at": self.created_at,
|
|
185
|
+
"gpu_name": self.gpu_name,
|
|
186
|
+
"gpu_count": self.gpu_count,
|
|
187
|
+
"hourly_cost_usd": self.hourly_cost_usd,
|
|
188
|
+
"host": self.host,
|
|
189
|
+
"ssh_host": self.ssh_host,
|
|
190
|
+
"ssh_port": self.ssh_port,
|
|
191
|
+
"ssh_user": self.ssh_user,
|
|
192
|
+
"label": self.label,
|
|
193
|
+
"host_id": self.host_id,
|
|
194
|
+
"endpoints": {
|
|
195
|
+
name: {
|
|
196
|
+
"url": ep.url,
|
|
197
|
+
"internal_port": ep.internal_port,
|
|
198
|
+
"external_port": ep.external_port,
|
|
199
|
+
}
|
|
200
|
+
for name, ep in self.endpoints.items()
|
|
201
|
+
},
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def from_dict(cls, data: dict[str, Any]) -> "Instance":
|
|
206
|
+
endpoints = {
|
|
207
|
+
name: Endpoint(
|
|
208
|
+
name=name,
|
|
209
|
+
url=ep["url"],
|
|
210
|
+
internal_port=int(ep.get("internal_port") or 0),
|
|
211
|
+
external_port=int(ep.get("external_port") or 0),
|
|
212
|
+
)
|
|
213
|
+
for name, ep in (data.get("endpoints") or {}).items()
|
|
214
|
+
}
|
|
215
|
+
return cls(
|
|
216
|
+
id=str(data["id"]),
|
|
217
|
+
provider=data["provider"],
|
|
218
|
+
state=InstanceState(data.get("state", "unknown")),
|
|
219
|
+
created_at=float(data.get("created_at") or time.time()),
|
|
220
|
+
gpu_name=data.get("gpu_name", ""),
|
|
221
|
+
gpu_count=int(data.get("gpu_count") or 0),
|
|
222
|
+
hourly_cost_usd=data.get("hourly_cost_usd"),
|
|
223
|
+
host=data.get("host"),
|
|
224
|
+
ssh_host=data.get("ssh_host"),
|
|
225
|
+
ssh_port=data.get("ssh_port"),
|
|
226
|
+
ssh_user=data.get("ssh_user", "root"),
|
|
227
|
+
endpoints=endpoints,
|
|
228
|
+
label=data.get("label", ""),
|
|
229
|
+
host_id=data.get("host_id"),
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@dataclass
|
|
234
|
+
class InstanceStatus:
|
|
235
|
+
"""What `status` reports after reconciliation (spec section 18)."""
|
|
236
|
+
|
|
237
|
+
state: InstanceState
|
|
238
|
+
instance: Optional[Instance] = None
|
|
239
|
+
exists: bool = True
|
|
240
|
+
hourly_cost_usd: Optional[float] = None
|
|
241
|
+
message: str = ""
|
|
242
|
+
raw: dict[str, Any] = field(default_factory=dict)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@dataclass
|
|
246
|
+
class Workload:
|
|
247
|
+
"""A unit of useful work the user asked for."""
|
|
248
|
+
|
|
249
|
+
id: str
|
|
250
|
+
command: list[str]
|
|
251
|
+
label: str = ""
|
|
252
|
+
cwd: Optional[str] = None
|
|
253
|
+
env: dict[str, str] = field(default_factory=dict)
|
|
254
|
+
mode: str = "local" # local | remote
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@dataclass
|
|
258
|
+
class Execution:
|
|
259
|
+
id: str
|
|
260
|
+
workload_id: str
|
|
261
|
+
started_at: float
|
|
262
|
+
finished_at: Optional[float] = None
|
|
263
|
+
returncode: Optional[int] = None
|
|
264
|
+
state: str = "running"
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def duration(self) -> float:
|
|
268
|
+
return (self.finished_at or time.time()) - self.started_at
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@dataclass
|
|
272
|
+
class LogStream:
|
|
273
|
+
"""Lazy line iterator over provider logs."""
|
|
274
|
+
|
|
275
|
+
lines: Iterator[str]
|
|
276
|
+
source: str = ""
|
|
277
|
+
|
|
278
|
+
def __iter__(self) -> Iterator[str]:
|
|
279
|
+
return iter(self.lines)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
@dataclass
|
|
283
|
+
class ValidationIssue:
|
|
284
|
+
level: str # error | warning
|
|
285
|
+
message: str
|
|
286
|
+
field: Optional[str] = None
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@dataclass
|
|
290
|
+
class ValidationResult:
|
|
291
|
+
issues: list[ValidationIssue] = field(default_factory=list)
|
|
292
|
+
|
|
293
|
+
@property
|
|
294
|
+
def ok(self) -> bool:
|
|
295
|
+
return not any(i.level == "error" for i in self.issues)
|
|
296
|
+
|
|
297
|
+
@property
|
|
298
|
+
def errors(self) -> list[ValidationIssue]:
|
|
299
|
+
return [i for i in self.issues if i.level == "error"]
|
|
300
|
+
|
|
301
|
+
@property
|
|
302
|
+
def warnings(self) -> list[ValidationIssue]:
|
|
303
|
+
return [i for i in self.issues if i.level == "warning"]
|
|
304
|
+
|
|
305
|
+
def error(self, message: str, field: Optional[str] = None) -> None:
|
|
306
|
+
self.issues.append(ValidationIssue("error", message, field))
|
|
307
|
+
|
|
308
|
+
def warn(self, message: str, field: Optional[str] = None) -> None:
|
|
309
|
+
self.issues.append(ValidationIssue("warning", message, field))
|
|
310
|
+
|
|
311
|
+
def merge(self, other: "ValidationResult") -> "ValidationResult":
|
|
312
|
+
self.issues.extend(other.issues)
|
|
313
|
+
return self
|