jambu-gpu 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jambu_gpu/__init__.py +5 -0
- jambu_gpu/__main__.py +4 -0
- jambu_gpu/cli/__init__.py +0 -0
- jambu_gpu/cli/_common.py +117 -0
- jambu_gpu/cli/inspect.py +207 -0
- jambu_gpu/cli/lifecycle.py +127 -0
- jambu_gpu/cli/main.py +103 -0
- jambu_gpu/cli/providers.py +96 -0
- jambu_gpu/cli/run.py +37 -0
- jambu_gpu/cli/setup.py +165 -0
- jambu_gpu/cli/status.py +214 -0
- jambu_gpu/cli/validate.py +57 -0
- jambu_gpu/core/__init__.py +0 -0
- jambu_gpu/core/beacon.py +124 -0
- jambu_gpu/core/config.py +596 -0
- jambu_gpu/core/credentials.py +75 -0
- jambu_gpu/core/durations.py +47 -0
- jambu_gpu/core/engine.py +951 -0
- jambu_gpu/core/errors.py +68 -0
- jambu_gpu/core/events.py +115 -0
- jambu_gpu/core/execution.py +169 -0
- jambu_gpu/core/health.py +62 -0
- jambu_gpu/core/model_inspect.py +368 -0
- jambu_gpu/core/models.py +283 -0
- jambu_gpu/core/policy.py +233 -0
- jambu_gpu/core/state.py +361 -0
- jambu_gpu/core/tool_parsers.py +63 -0
- jambu_gpu/providers/__init__.py +4 -0
- jambu_gpu/providers/base.py +125 -0
- jambu_gpu/providers/registry.py +63 -0
- jambu_gpu/providers/vast/__init__.py +3 -0
- jambu_gpu/providers/vast/client.py +230 -0
- jambu_gpu/providers/vast/config.py +51 -0
- jambu_gpu/providers/vast/mapper.py +221 -0
- jambu_gpu/providers/vast/provider.py +422 -0
- jambu_gpu/runtime/__init__.py +6 -0
- jambu_gpu/runtime/base.py +99 -0
- jambu_gpu/runtime/vllm.py +224 -0
- jambu_gpu/watchdog/__init__.py +3 -0
- jambu_gpu/watchdog/agent_body.py +547 -0
- jambu_gpu/watchdog/builder.py +178 -0
- jambu_gpu-0.1.0.dist-info/METADATA +82 -0
- jambu_gpu-0.1.0.dist-info/RECORD +46 -0
- jambu_gpu-0.1.0.dist-info/WHEEL +4 -0
- jambu_gpu-0.1.0.dist-info/entry_points.txt +2 -0
- jambu_gpu-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Runtime contract (spec section 19).
|
|
2
|
+
|
|
3
|
+
Provider answers "where does the compute run?".
|
|
4
|
+
Runtime answers "how does the model run?".
|
|
5
|
+
Model answers "what model runs?".
|
|
6
|
+
These stay independent; a runtime never talks to a provider API.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from typing import Optional, Type
|
|
13
|
+
|
|
14
|
+
from ..core.config import RuntimeConfig
|
|
15
|
+
from ..core.errors import ConfigError
|
|
16
|
+
from ..core.models import ValidationResult
|
|
17
|
+
|
|
18
|
+
ENDPOINT_NAME = "model"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ModelRuntime(ABC):
|
|
22
|
+
"""Describes how to boot a model server inside the provisioned container."""
|
|
23
|
+
|
|
24
|
+
engine: str = "abstract"
|
|
25
|
+
default_image: str = ""
|
|
26
|
+
|
|
27
|
+
def __init__(self, config: RuntimeConfig) -> None:
|
|
28
|
+
self.config = config
|
|
29
|
+
|
|
30
|
+
# -- container shape ----------------------------------------------------
|
|
31
|
+
|
|
32
|
+
def image(self) -> str:
|
|
33
|
+
return self.config.compute.image or self.default_image
|
|
34
|
+
|
|
35
|
+
def ports(self) -> list[int]:
|
|
36
|
+
return [self.config.runtime.port]
|
|
37
|
+
|
|
38
|
+
def env(self) -> dict[str, str]:
|
|
39
|
+
return {}
|
|
40
|
+
|
|
41
|
+
@abstractmethod
|
|
42
|
+
def start_command(self) -> str:
|
|
43
|
+
"""Shell command that launches the model server in the background."""
|
|
44
|
+
|
|
45
|
+
# -- health -------------------------------------------------------------
|
|
46
|
+
|
|
47
|
+
def health_path(self) -> str:
|
|
48
|
+
return self.config.health.path
|
|
49
|
+
|
|
50
|
+
def health_url(self, base_url: str) -> str:
|
|
51
|
+
return base_url.rstrip("/") + self.health_path()
|
|
52
|
+
|
|
53
|
+
def base_url(self, endpoint_url: str) -> str:
|
|
54
|
+
return endpoint_url.rstrip("/")
|
|
55
|
+
|
|
56
|
+
def validate(self) -> ValidationResult:
|
|
57
|
+
return ValidationResult()
|
|
58
|
+
|
|
59
|
+
def describe(self) -> str:
|
|
60
|
+
return f"{self.engine} :{self.config.runtime.port}"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class NoRuntime(ModelRuntime):
|
|
64
|
+
"""`runtime.engine: none` - provision bare compute, start nothing."""
|
|
65
|
+
|
|
66
|
+
engine = "none"
|
|
67
|
+
default_image = "pytorch/pytorch:2.4.0-cuda12.1-cudnn9-runtime"
|
|
68
|
+
|
|
69
|
+
def ports(self) -> list[int]:
|
|
70
|
+
return []
|
|
71
|
+
|
|
72
|
+
def start_command(self) -> str:
|
|
73
|
+
return "true"
|
|
74
|
+
|
|
75
|
+
def health_url(self, base_url: str) -> str: # pragma: no cover - unused
|
|
76
|
+
return base_url
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class RuntimeRegistry:
|
|
80
|
+
def __init__(self) -> None:
|
|
81
|
+
self._runtimes: dict[str, Type[ModelRuntime]] = {"none": NoRuntime}
|
|
82
|
+
|
|
83
|
+
def register(self, runtime_cls: Type[ModelRuntime]) -> Type[ModelRuntime]:
|
|
84
|
+
self._runtimes[runtime_cls.engine] = runtime_cls
|
|
85
|
+
return runtime_cls
|
|
86
|
+
|
|
87
|
+
def names(self) -> list[str]:
|
|
88
|
+
return sorted(self._runtimes)
|
|
89
|
+
|
|
90
|
+
def get(self, config: RuntimeConfig, engine: Optional[str] = None) -> ModelRuntime:
|
|
91
|
+
key = engine or config.runtime.engine
|
|
92
|
+
try:
|
|
93
|
+
return self._runtimes[key](config)
|
|
94
|
+
except KeyError:
|
|
95
|
+
known = ", ".join(self.names())
|
|
96
|
+
raise ConfigError(f"unknown runtime engine '{key}' (available: {known})") from None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
runtime_registry = RuntimeRegistry()
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""vLLM OpenAI-compatible server runtime.
|
|
2
|
+
|
|
3
|
+
This is where "does it support agentic flows?" gets answered: vLLM's server
|
|
4
|
+
already implements both ``/v1/chat/completions`` (with tool/function calling)
|
|
5
|
+
and ``/v1/responses`` (the OpenAI Responses API, MCP-tool-routing included).
|
|
6
|
+
gpu does not implement any of that - it only turns on the right vLLM
|
|
7
|
+
flags and forwards the same endpoint/credentials it always has, so any
|
|
8
|
+
OpenAI-SDK-compatible agent framework (LangChain, LangGraph, CrewAI, the
|
|
9
|
+
`openai` SDK's own `responses.create`, ...) can talk to the instance with no
|
|
10
|
+
extra layer in between.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import shlex
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from ..core.models import ValidationResult
|
|
20
|
+
from ..core.tool_parsers import guess_tool_call_parser # noqa: F401 (re-exported)
|
|
21
|
+
from .base import ModelRuntime
|
|
22
|
+
|
|
23
|
+
# vLLM flags gpu already manages explicitly - collided with via
|
|
24
|
+
# runtime.vllm_args, these would fight the config fields meant to control
|
|
25
|
+
# them (compute.gpu, model.id, lifecycle port wiring, ...). Not a hard block
|
|
26
|
+
# (power users can still reach them through raw extra_args if they really
|
|
27
|
+
# need to), just something validate() warns about.
|
|
28
|
+
_MANAGED_VLLM_FLAGS = frozenset(
|
|
29
|
+
{
|
|
30
|
+
"model",
|
|
31
|
+
"host",
|
|
32
|
+
"port",
|
|
33
|
+
"served-model-name",
|
|
34
|
+
"served_model_name",
|
|
35
|
+
"revision",
|
|
36
|
+
"trust-remote-code",
|
|
37
|
+
"trust_remote_code",
|
|
38
|
+
"enable-auto-tool-choice",
|
|
39
|
+
"enable_auto_tool_choice",
|
|
40
|
+
"tool-call-parser",
|
|
41
|
+
"tool_call_parser",
|
|
42
|
+
"tool-server",
|
|
43
|
+
"tool_server",
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def flatten_vllm_args(mapping: dict[str, Any]) -> list[str]:
|
|
49
|
+
"""Turn a ``{flag: value}`` mapping into vLLM CLI argv.
|
|
50
|
+
|
|
51
|
+
Rules, matched to how vLLM's own argparse-based CLI takes flags:
|
|
52
|
+
- key: underscores become dashes (``max_num_seqs`` -> ``--max-num-seqs``),
|
|
53
|
+
already-dashed keys pass through unchanged. A leading ``--`` in the
|
|
54
|
+
key is tolerated (stripped) so either spelling works.
|
|
55
|
+
- ``True`` -> a bare flag (``--enable-prefix-caching``)
|
|
56
|
+
- ``False`` -> the negated flag (``--no-enable-prefix-caching``), matching
|
|
57
|
+
vLLM's own store_true/store_false flag-pair convention
|
|
58
|
+
- ``None`` -> the flag is skipped entirely (lets a value come from
|
|
59
|
+
elsewhere, e.g. an env var, without an empty CLI arg)
|
|
60
|
+
- dict/list -> JSON-encoded (several vLLM flags, e.g.
|
|
61
|
+
``--limit-mm-per-prompt``, take a JSON object as their value)
|
|
62
|
+
- anything else -> ``str(value)``
|
|
63
|
+
"""
|
|
64
|
+
args: list[str] = []
|
|
65
|
+
for raw_key, value in mapping.items():
|
|
66
|
+
key = raw_key.lstrip("-").replace("_", "-")
|
|
67
|
+
flag = f"--{key}"
|
|
68
|
+
if value is None:
|
|
69
|
+
continue
|
|
70
|
+
if value is True:
|
|
71
|
+
args.append(flag)
|
|
72
|
+
elif value is False:
|
|
73
|
+
args.append(f"--no-{key}")
|
|
74
|
+
elif isinstance(value, (dict, list)):
|
|
75
|
+
args += [flag, json.dumps(value)]
|
|
76
|
+
else:
|
|
77
|
+
args += [flag, str(value)]
|
|
78
|
+
return args
|
|
79
|
+
|
|
80
|
+
LOG_PATH = "/var/log/jambu/vllm.log"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class VllmRuntime(ModelRuntime):
|
|
84
|
+
engine = "vllm"
|
|
85
|
+
default_image = "vllm/vllm-openai:latest"
|
|
86
|
+
|
|
87
|
+
def env(self) -> dict[str, str]:
|
|
88
|
+
env: dict[str, str] = {
|
|
89
|
+
"HF_HOME": "/workspace/hf",
|
|
90
|
+
"HUGGING_FACE_HUB_CACHE": "/workspace/hf/hub",
|
|
91
|
+
"VLLM_LOGGING_LEVEL": "INFO",
|
|
92
|
+
}
|
|
93
|
+
if self.config.model.trust_remote_code:
|
|
94
|
+
env["VLLM_TRUST_REMOTE_CODE"] = "1"
|
|
95
|
+
return env
|
|
96
|
+
|
|
97
|
+
def resolve_tool_call_parser(self) -> "str | None":
|
|
98
|
+
cfg = self.config.runtime
|
|
99
|
+
if cfg.tool_calling == "off":
|
|
100
|
+
return None
|
|
101
|
+
if cfg.tool_call_parser:
|
|
102
|
+
return cfg.tool_call_parser
|
|
103
|
+
return guess_tool_call_parser(self.config.model.id)
|
|
104
|
+
|
|
105
|
+
def server_args(self) -> list[str]:
|
|
106
|
+
cfg = self.config
|
|
107
|
+
args: list[str] = [
|
|
108
|
+
"--model",
|
|
109
|
+
cfg.model.id,
|
|
110
|
+
"--host",
|
|
111
|
+
"0.0.0.0",
|
|
112
|
+
"--port",
|
|
113
|
+
str(cfg.runtime.port),
|
|
114
|
+
"--dtype",
|
|
115
|
+
cfg.runtime.dtype,
|
|
116
|
+
"--gpu-memory-utilization",
|
|
117
|
+
str(cfg.runtime.gpu_memory_utilization),
|
|
118
|
+
]
|
|
119
|
+
if cfg.model.revision:
|
|
120
|
+
args += ["--revision", cfg.model.revision]
|
|
121
|
+
if cfg.model.served_name:
|
|
122
|
+
args += ["--served-model-name", cfg.model.served_name]
|
|
123
|
+
if cfg.runtime.context_length:
|
|
124
|
+
args += ["--max-model-len", str(cfg.runtime.context_length)]
|
|
125
|
+
tp = cfg.runtime.tensor_parallel_size or cfg.compute.gpu.count
|
|
126
|
+
if tp and tp > 1:
|
|
127
|
+
args += ["--tensor-parallel-size", str(tp)]
|
|
128
|
+
if cfg.model.trust_remote_code:
|
|
129
|
+
args.append("--trust-remote-code")
|
|
130
|
+
if cfg.runtime.api_key:
|
|
131
|
+
args += ["--api-key", cfg.runtime.api_key]
|
|
132
|
+
|
|
133
|
+
# Agentic flows: /v1/chat/completions and /v1/responses both need the
|
|
134
|
+
# model to actually be able to emit tool calls, which needs a parser
|
|
135
|
+
# matched to the model family.
|
|
136
|
+
parser = self.resolve_tool_call_parser()
|
|
137
|
+
if parser:
|
|
138
|
+
args += ["--enable-auto-tool-choice", "--tool-call-parser", parser]
|
|
139
|
+
if cfg.runtime.tool_server:
|
|
140
|
+
# Lets /v1/responses execute MCP tools server-side instead of the
|
|
141
|
+
# calling agent framework doing it client-side.
|
|
142
|
+
args += ["--tool-server", cfg.runtime.tool_server]
|
|
143
|
+
|
|
144
|
+
# Escape hatch for vLLM flags this schema doesn't model explicitly -
|
|
145
|
+
# structured form first, then the raw form, each able to override
|
|
146
|
+
# what came before it (vLLM's argparse takes the last occurrence of
|
|
147
|
+
# a repeated flag).
|
|
148
|
+
args += flatten_vllm_args(cfg.runtime.vllm_args)
|
|
149
|
+
args += list(cfg.runtime.extra_args)
|
|
150
|
+
return args
|
|
151
|
+
|
|
152
|
+
def start_command(self) -> str:
|
|
153
|
+
args = " ".join(shlex.quote(a) for a in self.server_args())
|
|
154
|
+
return (
|
|
155
|
+
f"mkdir -p $(dirname {LOG_PATH}) && "
|
|
156
|
+
f"nohup python3 -m vllm.entrypoints.openai.api_server {args} "
|
|
157
|
+
f">> {LOG_PATH} 2>&1 & echo $! > /var/run/jambu/vllm.pid"
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
def validate(self) -> ValidationResult:
|
|
161
|
+
result = ValidationResult()
|
|
162
|
+
cfg = self.config
|
|
163
|
+
if not (0.1 <= cfg.runtime.gpu_memory_utilization <= 1.0):
|
|
164
|
+
result.error(
|
|
165
|
+
"runtime.gpu_memory_utilization must be between 0.1 and 1.0",
|
|
166
|
+
"runtime.gpu_memory_utilization",
|
|
167
|
+
)
|
|
168
|
+
tp = cfg.runtime.tensor_parallel_size
|
|
169
|
+
if tp and tp > cfg.compute.gpu.count:
|
|
170
|
+
result.error(
|
|
171
|
+
f"runtime.tensor_parallel_size ({tp}) exceeds compute.gpu.count "
|
|
172
|
+
f"({cfg.compute.gpu.count})",
|
|
173
|
+
"runtime.tensor_parallel_size",
|
|
174
|
+
)
|
|
175
|
+
if cfg.runtime.context_length and cfg.runtime.context_length > 131072:
|
|
176
|
+
result.warn(
|
|
177
|
+
"runtime.context_length above 128k needs a lot of KV cache VRAM",
|
|
178
|
+
"runtime.context_length",
|
|
179
|
+
)
|
|
180
|
+
if "/" not in cfg.model.id:
|
|
181
|
+
result.warn(
|
|
182
|
+
f"model.id '{cfg.model.id}' does not look like a HuggingFace repo id",
|
|
183
|
+
"model.id",
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
collisions = {
|
|
187
|
+
key.lstrip("-").replace("_", "-") for key in cfg.runtime.vllm_args
|
|
188
|
+
} & _MANAGED_VLLM_FLAGS
|
|
189
|
+
if collisions:
|
|
190
|
+
result.warn(
|
|
191
|
+
f"runtime.vllm_args sets {sorted(collisions)}, which gpu "
|
|
192
|
+
"already manages via other config fields (compute.gpu, model.id, "
|
|
193
|
+
"runtime.tool_calling, ...) - it will still be passed through and "
|
|
194
|
+
"win (last flag wins), but the field meant to control it won't "
|
|
195
|
+
"reflect what's actually running",
|
|
196
|
+
"runtime.vllm_args",
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
if cfg.runtime.tool_calling == "on" and not self.resolve_tool_call_parser():
|
|
200
|
+
result.error(
|
|
201
|
+
"runtime.tool_calling is 'on' but no --tool-call-parser could be "
|
|
202
|
+
f"resolved for '{cfg.model.id}'. Set runtime.tool_call_parser "
|
|
203
|
+
"explicitly (see docs/CONFIG.md for the list vLLM supports).",
|
|
204
|
+
"runtime.tool_call_parser",
|
|
205
|
+
)
|
|
206
|
+
elif cfg.runtime.tool_calling == "auto" and not self.resolve_tool_call_parser():
|
|
207
|
+
result.warn(
|
|
208
|
+
f"could not guess a --tool-call-parser for '{cfg.model.id}'; "
|
|
209
|
+
"tool/function calling will be OFF on both /v1/chat/completions "
|
|
210
|
+
"and /v1/responses. Set runtime.tool_call_parser to turn it on.",
|
|
211
|
+
"runtime.tool_call_parser",
|
|
212
|
+
)
|
|
213
|
+
if cfg.runtime.tool_server and not self.resolve_tool_call_parser():
|
|
214
|
+
result.warn(
|
|
215
|
+
"runtime.tool_server is set but no tool-call parser is active; "
|
|
216
|
+
"the model will not be able to invoke it",
|
|
217
|
+
"runtime.tool_server",
|
|
218
|
+
)
|
|
219
|
+
return result
|
|
220
|
+
|
|
221
|
+
def describe(self) -> str:
|
|
222
|
+
parser = self.resolve_tool_call_parser()
|
|
223
|
+
tools = f", tools={parser}" if parser else ""
|
|
224
|
+
return f"vLLM :{self.config.runtime.port} ({self.config.model.id}{tools})"
|