jambu-gpu 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. jambu_gpu/__init__.py +5 -0
  2. jambu_gpu/__main__.py +4 -0
  3. jambu_gpu/cli/__init__.py +0 -0
  4. jambu_gpu/cli/_common.py +117 -0
  5. jambu_gpu/cli/inspect.py +207 -0
  6. jambu_gpu/cli/lifecycle.py +127 -0
  7. jambu_gpu/cli/main.py +103 -0
  8. jambu_gpu/cli/providers.py +96 -0
  9. jambu_gpu/cli/run.py +37 -0
  10. jambu_gpu/cli/setup.py +165 -0
  11. jambu_gpu/cli/status.py +214 -0
  12. jambu_gpu/cli/validate.py +57 -0
  13. jambu_gpu/core/__init__.py +0 -0
  14. jambu_gpu/core/beacon.py +124 -0
  15. jambu_gpu/core/config.py +596 -0
  16. jambu_gpu/core/credentials.py +75 -0
  17. jambu_gpu/core/durations.py +47 -0
  18. jambu_gpu/core/engine.py +951 -0
  19. jambu_gpu/core/errors.py +68 -0
  20. jambu_gpu/core/events.py +115 -0
  21. jambu_gpu/core/execution.py +169 -0
  22. jambu_gpu/core/health.py +62 -0
  23. jambu_gpu/core/model_inspect.py +368 -0
  24. jambu_gpu/core/models.py +283 -0
  25. jambu_gpu/core/policy.py +233 -0
  26. jambu_gpu/core/state.py +361 -0
  27. jambu_gpu/core/tool_parsers.py +63 -0
  28. jambu_gpu/providers/__init__.py +4 -0
  29. jambu_gpu/providers/base.py +125 -0
  30. jambu_gpu/providers/registry.py +63 -0
  31. jambu_gpu/providers/vast/__init__.py +3 -0
  32. jambu_gpu/providers/vast/client.py +230 -0
  33. jambu_gpu/providers/vast/config.py +51 -0
  34. jambu_gpu/providers/vast/mapper.py +221 -0
  35. jambu_gpu/providers/vast/provider.py +422 -0
  36. jambu_gpu/runtime/__init__.py +6 -0
  37. jambu_gpu/runtime/base.py +99 -0
  38. jambu_gpu/runtime/vllm.py +224 -0
  39. jambu_gpu/watchdog/__init__.py +3 -0
  40. jambu_gpu/watchdog/agent_body.py +547 -0
  41. jambu_gpu/watchdog/builder.py +178 -0
  42. jambu_gpu-0.1.0.dist-info/METADATA +82 -0
  43. jambu_gpu-0.1.0.dist-info/RECORD +46 -0
  44. jambu_gpu-0.1.0.dist-info/WHEEL +4 -0
  45. jambu_gpu-0.1.0.dist-info/entry_points.txt +2 -0
  46. jambu_gpu-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,99 @@
1
+ """Runtime contract (spec section 19).
2
+
3
+ Provider answers "where does the compute run?".
4
+ Runtime answers "how does the model run?".
5
+ Model answers "what model runs?".
6
+ These stay independent; a runtime never talks to a provider API.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from abc import ABC, abstractmethod
12
+ from typing import Optional, Type
13
+
14
+ from ..core.config import RuntimeConfig
15
+ from ..core.errors import ConfigError
16
+ from ..core.models import ValidationResult
17
+
18
+ ENDPOINT_NAME = "model"
19
+
20
+
21
+ class ModelRuntime(ABC):
22
+ """Describes how to boot a model server inside the provisioned container."""
23
+
24
+ engine: str = "abstract"
25
+ default_image: str = ""
26
+
27
+ def __init__(self, config: RuntimeConfig) -> None:
28
+ self.config = config
29
+
30
+ # -- container shape ----------------------------------------------------
31
+
32
+ def image(self) -> str:
33
+ return self.config.compute.image or self.default_image
34
+
35
+ def ports(self) -> list[int]:
36
+ return [self.config.runtime.port]
37
+
38
+ def env(self) -> dict[str, str]:
39
+ return {}
40
+
41
+ @abstractmethod
42
+ def start_command(self) -> str:
43
+ """Shell command that launches the model server in the background."""
44
+
45
+ # -- health -------------------------------------------------------------
46
+
47
+ def health_path(self) -> str:
48
+ return self.config.health.path
49
+
50
+ def health_url(self, base_url: str) -> str:
51
+ return base_url.rstrip("/") + self.health_path()
52
+
53
+ def base_url(self, endpoint_url: str) -> str:
54
+ return endpoint_url.rstrip("/")
55
+
56
+ def validate(self) -> ValidationResult:
57
+ return ValidationResult()
58
+
59
+ def describe(self) -> str:
60
+ return f"{self.engine} :{self.config.runtime.port}"
61
+
62
+
63
+ class NoRuntime(ModelRuntime):
64
+ """`runtime.engine: none` - provision bare compute, start nothing."""
65
+
66
+ engine = "none"
67
+ default_image = "pytorch/pytorch:2.4.0-cuda12.1-cudnn9-runtime"
68
+
69
+ def ports(self) -> list[int]:
70
+ return []
71
+
72
+ def start_command(self) -> str:
73
+ return "true"
74
+
75
+ def health_url(self, base_url: str) -> str: # pragma: no cover - unused
76
+ return base_url
77
+
78
+
79
+ class RuntimeRegistry:
80
+ def __init__(self) -> None:
81
+ self._runtimes: dict[str, Type[ModelRuntime]] = {"none": NoRuntime}
82
+
83
+ def register(self, runtime_cls: Type[ModelRuntime]) -> Type[ModelRuntime]:
84
+ self._runtimes[runtime_cls.engine] = runtime_cls
85
+ return runtime_cls
86
+
87
+ def names(self) -> list[str]:
88
+ return sorted(self._runtimes)
89
+
90
+ def get(self, config: RuntimeConfig, engine: Optional[str] = None) -> ModelRuntime:
91
+ key = engine or config.runtime.engine
92
+ try:
93
+ return self._runtimes[key](config)
94
+ except KeyError:
95
+ known = ", ".join(self.names())
96
+ raise ConfigError(f"unknown runtime engine '{key}' (available: {known})") from None
97
+
98
+
99
+ runtime_registry = RuntimeRegistry()
@@ -0,0 +1,224 @@
1
+ """vLLM OpenAI-compatible server runtime.
2
+
3
+ This is where "does it support agentic flows?" gets answered: vLLM's server
4
+ already implements both ``/v1/chat/completions`` (with tool/function calling)
5
+ and ``/v1/responses`` (the OpenAI Responses API, MCP-tool-routing included).
6
+ gpu does not implement any of that - it only turns on the right vLLM
7
+ flags and forwards the same endpoint/credentials it always has, so any
8
+ OpenAI-SDK-compatible agent framework (LangChain, LangGraph, CrewAI, the
9
+ `openai` SDK's own `responses.create`, ...) can talk to the instance with no
10
+ extra layer in between.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import shlex
17
+ from typing import Any
18
+
19
+ from ..core.models import ValidationResult
20
+ from ..core.tool_parsers import guess_tool_call_parser # noqa: F401 (re-exported)
21
+ from .base import ModelRuntime
22
+
23
+ # vLLM flags gpu already manages explicitly - collided with via
24
+ # runtime.vllm_args, these would fight the config fields meant to control
25
+ # them (compute.gpu, model.id, lifecycle port wiring, ...). Not a hard block
26
+ # (power users can still reach them through raw extra_args if they really
27
+ # need to), just something validate() warns about.
28
+ _MANAGED_VLLM_FLAGS = frozenset(
29
+ {
30
+ "model",
31
+ "host",
32
+ "port",
33
+ "served-model-name",
34
+ "served_model_name",
35
+ "revision",
36
+ "trust-remote-code",
37
+ "trust_remote_code",
38
+ "enable-auto-tool-choice",
39
+ "enable_auto_tool_choice",
40
+ "tool-call-parser",
41
+ "tool_call_parser",
42
+ "tool-server",
43
+ "tool_server",
44
+ }
45
+ )
46
+
47
+
48
+ def flatten_vllm_args(mapping: dict[str, Any]) -> list[str]:
49
+ """Turn a ``{flag: value}`` mapping into vLLM CLI argv.
50
+
51
+ Rules, matched to how vLLM's own argparse-based CLI takes flags:
52
+ - key: underscores become dashes (``max_num_seqs`` -> ``--max-num-seqs``),
53
+ already-dashed keys pass through unchanged. A leading ``--`` in the
54
+ key is tolerated (stripped) so either spelling works.
55
+ - ``True`` -> a bare flag (``--enable-prefix-caching``)
56
+ - ``False`` -> the negated flag (``--no-enable-prefix-caching``), matching
57
+ vLLM's own store_true/store_false flag-pair convention
58
+ - ``None`` -> the flag is skipped entirely (lets a value come from
59
+ elsewhere, e.g. an env var, without an empty CLI arg)
60
+ - dict/list -> JSON-encoded (several vLLM flags, e.g.
61
+ ``--limit-mm-per-prompt``, take a JSON object as their value)
62
+ - anything else -> ``str(value)``
63
+ """
64
+ args: list[str] = []
65
+ for raw_key, value in mapping.items():
66
+ key = raw_key.lstrip("-").replace("_", "-")
67
+ flag = f"--{key}"
68
+ if value is None:
69
+ continue
70
+ if value is True:
71
+ args.append(flag)
72
+ elif value is False:
73
+ args.append(f"--no-{key}")
74
+ elif isinstance(value, (dict, list)):
75
+ args += [flag, json.dumps(value)]
76
+ else:
77
+ args += [flag, str(value)]
78
+ return args
79
+
80
+ LOG_PATH = "/var/log/jambu/vllm.log"
81
+
82
+
83
+ class VllmRuntime(ModelRuntime):
84
+ engine = "vllm"
85
+ default_image = "vllm/vllm-openai:latest"
86
+
87
+ def env(self) -> dict[str, str]:
88
+ env: dict[str, str] = {
89
+ "HF_HOME": "/workspace/hf",
90
+ "HUGGING_FACE_HUB_CACHE": "/workspace/hf/hub",
91
+ "VLLM_LOGGING_LEVEL": "INFO",
92
+ }
93
+ if self.config.model.trust_remote_code:
94
+ env["VLLM_TRUST_REMOTE_CODE"] = "1"
95
+ return env
96
+
97
+ def resolve_tool_call_parser(self) -> "str | None":
98
+ cfg = self.config.runtime
99
+ if cfg.tool_calling == "off":
100
+ return None
101
+ if cfg.tool_call_parser:
102
+ return cfg.tool_call_parser
103
+ return guess_tool_call_parser(self.config.model.id)
104
+
105
+ def server_args(self) -> list[str]:
106
+ cfg = self.config
107
+ args: list[str] = [
108
+ "--model",
109
+ cfg.model.id,
110
+ "--host",
111
+ "0.0.0.0",
112
+ "--port",
113
+ str(cfg.runtime.port),
114
+ "--dtype",
115
+ cfg.runtime.dtype,
116
+ "--gpu-memory-utilization",
117
+ str(cfg.runtime.gpu_memory_utilization),
118
+ ]
119
+ if cfg.model.revision:
120
+ args += ["--revision", cfg.model.revision]
121
+ if cfg.model.served_name:
122
+ args += ["--served-model-name", cfg.model.served_name]
123
+ if cfg.runtime.context_length:
124
+ args += ["--max-model-len", str(cfg.runtime.context_length)]
125
+ tp = cfg.runtime.tensor_parallel_size or cfg.compute.gpu.count
126
+ if tp and tp > 1:
127
+ args += ["--tensor-parallel-size", str(tp)]
128
+ if cfg.model.trust_remote_code:
129
+ args.append("--trust-remote-code")
130
+ if cfg.runtime.api_key:
131
+ args += ["--api-key", cfg.runtime.api_key]
132
+
133
+ # Agentic flows: /v1/chat/completions and /v1/responses both need the
134
+ # model to actually be able to emit tool calls, which needs a parser
135
+ # matched to the model family.
136
+ parser = self.resolve_tool_call_parser()
137
+ if parser:
138
+ args += ["--enable-auto-tool-choice", "--tool-call-parser", parser]
139
+ if cfg.runtime.tool_server:
140
+ # Lets /v1/responses execute MCP tools server-side instead of the
141
+ # calling agent framework doing it client-side.
142
+ args += ["--tool-server", cfg.runtime.tool_server]
143
+
144
+ # Escape hatch for vLLM flags this schema doesn't model explicitly -
145
+ # structured form first, then the raw form, each able to override
146
+ # what came before it (vLLM's argparse takes the last occurrence of
147
+ # a repeated flag).
148
+ args += flatten_vllm_args(cfg.runtime.vllm_args)
149
+ args += list(cfg.runtime.extra_args)
150
+ return args
151
+
152
+ def start_command(self) -> str:
153
+ args = " ".join(shlex.quote(a) for a in self.server_args())
154
+ return (
155
+ f"mkdir -p $(dirname {LOG_PATH}) && "
156
+ f"nohup python3 -m vllm.entrypoints.openai.api_server {args} "
157
+ f">> {LOG_PATH} 2>&1 & echo $! > /var/run/jambu/vllm.pid"
158
+ )
159
+
160
+ def validate(self) -> ValidationResult:
161
+ result = ValidationResult()
162
+ cfg = self.config
163
+ if not (0.1 <= cfg.runtime.gpu_memory_utilization <= 1.0):
164
+ result.error(
165
+ "runtime.gpu_memory_utilization must be between 0.1 and 1.0",
166
+ "runtime.gpu_memory_utilization",
167
+ )
168
+ tp = cfg.runtime.tensor_parallel_size
169
+ if tp and tp > cfg.compute.gpu.count:
170
+ result.error(
171
+ f"runtime.tensor_parallel_size ({tp}) exceeds compute.gpu.count "
172
+ f"({cfg.compute.gpu.count})",
173
+ "runtime.tensor_parallel_size",
174
+ )
175
+ if cfg.runtime.context_length and cfg.runtime.context_length > 131072:
176
+ result.warn(
177
+ "runtime.context_length above 128k needs a lot of KV cache VRAM",
178
+ "runtime.context_length",
179
+ )
180
+ if "/" not in cfg.model.id:
181
+ result.warn(
182
+ f"model.id '{cfg.model.id}' does not look like a HuggingFace repo id",
183
+ "model.id",
184
+ )
185
+
186
+ collisions = {
187
+ key.lstrip("-").replace("_", "-") for key in cfg.runtime.vllm_args
188
+ } & _MANAGED_VLLM_FLAGS
189
+ if collisions:
190
+ result.warn(
191
+ f"runtime.vllm_args sets {sorted(collisions)}, which gpu "
192
+ "already manages via other config fields (compute.gpu, model.id, "
193
+ "runtime.tool_calling, ...) - it will still be passed through and "
194
+ "win (last flag wins), but the field meant to control it won't "
195
+ "reflect what's actually running",
196
+ "runtime.vllm_args",
197
+ )
198
+
199
+ if cfg.runtime.tool_calling == "on" and not self.resolve_tool_call_parser():
200
+ result.error(
201
+ "runtime.tool_calling is 'on' but no --tool-call-parser could be "
202
+ f"resolved for '{cfg.model.id}'. Set runtime.tool_call_parser "
203
+ "explicitly (see docs/CONFIG.md for the list vLLM supports).",
204
+ "runtime.tool_call_parser",
205
+ )
206
+ elif cfg.runtime.tool_calling == "auto" and not self.resolve_tool_call_parser():
207
+ result.warn(
208
+ f"could not guess a --tool-call-parser for '{cfg.model.id}'; "
209
+ "tool/function calling will be OFF on both /v1/chat/completions "
210
+ "and /v1/responses. Set runtime.tool_call_parser to turn it on.",
211
+ "runtime.tool_call_parser",
212
+ )
213
+ if cfg.runtime.tool_server and not self.resolve_tool_call_parser():
214
+ result.warn(
215
+ "runtime.tool_server is set but no tool-call parser is active; "
216
+ "the model will not be able to invoke it",
217
+ "runtime.tool_server",
218
+ )
219
+ return result
220
+
221
+ def describe(self) -> str:
222
+ parser = self.resolve_tool_call_parser()
223
+ tools = f", tools={parser}" if parser else ""
224
+ return f"vLLM :{self.config.runtime.port} ({self.config.model.id}{tools})"
@@ -0,0 +1,3 @@
1
+ from .builder import build_agent_source, build_onstart_script, generate_token
2
+
3
+ __all__ = ["build_agent_source", "build_onstart_script", "generate_token"]