bladekit 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bladekit/__init__.py +219 -0
- bladekit/_app.py +3241 -0
- bladekit/_banner.py +640 -0
- bladekit/_batch.py +150 -0
- bladekit/_bucket.py +211 -0
- bladekit/_build.py +337 -0
- bladekit/_bundle.py +459 -0
- bladekit/_client.py +1152 -0
- bladekit/_cls.py +574 -0
- bladekit/_constants.py +121 -0
- bladekit/_cron.py +88 -0
- bladekit/_dict.py +174 -0
- bladekit/_env.py +98 -0
- bladekit/_image.py +577 -0
- bladekit/_loader.py +79 -0
- bladekit/_map.py +555 -0
- bladekit/_preempt.py +94 -0
- bladekit/_queue.py +137 -0
- bladekit/_render.py +683 -0
- bladekit/_runner_api.py +148 -0
- bladekit/_sandbox.py +1051 -0
- bladekit/_secret.py +52 -0
- bladekit/_serialize.py +88 -0
- bladekit/_serve.py +188 -0
- bladekit/_volume.py +232 -0
- bladekit/_webhook.py +90 -0
- bladekit/cli.py +2282 -0
- bladekit/config.py +181 -0
- bladekit/exceptions.py +544 -0
- bladekit-4.5.0.dist-info/METADATA +373 -0
- bladekit-4.5.0.dist-info/RECORD +33 -0
- bladekit-4.5.0.dist-info/WHEEL +4 -0
- bladekit-4.5.0.dist-info/entry_points.txt +3 -0
bladekit/cli.py
ADDED
|
@@ -0,0 +1,2282 @@
|
|
|
1
|
+
"""``blade`` command-line interface.
|
|
2
|
+
|
|
3
|
+
Commands:
|
|
4
|
+
blade setup authenticate via the browser, save an API token
|
|
5
|
+
blade run <script.py> run the @app.local_entrypoint, stream logs live
|
|
6
|
+
blade logs <run_id> show / follow logs (with resume)
|
|
7
|
+
blade status <run_id> run state + queue position if pending
|
|
8
|
+
blade runs list runs
|
|
9
|
+
blade cancel <run_id> cancel a run
|
|
10
|
+
blade nodes fleet view (GPUs free/busy) — staff session token only
|
|
11
|
+
blade token set/show manage ~/.blade/config.toml
|
|
12
|
+
blade admin … contain-user / force-stop (staff session token)
|
|
13
|
+
blade --version print the installed bladekit version
|
|
14
|
+
|
|
15
|
+
``forge`` is the same command under its pre-PAAS-771 name (shipped by the
|
|
16
|
+
``forge-shadow`` compatibility distribution).
|
|
17
|
+
|
|
18
|
+
Heavy modules (httpx, the App machinery) are imported lazily inside commands so
|
|
19
|
+
``blade --help`` stays fast.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import sys
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Optional
|
|
28
|
+
|
|
29
|
+
import typer
|
|
30
|
+
|
|
31
|
+
app = typer.Typer(
|
|
32
|
+
add_completion=False,
|
|
33
|
+
no_args_is_help=True,
|
|
34
|
+
help="Blade AI Fabric — run Python functions on the GPU fleet.",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
token_app = typer.Typer(no_args_is_help=True, help="Manage API token / endpoint.")
|
|
38
|
+
app.add_typer(token_app, name="token")
|
|
39
|
+
|
|
40
|
+
volume_app = typer.Typer(no_args_is_help=True, help="Manage network volumes.")
|
|
41
|
+
app.add_typer(volume_app, name="volume")
|
|
42
|
+
|
|
43
|
+
secret_app = typer.Typer(no_args_is_help=True, help="Manage secrets.")
|
|
44
|
+
app.add_typer(secret_app, name="secret")
|
|
45
|
+
|
|
46
|
+
image_app = typer.Typer(no_args_is_help=True, help="Image catalogue / DSL builds.")
|
|
47
|
+
app.add_typer(image_app, name="image")
|
|
48
|
+
|
|
49
|
+
app_app = typer.Typer(no_args_is_help=True, help="Manage deployed apps.")
|
|
50
|
+
app.add_typer(app_app, name="app")
|
|
51
|
+
|
|
52
|
+
map_app = typer.Typer(no_args_is_help=True, help="Manage map batch invocations.")
|
|
53
|
+
app.add_typer(map_app, name="map")
|
|
54
|
+
|
|
55
|
+
cron_app = typer.Typer(no_args_is_help=True, help="Manage cron schedules.")
|
|
56
|
+
app.add_typer(cron_app, name="cron")
|
|
57
|
+
|
|
58
|
+
queue_app = typer.Typer(no_args_is_help=True, help="Manage distributed queues.")
|
|
59
|
+
app.add_typer(queue_app, name="queue")
|
|
60
|
+
|
|
61
|
+
dict_app = typer.Typer(no_args_is_help=True, help="Manage distributed key-value stores.")
|
|
62
|
+
app.add_typer(dict_app, name="dict")
|
|
63
|
+
|
|
64
|
+
sandbox_app = typer.Typer(no_args_is_help=True, help="Manage sandboxes.")
|
|
65
|
+
app.add_typer(sandbox_app, name="sandbox")
|
|
66
|
+
|
|
67
|
+
admin_app = typer.Typer(
|
|
68
|
+
no_args_is_help=True,
|
|
69
|
+
help="Platform-admin containment tools (requires a staff SESSION token: an fg_ key "
|
|
70
|
+
"carries no compute:admin and is refused 403 INSUFFICIENT_SCOPE).",
|
|
71
|
+
)
|
|
72
|
+
app.add_typer(admin_app, name="admin")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _env_flag(name: str) -> bool:
|
|
76
|
+
"""Read a boolean FABRIC_<name> env var, falling back to FORGE_<name> (repo
|
|
77
|
+
convention: manual os.environ lookup, see _env.py — not typer's envvar=)."""
|
|
78
|
+
from ._env import flag
|
|
79
|
+
|
|
80
|
+
return flag(name)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _version_callback(value: bool) -> None:
|
|
84
|
+
"""``blade --version``: the compact banner on a terminal, plain text otherwise."""
|
|
85
|
+
if not value:
|
|
86
|
+
return
|
|
87
|
+
from . import DISTRIBUTION, __version__
|
|
88
|
+
from ._render import print_version
|
|
89
|
+
|
|
90
|
+
print_version(f"{DISTRIBUTION} {__version__}")
|
|
91
|
+
raise typer.Exit()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@app.callback()
|
|
95
|
+
def _main(
|
|
96
|
+
verbose: bool = typer.Option(
|
|
97
|
+
False,
|
|
98
|
+
"--verbose",
|
|
99
|
+
"-v",
|
|
100
|
+
help="Verbose output: raw build steps and per-state transitions "
|
|
101
|
+
"(also enabled by FABRIC_VERBOSE). Default is a clean phase view.",
|
|
102
|
+
),
|
|
103
|
+
version: bool = typer.Option(
|
|
104
|
+
False,
|
|
105
|
+
"--version",
|
|
106
|
+
callback=_version_callback,
|
|
107
|
+
is_eager=True,
|
|
108
|
+
help="Show the installed bladekit version and exit.",
|
|
109
|
+
),
|
|
110
|
+
):
|
|
111
|
+
"""Blade AI Fabric — run Python functions on the GPU fleet."""
|
|
112
|
+
from ._render import set_verbose
|
|
113
|
+
|
|
114
|
+
set_verbose(verbose or _env_flag("VERBOSE"))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# --------------------------------------------------------------------------
|
|
118
|
+
# shared helpers
|
|
119
|
+
# --------------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _make_client():
|
|
123
|
+
from ._client import Client
|
|
124
|
+
from ._render import err_console
|
|
125
|
+
from .config import load_config
|
|
126
|
+
|
|
127
|
+
cfg = load_config()
|
|
128
|
+
if not cfg.token:
|
|
129
|
+
err_console.print(
|
|
130
|
+
"[red]No API token configured.[/red] Run `blade setup` or "
|
|
131
|
+
"`blade token set`, or set FABRIC_TOKEN."
|
|
132
|
+
)
|
|
133
|
+
raise typer.Exit(code=2)
|
|
134
|
+
client = Client(cfg.api_url, cfg.token)
|
|
135
|
+
client.warn_if_outdated()
|
|
136
|
+
return client, cfg
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _handle_fabric_error(e: Exception) -> "typer.Exit":
|
|
140
|
+
"""Render any FabricError with its machine code + human message, return Exit."""
|
|
141
|
+
from ._render import err_console
|
|
142
|
+
from . import exceptions as exc
|
|
143
|
+
|
|
144
|
+
if isinstance(e, exc.ImageBuildError):
|
|
145
|
+
# Failed DSL build: code + message + the actionable `blade image logs`
|
|
146
|
+
# hint (already embedded in the message).
|
|
147
|
+
err_console.print(f"[red]image build failed[/red] [bold]{e.code}[/bold]:")
|
|
148
|
+
err_console.print(str(e), markup=False, highlight=False)
|
|
149
|
+
elif isinstance(e, exc.VolumeCommitError):
|
|
150
|
+
err_console.print(f"[red]volume commit failed[/red] [bold]{e.code}[/bold]: {e}")
|
|
151
|
+
elif isinstance(e, exc.APIError):
|
|
152
|
+
err_console.print(f"[red]error[/red] [bold]{e.code}[/bold]: {e.api_message}")
|
|
153
|
+
if e.details:
|
|
154
|
+
err_console.print(f" details: {json.dumps(e.details)}")
|
|
155
|
+
elif isinstance(e, exc.RunError):
|
|
156
|
+
# Show code + full message (carries the actionable hint, exceptions.py).
|
|
157
|
+
err_console.print(f"[red]run failed[/red] [bold]{e.code}[/bold]:")
|
|
158
|
+
err_console.print(str(e), markup=False, highlight=False)
|
|
159
|
+
elif isinstance(e, exc.ConnectionFailed):
|
|
160
|
+
err_console.print(f"[red]connection failed[/red]: {e}")
|
|
161
|
+
else:
|
|
162
|
+
err_console.print(f"[red]error[/red]: {e}")
|
|
163
|
+
return typer.Exit(code=1)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# --------------------------------------------------------------------------
|
|
167
|
+
# token
|
|
168
|
+
# --------------------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@token_app.command("set")
|
|
172
|
+
def token_set(
|
|
173
|
+
api_url: Optional[str] = typer.Option(
|
|
174
|
+
None, "--api-url", help="Control plane base URL to persist."
|
|
175
|
+
),
|
|
176
|
+
from_env: Optional[str] = typer.Option(
|
|
177
|
+
None,
|
|
178
|
+
"--from-env",
|
|
179
|
+
help="Read the token from this environment variable (e.g. FABRIC_TOKEN).",
|
|
180
|
+
),
|
|
181
|
+
):
|
|
182
|
+
"""Write the token (and optionally api_url) to ~/.blade/config.toml.
|
|
183
|
+
|
|
184
|
+
The token is never accepted as a positional argument — that would leak it
|
|
185
|
+
into ``ps aux``, shell history, and process monitoring tools. Instead:
|
|
186
|
+
|
|
187
|
+
\\b
|
|
188
|
+
blade token set --from-env FABRIC_TOKEN # reads $FABRIC_TOKEN
|
|
189
|
+
echo $TOKEN | blade token set # reads from stdin
|
|
190
|
+
"""
|
|
191
|
+
from ._render import console
|
|
192
|
+
from .config import write_config
|
|
193
|
+
|
|
194
|
+
token: Optional[str] = None
|
|
195
|
+
if from_env:
|
|
196
|
+
import os
|
|
197
|
+
|
|
198
|
+
token = os.environ.get(from_env)
|
|
199
|
+
if not token:
|
|
200
|
+
console.print(f"[red]env var not set or empty[/red]: {from_env}")
|
|
201
|
+
raise typer.Exit(code=2)
|
|
202
|
+
else:
|
|
203
|
+
# Read from stdin so the token never appears in argv.
|
|
204
|
+
if sys.stdin.isatty():
|
|
205
|
+
console.print("[dim]paste your API token then press Enter (input not echoed)…[/dim]")
|
|
206
|
+
line = sys.stdin.readline().rstrip("\n\r")
|
|
207
|
+
if not line:
|
|
208
|
+
console.print("[red]no token provided[/red] (pass via stdin or --from-env)")
|
|
209
|
+
raise typer.Exit(code=2)
|
|
210
|
+
token = line
|
|
211
|
+
|
|
212
|
+
path = write_config(token=token, api_url=api_url)
|
|
213
|
+
console.print(f"[green]Saved[/green] token to {path}")
|
|
214
|
+
if api_url:
|
|
215
|
+
console.print(f"[green]Saved[/green] api_url = {api_url}")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@token_app.command("show")
|
|
219
|
+
def token_show():
|
|
220
|
+
"""Show the effective token/endpoint and where each came from."""
|
|
221
|
+
from ._render import console
|
|
222
|
+
from .config import load_config
|
|
223
|
+
|
|
224
|
+
cfg = load_config()
|
|
225
|
+
console.print(f"api_url : {cfg.api_url} [dim]({cfg.api_url_source})[/dim]")
|
|
226
|
+
if cfg.token:
|
|
227
|
+
masked = cfg.token[:6] + "…" + cfg.token[-4:] if len(cfg.token) > 12 else "set"
|
|
228
|
+
console.print(f"token : {masked} [dim]({cfg.token_source})[/dim]")
|
|
229
|
+
else:
|
|
230
|
+
console.print("token : [yellow]not set[/yellow]")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@app.command()
|
|
234
|
+
def setup(
|
|
235
|
+
api_url: Optional[str] = typer.Option(
|
|
236
|
+
None, "--api-url", help="Control plane base URL (default: configured value / FABRIC_API_URL)."
|
|
237
|
+
),
|
|
238
|
+
):
|
|
239
|
+
"""Authenticate this machine through the browser and save an API token.
|
|
240
|
+
|
|
241
|
+
Opens your browser to confirm your identity, mints a token bound to your
|
|
242
|
+
workspace, and writes it to ~/.blade/config.toml — like ``modal setup``.
|
|
243
|
+
"""
|
|
244
|
+
import time
|
|
245
|
+
import webbrowser
|
|
246
|
+
|
|
247
|
+
import httpx
|
|
248
|
+
|
|
249
|
+
from ._render import console, err_console, print_intro
|
|
250
|
+
from .config import load_config, write_config
|
|
251
|
+
|
|
252
|
+
print_intro()
|
|
253
|
+
cfg = load_config()
|
|
254
|
+
base = (api_url or cfg.api_url).rstrip("/")
|
|
255
|
+
|
|
256
|
+
token: Optional[str] = None
|
|
257
|
+
workspace: Optional[str] = None
|
|
258
|
+
try:
|
|
259
|
+
with httpx.Client(base_url=base, timeout=35.0) as http:
|
|
260
|
+
start = http.post("/v1/token-flow/start")
|
|
261
|
+
start.raise_for_status()
|
|
262
|
+
data = start.json()
|
|
263
|
+
flow_id = data["flow_id"]
|
|
264
|
+
url = data.get("url") or f"{base}/token-flow/{flow_id}"
|
|
265
|
+
|
|
266
|
+
console.print(
|
|
267
|
+
"The web browser should have opened for you to authenticate and "
|
|
268
|
+
"get an API token.\nIf it didn't, please copy this URL into your "
|
|
269
|
+
f"web browser manually:\n\n [cyan]{url}[/cyan]\n"
|
|
270
|
+
)
|
|
271
|
+
try:
|
|
272
|
+
webbrowser.open(url)
|
|
273
|
+
except Exception:
|
|
274
|
+
pass
|
|
275
|
+
|
|
276
|
+
# Each /wait long-polls ~20s server-side; loop until confirmed.
|
|
277
|
+
deadline = time.monotonic() + 300 # 5 min
|
|
278
|
+
with console.status("Waiting for web authentication…"):
|
|
279
|
+
while time.monotonic() < deadline:
|
|
280
|
+
resp = http.get(f"/v1/token-flow/{flow_id}/wait", timeout=35.0)
|
|
281
|
+
resp.raise_for_status()
|
|
282
|
+
body = resp.json()
|
|
283
|
+
if body.get("status") == "confirmed" and body.get("token"):
|
|
284
|
+
token = body["token"]
|
|
285
|
+
workspace = body.get("workspace")
|
|
286
|
+
break
|
|
287
|
+
except httpx.HTTPError as e:
|
|
288
|
+
err_console.print(f"[red]Could not reach the control plane at {base}[/red]: {e}")
|
|
289
|
+
raise typer.Exit(code=1)
|
|
290
|
+
|
|
291
|
+
if not token:
|
|
292
|
+
err_console.print(
|
|
293
|
+
"[red]Timed out waiting for web authentication.[/red] Re-run `blade setup`."
|
|
294
|
+
)
|
|
295
|
+
raise typer.Exit(code=1)
|
|
296
|
+
|
|
297
|
+
console.print("[green]Web authentication finished successfully![/green]")
|
|
298
|
+
if workspace:
|
|
299
|
+
console.print(f"Token is connected to the [bold]{workspace}[/bold] workspace.")
|
|
300
|
+
path = write_config(token=token, api_url=base)
|
|
301
|
+
console.print(f"Token written to [bold]{path}[/bold].")
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
# --------------------------------------------------------------------------
|
|
305
|
+
# run
|
|
306
|
+
# --------------------------------------------------------------------------
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
@app.command()
|
|
310
|
+
def run(
|
|
311
|
+
script: Path = typer.Argument(..., help="Python script defining the app."),
|
|
312
|
+
no_logs: bool = typer.Option(
|
|
313
|
+
False, "--no-logs", help="Do not stream logs (just run and report)."
|
|
314
|
+
),
|
|
315
|
+
verbose: bool = typer.Option(
|
|
316
|
+
False, "--verbose", "-v",
|
|
317
|
+
help="Verbose: per-state transition lines instead of the clean running phase.",
|
|
318
|
+
),
|
|
319
|
+
):
|
|
320
|
+
"""Execute the script's @app.local_entrypoint, streaming logs live.
|
|
321
|
+
|
|
322
|
+
Ctrl-C cancels the remote run(s). If a node is unreachable the run sits in
|
|
323
|
+
``cancelling`` and we print "cancellation pending on node".
|
|
324
|
+
"""
|
|
325
|
+
from ._loader import load_script, select_app_and_entrypoint
|
|
326
|
+
from ._render import console, err_console, set_verbose
|
|
327
|
+
from . import _app as app_mod
|
|
328
|
+
from . import exceptions as exc
|
|
329
|
+
|
|
330
|
+
if verbose:
|
|
331
|
+
set_verbose(True)
|
|
332
|
+
|
|
333
|
+
try:
|
|
334
|
+
client, cfg = _make_client()
|
|
335
|
+
except typer.Exit:
|
|
336
|
+
raise
|
|
337
|
+
|
|
338
|
+
try:
|
|
339
|
+
module, module_name, bundle_root = load_script(script)
|
|
340
|
+
forge_app, entrypoint = select_app_and_entrypoint(module)
|
|
341
|
+
except exc.FabricError as e:
|
|
342
|
+
raise _handle_fabric_error(e)
|
|
343
|
+
|
|
344
|
+
forge_app._bind_client(client)
|
|
345
|
+
forge_app._set_bundle_context(bundle_root, module_name)
|
|
346
|
+
|
|
347
|
+
# Background log streaming for every run this script opens. We start it lazily
|
|
348
|
+
# once the first run id is known. For simplicity (one function/run is the
|
|
349
|
+
# common case) we stream the first opened run.
|
|
350
|
+
import threading
|
|
351
|
+
|
|
352
|
+
stop_event = threading.Event()
|
|
353
|
+
streamed = {"run_id": None}
|
|
354
|
+
|
|
355
|
+
def _log_pump():
|
|
356
|
+
# Wait for a run to be registered, then stream it.
|
|
357
|
+
while not stop_event.is_set():
|
|
358
|
+
run_ids = list(app_mod._SESSION.open_run_ids)
|
|
359
|
+
if run_ids:
|
|
360
|
+
streamed["run_id"] = run_ids[0]
|
|
361
|
+
break
|
|
362
|
+
stop_event.wait(0.1)
|
|
363
|
+
rid = streamed["run_id"]
|
|
364
|
+
if not rid or no_logs:
|
|
365
|
+
return
|
|
366
|
+
_stream_run_logs(client, rid, stop_event, phase_ui=True)
|
|
367
|
+
|
|
368
|
+
log_thread = threading.Thread(target=_log_pump, daemon=True)
|
|
369
|
+
if not no_logs:
|
|
370
|
+
log_thread.start()
|
|
371
|
+
|
|
372
|
+
exit_code = 0
|
|
373
|
+
try:
|
|
374
|
+
result = entrypoint()
|
|
375
|
+
if result is not None:
|
|
376
|
+
console.print(f"[green]entrypoint returned:[/green] {result!r}")
|
|
377
|
+
except KeyboardInterrupt:
|
|
378
|
+
_on_ctrl_c(client, app_mod._SESSION.open_run_ids)
|
|
379
|
+
exit_code = 130
|
|
380
|
+
except exc.RunError as e:
|
|
381
|
+
# A run that FAILED for an infrastructure reason. Pilot UX fix
|
|
382
|
+
# (GPU-PILOT-REPORT §"Findings UX additionnels"): show the RunError
|
|
383
|
+
# (code + message + hint) and exit 1 — never "run cancelled"/130.
|
|
384
|
+
# RunError is checked BEFORE RunCancelled because RunLostError, etc. are
|
|
385
|
+
# RunError subclasses and must not be mistaken for a cancellation.
|
|
386
|
+
_handle_fabric_error(e)
|
|
387
|
+
exit_code = 1
|
|
388
|
+
except exc.RunCancelled as e:
|
|
389
|
+
err_console.print(f"[magenta]run cancelled[/magenta]: {e.run_id}")
|
|
390
|
+
exit_code = 130
|
|
391
|
+
except exc.RemoteFunctionError as e:
|
|
392
|
+
# A remote *user* exception that bubbled out of the entrypoint.
|
|
393
|
+
# RemoteFunctionError (RemoteError) is a FabricError subclass, so this
|
|
394
|
+
# must be checked BEFORE the generic FabricError branch or it would
|
|
395
|
+
# never be reached (first-match except semantics).
|
|
396
|
+
err_console.print(f"[red]remote function raised:[/red]\n{e}")
|
|
397
|
+
exit_code = 1
|
|
398
|
+
except exc.FabricError as e:
|
|
399
|
+
_handle_fabric_error(e)
|
|
400
|
+
exit_code = 1
|
|
401
|
+
except BaseException as e: # user exception re-raised from remote
|
|
402
|
+
# When the SDK re-raised the ORIGINAL user exception, the remote
|
|
403
|
+
# traceback travels on `_forge_remote_traceback` (set by
|
|
404
|
+
# Function._build_remote_exception). Print it verbatim so the remote
|
|
405
|
+
# stacktrace reads "as if local" (criterion) — without it the
|
|
406
|
+
# user would only see `type: message`.
|
|
407
|
+
remote_tb = getattr(e, "_forge_remote_traceback", None)
|
|
408
|
+
if remote_tb:
|
|
409
|
+
err_console.print("[red]remote function raised:[/red]")
|
|
410
|
+
err_console.print(str(remote_tb).rstrip("\n"), markup=False, highlight=False)
|
|
411
|
+
else:
|
|
412
|
+
err_console.print(f"[red]{type(e).__name__}[/red]: {e}")
|
|
413
|
+
exit_code = 1
|
|
414
|
+
finally:
|
|
415
|
+
if exit_code == 0:
|
|
416
|
+
# Drain hot containers promptly.
|
|
417
|
+
try:
|
|
418
|
+
app_mod.complete_session()
|
|
419
|
+
except exc.FabricError:
|
|
420
|
+
pass
|
|
421
|
+
# Let the log pump flush the tail, then stop.
|
|
422
|
+
if not no_logs:
|
|
423
|
+
_join_log_thread(log_thread, stop_event)
|
|
424
|
+
client.close()
|
|
425
|
+
|
|
426
|
+
raise typer.Exit(code=exit_code)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _on_ctrl_c(client, run_ids) -> None:
|
|
430
|
+
"""Cancel open runs; report nodes that are unreachable."""
|
|
431
|
+
from ._render import err_console
|
|
432
|
+
from . import exceptions as exc
|
|
433
|
+
|
|
434
|
+
err_console.print("\n[magenta]Ctrl-C — cancelling remote run(s)…[/magenta]")
|
|
435
|
+
for run_id in list(run_ids):
|
|
436
|
+
try:
|
|
437
|
+
run = client.cancel_run(run_id)
|
|
438
|
+
except exc.FabricError as e:
|
|
439
|
+
err_console.print(f" cancel {run_id}: [red]{e}[/red]")
|
|
440
|
+
continue
|
|
441
|
+
state = run.get("state")
|
|
442
|
+
if state == "cancelling":
|
|
443
|
+
node_id = run.get("node_id")
|
|
444
|
+
unreachable = _node_unreachable(client, node_id)
|
|
445
|
+
if unreachable:
|
|
446
|
+
err_console.print(
|
|
447
|
+
f" {run_id}: [yellow]cancellation pending on node[/yellow] "
|
|
448
|
+
f"(node {node_id} unreachable)"
|
|
449
|
+
)
|
|
450
|
+
else:
|
|
451
|
+
err_console.print(f" {run_id}: cancelling…")
|
|
452
|
+
else:
|
|
453
|
+
err_console.print(f" {run_id}: {state}")
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _node_unreachable(client, node_id: Optional[str]) -> bool:
|
|
457
|
+
"""Is the node holding a cancelling run unreachable? Best effort.
|
|
458
|
+
|
|
459
|
+
Degrades quietly by design: this only decorates a cancellation message, and
|
|
460
|
+
failing the cancel because the decoration could not be fetched would be
|
|
461
|
+
worse than the message being vague.
|
|
462
|
+
|
|
463
|
+
KNOWN GAP, since `GET /v1/nodes/{node_id}` became staff-only: an ordinary
|
|
464
|
+
user token now gets `403 PERMISSION_DENIED` here, which is swallowed the
|
|
465
|
+
same as any other FabricError, so the caller prints "cancelling…" even when
|
|
466
|
+
the node really is unreachable and the cancellation really is stuck. The
|
|
467
|
+
answer is wrong in the safe direction (it never claims a node IS
|
|
468
|
+
unreachable), and it is now wrong for EVERY non-staff caller rather than
|
|
469
|
+
only when the lookup fails. Not redesigned here — telling the two apart
|
|
470
|
+
means either a per-run node-health field on `GET /v1/runs/{id}` or
|
|
471
|
+
distinguishing 403 from the rest, and both are their own change.
|
|
472
|
+
"""
|
|
473
|
+
from . import exceptions as exc
|
|
474
|
+
|
|
475
|
+
if not node_id:
|
|
476
|
+
return False
|
|
477
|
+
try:
|
|
478
|
+
node = client._request("GET", f"/v1/nodes/{node_id}")
|
|
479
|
+
except exc.FabricError:
|
|
480
|
+
return False
|
|
481
|
+
return node.get("state") == "unreachable"
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _join_log_thread(thread, stop_event) -> None:
|
|
485
|
+
# Give the stream a moment to deliver the eof event after the run finishes.
|
|
486
|
+
thread.join(timeout=5.0)
|
|
487
|
+
stop_event.set()
|
|
488
|
+
thread.join(timeout=2.0)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
# --------------------------------------------------------------------------
|
|
492
|
+
# log streaming (shared by `run` and `logs --follow`)
|
|
493
|
+
# --------------------------------------------------------------------------
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _stream_run_logs(
|
|
497
|
+
client, run_id: str, stop_event=None, phase_ui: bool = False
|
|
498
|
+
) -> Optional[str]:
|
|
499
|
+
"""Stream SSE logs + state events until eof. Returns final run state.
|
|
500
|
+
|
|
501
|
+
Resilient to disconnects: on a transport error we reconnect from the last
|
|
502
|
+
seen seq (resume cursor).
|
|
503
|
+
|
|
504
|
+
``phase_ui=True`` (``blade run``, clean mode) renders a braille spinner
|
|
505
|
+
while the run is pending/scheduled/starting and a resolved
|
|
506
|
+
``✓ succeeded (dur)`` / ``✗ failed (dur)`` line at the end, INSTEAD of the
|
|
507
|
+
per-transition ``[clock] • state`` lines. The user program's stdout/stderr
|
|
508
|
+
is streamed unchanged in every mode. In verbose mode (or for ``blade logs
|
|
509
|
+
-f``, which passes ``phase_ui=False``) the classic state-transition feed is
|
|
510
|
+
kept.
|
|
511
|
+
"""
|
|
512
|
+
from ._render import (
|
|
513
|
+
print_log_chunk,
|
|
514
|
+
print_state_transition,
|
|
515
|
+
err_console,
|
|
516
|
+
flush_log_tail,
|
|
517
|
+
is_verbose,
|
|
518
|
+
RunProgress,
|
|
519
|
+
)
|
|
520
|
+
from ._constants import RUN_TERMINAL_STATES
|
|
521
|
+
from . import exceptions as exc
|
|
522
|
+
|
|
523
|
+
progress = None
|
|
524
|
+
if phase_ui and not is_verbose():
|
|
525
|
+
progress = RunProgress(_run_progress_meta(client, run_id), run_id=run_id)
|
|
526
|
+
progress.start()
|
|
527
|
+
|
|
528
|
+
cursor: Optional[int] = None
|
|
529
|
+
final_state: Optional[str] = None
|
|
530
|
+
|
|
531
|
+
try:
|
|
532
|
+
while stop_event is None or not stop_event.is_set():
|
|
533
|
+
try:
|
|
534
|
+
for event, data, event_id in client.stream_logs(run_id, cursor=cursor):
|
|
535
|
+
if stop_event is not None and stop_event.is_set():
|
|
536
|
+
return final_state
|
|
537
|
+
if event == "log":
|
|
538
|
+
# The server is trusted, but a truncated SSE chunk on a flaky
|
|
539
|
+
# connection can still yield a non-JSON `data:` or a non-numeric
|
|
540
|
+
# `id:`. Skip the unreadable event rather than crash the whole
|
|
541
|
+
# `blade run` / `blade logs -f` with a raw stacktrace.
|
|
542
|
+
try:
|
|
543
|
+
chunk = json.loads(data)
|
|
544
|
+
except ValueError:
|
|
545
|
+
continue
|
|
546
|
+
# Retire the spinner before the first user output so it
|
|
547
|
+
# never fights the streamed stdout (never suppress it).
|
|
548
|
+
if progress is not None:
|
|
549
|
+
progress.before_log()
|
|
550
|
+
print_log_chunk(chunk)
|
|
551
|
+
if isinstance(chunk, dict) and "seq" in chunk:
|
|
552
|
+
cursor = chunk["seq"]
|
|
553
|
+
elif event_id is not None:
|
|
554
|
+
try:
|
|
555
|
+
cursor = int(event_id)
|
|
556
|
+
except ValueError:
|
|
557
|
+
pass
|
|
558
|
+
elif event == "state":
|
|
559
|
+
try:
|
|
560
|
+
payload = json.loads(data)
|
|
561
|
+
except ValueError:
|
|
562
|
+
continue
|
|
563
|
+
final_state = payload.get("state")
|
|
564
|
+
if progress is not None:
|
|
565
|
+
progress.on_state(final_state)
|
|
566
|
+
else:
|
|
567
|
+
print_state_transition(final_state, payload.get("ts"))
|
|
568
|
+
elif event == "eof":
|
|
569
|
+
return final_state
|
|
570
|
+
elif event == "ping":
|
|
571
|
+
continue
|
|
572
|
+
# Generator ended without eof (server closed): stop.
|
|
573
|
+
return final_state
|
|
574
|
+
except exc.ConnectionFailed:
|
|
575
|
+
if stop_event is not None and stop_event.is_set():
|
|
576
|
+
return final_state
|
|
577
|
+
# Don't reconnect forever: if the run is already terminal the server
|
|
578
|
+
# may have closed the socket without an `eof` (e.g. control-plane
|
|
579
|
+
# restart / proxy drop after the run finished). Without this guard
|
|
580
|
+
# `blade logs -f <terminal_run>` (stop_event is None) spins at 2 req/s
|
|
581
|
+
# for ever. `eof` stays the clean terminator; this is the fallback.
|
|
582
|
+
try:
|
|
583
|
+
run = client.get_run(run_id)
|
|
584
|
+
except exc.FabricError:
|
|
585
|
+
run = None
|
|
586
|
+
if run is not None:
|
|
587
|
+
state = run.get("state")
|
|
588
|
+
if state in RUN_TERMINAL_STATES:
|
|
589
|
+
final_state = state
|
|
590
|
+
return state
|
|
591
|
+
err_console.print("[dim]log stream dropped, reconnecting…[/dim]")
|
|
592
|
+
import time
|
|
593
|
+
|
|
594
|
+
time.sleep(0.5)
|
|
595
|
+
continue
|
|
596
|
+
except exc.FabricError as e:
|
|
597
|
+
raise _handle_fabric_error(e)
|
|
598
|
+
|
|
599
|
+
return final_state
|
|
600
|
+
finally:
|
|
601
|
+
if progress is not None:
|
|
602
|
+
progress.finish(final_state)
|
|
603
|
+
# Emit any residual unterminated line (chunk boundary mid-line) so it
|
|
604
|
+
# isn't silently dropped once the stream ends.
|
|
605
|
+
flush_log_tail()
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _run_progress_meta(client, run_id: str) -> str:
|
|
609
|
+
"""Best-effort spinner metadata: ``<gpu> · <short run id>``. The 'state'
|
|
610
|
+
SSE payload carries no resources, so the GPU type comes from one get_run
|
|
611
|
+
(recon anchor dto.go:65); a failure just drops the GPU part."""
|
|
612
|
+
from . import exceptions as exc
|
|
613
|
+
|
|
614
|
+
short = run_id[:12] + "…" if len(run_id) > 12 else run_id
|
|
615
|
+
try:
|
|
616
|
+
run = client.get_run(run_id)
|
|
617
|
+
except exc.FabricError:
|
|
618
|
+
return short
|
|
619
|
+
gpu = (run.get("resources") or {}).get("gpu_type")
|
|
620
|
+
if gpu and gpu != "none":
|
|
621
|
+
return f"{gpu} · {short}"
|
|
622
|
+
return short
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
# --------------------------------------------------------------------------
|
|
626
|
+
# logs
|
|
627
|
+
# --------------------------------------------------------------------------
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
@app.command()
|
|
631
|
+
def logs(
|
|
632
|
+
run_id: str = typer.Argument(..., help="Run id."),
|
|
633
|
+
follow: bool = typer.Option(
|
|
634
|
+
False, "-f", "--follow", help="Stream live until the run finishes."
|
|
635
|
+
),
|
|
636
|
+
since: Optional[int] = typer.Option(
|
|
637
|
+
None, "--since", help="Resume from this seq (cursor)."
|
|
638
|
+
),
|
|
639
|
+
):
|
|
640
|
+
"""Show a run's logs, optionally following live with resume support."""
|
|
641
|
+
from ._render import print_log_chunk, flush_log_tail, warn_logs_incomplete
|
|
642
|
+
from . import exceptions as exc
|
|
643
|
+
|
|
644
|
+
client, _ = _make_client()
|
|
645
|
+
try:
|
|
646
|
+
if follow:
|
|
647
|
+
_stream_run_logs(client, run_id)
|
|
648
|
+
else:
|
|
649
|
+
cursor = since
|
|
650
|
+
incomplete = False
|
|
651
|
+
try:
|
|
652
|
+
while True:
|
|
653
|
+
page = client.get_logs(run_id, cursor=cursor)
|
|
654
|
+
for chunk in page.get("items", []):
|
|
655
|
+
print_log_chunk(chunk)
|
|
656
|
+
incomplete = incomplete or bool(page.get("incomplete"))
|
|
657
|
+
nxt = page.get("next_cursor")
|
|
658
|
+
if nxt is None:
|
|
659
|
+
break
|
|
660
|
+
cursor = nxt
|
|
661
|
+
finally:
|
|
662
|
+
flush_log_tail()
|
|
663
|
+
if incomplete:
|
|
664
|
+
warn_logs_incomplete()
|
|
665
|
+
except exc.FabricError as e:
|
|
666
|
+
raise _handle_fabric_error(e)
|
|
667
|
+
finally:
|
|
668
|
+
client.close()
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
# --------------------------------------------------------------------------
|
|
672
|
+
# status
|
|
673
|
+
# --------------------------------------------------------------------------
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
@app.command()
|
|
677
|
+
def status(run_id: str = typer.Argument(..., help="Run id.")):
|
|
678
|
+
"""Show a run's state; if pending, its position in the scheduler queue."""
|
|
679
|
+
from ._render import console, style_state, duration_str
|
|
680
|
+
from . import exceptions as exc
|
|
681
|
+
|
|
682
|
+
client, _ = _make_client()
|
|
683
|
+
try:
|
|
684
|
+
run = client.get_run(run_id)
|
|
685
|
+
except exc.FabricError as e:
|
|
686
|
+
raise _handle_fabric_error(e)
|
|
687
|
+
try:
|
|
688
|
+
state = run.get("state")
|
|
689
|
+
console.print(f"run : {run_id}")
|
|
690
|
+
console.print("state : ", style_state(state))
|
|
691
|
+
if run.get("node_id"):
|
|
692
|
+
console.print(f"node : {run['node_id']}")
|
|
693
|
+
res = run.get("resources") or {}
|
|
694
|
+
console.print(f"gpu : {res.get('gpu_type', '-')}")
|
|
695
|
+
console.print(
|
|
696
|
+
f"duration: {duration_str(run.get('started_at'), run.get('finished_at'))}"
|
|
697
|
+
)
|
|
698
|
+
console.print(
|
|
699
|
+
f"calls : {run.get('calls_finished', 0)}/{run.get('calls_total', 0)}"
|
|
700
|
+
)
|
|
701
|
+
if state == "pending":
|
|
702
|
+
pos = _queue_position(client, run)
|
|
703
|
+
if pos is not None:
|
|
704
|
+
console.print(f"queue : position {pos} (waiting for capacity)")
|
|
705
|
+
err = run.get("error")
|
|
706
|
+
if err:
|
|
707
|
+
console.print(
|
|
708
|
+
f"error : [red]{err.get('code')}[/red]: {err.get('message')}"
|
|
709
|
+
)
|
|
710
|
+
finally:
|
|
711
|
+
client.close()
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _queue_position(client, run: dict) -> Optional[int]:
|
|
715
|
+
"""1-based position of this pending run in the FIFO queue.
|
|
716
|
+
|
|
717
|
+
The scheduler queue is FIFO by created_at (schema.sql runs_scheduler_queue);
|
|
718
|
+
position = number of pending runs created at-or-before this one, including
|
|
719
|
+
itself. ``GET /v1/runs?state=pending`` returns runs most-recent-first; we
|
|
720
|
+
page through and count those with created_at <= ours.
|
|
721
|
+
"""
|
|
722
|
+
target_id = run.get("run_id")
|
|
723
|
+
target_created = run.get("created_at")
|
|
724
|
+
if not target_created:
|
|
725
|
+
return None
|
|
726
|
+
count = 0
|
|
727
|
+
cursor = None
|
|
728
|
+
seen_self = False
|
|
729
|
+
while True:
|
|
730
|
+
page = client.list_runs(state=["pending"], limit=200, cursor=cursor)
|
|
731
|
+
for r in page.get("items", []):
|
|
732
|
+
if r.get("run_id") == target_id:
|
|
733
|
+
seen_self = True
|
|
734
|
+
# Count pending runs ahead (created earlier) plus self.
|
|
735
|
+
if (r.get("created_at") or "") <= target_created:
|
|
736
|
+
count += 1
|
|
737
|
+
cursor = page.get("next_cursor")
|
|
738
|
+
if cursor is None:
|
|
739
|
+
break
|
|
740
|
+
if not seen_self:
|
|
741
|
+
# Run already left the pending queue between get_run and the list.
|
|
742
|
+
return None
|
|
743
|
+
return count
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
# --------------------------------------------------------------------------
|
|
747
|
+
# runs (list)
|
|
748
|
+
# --------------------------------------------------------------------------
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
@app.command()
|
|
752
|
+
def runs(
|
|
753
|
+
state: Optional[list[str]] = typer.Option(
|
|
754
|
+
None, "--state", help="Filter by state (repeatable)."
|
|
755
|
+
),
|
|
756
|
+
limit: int = typer.Option(50, "--limit", min=1, max=200),
|
|
757
|
+
):
|
|
758
|
+
"""List runs (most recent first)."""
|
|
759
|
+
from ._render import console, runs_table
|
|
760
|
+
from . import exceptions as exc
|
|
761
|
+
|
|
762
|
+
client, _ = _make_client()
|
|
763
|
+
try:
|
|
764
|
+
page = client.list_runs(state=state, limit=limit)
|
|
765
|
+
except exc.FabricError as e:
|
|
766
|
+
raise _handle_fabric_error(e)
|
|
767
|
+
try:
|
|
768
|
+
items = page.get("items", [])
|
|
769
|
+
if not items:
|
|
770
|
+
console.print("[dim]no runs[/dim]")
|
|
771
|
+
return
|
|
772
|
+
console.print(runs_table(items))
|
|
773
|
+
finally:
|
|
774
|
+
client.close()
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
# --------------------------------------------------------------------------
|
|
778
|
+
# cancel
|
|
779
|
+
# --------------------------------------------------------------------------
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
@app.command()
|
|
783
|
+
def cancel(run_id: str = typer.Argument(..., help="Run id.")):
|
|
784
|
+
"""Cancel a run."""
|
|
785
|
+
from ._render import console, err_console
|
|
786
|
+
from . import exceptions as exc
|
|
787
|
+
|
|
788
|
+
client, _ = _make_client()
|
|
789
|
+
try:
|
|
790
|
+
run = client.cancel_run(run_id)
|
|
791
|
+
except exc.FabricError as e:
|
|
792
|
+
raise _handle_fabric_error(e)
|
|
793
|
+
try:
|
|
794
|
+
state = run.get("state")
|
|
795
|
+
if state == "cancelling" and _node_unreachable(client, run.get("node_id")):
|
|
796
|
+
err_console.print(
|
|
797
|
+
f"[yellow]cancellation pending on node[/yellow] "
|
|
798
|
+
f"(node {run.get('node_id')} unreachable)"
|
|
799
|
+
)
|
|
800
|
+
else:
|
|
801
|
+
console.print(f"[magenta]cancel[/magenta] {run_id}: {state}")
|
|
802
|
+
finally:
|
|
803
|
+
client.close()
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
# --------------------------------------------------------------------------
|
|
807
|
+
# admin (PAAS-24): contain-user / force-stop
|
|
808
|
+
# --------------------------------------------------------------------------
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
_FORCE_STOP_KINDS = ("app", "deployment", "run", "sandbox", "map")
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
@admin_app.command("contain-user")
|
|
815
|
+
def admin_contain_user(
|
|
816
|
+
user_id: str = typer.Argument(..., help="User id (user_…)."),
|
|
817
|
+
reason: str = typer.Option(
|
|
818
|
+
"", "--reason", "-r", help="Operator note, recorded in the audit trail."
|
|
819
|
+
),
|
|
820
|
+
):
|
|
821
|
+
"""Stop every workload of an account (graceful, then hard): runs, apps,
|
|
822
|
+
sandboxes and maps. Idempotent; nothing is restarted later.
|
|
823
|
+
|
|
824
|
+
This does NOT suspend the account. Suspension is iam's and is done in the
|
|
825
|
+
backoffice (the account page's Suspend action), which also calls this;
|
|
826
|
+
AI Fabric contains every account iam lists as suspended on its own within one
|
|
827
|
+
tick anyway. Use this command to stop an account's workloads now."""
|
|
828
|
+
from ._render import console
|
|
829
|
+
from . import exceptions as exc
|
|
830
|
+
|
|
831
|
+
client, _ = _make_client()
|
|
832
|
+
try:
|
|
833
|
+
out = client.admin_contain_user(user_id, reason)
|
|
834
|
+
except exc.FabricError as e:
|
|
835
|
+
raise _handle_fabric_error(e)
|
|
836
|
+
finally:
|
|
837
|
+
client.close()
|
|
838
|
+
c = out.get("contained", {})
|
|
839
|
+
console.print(
|
|
840
|
+
f"[magenta]contained[/magenta] {user_id}: "
|
|
841
|
+
f"{c.get('runs_stopped', 0)} run(s), {c.get('apps_stopped', 0)} app(s), "
|
|
842
|
+
f"{c.get('sandboxes_stopped', 0)} sandbox(es) stopped, "
|
|
843
|
+
f"{c.get('maps_cancelled', 0)} map(s) cancelled"
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
@admin_app.command("force-stop")
|
|
848
|
+
def admin_force_stop(
|
|
849
|
+
kind: str = typer.Argument(..., help="One of: app, deployment, run, sandbox, map."),
|
|
850
|
+
ref: str = typer.Argument(..., help="Id (or app name) of the workload."),
|
|
851
|
+
):
|
|
852
|
+
"""Force-stop ANY tenant's workload regardless of ownership. Audited."""
|
|
853
|
+
from ._render import console, err_console
|
|
854
|
+
from . import exceptions as exc
|
|
855
|
+
|
|
856
|
+
if kind not in _FORCE_STOP_KINDS:
|
|
857
|
+
err_console.print(
|
|
858
|
+
f"[red]unknown kind[/red] {kind!r}; expected one of {', '.join(_FORCE_STOP_KINDS)}"
|
|
859
|
+
)
|
|
860
|
+
raise typer.Exit(2)
|
|
861
|
+
client, _ = _make_client()
|
|
862
|
+
try:
|
|
863
|
+
out = client.admin_force_stop(kind, ref)
|
|
864
|
+
except exc.FabricError as e:
|
|
865
|
+
raise _handle_fabric_error(e)
|
|
866
|
+
finally:
|
|
867
|
+
client.close()
|
|
868
|
+
console.print(f"[magenta]force-stop[/magenta] {kind} {ref}: {out.get('state')}")
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
# --------------------------------------------------------------------------
|
|
872
|
+
# nodes
|
|
873
|
+
# --------------------------------------------------------------------------
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
@app.command()
|
|
877
|
+
def nodes(
|
|
878
|
+
state: Optional[list[str]] = typer.Option(
|
|
879
|
+
None, "--state", help="Filter by node state (repeatable)."
|
|
880
|
+
),
|
|
881
|
+
):
|
|
882
|
+
"""Fleet view: GPUs free/busy per node (requires a staff session token).
|
|
883
|
+
|
|
884
|
+
STAFF ONLY. `GET /v1/nodes` is the whole physical fleet with no tenant
|
|
885
|
+
predicate — it names every node and, per GPU, the run id holding it, which
|
|
886
|
+
belongs to whichever tenant is on that GPU. It is gated on the `ops` or
|
|
887
|
+
`support` staff persona (contracts/admin-route-policy.json); an ordinary
|
|
888
|
+
user token gets `403 PERMISSION_DENIED`, and a staff `fg_` key
|
|
889
|
+
`403 INSUFFICIENT_SCOPE` (it carries no compute:admin, PAAS-630).
|
|
890
|
+
|
|
891
|
+
Kept as a top-level command rather than moved under `blade admin` (whose
|
|
892
|
+
help already says "requires a staff SESSION token"): `blade admin` is the
|
|
893
|
+
containment surface — contain-user, force-stop — and renaming a shipped
|
|
894
|
+
command is a breaking change for the operators who run this one. The help
|
|
895
|
+
text carries the requirement instead.
|
|
896
|
+
"""
|
|
897
|
+
from ._render import console, nodes_table
|
|
898
|
+
from . import exceptions as exc
|
|
899
|
+
|
|
900
|
+
client, _ = _make_client()
|
|
901
|
+
try:
|
|
902
|
+
page = client._request(
|
|
903
|
+
"GET",
|
|
904
|
+
"/v1/nodes",
|
|
905
|
+
params={"limit": 200, **({"state": state} if state else {})},
|
|
906
|
+
)
|
|
907
|
+
except exc.FabricError as e:
|
|
908
|
+
raise _handle_fabric_error(e)
|
|
909
|
+
try:
|
|
910
|
+
items = page.get("items", [])
|
|
911
|
+
if not items:
|
|
912
|
+
console.print("[dim]no nodes registered[/dim]")
|
|
913
|
+
return
|
|
914
|
+
console.print(nodes_table(items))
|
|
915
|
+
finally:
|
|
916
|
+
client.close()
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
# --------------------------------------------------------------------------
|
|
920
|
+
# serve (dev loop)
|
|
921
|
+
# --------------------------------------------------------------------------
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
@app.command()
|
|
925
|
+
def serve(script: Path = typer.Argument(..., help="Python script defining the app.")):
|
|
926
|
+
"""Dev loop: run the entrypoint, watch files, hot-reload the bundle (< 5s).
|
|
927
|
+
|
|
928
|
+
The run is created with serve=true so its container never drains on idle.
|
|
929
|
+
Edit a file → the bundle is re-pushed (deduplicated by sha256) and the
|
|
930
|
+
container reloads the code while keeping the loaded model hot. Ctrl-C stops.
|
|
931
|
+
"""
|
|
932
|
+
from ._loader import load_script, select_app_and_entrypoint
|
|
933
|
+
from ._serve import serve as run_serve
|
|
934
|
+
from . import exceptions as exc
|
|
935
|
+
|
|
936
|
+
client, cfg = _make_client()
|
|
937
|
+
try:
|
|
938
|
+
module, module_name, bundle_root = load_script(script)
|
|
939
|
+
forge_app, entrypoint = select_app_and_entrypoint(module)
|
|
940
|
+
except exc.FabricError as e:
|
|
941
|
+
client.close()
|
|
942
|
+
raise _handle_fabric_error(e)
|
|
943
|
+
|
|
944
|
+
forge_app._bind_client(client)
|
|
945
|
+
forge_app._set_bundle_context(bundle_root, module_name)
|
|
946
|
+
try:
|
|
947
|
+
code = run_serve(client, forge_app, entrypoint, bundle_root)
|
|
948
|
+
finally:
|
|
949
|
+
client.close()
|
|
950
|
+
raise typer.Exit(code=code)
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
# --------------------------------------------------------------------------
|
|
954
|
+
# deploy / app
|
|
955
|
+
# --------------------------------------------------------------------------
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
def _parse_labels(labels: Optional[list[str]]) -> dict[str, str]:
|
|
959
|
+
parsed: dict[str, str] = {}
|
|
960
|
+
for item in labels or []:
|
|
961
|
+
if "=" not in item:
|
|
962
|
+
from ._render import err_console
|
|
963
|
+
|
|
964
|
+
err_console.print(f"[red]bad label[/red] {item!r} (expected KEY=VALUE)")
|
|
965
|
+
raise typer.Exit(code=2)
|
|
966
|
+
k, v = item.split("=", 1)
|
|
967
|
+
if not k:
|
|
968
|
+
from ._render import err_console
|
|
969
|
+
|
|
970
|
+
err_console.print(f"[red]bad label[/red] {item!r} (empty key)")
|
|
971
|
+
raise typer.Exit(code=2)
|
|
972
|
+
parsed[k] = v
|
|
973
|
+
return parsed
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _select_single_app(module):
|
|
977
|
+
from ._loader import find_apps
|
|
978
|
+
from . import exceptions as exc
|
|
979
|
+
|
|
980
|
+
apps = find_apps(module)
|
|
981
|
+
if not apps:
|
|
982
|
+
raise exc.FabricError("no bladekit.App found in script")
|
|
983
|
+
if len(apps) > 1:
|
|
984
|
+
raise exc.FabricError(
|
|
985
|
+
f"multiple bladekit.App objects found ({len(apps)}); deploy expects one"
|
|
986
|
+
)
|
|
987
|
+
return apps[0]
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
@app.command()
|
|
991
|
+
def deploy(
|
|
992
|
+
script: Path = typer.Argument(..., help="Python script defining the app."),
|
|
993
|
+
label: Optional[list[str]] = typer.Option(
|
|
994
|
+
None, "--label", help="Deployment label KEY=VALUE (repeatable)."
|
|
995
|
+
),
|
|
996
|
+
verbose: bool = typer.Option(
|
|
997
|
+
False, "--verbose", "-v",
|
|
998
|
+
help="Verbose: raw build steps instead of the clean phase view.",
|
|
999
|
+
),
|
|
1000
|
+
):
|
|
1001
|
+
"""Create a versioned deployment from @app.web_endpoint()/@app.asgi_app().
|
|
1002
|
+
|
|
1003
|
+
Clean mode renders one status line per phase (registering, building,
|
|
1004
|
+
deployed), a gold-arrow line per endpoint route, and an indented dim
|
|
1005
|
+
summary (autoscaling bounds, best-effort cold start). ``--verbose`` keeps
|
|
1006
|
+
the raw build-step stream.
|
|
1007
|
+
"""
|
|
1008
|
+
from ._loader import load_script
|
|
1009
|
+
from ._render import phase, print_deployed, result_line, summary_line, set_verbose
|
|
1010
|
+
from rich.text import Text
|
|
1011
|
+
from . import exceptions as exc
|
|
1012
|
+
|
|
1013
|
+
if verbose:
|
|
1014
|
+
set_verbose(True)
|
|
1015
|
+
|
|
1016
|
+
client, _ = _make_client()
|
|
1017
|
+
try:
|
|
1018
|
+
module, module_name, bundle_root = load_script(script)
|
|
1019
|
+
forge_app = _select_single_app(module)
|
|
1020
|
+
forge_app._bind_client(client)
|
|
1021
|
+
forge_app._set_bundle_context(bundle_root, module_name)
|
|
1022
|
+
|
|
1023
|
+
labels = _parse_labels(label)
|
|
1024
|
+
# Resolve the manifest first — this validates endpoints/crons and (if the
|
|
1025
|
+
# image is a bladekit.Image DSL build) emits the `building` phase — before
|
|
1026
|
+
# any app is registered, so a validation error never leaves an orphan app.
|
|
1027
|
+
manifest = forge_app._deployment_manifest(
|
|
1028
|
+
resolve=True,
|
|
1029
|
+
require_endpoints=True,
|
|
1030
|
+
labels=labels or None,
|
|
1031
|
+
)
|
|
1032
|
+
|
|
1033
|
+
gpu = _manifest_gpu(manifest)
|
|
1034
|
+
reg_meta = forge_app.name if not gpu else f"{forge_app.name} ({gpu})"
|
|
1035
|
+
with phase("registering", reg_meta):
|
|
1036
|
+
app_resp = client.create_app(forge_app.name)
|
|
1037
|
+
app_obj = app_resp["app"]
|
|
1038
|
+
|
|
1039
|
+
url_text = Text(app_obj["url"], style="cyan") if app_obj.get("url") else None
|
|
1040
|
+
with phase("deployed", url_text) as ph:
|
|
1041
|
+
deployment = client.create_deployment(
|
|
1042
|
+
app_obj["app_id"], manifest.to_deployment_request()
|
|
1043
|
+
)
|
|
1044
|
+
ph.update(meta=url_text)
|
|
1045
|
+
|
|
1046
|
+
endpoints = deployment.get("endpoints") or []
|
|
1047
|
+
want_coldstart = any(
|
|
1048
|
+
((e.get("autoscaling") or {}).get("min_replicas") or 0) > 0
|
|
1049
|
+
for e in endpoints
|
|
1050
|
+
)
|
|
1051
|
+
coldstart = (
|
|
1052
|
+
_best_effort_coldstart(client, app_obj["app_id"]) if want_coldstart else None
|
|
1053
|
+
)
|
|
1054
|
+
for endpoint in endpoints:
|
|
1055
|
+
methods = ",".join(endpoint.get("methods") or [])
|
|
1056
|
+
kind = endpoint.get("kind")
|
|
1057
|
+
route = f"{methods} {endpoint.get('path')}".strip()
|
|
1058
|
+
result_line(f"{route} {kind}" if kind else route)
|
|
1059
|
+
summary = _endpoint_summary(endpoint, coldstart)
|
|
1060
|
+
if summary:
|
|
1061
|
+
summary_line(summary)
|
|
1062
|
+
print_deployed(forge_app.name, deployment.get("version"))
|
|
1063
|
+
except exc.FabricError as e:
|
|
1064
|
+
raise _handle_fabric_error(e)
|
|
1065
|
+
finally:
|
|
1066
|
+
client.close()
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def _manifest_gpu(manifest) -> Optional[str]:
|
|
1070
|
+
"""The single GPU type shared by every deployment endpoint, or None (mixed
|
|
1071
|
+
or CPU-only) — used for the `registering` phase metadata."""
|
|
1072
|
+
gpus = set()
|
|
1073
|
+
for ep in getattr(manifest, "endpoints", None) or []:
|
|
1074
|
+
gt = (ep.get("resources") or {}).get("gpu_type")
|
|
1075
|
+
if gt and gt != "none":
|
|
1076
|
+
gpus.add(gt)
|
|
1077
|
+
return next(iter(gpus)) if len(gpus) == 1 else None
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def _endpoint_summary(endpoint: dict, coldstart: Optional[str]) -> str:
|
|
1081
|
+
"""Indented dim summary for one endpoint: autoscaling bounds + optional
|
|
1082
|
+
cold start. Only fields actually present are shown (middot-separated)."""
|
|
1083
|
+
a = endpoint.get("autoscaling") or {}
|
|
1084
|
+
parts: list[str] = []
|
|
1085
|
+
lo, hi = a.get("min_replicas"), a.get("max_replicas")
|
|
1086
|
+
if lo is not None and hi is not None:
|
|
1087
|
+
parts.append(f"min {lo} → max {hi}")
|
|
1088
|
+
if a.get("reserved_replicas"):
|
|
1089
|
+
parts.append(f"reserved {a['reserved_replicas']}")
|
|
1090
|
+
if a.get("max_concurrent_inputs"):
|
|
1091
|
+
parts.append(f"concurrency {a['max_concurrent_inputs']}")
|
|
1092
|
+
if coldstart:
|
|
1093
|
+
parts.append(f"cold-start {coldstart}")
|
|
1094
|
+
return " · ".join(parts)
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def _best_effort_coldstart(client, app_id: str) -> Optional[str]:
|
|
1098
|
+
"""Newest cold-start total for an app as e.g. ``0.41s``; None when there is
|
|
1099
|
+
no ready replica yet (the common case right after deploy) or on any error —
|
|
1100
|
+
the cold-start line is optional, never blocking."""
|
|
1101
|
+
try:
|
|
1102
|
+
resp = client.get_app_coldstarts(app_id, limit=1)
|
|
1103
|
+
except Exception: # noqa: BLE001 — best-effort, optional summary field
|
|
1104
|
+
return None
|
|
1105
|
+
items = (resp or {}).get("items") or []
|
|
1106
|
+
if not items:
|
|
1107
|
+
return None
|
|
1108
|
+
total = ((items[0] or {}).get("phases") or {}).get("total_s")
|
|
1109
|
+
if not isinstance(total, (int, float)):
|
|
1110
|
+
return None
|
|
1111
|
+
return f"{total:.2f}s"
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def _resolve_app_id(client, name_or_id: str) -> str:
|
|
1115
|
+
from . import exceptions as exc
|
|
1116
|
+
|
|
1117
|
+
if name_or_id.startswith("app_"):
|
|
1118
|
+
return name_or_id
|
|
1119
|
+
page = client.list_apps(name=name_or_id, limit=1)
|
|
1120
|
+
items = page.get("items", [])
|
|
1121
|
+
if not items:
|
|
1122
|
+
raise exc.NotFoundError("APP_NOT_FOUND", f"no app named {name_or_id!r}")
|
|
1123
|
+
return items[0]["app_id"]
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
@app_app.command("list")
|
|
1127
|
+
def app_list(
|
|
1128
|
+
state: Optional[str] = typer.Option(None, "--state", help="Filter by state."),
|
|
1129
|
+
limit: int = typer.Option(50, "--limit", min=1, max=200),
|
|
1130
|
+
):
|
|
1131
|
+
"""List deployed apps."""
|
|
1132
|
+
from rich.table import Table
|
|
1133
|
+
from ._render import console, clock_from_ts
|
|
1134
|
+
from . import exceptions as exc
|
|
1135
|
+
|
|
1136
|
+
client, _ = _make_client()
|
|
1137
|
+
try:
|
|
1138
|
+
page = client.list_apps(state=state, limit=limit)
|
|
1139
|
+
items = page.get("items", [])
|
|
1140
|
+
if not items:
|
|
1141
|
+
console.print("[dim]no apps[/dim]")
|
|
1142
|
+
return
|
|
1143
|
+
table = Table(title="apps", header_style="bold")
|
|
1144
|
+
table.add_column("app_id", overflow="fold")
|
|
1145
|
+
table.add_column("name")
|
|
1146
|
+
table.add_column("state")
|
|
1147
|
+
table.add_column("active_deployment")
|
|
1148
|
+
table.add_column("created")
|
|
1149
|
+
for item in items:
|
|
1150
|
+
table.add_row(
|
|
1151
|
+
item.get("app_id", "-"),
|
|
1152
|
+
item.get("name", "-"),
|
|
1153
|
+
item.get("state", "-"),
|
|
1154
|
+
item.get("active_deployment_id") or "-",
|
|
1155
|
+
clock_from_ts(item.get("created_at")),
|
|
1156
|
+
)
|
|
1157
|
+
console.print(table)
|
|
1158
|
+
except exc.FabricError as e:
|
|
1159
|
+
raise _handle_fabric_error(e)
|
|
1160
|
+
finally:
|
|
1161
|
+
client.close()
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
@app_app.command("history")
|
|
1165
|
+
def app_history(
|
|
1166
|
+
app_name_or_id: str = typer.Argument(..., help="App name or app_ id."),
|
|
1167
|
+
limit: int = typer.Option(20, "--limit", min=1, max=200),
|
|
1168
|
+
):
|
|
1169
|
+
"""Show deployment history for an app."""
|
|
1170
|
+
from rich.table import Table
|
|
1171
|
+
from ._render import console, clock_from_ts
|
|
1172
|
+
from . import exceptions as exc
|
|
1173
|
+
|
|
1174
|
+
client, _ = _make_client()
|
|
1175
|
+
try:
|
|
1176
|
+
app_id = _resolve_app_id(client, app_name_or_id)
|
|
1177
|
+
page = client.list_deployments(app_id, limit=limit)
|
|
1178
|
+
items = page.get("items", [])
|
|
1179
|
+
if not items:
|
|
1180
|
+
console.print("[dim]no deployments[/dim]")
|
|
1181
|
+
return
|
|
1182
|
+
table = Table(title="deployments", header_style="bold")
|
|
1183
|
+
table.add_column("deployment_id", overflow="fold")
|
|
1184
|
+
table.add_column("version")
|
|
1185
|
+
table.add_column("state")
|
|
1186
|
+
table.add_column("endpoints")
|
|
1187
|
+
table.add_column("created")
|
|
1188
|
+
for dep in items:
|
|
1189
|
+
table.add_row(
|
|
1190
|
+
dep.get("deployment_id", "-"),
|
|
1191
|
+
str(dep.get("version", "-")),
|
|
1192
|
+
dep.get("state", "-"),
|
|
1193
|
+
str(len(dep.get("endpoints") or [])),
|
|
1194
|
+
clock_from_ts(dep.get("created_at")),
|
|
1195
|
+
)
|
|
1196
|
+
console.print(table)
|
|
1197
|
+
except exc.FabricError as e:
|
|
1198
|
+
raise _handle_fabric_error(e)
|
|
1199
|
+
finally:
|
|
1200
|
+
client.close()
|
|
1201
|
+
|
|
1202
|
+
|
|
1203
|
+
@app_app.command("rollback")
|
|
1204
|
+
def app_rollback(
|
|
1205
|
+
app_name_or_id: str = typer.Argument(..., help="App name or app_ id."),
|
|
1206
|
+
deployment_id: Optional[str] = typer.Option(None, "--deployment-id"),
|
|
1207
|
+
version: Optional[int] = typer.Option(None, "--version", min=1),
|
|
1208
|
+
):
|
|
1209
|
+
"""Rollback an app to a previous deployment id or version."""
|
|
1210
|
+
from ._render import console, err_console
|
|
1211
|
+
from . import exceptions as exc
|
|
1212
|
+
|
|
1213
|
+
if (deployment_id is None) == (version is None):
|
|
1214
|
+
err_console.print("[red]choose exactly one[/red]: --deployment-id or --version")
|
|
1215
|
+
raise typer.Exit(code=2)
|
|
1216
|
+
client, _ = _make_client()
|
|
1217
|
+
try:
|
|
1218
|
+
app_id = _resolve_app_id(client, app_name_or_id)
|
|
1219
|
+
app_obj = client.rollback_app(
|
|
1220
|
+
app_id, deployment_id=deployment_id, version=version
|
|
1221
|
+
)
|
|
1222
|
+
console.print(
|
|
1223
|
+
f"[green]rolled back[/green] {app_obj.get('name', app_id)} -> "
|
|
1224
|
+
f"{app_obj.get('active_deployment_id') or '-'}"
|
|
1225
|
+
)
|
|
1226
|
+
except exc.FabricError as e:
|
|
1227
|
+
raise _handle_fabric_error(e)
|
|
1228
|
+
finally:
|
|
1229
|
+
client.close()
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
@app_app.command("stop")
|
|
1233
|
+
def app_stop(app_name_or_id: str = typer.Argument(..., help="App name or app_ id.")):
|
|
1234
|
+
"""Stop an app and drain its active deployment replicas."""
|
|
1235
|
+
from ._render import console
|
|
1236
|
+
from . import exceptions as exc
|
|
1237
|
+
|
|
1238
|
+
client, _ = _make_client()
|
|
1239
|
+
try:
|
|
1240
|
+
app_id = _resolve_app_id(client, app_name_or_id)
|
|
1241
|
+
app_obj = client.stop_app(app_id)
|
|
1242
|
+
console.print(f"[magenta]stopped[/magenta] {app_obj.get('name', app_id)}")
|
|
1243
|
+
except exc.FabricError as e:
|
|
1244
|
+
raise _handle_fabric_error(e)
|
|
1245
|
+
finally:
|
|
1246
|
+
client.close()
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
@app_app.command("scale")
|
|
1250
|
+
def app_scale(
|
|
1251
|
+
app_name_or_id: str = typer.Argument(..., help="App name or app_ id."),
|
|
1252
|
+
endpoint: Optional[str] = typer.Option(
|
|
1253
|
+
None, "--endpoint", "-e",
|
|
1254
|
+
help="Endpoint name to scale. Defaults to the only endpoint if there is one.",
|
|
1255
|
+
),
|
|
1256
|
+
min_replicas: Optional[int] = typer.Option(None, "--min", help="Minimum replicas (0-50)."),
|
|
1257
|
+
max_replicas: Optional[int] = typer.Option(None, "--max", help="Maximum replicas (1-50)."),
|
|
1258
|
+
concurrency: Optional[int] = typer.Option(
|
|
1259
|
+
None, "--concurrency", help="Max concurrent inputs per replica (1-100)."
|
|
1260
|
+
),
|
|
1261
|
+
idle: Optional[int] = typer.Option(
|
|
1262
|
+
None, "--idle", help="Scale-down idle seconds (30-3600)."
|
|
1263
|
+
),
|
|
1264
|
+
reserved: Optional[int] = typer.Option(
|
|
1265
|
+
None, "--reserved",
|
|
1266
|
+
help="Hybrid: replicas kept on reserved (always-on); the rest burst on the endpoint's class.",
|
|
1267
|
+
),
|
|
1268
|
+
target: Optional[int] = typer.Option(
|
|
1269
|
+
None, "--target",
|
|
1270
|
+
help="Autoscaling v2: per-replica concurrency the autoscaler AIMS for (1-100, <= concurrency).",
|
|
1271
|
+
),
|
|
1272
|
+
buffer: Optional[int] = typer.Option(
|
|
1273
|
+
None, "--buffer",
|
|
1274
|
+
help="Autoscaling v2: warm spare replicas kept while active (0-10; 0 removes the buffer).",
|
|
1275
|
+
),
|
|
1276
|
+
vllm_waiting: Optional[int] = typer.Option(
|
|
1277
|
+
None, "--vllm-waiting",
|
|
1278
|
+
help="vLLM preset: target requests waiting in vLLM's queue per replica (1-1000; 0 removes the preset; web_server only).",
|
|
1279
|
+
),
|
|
1280
|
+
clear_policy: bool = typer.Option(
|
|
1281
|
+
False, "--clear-policy",
|
|
1282
|
+
help="Clear the scaling policy entirely (back to default concurrency autoscaling).",
|
|
1283
|
+
),
|
|
1284
|
+
):
|
|
1285
|
+
"""Update a deployed endpoint's autoscaling bounds live (no redeploy).
|
|
1286
|
+
|
|
1287
|
+
Only the flags you pass change; the autoscaler applies the new bounds within
|
|
1288
|
+
seconds. Example: blade app scale my-api --min 2 --max 8 --reserved 2
|
|
1289
|
+
(2 always-on reserved replicas, bursting up to 6 more on spot.)
|
|
1290
|
+
|
|
1291
|
+
The v2 policy flags (--target/--buffer/--vllm-waiting) are merged into the
|
|
1292
|
+
endpoint's current policy client-side, then sent as a full replacement.
|
|
1293
|
+
Like every scale edit, they last until the next deploy (code wins).
|
|
1294
|
+
"""
|
|
1295
|
+
from ._render import console
|
|
1296
|
+
from . import exceptions as exc
|
|
1297
|
+
|
|
1298
|
+
if (min_replicas is None and max_replicas is None and concurrency is None
|
|
1299
|
+
and idle is None and reserved is None
|
|
1300
|
+
and target is None and buffer is None and vllm_waiting is None
|
|
1301
|
+
and not clear_policy):
|
|
1302
|
+
console.print(
|
|
1303
|
+
"[red]Nothing to change.[/red] Pass at least one of "
|
|
1304
|
+
"--min/--max/--concurrency/--idle/--reserved/--target/--buffer/--vllm-waiting/--clear-policy."
|
|
1305
|
+
)
|
|
1306
|
+
raise typer.Exit(2)
|
|
1307
|
+
if clear_policy and (target is not None or buffer is not None or vllm_waiting is not None):
|
|
1308
|
+
console.print("[red]--clear-policy cannot be combined with --target/--buffer/--vllm-waiting.[/red]")
|
|
1309
|
+
raise typer.Exit(2)
|
|
1310
|
+
|
|
1311
|
+
client, _ = _make_client()
|
|
1312
|
+
try:
|
|
1313
|
+
app_id = _resolve_app_id(client, app_name_or_id)
|
|
1314
|
+
app_obj = client.get_app(app_id)
|
|
1315
|
+
endpoints = (app_obj.get("active_deployment") or {}).get("endpoints") or []
|
|
1316
|
+
if not endpoints:
|
|
1317
|
+
console.print("[red]No active deployment / endpoints to scale.[/red]")
|
|
1318
|
+
raise typer.Exit(1)
|
|
1319
|
+
target_ep = endpoint
|
|
1320
|
+
if target_ep is None:
|
|
1321
|
+
if len(endpoints) == 1:
|
|
1322
|
+
target_ep = endpoints[0]["name"]
|
|
1323
|
+
else:
|
|
1324
|
+
names = ", ".join(e["name"] for e in endpoints)
|
|
1325
|
+
console.print(f"[red]Multiple endpoints[/red] — pass --endpoint <name>: {names}")
|
|
1326
|
+
raise typer.Exit(2)
|
|
1327
|
+
|
|
1328
|
+
# v2 policy: read-modify-write. The server replaces the policy wholesale
|
|
1329
|
+
# (never merges), so the CLI merges the flags into the CURRENT policy.
|
|
1330
|
+
policy: Optional[dict] = None
|
|
1331
|
+
if target is not None or buffer is not None or vllm_waiting is not None:
|
|
1332
|
+
cur = next((e for e in endpoints if e["name"] == target_ep), None)
|
|
1333
|
+
policy = dict(((cur or {}).get("autoscaling") or {}).get("scaling_policy") or {})
|
|
1334
|
+
policy["v"] = 1
|
|
1335
|
+
if target is not None:
|
|
1336
|
+
policy["target_inputs"] = target
|
|
1337
|
+
if buffer is not None:
|
|
1338
|
+
if buffer > 0:
|
|
1339
|
+
policy["buffer_replicas"] = buffer
|
|
1340
|
+
else:
|
|
1341
|
+
policy.pop("buffer_replicas", None)
|
|
1342
|
+
if vllm_waiting is not None:
|
|
1343
|
+
if vllm_waiting > 0:
|
|
1344
|
+
policy["vllm"] = {"target_waiting": vllm_waiting}
|
|
1345
|
+
else:
|
|
1346
|
+
policy.pop("vllm", None)
|
|
1347
|
+
if set(policy.keys()) == {"v"}:
|
|
1348
|
+
# Everything removed → clear rather than send an empty policy.
|
|
1349
|
+
policy = None
|
|
1350
|
+
clear_policy = True
|
|
1351
|
+
|
|
1352
|
+
updated = client.scale_endpoint(
|
|
1353
|
+
app_id,
|
|
1354
|
+
target_ep,
|
|
1355
|
+
min_replicas=min_replicas,
|
|
1356
|
+
max_replicas=max_replicas,
|
|
1357
|
+
max_concurrent_inputs=concurrency,
|
|
1358
|
+
scale_down_idle_seconds=idle,
|
|
1359
|
+
reserved_replicas=reserved,
|
|
1360
|
+
scaling_policy=policy,
|
|
1361
|
+
clear_scaling_policy=clear_policy,
|
|
1362
|
+
)
|
|
1363
|
+
ep = next(
|
|
1364
|
+
(e for e in (updated.get("active_deployment") or {}).get("endpoints", []) if e["name"] == target_ep),
|
|
1365
|
+
None,
|
|
1366
|
+
)
|
|
1367
|
+
a = ep["autoscaling"] if ep else {}
|
|
1368
|
+
rsv = a.get("reserved_replicas") or 0
|
|
1369
|
+
hybrid = f", reserved {rsv}" if rsv else ""
|
|
1370
|
+
pol = a.get("scaling_policy") or {}
|
|
1371
|
+
extras = ""
|
|
1372
|
+
if pol.get("target_inputs"):
|
|
1373
|
+
extras += f", target {pol['target_inputs']}"
|
|
1374
|
+
if pol.get("buffer_replicas"):
|
|
1375
|
+
extras += f", buffer {pol['buffer_replicas']}"
|
|
1376
|
+
if (pol.get("vllm") or {}).get("target_waiting"):
|
|
1377
|
+
extras += f", vllm-waiting {pol['vllm']['target_waiting']}"
|
|
1378
|
+
console.print(
|
|
1379
|
+
f"[green]scaled[/green] {app_obj.get('name', app_id)} / {target_ep} → "
|
|
1380
|
+
f"replicas {a.get('min_replicas')}-{a.get('max_replicas')}{hybrid}, "
|
|
1381
|
+
f"concurrency {a.get('max_concurrent_inputs')}, idle {a.get('scale_down_idle_seconds')}s{extras}"
|
|
1382
|
+
)
|
|
1383
|
+
except exc.FabricError as e:
|
|
1384
|
+
raise _handle_fabric_error(e)
|
|
1385
|
+
finally:
|
|
1386
|
+
client.close()
|
|
1387
|
+
|
|
1388
|
+
|
|
1389
|
+
# --------------------------------------------------------------------------
|
|
1390
|
+
# volume
|
|
1391
|
+
# --------------------------------------------------------------------------
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
@volume_app.command("list")
|
|
1395
|
+
def volume_list():
|
|
1396
|
+
"""List volumes (generation, size, file count)."""
|
|
1397
|
+
from ._render import console, volumes_table
|
|
1398
|
+
from . import exceptions as exc
|
|
1399
|
+
|
|
1400
|
+
client, _ = _make_client()
|
|
1401
|
+
try:
|
|
1402
|
+
page = client.list_volumes(limit=200)
|
|
1403
|
+
except exc.FabricError as e:
|
|
1404
|
+
raise _handle_fabric_error(e)
|
|
1405
|
+
try:
|
|
1406
|
+
items = page.get("items", [])
|
|
1407
|
+
if not items:
|
|
1408
|
+
console.print("[dim]no volumes[/dim]")
|
|
1409
|
+
return
|
|
1410
|
+
console.print(volumes_table(items))
|
|
1411
|
+
finally:
|
|
1412
|
+
client.close()
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
def _resolve_volume_id(client, name_or_id: str) -> str:
|
|
1416
|
+
"""Accept a volume name or a vol_ id; return the id (404 if unknown name)."""
|
|
1417
|
+
from . import exceptions as exc
|
|
1418
|
+
|
|
1419
|
+
if name_or_id.startswith("vol_"):
|
|
1420
|
+
return name_or_id
|
|
1421
|
+
page = client.list_volumes(name=name_or_id, limit=1)
|
|
1422
|
+
items = page.get("items", [])
|
|
1423
|
+
if not items:
|
|
1424
|
+
raise exc.NotFoundError("VOLUME_NOT_FOUND", f"no volume named {name_or_id!r}")
|
|
1425
|
+
return items[0]["volume_id"]
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
@volume_app.command("put")
|
|
1429
|
+
def volume_put(
|
|
1430
|
+
volume: str = typer.Argument(..., help="Volume name or id."),
|
|
1431
|
+
local_path: Path = typer.Argument(..., help="Local file to upload."),
|
|
1432
|
+
remote_path: str = typer.Argument(..., help="Destination path inside the volume."),
|
|
1433
|
+
):
|
|
1434
|
+
"""Upload a single file into the volume as a new generation (presigned)."""
|
|
1435
|
+
import hashlib
|
|
1436
|
+
|
|
1437
|
+
from ._render import console
|
|
1438
|
+
from . import exceptions as exc
|
|
1439
|
+
|
|
1440
|
+
client, _ = _make_client()
|
|
1441
|
+
try:
|
|
1442
|
+
if not local_path.is_file():
|
|
1443
|
+
console.print(f"[red]not a file[/red]: {local_path}")
|
|
1444
|
+
raise typer.Exit(code=1)
|
|
1445
|
+
data = local_path.read_bytes()
|
|
1446
|
+
sha = hashlib.sha256(data).hexdigest()
|
|
1447
|
+
rel = remote_path.lstrip("/")
|
|
1448
|
+
if ".." in Path(rel).parts:
|
|
1449
|
+
console.print(
|
|
1450
|
+
f"[red]invalid path[/red]: {remote_path!r} — '..' components are not allowed"
|
|
1451
|
+
)
|
|
1452
|
+
raise typer.Exit(code=2)
|
|
1453
|
+
volume_id = _resolve_volume_id(client, volume)
|
|
1454
|
+
body = {
|
|
1455
|
+
"entries": [
|
|
1456
|
+
{"path": rel, "type": "file", "size_bytes": len(data), "sha256": sha}
|
|
1457
|
+
]
|
|
1458
|
+
}
|
|
1459
|
+
resp = client.create_volume_commit(volume_id, body)
|
|
1460
|
+
commit = resp["commit"]
|
|
1461
|
+
for up in resp.get("missing_uploads", []):
|
|
1462
|
+
if up["sha256"] == sha:
|
|
1463
|
+
client.upload_to_presigned(
|
|
1464
|
+
up["put_url"], data, "application/octet-stream"
|
|
1465
|
+
)
|
|
1466
|
+
final = client.finalize_volume_commit(volume_id, commit["commit_id"])
|
|
1467
|
+
gen = final.get("generation")
|
|
1468
|
+
console.print(
|
|
1469
|
+
f"[green]put[/green] {rel} → generation {gen} "
|
|
1470
|
+
f"[dim]({len(data)} bytes)[/dim]"
|
|
1471
|
+
)
|
|
1472
|
+
except exc.FabricError as e:
|
|
1473
|
+
raise _handle_fabric_error(e)
|
|
1474
|
+
finally:
|
|
1475
|
+
client.close()
|
|
1476
|
+
|
|
1477
|
+
|
|
1478
|
+
@volume_app.command("get")
|
|
1479
|
+
def volume_get(
|
|
1480
|
+
volume: str = typer.Argument(..., help="Volume name or id."),
|
|
1481
|
+
remote_path: str = typer.Argument(..., help="Path inside the volume."),
|
|
1482
|
+
out: Path = typer.Argument(..., help="Local destination file."),
|
|
1483
|
+
generation: Optional[int] = typer.Option(None, "--generation"),
|
|
1484
|
+
):
|
|
1485
|
+
"""Download a single file from the volume (presigned)."""
|
|
1486
|
+
from ._render import console
|
|
1487
|
+
from . import exceptions as exc
|
|
1488
|
+
|
|
1489
|
+
client, _ = _make_client()
|
|
1490
|
+
try:
|
|
1491
|
+
rel = remote_path.lstrip("/")
|
|
1492
|
+
if ".." in Path(rel).parts:
|
|
1493
|
+
from ._render import err_console as _ec
|
|
1494
|
+
|
|
1495
|
+
_ec.print(
|
|
1496
|
+
f"[red]invalid path[/red]: {remote_path!r} — '..' components are not allowed"
|
|
1497
|
+
)
|
|
1498
|
+
raise typer.Exit(code=2)
|
|
1499
|
+
volume_id = _resolve_volume_id(client, volume)
|
|
1500
|
+
info = client.get_volume_file_content(
|
|
1501
|
+
volume_id, rel, generation=generation
|
|
1502
|
+
)
|
|
1503
|
+
data = client.download_presigned(info["get_url"])
|
|
1504
|
+
out.write_bytes(data)
|
|
1505
|
+
console.print(
|
|
1506
|
+
f"[green]got[/green] {remote_path} → {out} [dim]({len(data)} bytes)[/dim]"
|
|
1507
|
+
)
|
|
1508
|
+
except exc.FabricError as e:
|
|
1509
|
+
raise _handle_fabric_error(e)
|
|
1510
|
+
finally:
|
|
1511
|
+
client.close()
|
|
1512
|
+
|
|
1513
|
+
|
|
1514
|
+
@volume_app.command("ls")
|
|
1515
|
+
def volume_ls(
|
|
1516
|
+
volume: str = typer.Argument(..., help="Volume name or id."),
|
|
1517
|
+
path: str = typer.Option("/", "--path", help="Directory to list."),
|
|
1518
|
+
generation: Optional[int] = typer.Option(None, "--generation"),
|
|
1519
|
+
):
|
|
1520
|
+
"""List entries under a path in the volume (from the manifest)."""
|
|
1521
|
+
from ._render import console, volume_files_table
|
|
1522
|
+
from . import exceptions as exc
|
|
1523
|
+
|
|
1524
|
+
client, _ = _make_client()
|
|
1525
|
+
try:
|
|
1526
|
+
volume_id = _resolve_volume_id(client, volume)
|
|
1527
|
+
page = client.list_volume_files(
|
|
1528
|
+
volume_id, path=path, generation=generation, limit=200
|
|
1529
|
+
)
|
|
1530
|
+
except exc.FabricError as e:
|
|
1531
|
+
raise _handle_fabric_error(e)
|
|
1532
|
+
try:
|
|
1533
|
+
items = page.get("items", [])
|
|
1534
|
+
console.print(f"[dim]generation {page.get('generation')}[/dim]")
|
|
1535
|
+
if not items:
|
|
1536
|
+
console.print("[dim](empty)[/dim]")
|
|
1537
|
+
return
|
|
1538
|
+
console.print(volume_files_table(items))
|
|
1539
|
+
finally:
|
|
1540
|
+
client.close()
|
|
1541
|
+
|
|
1542
|
+
|
|
1543
|
+
@volume_app.command("rm")
|
|
1544
|
+
def volume_rm(volume: str = typer.Argument(..., help="Volume name or id.")):
|
|
1545
|
+
"""Delete a volume (refused if mounted by an active run: VOLUME_IN_USE)."""
|
|
1546
|
+
from ._render import console
|
|
1547
|
+
from . import exceptions as exc
|
|
1548
|
+
|
|
1549
|
+
client, _ = _make_client()
|
|
1550
|
+
try:
|
|
1551
|
+
volume_id = _resolve_volume_id(client, volume)
|
|
1552
|
+
client.delete_volume(volume_id)
|
|
1553
|
+
console.print(f"[magenta]deleted[/magenta] volume {volume}")
|
|
1554
|
+
except exc.FabricError as e:
|
|
1555
|
+
raise _handle_fabric_error(e)
|
|
1556
|
+
finally:
|
|
1557
|
+
client.close()
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
# --------------------------------------------------------------------------
|
|
1561
|
+
# secret
|
|
1562
|
+
# --------------------------------------------------------------------------
|
|
1563
|
+
|
|
1564
|
+
|
|
1565
|
+
@secret_app.command("set")
|
|
1566
|
+
def secret_set(
|
|
1567
|
+
name: str = typer.Argument(..., help="Secret name."),
|
|
1568
|
+
from_env: Optional[list[str]] = typer.Option(
|
|
1569
|
+
None,
|
|
1570
|
+
"--from-env",
|
|
1571
|
+
help="Read KEY from the environment (repeatable: --from-env HF_TOKEN).",
|
|
1572
|
+
),
|
|
1573
|
+
):
|
|
1574
|
+
"""Create or replace a secret. Values come from --from-env or stdin — NEVER argv.
|
|
1575
|
+
|
|
1576
|
+
Reading values from a positional argument would leak them into ``ps``/shell
|
|
1577
|
+
history, so it is intentionally impossible. With no --from-env, the secret is
|
|
1578
|
+
read from stdin as ``KEY=VALUE`` lines (one per line).
|
|
1579
|
+
"""
|
|
1580
|
+
import os
|
|
1581
|
+
|
|
1582
|
+
from ._render import console
|
|
1583
|
+
from . import exceptions as exc
|
|
1584
|
+
|
|
1585
|
+
values: dict[str, str] = {}
|
|
1586
|
+
if from_env:
|
|
1587
|
+
for key in from_env:
|
|
1588
|
+
if key not in os.environ:
|
|
1589
|
+
console.print(f"[red]env var not set[/red]: {key}")
|
|
1590
|
+
raise typer.Exit(code=2)
|
|
1591
|
+
values[key] = os.environ[key]
|
|
1592
|
+
else:
|
|
1593
|
+
# Read KEY=VALUE lines from stdin (so the value never appears in argv).
|
|
1594
|
+
if sys.stdin.isatty():
|
|
1595
|
+
console.print(
|
|
1596
|
+
"[dim]reading KEY=VALUE lines from stdin (Ctrl-D to finish)…[/dim]"
|
|
1597
|
+
)
|
|
1598
|
+
for line in sys.stdin:
|
|
1599
|
+
line = line.rstrip("\n")
|
|
1600
|
+
if not line or line.startswith("#"):
|
|
1601
|
+
continue
|
|
1602
|
+
if "=" not in line:
|
|
1603
|
+
console.print(f"[red]bad line[/red] (expected KEY=VALUE): {line!r}")
|
|
1604
|
+
raise typer.Exit(code=2)
|
|
1605
|
+
k, v = line.split("=", 1)
|
|
1606
|
+
values[k.strip()] = v
|
|
1607
|
+
if not values:
|
|
1608
|
+
console.print("[red]no values provided[/red]")
|
|
1609
|
+
raise typer.Exit(code=2)
|
|
1610
|
+
|
|
1611
|
+
client, _ = _make_client()
|
|
1612
|
+
try:
|
|
1613
|
+
secret = client.set_secret(name, values)
|
|
1614
|
+
console.print(
|
|
1615
|
+
f"[green]set[/green] secret {secret['name']} "
|
|
1616
|
+
f"[dim](keys: {', '.join(secret.get('env_keys', []))})[/dim]"
|
|
1617
|
+
)
|
|
1618
|
+
except exc.FabricError as e:
|
|
1619
|
+
raise _handle_fabric_error(e)
|
|
1620
|
+
finally:
|
|
1621
|
+
client.close()
|
|
1622
|
+
|
|
1623
|
+
|
|
1624
|
+
@secret_app.command("list")
|
|
1625
|
+
def secret_list():
|
|
1626
|
+
"""List secrets — names and env key names only (never values)."""
|
|
1627
|
+
from ._render import console, secrets_table
|
|
1628
|
+
from . import exceptions as exc
|
|
1629
|
+
|
|
1630
|
+
client, _ = _make_client()
|
|
1631
|
+
try:
|
|
1632
|
+
page = client.list_secrets(limit=200)
|
|
1633
|
+
except exc.FabricError as e:
|
|
1634
|
+
raise _handle_fabric_error(e)
|
|
1635
|
+
try:
|
|
1636
|
+
items = page.get("items", [])
|
|
1637
|
+
if not items:
|
|
1638
|
+
console.print("[dim]no secrets[/dim]")
|
|
1639
|
+
return
|
|
1640
|
+
console.print(secrets_table(items))
|
|
1641
|
+
finally:
|
|
1642
|
+
client.close()
|
|
1643
|
+
|
|
1644
|
+
|
|
1645
|
+
@secret_app.command("rm")
|
|
1646
|
+
def secret_rm(name: str = typer.Argument(..., help="Secret name or id.")):
|
|
1647
|
+
"""Delete a secret (never blocked; pending runs referencing it will fail)."""
|
|
1648
|
+
from ._render import console
|
|
1649
|
+
from . import exceptions as exc
|
|
1650
|
+
|
|
1651
|
+
client, _ = _make_client()
|
|
1652
|
+
try:
|
|
1653
|
+
secret_id = name
|
|
1654
|
+
if not name.startswith("scr_"):
|
|
1655
|
+
page = client.list_secrets(name=name, limit=1)
|
|
1656
|
+
items = page.get("items", [])
|
|
1657
|
+
if not items:
|
|
1658
|
+
raise exc.NotFoundError(
|
|
1659
|
+
"SECRET_NOT_FOUND", f"no secret named {name!r}"
|
|
1660
|
+
)
|
|
1661
|
+
secret_id = items[0]["secret_id"]
|
|
1662
|
+
client.delete_secret(secret_id)
|
|
1663
|
+
console.print(f"[magenta]deleted[/magenta] secret {name}")
|
|
1664
|
+
except exc.FabricError as e:
|
|
1665
|
+
raise _handle_fabric_error(e)
|
|
1666
|
+
finally:
|
|
1667
|
+
client.close()
|
|
1668
|
+
|
|
1669
|
+
|
|
1670
|
+
# --------------------------------------------------------------------------
|
|
1671
|
+
# image
|
|
1672
|
+
# --------------------------------------------------------------------------
|
|
1673
|
+
|
|
1674
|
+
|
|
1675
|
+
def _load_image_from_script(module, script: Path):
|
|
1676
|
+
"""Return the module-level ``image`` (a ``bladekit.Image``) that ``blade image
|
|
1677
|
+
build`` needs, or a clear :class:`bladekit.FabricError` otherwise.
|
|
1678
|
+
|
|
1679
|
+
Mirrors ``_select_single_app``'s "clear error, not a raw AttributeError/
|
|
1680
|
+
TypeError" pattern so a missing/wrong-type ``image`` hits the standard
|
|
1681
|
+
``_handle_fabric_error`` rendering like every other script-loading error.
|
|
1682
|
+
"""
|
|
1683
|
+
from ._image import Image
|
|
1684
|
+
from . import exceptions as exc
|
|
1685
|
+
|
|
1686
|
+
image_obj = getattr(module, "image", None)
|
|
1687
|
+
if image_obj is None:
|
|
1688
|
+
raise exc.FabricError(
|
|
1689
|
+
f"no module-level `image` found in {script} "
|
|
1690
|
+
"(expected `image = bladekit.Image(...)`)"
|
|
1691
|
+
)
|
|
1692
|
+
if not isinstance(image_obj, Image):
|
|
1693
|
+
raise exc.FabricError(
|
|
1694
|
+
f"`image` in {script} must be a bladekit.Image, got "
|
|
1695
|
+
f"{type(image_obj).__name__}"
|
|
1696
|
+
)
|
|
1697
|
+
return image_obj
|
|
1698
|
+
|
|
1699
|
+
|
|
1700
|
+
@image_app.command("build")
|
|
1701
|
+
def image_build(
|
|
1702
|
+
script: Path = typer.Argument(
|
|
1703
|
+
..., help="Python script with a module-level `image = bladekit.Image(...)`."
|
|
1704
|
+
),
|
|
1705
|
+
force: bool = typer.Option(
|
|
1706
|
+
False,
|
|
1707
|
+
"--force",
|
|
1708
|
+
help="Force a fresh build even if a succeeded build for this definition "
|
|
1709
|
+
"hash is already cached. Needed after a catalog base re-pin: the "
|
|
1710
|
+
"definition hash covers the alias, not the resolved digest, so an "
|
|
1711
|
+
"unchanged definition would otherwise dedup to the pre-re-pin build.",
|
|
1712
|
+
),
|
|
1713
|
+
force_compression: bool = typer.Option(
|
|
1714
|
+
False,
|
|
1715
|
+
"--force-compression",
|
|
1716
|
+
help="Ask the control plane to force zstd recompression of inherited "
|
|
1717
|
+
"layers for this build (admin-authed callers only). Always set by the "
|
|
1718
|
+
"catalog:build:* CI jobs.",
|
|
1719
|
+
),
|
|
1720
|
+
json_output: bool = typer.Option(
|
|
1721
|
+
False,
|
|
1722
|
+
"--json",
|
|
1723
|
+
help="Print a single JSON object to stdout "
|
|
1724
|
+
"({build_id, state, image_ref, definition_hash}) and nothing else — "
|
|
1725
|
+
"for CI to parse. Build logs still stream to stderr.",
|
|
1726
|
+
),
|
|
1727
|
+
):
|
|
1728
|
+
"""Build a standalone bladekit.Image, outside of run/deploy.
|
|
1729
|
+
|
|
1730
|
+
Imports <script.py> as a module and requires a module-level `image`
|
|
1731
|
+
attribute (a bladekit.Image). Computes its definition + definition_hash the
|
|
1732
|
+
same way the implicit build path (run/deploy) does, then drives it
|
|
1733
|
+
through POST /v1/images/builds to a terminal state — its own command so
|
|
1734
|
+
CI can build (and promote) catalog base images
|
|
1735
|
+
(docs/specs/phase2-dogfood-catalog-build.md §4b).
|
|
1736
|
+
"""
|
|
1737
|
+
from ._build import run_standalone_build
|
|
1738
|
+
from ._loader import load_script
|
|
1739
|
+
from ._render import console, result_line
|
|
1740
|
+
from . import exceptions as exc
|
|
1741
|
+
|
|
1742
|
+
try:
|
|
1743
|
+
module, _module_name, _bundle_root = load_script(script)
|
|
1744
|
+
image_obj = _load_image_from_script(module, script)
|
|
1745
|
+
definition = image_obj.to_definition()
|
|
1746
|
+
definition_hash = image_obj.definition_hash()
|
|
1747
|
+
except exc.FabricError as e:
|
|
1748
|
+
raise _handle_fabric_error(e)
|
|
1749
|
+
|
|
1750
|
+
client, _ = _make_client()
|
|
1751
|
+
try:
|
|
1752
|
+
build = run_standalone_build(
|
|
1753
|
+
client,
|
|
1754
|
+
definition,
|
|
1755
|
+
definition_hash,
|
|
1756
|
+
force=force,
|
|
1757
|
+
force_compression=force_compression,
|
|
1758
|
+
context_data=getattr(image_obj, "_context_data", None),
|
|
1759
|
+
)
|
|
1760
|
+
except exc.FabricError as e:
|
|
1761
|
+
raise _handle_fabric_error(e)
|
|
1762
|
+
finally:
|
|
1763
|
+
client.close()
|
|
1764
|
+
|
|
1765
|
+
image_ref = build.get("image_ref")
|
|
1766
|
+
if json_output:
|
|
1767
|
+
# Plain print, not console.print: a rich Console can soft-wrap long
|
|
1768
|
+
# lines to the terminal width, which would corrupt this as JSON once
|
|
1769
|
+
# piped/redirected (the CI case) — stdout must be EXACTLY one
|
|
1770
|
+
# parseable line, nothing else.
|
|
1771
|
+
print(
|
|
1772
|
+
json.dumps(
|
|
1773
|
+
{
|
|
1774
|
+
"build_id": build.get("build_id"),
|
|
1775
|
+
"state": build.get("state"),
|
|
1776
|
+
"image_ref": image_ref,
|
|
1777
|
+
"definition_hash": build.get("definition_hash", definition_hash),
|
|
1778
|
+
}
|
|
1779
|
+
)
|
|
1780
|
+
)
|
|
1781
|
+
return
|
|
1782
|
+
|
|
1783
|
+
console.print(f"[green]image built[/green] [dim]{definition_hash[:12]}…[/dim]")
|
|
1784
|
+
if image_ref:
|
|
1785
|
+
result_line(image_ref)
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
@image_app.command("list")
|
|
1789
|
+
def image_list():
|
|
1790
|
+
"""List the image catalogue aliases (maintained images)."""
|
|
1791
|
+
from ._render import console, images_table
|
|
1792
|
+
from . import exceptions as exc
|
|
1793
|
+
|
|
1794
|
+
client, _ = _make_client()
|
|
1795
|
+
try:
|
|
1796
|
+
page = client.list_images(limit=200)
|
|
1797
|
+
except exc.FabricError as e:
|
|
1798
|
+
raise _handle_fabric_error(e)
|
|
1799
|
+
try:
|
|
1800
|
+
items = page.get("items", [])
|
|
1801
|
+
if not items:
|
|
1802
|
+
console.print("[dim]no image aliases[/dim]")
|
|
1803
|
+
return
|
|
1804
|
+
console.print(images_table(items))
|
|
1805
|
+
finally:
|
|
1806
|
+
client.close()
|
|
1807
|
+
|
|
1808
|
+
|
|
1809
|
+
@image_app.command("builds")
|
|
1810
|
+
def image_builds(limit: int = typer.Option(50, "--limit", min=1, max=200)):
|
|
1811
|
+
"""List recent DSL image builds (state, hash)."""
|
|
1812
|
+
from ._render import console, builds_table
|
|
1813
|
+
from . import exceptions as exc
|
|
1814
|
+
|
|
1815
|
+
client, _ = _make_client()
|
|
1816
|
+
try:
|
|
1817
|
+
page = client.list_image_builds(limit=limit)
|
|
1818
|
+
except exc.FabricError as e:
|
|
1819
|
+
raise _handle_fabric_error(e)
|
|
1820
|
+
try:
|
|
1821
|
+
items = page.get("items", [])
|
|
1822
|
+
if not items:
|
|
1823
|
+
console.print("[dim]no builds[/dim]")
|
|
1824
|
+
return
|
|
1825
|
+
console.print(builds_table(items))
|
|
1826
|
+
finally:
|
|
1827
|
+
client.close()
|
|
1828
|
+
|
|
1829
|
+
|
|
1830
|
+
@image_app.command("logs")
|
|
1831
|
+
def image_logs(
|
|
1832
|
+
build_id: str = typer.Argument(..., help="Image build id (ibld_…)."),
|
|
1833
|
+
follow: bool = typer.Option(False, "-f", "--follow", help="Stream live."),
|
|
1834
|
+
):
|
|
1835
|
+
"""Show an image build's logs (the failed-build hint points here)."""
|
|
1836
|
+
from ._render import print_log_chunk, flush_log_tail
|
|
1837
|
+
from ._build import _stream_build_logs
|
|
1838
|
+
from . import exceptions as exc
|
|
1839
|
+
|
|
1840
|
+
client, _ = _make_client()
|
|
1841
|
+
try:
|
|
1842
|
+
if follow:
|
|
1843
|
+
_stream_build_logs(client, build_id)
|
|
1844
|
+
else:
|
|
1845
|
+
cursor = None
|
|
1846
|
+
try:
|
|
1847
|
+
while True:
|
|
1848
|
+
page = client.get_build_logs(build_id, cursor=cursor)
|
|
1849
|
+
for chunk in page.get("items", []):
|
|
1850
|
+
print_log_chunk(chunk)
|
|
1851
|
+
nxt = page.get("next_cursor")
|
|
1852
|
+
if nxt is None:
|
|
1853
|
+
break
|
|
1854
|
+
cursor = nxt
|
|
1855
|
+
finally:
|
|
1856
|
+
flush_log_tail()
|
|
1857
|
+
except exc.FabricError as e:
|
|
1858
|
+
raise _handle_fabric_error(e)
|
|
1859
|
+
finally:
|
|
1860
|
+
client.close()
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
# ---------------------------------------------------------------------------
|
|
1864
|
+
# map commands
|
|
1865
|
+
# ---------------------------------------------------------------------------
|
|
1866
|
+
|
|
1867
|
+
|
|
1868
|
+
@map_app.command("status")
|
|
1869
|
+
def map_status(
|
|
1870
|
+
map_id: str = typer.Argument(..., help="Map invocation ID (map_…)."),
|
|
1871
|
+
):
|
|
1872
|
+
"""Show the status of a map batch invocation."""
|
|
1873
|
+
from ._render import console
|
|
1874
|
+
from . import exceptions as exc
|
|
1875
|
+
|
|
1876
|
+
client, _ = _make_client()
|
|
1877
|
+
try:
|
|
1878
|
+
m = client.get_map(map_id)
|
|
1879
|
+
state = m.get("state", "unknown")
|
|
1880
|
+
input_count = m.get("input_count", 0)
|
|
1881
|
+
result_count = m.get("result_count", 0)
|
|
1882
|
+
console.print(f"[bold]{map_id}[/bold] state=[cyan]{state}[/cyan]")
|
|
1883
|
+
console.print(f" inputs: {input_count} completed: {result_count}")
|
|
1884
|
+
if m.get("finalized_at"):
|
|
1885
|
+
console.print(f" finalized_at: {m['finalized_at']}")
|
|
1886
|
+
if m.get("completed_at"):
|
|
1887
|
+
console.print(f" completed_at: {m['completed_at']}")
|
|
1888
|
+
if m.get("error"):
|
|
1889
|
+
err = m["error"]
|
|
1890
|
+
console.print(
|
|
1891
|
+
f" [red]error[/red]: [{err.get('code')}] {err.get('message')}"
|
|
1892
|
+
)
|
|
1893
|
+
except exc.FabricError as e:
|
|
1894
|
+
raise _handle_fabric_error(e)
|
|
1895
|
+
finally:
|
|
1896
|
+
client.close()
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
@map_app.command("results")
|
|
1900
|
+
def map_results(
|
|
1901
|
+
map_id: str = typer.Argument(..., help="Map invocation ID (map_…)."),
|
|
1902
|
+
cursor: Optional[str] = typer.Option(None, "--cursor", help="Resume from cursor."),
|
|
1903
|
+
order: str = typer.Option("index", "--order", help="index or arrival."),
|
|
1904
|
+
return_exceptions: bool = typer.Option(
|
|
1905
|
+
False, "--return-exceptions", help="Yield exceptions instead of raising."
|
|
1906
|
+
),
|
|
1907
|
+
limit: int = typer.Option(200, "--limit", help="Results per page."),
|
|
1908
|
+
):
|
|
1909
|
+
"""Stream results from a map batch (or resume from a cursor)."""
|
|
1910
|
+
from ._render import console
|
|
1911
|
+
from ._map import _stream_results
|
|
1912
|
+
from . import exceptions as exc
|
|
1913
|
+
|
|
1914
|
+
client, _ = _make_client()
|
|
1915
|
+
try:
|
|
1916
|
+
for i, result in enumerate(_stream_results(
|
|
1917
|
+
client,
|
|
1918
|
+
map_id,
|
|
1919
|
+
order=order,
|
|
1920
|
+
return_exceptions=return_exceptions,
|
|
1921
|
+
cursor=cursor,
|
|
1922
|
+
)):
|
|
1923
|
+
if isinstance(result, BaseException):
|
|
1924
|
+
console.print(f"[{i}] [red]exception[/red]: {result}")
|
|
1925
|
+
else:
|
|
1926
|
+
console.print(f"[{i}] {result!r}")
|
|
1927
|
+
except exc.FabricError as e:
|
|
1928
|
+
raise _handle_fabric_error(e)
|
|
1929
|
+
finally:
|
|
1930
|
+
client.close()
|
|
1931
|
+
|
|
1932
|
+
|
|
1933
|
+
# ---------------------------------------------------------------------------
|
|
1934
|
+
# cron commands
|
|
1935
|
+
# ---------------------------------------------------------------------------
|
|
1936
|
+
|
|
1937
|
+
|
|
1938
|
+
@cron_app.command("list")
|
|
1939
|
+
def cron_list(
|
|
1940
|
+
app_id: str = typer.Argument(..., help="App ID (app_…)."),
|
|
1941
|
+
):
|
|
1942
|
+
"""List cron schedules for an app."""
|
|
1943
|
+
from ._render import console
|
|
1944
|
+
from . import exceptions as exc
|
|
1945
|
+
|
|
1946
|
+
client, _ = _make_client()
|
|
1947
|
+
try:
|
|
1948
|
+
resp = client.list_crons(app_id)
|
|
1949
|
+
items = resp.get("items", [])
|
|
1950
|
+
if not items:
|
|
1951
|
+
console.print("[dim]no crons[/dim]")
|
|
1952
|
+
return
|
|
1953
|
+
for c in items:
|
|
1954
|
+
state = c.get("state", "unknown")
|
|
1955
|
+
expr = c.get("cron_expr", "?")
|
|
1956
|
+
cid = c.get("cron_id", "?")
|
|
1957
|
+
console.print(
|
|
1958
|
+
f"[bold]{cid}[/bold] expr=[cyan]{expr}[/cyan] state={state}"
|
|
1959
|
+
)
|
|
1960
|
+
except exc.FabricError as e:
|
|
1961
|
+
raise _handle_fabric_error(e)
|
|
1962
|
+
finally:
|
|
1963
|
+
client.close()
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
@cron_app.command("history")
|
|
1967
|
+
def cron_history(
|
|
1968
|
+
cron_id: str = typer.Argument(..., help="Cron ID (cron_…)."),
|
|
1969
|
+
limit: int = typer.Option(20, "--limit", help="Max executions to show."),
|
|
1970
|
+
):
|
|
1971
|
+
"""Show recent cron executions."""
|
|
1972
|
+
from ._render import console
|
|
1973
|
+
from . import exceptions as exc
|
|
1974
|
+
|
|
1975
|
+
client, _ = _make_client()
|
|
1976
|
+
try:
|
|
1977
|
+
resp = client.list_cron_executions(cron_id, limit=limit)
|
|
1978
|
+
items = resp.get("items", [])
|
|
1979
|
+
if not items:
|
|
1980
|
+
console.print("[dim]no executions[/dim]")
|
|
1981
|
+
return
|
|
1982
|
+
for ex in items:
|
|
1983
|
+
eid = ex.get("execution_id", "?")
|
|
1984
|
+
state = ex.get("state", "?")
|
|
1985
|
+
sched = ex.get("scheduled_at", "?")
|
|
1986
|
+
console.print(
|
|
1987
|
+
f"[bold]{eid}[/bold] scheduled={sched} state=[cyan]{state}[/cyan]"
|
|
1988
|
+
)
|
|
1989
|
+
except exc.FabricError as e:
|
|
1990
|
+
raise _handle_fabric_error(e)
|
|
1991
|
+
finally:
|
|
1992
|
+
client.close()
|
|
1993
|
+
|
|
1994
|
+
|
|
1995
|
+
@cron_app.command("trigger")
|
|
1996
|
+
def cron_trigger(
|
|
1997
|
+
cron_id: str = typer.Argument(..., help="Cron ID (cron_…)."),
|
|
1998
|
+
):
|
|
1999
|
+
"""Trigger a one-off cron execution immediately."""
|
|
2000
|
+
from ._render import console
|
|
2001
|
+
from . import exceptions as exc
|
|
2002
|
+
|
|
2003
|
+
client, _ = _make_client()
|
|
2004
|
+
try:
|
|
2005
|
+
ex = client.trigger_cron(cron_id)
|
|
2006
|
+
eid = ex.get("execution_id", "?")
|
|
2007
|
+
console.print(
|
|
2008
|
+
f"[green]triggered[/green] execution [bold]{eid}[/bold]"
|
|
2009
|
+
)
|
|
2010
|
+
except exc.FabricError as e:
|
|
2011
|
+
raise _handle_fabric_error(e)
|
|
2012
|
+
finally:
|
|
2013
|
+
client.close()
|
|
2014
|
+
|
|
2015
|
+
|
|
2016
|
+
# ---------------------------------------------------------------------------
|
|
2017
|
+
# queue commands
|
|
2018
|
+
# ---------------------------------------------------------------------------
|
|
2019
|
+
|
|
2020
|
+
|
|
2021
|
+
@queue_app.command("len")
|
|
2022
|
+
def queue_len(
|
|
2023
|
+
name: str = typer.Argument(..., help="Queue name."),
|
|
2024
|
+
):
|
|
2025
|
+
"""Show the number of items in a queue."""
|
|
2026
|
+
from ._render import console
|
|
2027
|
+
from . import exceptions as exc
|
|
2028
|
+
|
|
2029
|
+
client, _ = _make_client()
|
|
2030
|
+
try:
|
|
2031
|
+
resp = client.get_queue(name)
|
|
2032
|
+
console.print(f"[cyan]{name}[/cyan]: {resp.get('len', 0)} items")
|
|
2033
|
+
except exc.FabricError as e:
|
|
2034
|
+
raise _handle_fabric_error(e)
|
|
2035
|
+
finally:
|
|
2036
|
+
client.close()
|
|
2037
|
+
|
|
2038
|
+
|
|
2039
|
+
# ---------------------------------------------------------------------------
|
|
2040
|
+
# dict commands
|
|
2041
|
+
# ---------------------------------------------------------------------------
|
|
2042
|
+
|
|
2043
|
+
|
|
2044
|
+
@dict_app.command("get")
|
|
2045
|
+
def dict_get_cmd(
|
|
2046
|
+
name: str = typer.Argument(..., help="Dict name."),
|
|
2047
|
+
key: str = typer.Argument(..., help="Key to retrieve."),
|
|
2048
|
+
):
|
|
2049
|
+
"""Get a value from a distributed dict."""
|
|
2050
|
+
from ._render import console
|
|
2051
|
+
from ._dict import Dict as DictClass
|
|
2052
|
+
from . import exceptions as exc
|
|
2053
|
+
|
|
2054
|
+
client, _ = _make_client()
|
|
2055
|
+
try:
|
|
2056
|
+
d = DictClass._from_client(name, client)
|
|
2057
|
+
value = d.get(key)
|
|
2058
|
+
if value is None:
|
|
2059
|
+
console.print(f"[dim]key {key!r} not found[/dim]")
|
|
2060
|
+
else:
|
|
2061
|
+
console.print(repr(value))
|
|
2062
|
+
except exc.FabricError as e:
|
|
2063
|
+
raise _handle_fabric_error(e)
|
|
2064
|
+
finally:
|
|
2065
|
+
client.close()
|
|
2066
|
+
|
|
2067
|
+
|
|
2068
|
+
@dict_app.command("set")
|
|
2069
|
+
def dict_set_cmd(
|
|
2070
|
+
name: str = typer.Argument(..., help="Dict name."),
|
|
2071
|
+
key: str = typer.Argument(..., help="Key to set."),
|
|
2072
|
+
value: str = typer.Argument(..., help="JSON-encoded value."),
|
|
2073
|
+
):
|
|
2074
|
+
"""Set a value in a distributed dict (value must be JSON)."""
|
|
2075
|
+
from ._render import console
|
|
2076
|
+
from ._dict import Dict as DictClass
|
|
2077
|
+
from . import exceptions as exc
|
|
2078
|
+
|
|
2079
|
+
try:
|
|
2080
|
+
import json as json_mod
|
|
2081
|
+
parsed = json_mod.loads(value)
|
|
2082
|
+
except json.JSONDecodeError as e:
|
|
2083
|
+
from ._render import err_console
|
|
2084
|
+
err_console.print(f"[red]error[/red]: value must be valid JSON: {e}")
|
|
2085
|
+
raise typer.Exit(code=1)
|
|
2086
|
+
|
|
2087
|
+
client, _ = _make_client()
|
|
2088
|
+
try:
|
|
2089
|
+
d = DictClass._from_client(name, client)
|
|
2090
|
+
d.put(key, parsed)
|
|
2091
|
+
console.print(f"[green]ok[/green] set [bold]{key}[/bold]")
|
|
2092
|
+
except exc.FabricError as e:
|
|
2093
|
+
raise _handle_fabric_error(e)
|
|
2094
|
+
finally:
|
|
2095
|
+
client.close()
|
|
2096
|
+
|
|
2097
|
+
|
|
2098
|
+
# ---------------------------------------------------------------------------
|
|
2099
|
+
# sandbox commands
|
|
2100
|
+
# ---------------------------------------------------------------------------
|
|
2101
|
+
|
|
2102
|
+
|
|
2103
|
+
def _fmt_sandbox_row(sb: dict) -> str:
|
|
2104
|
+
"""One-line summary for a sandbox object."""
|
|
2105
|
+
sid = sb.get("sandbox_id", "?")
|
|
2106
|
+
state = sb.get("state", "?")
|
|
2107
|
+
rt = sb.get("runtime", "?")
|
|
2108
|
+
image = sb.get("image_alias") or sb.get("image_ref", "?")
|
|
2109
|
+
created = sb.get("created_at", "")[:19].replace("T", " ")
|
|
2110
|
+
# A sealed sandbox looks identical to any other otherwise, and "why can't this
|
|
2111
|
+
# thing resolve anything" is the first question it raises.
|
|
2112
|
+
net = " no-network" if sb.get("block_network") else ""
|
|
2113
|
+
return f"{sid} {state:<12} {rt:<6} {image:<30} {created}{net}"
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
@sandbox_app.command("list")
|
|
2117
|
+
def sandbox_list(
|
|
2118
|
+
state: Optional[str] = typer.Option(
|
|
2119
|
+
None, "--state", help="Filter by state (creating/running/terminating/terminated/failed)."
|
|
2120
|
+
),
|
|
2121
|
+
limit: int = typer.Option(50, "--limit", min=1, max=200),
|
|
2122
|
+
):
|
|
2123
|
+
"""List sandboxes (most recent first)."""
|
|
2124
|
+
from ._render import console
|
|
2125
|
+
from . import exceptions as exc
|
|
2126
|
+
|
|
2127
|
+
client, _ = _make_client()
|
|
2128
|
+
try:
|
|
2129
|
+
states = [state] if state else None
|
|
2130
|
+
page = client.list_sandboxes(state=states, limit=limit)
|
|
2131
|
+
items = page.get("items", [])
|
|
2132
|
+
if not items:
|
|
2133
|
+
console.print("[dim]no sandboxes[/dim]")
|
|
2134
|
+
return
|
|
2135
|
+
console.print(f"{'SANDBOX ID':<32} {'STATE':<12} {'RUNTIME':<6} {'IMAGE':<30} CREATED")
|
|
2136
|
+
console.print("-" * 100)
|
|
2137
|
+
for sb in items:
|
|
2138
|
+
console.print(_fmt_sandbox_row(sb))
|
|
2139
|
+
except exc.FabricError as e:
|
|
2140
|
+
raise _handle_fabric_error(e)
|
|
2141
|
+
finally:
|
|
2142
|
+
client.close()
|
|
2143
|
+
|
|
2144
|
+
|
|
2145
|
+
@sandbox_app.command("terminate")
|
|
2146
|
+
def sandbox_terminate(
|
|
2147
|
+
sandbox_id: str = typer.Argument(..., help="Sandbox ID (sb_…)."),
|
|
2148
|
+
):
|
|
2149
|
+
"""Terminate a running sandbox (SIGTERM -> grace -> SIGKILL)."""
|
|
2150
|
+
from ._render import console
|
|
2151
|
+
from . import exceptions as exc
|
|
2152
|
+
|
|
2153
|
+
client, _ = _make_client()
|
|
2154
|
+
try:
|
|
2155
|
+
client.sandbox_terminate(sandbox_id)
|
|
2156
|
+
console.print(f"[magenta]terminating[/magenta] sandbox {sandbox_id}")
|
|
2157
|
+
except exc.NotFoundError:
|
|
2158
|
+
console.print(f"[yellow]not found[/yellow]: sandbox {sandbox_id} (already gone?)")
|
|
2159
|
+
except exc.FabricError as e:
|
|
2160
|
+
raise _handle_fabric_error(e)
|
|
2161
|
+
finally:
|
|
2162
|
+
client.close()
|
|
2163
|
+
|
|
2164
|
+
|
|
2165
|
+
# ---------------------------------------------------------------------------
|
|
2166
|
+
# blade shell — interactive shell inside a sandbox
|
|
2167
|
+
# ---------------------------------------------------------------------------
|
|
2168
|
+
|
|
2169
|
+
|
|
2170
|
+
@app.command("shell")
|
|
2171
|
+
def shell(
|
|
2172
|
+
image: str = typer.Option("fabric/base", "--image", "-i", help="Image alias or ref."),
|
|
2173
|
+
gpu: Optional[str] = typer.Option(None, "--gpu", "-g", help="GPU type (e.g. RTX-A4500)."),
|
|
2174
|
+
runtime: str = typer.Option("runsc", "--runtime", help="Runtime: runsc (default) or runc."),
|
|
2175
|
+
timeout: int = typer.Option(3600, "--timeout", help="Sandbox lifetime in seconds."),
|
|
2176
|
+
block_network: bool = typer.Option(
|
|
2177
|
+
False,
|
|
2178
|
+
"--block-network",
|
|
2179
|
+
help="No network at all: loopback only, no DNS, no Internet (image/volumes still work).",
|
|
2180
|
+
),
|
|
2181
|
+
cmd: Optional[list[str]] = typer.Argument(None, help="Command to run (default: /bin/bash)."),
|
|
2182
|
+
):
|
|
2183
|
+
"""Open an interactive shell inside a GPU sandbox.
|
|
2184
|
+
|
|
2185
|
+
Equivalent to: Sandbox.create(...) then exec /bin/bash (or the given cmd).
|
|
2186
|
+
Streams stdin/stdout/stderr interactively via the sandbox exec WebSocket.
|
|
2187
|
+
|
|
2188
|
+
\\b
|
|
2189
|
+
blade shell # /bin/bash in a CPU sandbox (runsc)
|
|
2190
|
+
blade shell --gpu RTX-A4500 # GPU sandbox
|
|
2191
|
+
blade shell --runtime runc # trusted-only debug (runc)
|
|
2192
|
+
blade shell --block-network # sealed: no Internet from inside
|
|
2193
|
+
blade shell -- python3 script.py # run a command and stream output
|
|
2194
|
+
"""
|
|
2195
|
+
from ._render import err_console
|
|
2196
|
+
from ._sandbox import Sandbox
|
|
2197
|
+
from . import exceptions as exc
|
|
2198
|
+
|
|
2199
|
+
argv = list(cmd) if cmd else ["/bin/bash"]
|
|
2200
|
+
|
|
2201
|
+
client, _ = _make_client()
|
|
2202
|
+
|
|
2203
|
+
try:
|
|
2204
|
+
net = ", network=blocked" if block_network else ""
|
|
2205
|
+
err_console.print(
|
|
2206
|
+
f"[dim]Creating sandbox (image={image!r}, runtime={runtime!r}{net})...[/dim]"
|
|
2207
|
+
)
|
|
2208
|
+
sb = Sandbox.create(
|
|
2209
|
+
image,
|
|
2210
|
+
gpu=gpu,
|
|
2211
|
+
timeout=timeout,
|
|
2212
|
+
runtime=runtime,
|
|
2213
|
+
block_network=block_network,
|
|
2214
|
+
client=client,
|
|
2215
|
+
)
|
|
2216
|
+
err_console.print(f"[dim]Sandbox {sb.sandbox_id} created. Waiting for running...[/dim]")
|
|
2217
|
+
try:
|
|
2218
|
+
sb.wait_for_running(timeout=120.0)
|
|
2219
|
+
except (TimeoutError, exc.SandboxFailedError) as e:
|
|
2220
|
+
err_console.print(f"[red]sandbox failed to start[/red]: {e}")
|
|
2221
|
+
raise typer.Exit(code=1)
|
|
2222
|
+
|
|
2223
|
+
err_console.print(f"[green]Connected[/green] ({sb.sandbox_id}). Type 'exit' to quit.")
|
|
2224
|
+
|
|
2225
|
+
# Use streaming exec for interactive shell
|
|
2226
|
+
proc = sb.exec(*argv, stream=True)
|
|
2227
|
+
|
|
2228
|
+
# Interactive I/O pump: forward local stdin to the WebSocket and print
|
|
2229
|
+
# remote stdout/stderr. The websockets library is required for streaming.
|
|
2230
|
+
# For simplicity in CI/headless environments, fall back to one-shot mode.
|
|
2231
|
+
if proc.stdin is not None:
|
|
2232
|
+
# Streaming mode: drive stdin from the local terminal in a thread.
|
|
2233
|
+
import threading as _threading
|
|
2234
|
+
|
|
2235
|
+
def _feed_stdin() -> None:
|
|
2236
|
+
import sys as _sys
|
|
2237
|
+
stdin = proc.stdin
|
|
2238
|
+
assert stdin is not None # guarded by `if proc.stdin is not None`
|
|
2239
|
+
try:
|
|
2240
|
+
while True:
|
|
2241
|
+
line = _sys.stdin.readline()
|
|
2242
|
+
if not line:
|
|
2243
|
+
stdin.close()
|
|
2244
|
+
break
|
|
2245
|
+
stdin.write(line.encode())
|
|
2246
|
+
except Exception: # noqa: BLE001
|
|
2247
|
+
try:
|
|
2248
|
+
stdin.close()
|
|
2249
|
+
except Exception: # noqa: BLE001
|
|
2250
|
+
pass
|
|
2251
|
+
|
|
2252
|
+
feeder = _threading.Thread(target=_feed_stdin, daemon=True)
|
|
2253
|
+
feeder.start()
|
|
2254
|
+
|
|
2255
|
+
# Print stdout/stderr as they arrive.
|
|
2256
|
+
import sys as _sys
|
|
2257
|
+
for chunk in proc.stdout:
|
|
2258
|
+
_sys.stdout.buffer.write(chunk)
|
|
2259
|
+
_sys.stdout.buffer.flush()
|
|
2260
|
+
for chunk in proc.stderr:
|
|
2261
|
+
_sys.stderr.buffer.write(chunk)
|
|
2262
|
+
_sys.stderr.buffer.flush()
|
|
2263
|
+
|
|
2264
|
+
rc = proc.wait(timeout=float(timeout))
|
|
2265
|
+
raise typer.Exit(code=rc)
|
|
2266
|
+
|
|
2267
|
+
except exc.FabricError as e:
|
|
2268
|
+
raise _handle_fabric_error(e)
|
|
2269
|
+
finally:
|
|
2270
|
+
try:
|
|
2271
|
+
sb.terminate() # type: ignore[possibly-undefined]
|
|
2272
|
+
except Exception: # noqa: BLE001
|
|
2273
|
+
pass
|
|
2274
|
+
client.close()
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
def main() -> None: # console_script entry indirection if ever needed
|
|
2278
|
+
app()
|
|
2279
|
+
|
|
2280
|
+
|
|
2281
|
+
if __name__ == "__main__": # pragma: no cover
|
|
2282
|
+
main()
|