bladekit 4.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
bladekit/cli.py ADDED
@@ -0,0 +1,2282 @@
1
+ """``blade`` command-line interface.
2
+
3
+ Commands:
4
+ blade setup authenticate via the browser, save an API token
5
+ blade run <script.py> run the @app.local_entrypoint, stream logs live
6
+ blade logs <run_id> show / follow logs (with resume)
7
+ blade status <run_id> run state + queue position if pending
8
+ blade runs list runs
9
+ blade cancel <run_id> cancel a run
10
+ blade nodes fleet view (GPUs free/busy) — staff session token only
11
+ blade token set/show manage ~/.blade/config.toml
12
+ blade admin … contain-user / force-stop (staff session token)
13
+ blade --version print the installed bladekit version
14
+
15
+ ``forge`` is the same command under its pre-PAAS-771 name (shipped by the
16
+ ``forge-shadow`` compatibility distribution).
17
+
18
+ Heavy modules (httpx, the App machinery) are imported lazily inside commands so
19
+ ``blade --help`` stays fast.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import json
25
+ import sys
26
+ from pathlib import Path
27
+ from typing import Optional
28
+
29
+ import typer
30
+
31
+ app = typer.Typer(
32
+ add_completion=False,
33
+ no_args_is_help=True,
34
+ help="Blade AI Fabric — run Python functions on the GPU fleet.",
35
+ )
36
+
37
+ token_app = typer.Typer(no_args_is_help=True, help="Manage API token / endpoint.")
38
+ app.add_typer(token_app, name="token")
39
+
40
+ volume_app = typer.Typer(no_args_is_help=True, help="Manage network volumes.")
41
+ app.add_typer(volume_app, name="volume")
42
+
43
+ secret_app = typer.Typer(no_args_is_help=True, help="Manage secrets.")
44
+ app.add_typer(secret_app, name="secret")
45
+
46
+ image_app = typer.Typer(no_args_is_help=True, help="Image catalogue / DSL builds.")
47
+ app.add_typer(image_app, name="image")
48
+
49
+ app_app = typer.Typer(no_args_is_help=True, help="Manage deployed apps.")
50
+ app.add_typer(app_app, name="app")
51
+
52
+ map_app = typer.Typer(no_args_is_help=True, help="Manage map batch invocations.")
53
+ app.add_typer(map_app, name="map")
54
+
55
+ cron_app = typer.Typer(no_args_is_help=True, help="Manage cron schedules.")
56
+ app.add_typer(cron_app, name="cron")
57
+
58
+ queue_app = typer.Typer(no_args_is_help=True, help="Manage distributed queues.")
59
+ app.add_typer(queue_app, name="queue")
60
+
61
+ dict_app = typer.Typer(no_args_is_help=True, help="Manage distributed key-value stores.")
62
+ app.add_typer(dict_app, name="dict")
63
+
64
+ sandbox_app = typer.Typer(no_args_is_help=True, help="Manage sandboxes.")
65
+ app.add_typer(sandbox_app, name="sandbox")
66
+
67
+ admin_app = typer.Typer(
68
+ no_args_is_help=True,
69
+ help="Platform-admin containment tools (requires a staff SESSION token: an fg_ key "
70
+ "carries no compute:admin and is refused 403 INSUFFICIENT_SCOPE).",
71
+ )
72
+ app.add_typer(admin_app, name="admin")
73
+
74
+
75
+ def _env_flag(name: str) -> bool:
76
+ """Read a boolean FABRIC_<name> env var, falling back to FORGE_<name> (repo
77
+ convention: manual os.environ lookup, see _env.py — not typer's envvar=)."""
78
+ from ._env import flag
79
+
80
+ return flag(name)
81
+
82
+
83
+ def _version_callback(value: bool) -> None:
84
+ """``blade --version``: the compact banner on a terminal, plain text otherwise."""
85
+ if not value:
86
+ return
87
+ from . import DISTRIBUTION, __version__
88
+ from ._render import print_version
89
+
90
+ print_version(f"{DISTRIBUTION} {__version__}")
91
+ raise typer.Exit()
92
+
93
+
94
+ @app.callback()
95
+ def _main(
96
+ verbose: bool = typer.Option(
97
+ False,
98
+ "--verbose",
99
+ "-v",
100
+ help="Verbose output: raw build steps and per-state transitions "
101
+ "(also enabled by FABRIC_VERBOSE). Default is a clean phase view.",
102
+ ),
103
+ version: bool = typer.Option(
104
+ False,
105
+ "--version",
106
+ callback=_version_callback,
107
+ is_eager=True,
108
+ help="Show the installed bladekit version and exit.",
109
+ ),
110
+ ):
111
+ """Blade AI Fabric — run Python functions on the GPU fleet."""
112
+ from ._render import set_verbose
113
+
114
+ set_verbose(verbose or _env_flag("VERBOSE"))
115
+
116
+
117
+ # --------------------------------------------------------------------------
118
+ # shared helpers
119
+ # --------------------------------------------------------------------------
120
+
121
+
122
+ def _make_client():
123
+ from ._client import Client
124
+ from ._render import err_console
125
+ from .config import load_config
126
+
127
+ cfg = load_config()
128
+ if not cfg.token:
129
+ err_console.print(
130
+ "[red]No API token configured.[/red] Run `blade setup` or "
131
+ "`blade token set`, or set FABRIC_TOKEN."
132
+ )
133
+ raise typer.Exit(code=2)
134
+ client = Client(cfg.api_url, cfg.token)
135
+ client.warn_if_outdated()
136
+ return client, cfg
137
+
138
+
139
+ def _handle_fabric_error(e: Exception) -> "typer.Exit":
140
+ """Render any FabricError with its machine code + human message, return Exit."""
141
+ from ._render import err_console
142
+ from . import exceptions as exc
143
+
144
+ if isinstance(e, exc.ImageBuildError):
145
+ # Failed DSL build: code + message + the actionable `blade image logs`
146
+ # hint (already embedded in the message).
147
+ err_console.print(f"[red]image build failed[/red] [bold]{e.code}[/bold]:")
148
+ err_console.print(str(e), markup=False, highlight=False)
149
+ elif isinstance(e, exc.VolumeCommitError):
150
+ err_console.print(f"[red]volume commit failed[/red] [bold]{e.code}[/bold]: {e}")
151
+ elif isinstance(e, exc.APIError):
152
+ err_console.print(f"[red]error[/red] [bold]{e.code}[/bold]: {e.api_message}")
153
+ if e.details:
154
+ err_console.print(f" details: {json.dumps(e.details)}")
155
+ elif isinstance(e, exc.RunError):
156
+ # Show code + full message (carries the actionable hint, exceptions.py).
157
+ err_console.print(f"[red]run failed[/red] [bold]{e.code}[/bold]:")
158
+ err_console.print(str(e), markup=False, highlight=False)
159
+ elif isinstance(e, exc.ConnectionFailed):
160
+ err_console.print(f"[red]connection failed[/red]: {e}")
161
+ else:
162
+ err_console.print(f"[red]error[/red]: {e}")
163
+ return typer.Exit(code=1)
164
+
165
+
166
+ # --------------------------------------------------------------------------
167
+ # token
168
+ # --------------------------------------------------------------------------
169
+
170
+
171
+ @token_app.command("set")
172
+ def token_set(
173
+ api_url: Optional[str] = typer.Option(
174
+ None, "--api-url", help="Control plane base URL to persist."
175
+ ),
176
+ from_env: Optional[str] = typer.Option(
177
+ None,
178
+ "--from-env",
179
+ help="Read the token from this environment variable (e.g. FABRIC_TOKEN).",
180
+ ),
181
+ ):
182
+ """Write the token (and optionally api_url) to ~/.blade/config.toml.
183
+
184
+ The token is never accepted as a positional argument — that would leak it
185
+ into ``ps aux``, shell history, and process monitoring tools. Instead:
186
+
187
+ \\b
188
+ blade token set --from-env FABRIC_TOKEN # reads $FABRIC_TOKEN
189
+ echo $TOKEN | blade token set # reads from stdin
190
+ """
191
+ from ._render import console
192
+ from .config import write_config
193
+
194
+ token: Optional[str] = None
195
+ if from_env:
196
+ import os
197
+
198
+ token = os.environ.get(from_env)
199
+ if not token:
200
+ console.print(f"[red]env var not set or empty[/red]: {from_env}")
201
+ raise typer.Exit(code=2)
202
+ else:
203
+ # Read from stdin so the token never appears in argv.
204
+ if sys.stdin.isatty():
205
+ console.print("[dim]paste your API token then press Enter (input not echoed)…[/dim]")
206
+ line = sys.stdin.readline().rstrip("\n\r")
207
+ if not line:
208
+ console.print("[red]no token provided[/red] (pass via stdin or --from-env)")
209
+ raise typer.Exit(code=2)
210
+ token = line
211
+
212
+ path = write_config(token=token, api_url=api_url)
213
+ console.print(f"[green]Saved[/green] token to {path}")
214
+ if api_url:
215
+ console.print(f"[green]Saved[/green] api_url = {api_url}")
216
+
217
+
218
+ @token_app.command("show")
219
+ def token_show():
220
+ """Show the effective token/endpoint and where each came from."""
221
+ from ._render import console
222
+ from .config import load_config
223
+
224
+ cfg = load_config()
225
+ console.print(f"api_url : {cfg.api_url} [dim]({cfg.api_url_source})[/dim]")
226
+ if cfg.token:
227
+ masked = cfg.token[:6] + "…" + cfg.token[-4:] if len(cfg.token) > 12 else "set"
228
+ console.print(f"token : {masked} [dim]({cfg.token_source})[/dim]")
229
+ else:
230
+ console.print("token : [yellow]not set[/yellow]")
231
+
232
+
233
+ @app.command()
234
+ def setup(
235
+ api_url: Optional[str] = typer.Option(
236
+ None, "--api-url", help="Control plane base URL (default: configured value / FABRIC_API_URL)."
237
+ ),
238
+ ):
239
+ """Authenticate this machine through the browser and save an API token.
240
+
241
+ Opens your browser to confirm your identity, mints a token bound to your
242
+ workspace, and writes it to ~/.blade/config.toml — like ``modal setup``.
243
+ """
244
+ import time
245
+ import webbrowser
246
+
247
+ import httpx
248
+
249
+ from ._render import console, err_console, print_intro
250
+ from .config import load_config, write_config
251
+
252
+ print_intro()
253
+ cfg = load_config()
254
+ base = (api_url or cfg.api_url).rstrip("/")
255
+
256
+ token: Optional[str] = None
257
+ workspace: Optional[str] = None
258
+ try:
259
+ with httpx.Client(base_url=base, timeout=35.0) as http:
260
+ start = http.post("/v1/token-flow/start")
261
+ start.raise_for_status()
262
+ data = start.json()
263
+ flow_id = data["flow_id"]
264
+ url = data.get("url") or f"{base}/token-flow/{flow_id}"
265
+
266
+ console.print(
267
+ "The web browser should have opened for you to authenticate and "
268
+ "get an API token.\nIf it didn't, please copy this URL into your "
269
+ f"web browser manually:\n\n [cyan]{url}[/cyan]\n"
270
+ )
271
+ try:
272
+ webbrowser.open(url)
273
+ except Exception:
274
+ pass
275
+
276
+ # Each /wait long-polls ~20s server-side; loop until confirmed.
277
+ deadline = time.monotonic() + 300 # 5 min
278
+ with console.status("Waiting for web authentication…"):
279
+ while time.monotonic() < deadline:
280
+ resp = http.get(f"/v1/token-flow/{flow_id}/wait", timeout=35.0)
281
+ resp.raise_for_status()
282
+ body = resp.json()
283
+ if body.get("status") == "confirmed" and body.get("token"):
284
+ token = body["token"]
285
+ workspace = body.get("workspace")
286
+ break
287
+ except httpx.HTTPError as e:
288
+ err_console.print(f"[red]Could not reach the control plane at {base}[/red]: {e}")
289
+ raise typer.Exit(code=1)
290
+
291
+ if not token:
292
+ err_console.print(
293
+ "[red]Timed out waiting for web authentication.[/red] Re-run `blade setup`."
294
+ )
295
+ raise typer.Exit(code=1)
296
+
297
+ console.print("[green]Web authentication finished successfully![/green]")
298
+ if workspace:
299
+ console.print(f"Token is connected to the [bold]{workspace}[/bold] workspace.")
300
+ path = write_config(token=token, api_url=base)
301
+ console.print(f"Token written to [bold]{path}[/bold].")
302
+
303
+
304
+ # --------------------------------------------------------------------------
305
+ # run
306
+ # --------------------------------------------------------------------------
307
+
308
+
309
+ @app.command()
310
+ def run(
311
+ script: Path = typer.Argument(..., help="Python script defining the app."),
312
+ no_logs: bool = typer.Option(
313
+ False, "--no-logs", help="Do not stream logs (just run and report)."
314
+ ),
315
+ verbose: bool = typer.Option(
316
+ False, "--verbose", "-v",
317
+ help="Verbose: per-state transition lines instead of the clean running phase.",
318
+ ),
319
+ ):
320
+ """Execute the script's @app.local_entrypoint, streaming logs live.
321
+
322
+ Ctrl-C cancels the remote run(s). If a node is unreachable the run sits in
323
+ ``cancelling`` and we print "cancellation pending on node".
324
+ """
325
+ from ._loader import load_script, select_app_and_entrypoint
326
+ from ._render import console, err_console, set_verbose
327
+ from . import _app as app_mod
328
+ from . import exceptions as exc
329
+
330
+ if verbose:
331
+ set_verbose(True)
332
+
333
+ try:
334
+ client, cfg = _make_client()
335
+ except typer.Exit:
336
+ raise
337
+
338
+ try:
339
+ module, module_name, bundle_root = load_script(script)
340
+ forge_app, entrypoint = select_app_and_entrypoint(module)
341
+ except exc.FabricError as e:
342
+ raise _handle_fabric_error(e)
343
+
344
+ forge_app._bind_client(client)
345
+ forge_app._set_bundle_context(bundle_root, module_name)
346
+
347
+ # Background log streaming for every run this script opens. We start it lazily
348
+ # once the first run id is known. For simplicity (one function/run is the
349
+ # common case) we stream the first opened run.
350
+ import threading
351
+
352
+ stop_event = threading.Event()
353
+ streamed = {"run_id": None}
354
+
355
+ def _log_pump():
356
+ # Wait for a run to be registered, then stream it.
357
+ while not stop_event.is_set():
358
+ run_ids = list(app_mod._SESSION.open_run_ids)
359
+ if run_ids:
360
+ streamed["run_id"] = run_ids[0]
361
+ break
362
+ stop_event.wait(0.1)
363
+ rid = streamed["run_id"]
364
+ if not rid or no_logs:
365
+ return
366
+ _stream_run_logs(client, rid, stop_event, phase_ui=True)
367
+
368
+ log_thread = threading.Thread(target=_log_pump, daemon=True)
369
+ if not no_logs:
370
+ log_thread.start()
371
+
372
+ exit_code = 0
373
+ try:
374
+ result = entrypoint()
375
+ if result is not None:
376
+ console.print(f"[green]entrypoint returned:[/green] {result!r}")
377
+ except KeyboardInterrupt:
378
+ _on_ctrl_c(client, app_mod._SESSION.open_run_ids)
379
+ exit_code = 130
380
+ except exc.RunError as e:
381
+ # A run that FAILED for an infrastructure reason. Pilot UX fix
382
+ # (GPU-PILOT-REPORT §"Findings UX additionnels"): show the RunError
383
+ # (code + message + hint) and exit 1 — never "run cancelled"/130.
384
+ # RunError is checked BEFORE RunCancelled because RunLostError, etc. are
385
+ # RunError subclasses and must not be mistaken for a cancellation.
386
+ _handle_fabric_error(e)
387
+ exit_code = 1
388
+ except exc.RunCancelled as e:
389
+ err_console.print(f"[magenta]run cancelled[/magenta]: {e.run_id}")
390
+ exit_code = 130
391
+ except exc.RemoteFunctionError as e:
392
+ # A remote *user* exception that bubbled out of the entrypoint.
393
+ # RemoteFunctionError (RemoteError) is a FabricError subclass, so this
394
+ # must be checked BEFORE the generic FabricError branch or it would
395
+ # never be reached (first-match except semantics).
396
+ err_console.print(f"[red]remote function raised:[/red]\n{e}")
397
+ exit_code = 1
398
+ except exc.FabricError as e:
399
+ _handle_fabric_error(e)
400
+ exit_code = 1
401
+ except BaseException as e: # user exception re-raised from remote
402
+ # When the SDK re-raised the ORIGINAL user exception, the remote
403
+ # traceback travels on `_forge_remote_traceback` (set by
404
+ # Function._build_remote_exception). Print it verbatim so the remote
405
+ # stacktrace reads "as if local" (criterion) — without it the
406
+ # user would only see `type: message`.
407
+ remote_tb = getattr(e, "_forge_remote_traceback", None)
408
+ if remote_tb:
409
+ err_console.print("[red]remote function raised:[/red]")
410
+ err_console.print(str(remote_tb).rstrip("\n"), markup=False, highlight=False)
411
+ else:
412
+ err_console.print(f"[red]{type(e).__name__}[/red]: {e}")
413
+ exit_code = 1
414
+ finally:
415
+ if exit_code == 0:
416
+ # Drain hot containers promptly.
417
+ try:
418
+ app_mod.complete_session()
419
+ except exc.FabricError:
420
+ pass
421
+ # Let the log pump flush the tail, then stop.
422
+ if not no_logs:
423
+ _join_log_thread(log_thread, stop_event)
424
+ client.close()
425
+
426
+ raise typer.Exit(code=exit_code)
427
+
428
+
429
+ def _on_ctrl_c(client, run_ids) -> None:
430
+ """Cancel open runs; report nodes that are unreachable."""
431
+ from ._render import err_console
432
+ from . import exceptions as exc
433
+
434
+ err_console.print("\n[magenta]Ctrl-C — cancelling remote run(s)…[/magenta]")
435
+ for run_id in list(run_ids):
436
+ try:
437
+ run = client.cancel_run(run_id)
438
+ except exc.FabricError as e:
439
+ err_console.print(f" cancel {run_id}: [red]{e}[/red]")
440
+ continue
441
+ state = run.get("state")
442
+ if state == "cancelling":
443
+ node_id = run.get("node_id")
444
+ unreachable = _node_unreachable(client, node_id)
445
+ if unreachable:
446
+ err_console.print(
447
+ f" {run_id}: [yellow]cancellation pending on node[/yellow] "
448
+ f"(node {node_id} unreachable)"
449
+ )
450
+ else:
451
+ err_console.print(f" {run_id}: cancelling…")
452
+ else:
453
+ err_console.print(f" {run_id}: {state}")
454
+
455
+
456
+ def _node_unreachable(client, node_id: Optional[str]) -> bool:
457
+ """Is the node holding a cancelling run unreachable? Best effort.
458
+
459
+ Degrades quietly by design: this only decorates a cancellation message, and
460
+ failing the cancel because the decoration could not be fetched would be
461
+ worse than the message being vague.
462
+
463
+ KNOWN GAP, since `GET /v1/nodes/{node_id}` became staff-only: an ordinary
464
+ user token now gets `403 PERMISSION_DENIED` here, which is swallowed the
465
+ same as any other FabricError, so the caller prints "cancelling…" even when
466
+ the node really is unreachable and the cancellation really is stuck. The
467
+ answer is wrong in the safe direction (it never claims a node IS
468
+ unreachable), and it is now wrong for EVERY non-staff caller rather than
469
+ only when the lookup fails. Not redesigned here — telling the two apart
470
+ means either a per-run node-health field on `GET /v1/runs/{id}` or
471
+ distinguishing 403 from the rest, and both are their own change.
472
+ """
473
+ from . import exceptions as exc
474
+
475
+ if not node_id:
476
+ return False
477
+ try:
478
+ node = client._request("GET", f"/v1/nodes/{node_id}")
479
+ except exc.FabricError:
480
+ return False
481
+ return node.get("state") == "unreachable"
482
+
483
+
484
+ def _join_log_thread(thread, stop_event) -> None:
485
+ # Give the stream a moment to deliver the eof event after the run finishes.
486
+ thread.join(timeout=5.0)
487
+ stop_event.set()
488
+ thread.join(timeout=2.0)
489
+
490
+
491
+ # --------------------------------------------------------------------------
492
+ # log streaming (shared by `run` and `logs --follow`)
493
+ # --------------------------------------------------------------------------
494
+
495
+
496
+ def _stream_run_logs(
497
+ client, run_id: str, stop_event=None, phase_ui: bool = False
498
+ ) -> Optional[str]:
499
+ """Stream SSE logs + state events until eof. Returns final run state.
500
+
501
+ Resilient to disconnects: on a transport error we reconnect from the last
502
+ seen seq (resume cursor).
503
+
504
+ ``phase_ui=True`` (``blade run``, clean mode) renders a braille spinner
505
+ while the run is pending/scheduled/starting and a resolved
506
+ ``✓ succeeded (dur)`` / ``✗ failed (dur)`` line at the end, INSTEAD of the
507
+ per-transition ``[clock] • state`` lines. The user program's stdout/stderr
508
+ is streamed unchanged in every mode. In verbose mode (or for ``blade logs
509
+ -f``, which passes ``phase_ui=False``) the classic state-transition feed is
510
+ kept.
511
+ """
512
+ from ._render import (
513
+ print_log_chunk,
514
+ print_state_transition,
515
+ err_console,
516
+ flush_log_tail,
517
+ is_verbose,
518
+ RunProgress,
519
+ )
520
+ from ._constants import RUN_TERMINAL_STATES
521
+ from . import exceptions as exc
522
+
523
+ progress = None
524
+ if phase_ui and not is_verbose():
525
+ progress = RunProgress(_run_progress_meta(client, run_id), run_id=run_id)
526
+ progress.start()
527
+
528
+ cursor: Optional[int] = None
529
+ final_state: Optional[str] = None
530
+
531
+ try:
532
+ while stop_event is None or not stop_event.is_set():
533
+ try:
534
+ for event, data, event_id in client.stream_logs(run_id, cursor=cursor):
535
+ if stop_event is not None and stop_event.is_set():
536
+ return final_state
537
+ if event == "log":
538
+ # The server is trusted, but a truncated SSE chunk on a flaky
539
+ # connection can still yield a non-JSON `data:` or a non-numeric
540
+ # `id:`. Skip the unreadable event rather than crash the whole
541
+ # `blade run` / `blade logs -f` with a raw stacktrace.
542
+ try:
543
+ chunk = json.loads(data)
544
+ except ValueError:
545
+ continue
546
+ # Retire the spinner before the first user output so it
547
+ # never fights the streamed stdout (never suppress it).
548
+ if progress is not None:
549
+ progress.before_log()
550
+ print_log_chunk(chunk)
551
+ if isinstance(chunk, dict) and "seq" in chunk:
552
+ cursor = chunk["seq"]
553
+ elif event_id is not None:
554
+ try:
555
+ cursor = int(event_id)
556
+ except ValueError:
557
+ pass
558
+ elif event == "state":
559
+ try:
560
+ payload = json.loads(data)
561
+ except ValueError:
562
+ continue
563
+ final_state = payload.get("state")
564
+ if progress is not None:
565
+ progress.on_state(final_state)
566
+ else:
567
+ print_state_transition(final_state, payload.get("ts"))
568
+ elif event == "eof":
569
+ return final_state
570
+ elif event == "ping":
571
+ continue
572
+ # Generator ended without eof (server closed): stop.
573
+ return final_state
574
+ except exc.ConnectionFailed:
575
+ if stop_event is not None and stop_event.is_set():
576
+ return final_state
577
+ # Don't reconnect forever: if the run is already terminal the server
578
+ # may have closed the socket without an `eof` (e.g. control-plane
579
+ # restart / proxy drop after the run finished). Without this guard
580
+ # `blade logs -f <terminal_run>` (stop_event is None) spins at 2 req/s
581
+ # for ever. `eof` stays the clean terminator; this is the fallback.
582
+ try:
583
+ run = client.get_run(run_id)
584
+ except exc.FabricError:
585
+ run = None
586
+ if run is not None:
587
+ state = run.get("state")
588
+ if state in RUN_TERMINAL_STATES:
589
+ final_state = state
590
+ return state
591
+ err_console.print("[dim]log stream dropped, reconnecting…[/dim]")
592
+ import time
593
+
594
+ time.sleep(0.5)
595
+ continue
596
+ except exc.FabricError as e:
597
+ raise _handle_fabric_error(e)
598
+
599
+ return final_state
600
+ finally:
601
+ if progress is not None:
602
+ progress.finish(final_state)
603
+ # Emit any residual unterminated line (chunk boundary mid-line) so it
604
+ # isn't silently dropped once the stream ends.
605
+ flush_log_tail()
606
+
607
+
608
+ def _run_progress_meta(client, run_id: str) -> str:
609
+ """Best-effort spinner metadata: ``<gpu> · <short run id>``. The 'state'
610
+ SSE payload carries no resources, so the GPU type comes from one get_run
611
+ (recon anchor dto.go:65); a failure just drops the GPU part."""
612
+ from . import exceptions as exc
613
+
614
+ short = run_id[:12] + "…" if len(run_id) > 12 else run_id
615
+ try:
616
+ run = client.get_run(run_id)
617
+ except exc.FabricError:
618
+ return short
619
+ gpu = (run.get("resources") or {}).get("gpu_type")
620
+ if gpu and gpu != "none":
621
+ return f"{gpu} · {short}"
622
+ return short
623
+
624
+
625
+ # --------------------------------------------------------------------------
626
+ # logs
627
+ # --------------------------------------------------------------------------
628
+
629
+
630
+ @app.command()
631
+ def logs(
632
+ run_id: str = typer.Argument(..., help="Run id."),
633
+ follow: bool = typer.Option(
634
+ False, "-f", "--follow", help="Stream live until the run finishes."
635
+ ),
636
+ since: Optional[int] = typer.Option(
637
+ None, "--since", help="Resume from this seq (cursor)."
638
+ ),
639
+ ):
640
+ """Show a run's logs, optionally following live with resume support."""
641
+ from ._render import print_log_chunk, flush_log_tail, warn_logs_incomplete
642
+ from . import exceptions as exc
643
+
644
+ client, _ = _make_client()
645
+ try:
646
+ if follow:
647
+ _stream_run_logs(client, run_id)
648
+ else:
649
+ cursor = since
650
+ incomplete = False
651
+ try:
652
+ while True:
653
+ page = client.get_logs(run_id, cursor=cursor)
654
+ for chunk in page.get("items", []):
655
+ print_log_chunk(chunk)
656
+ incomplete = incomplete or bool(page.get("incomplete"))
657
+ nxt = page.get("next_cursor")
658
+ if nxt is None:
659
+ break
660
+ cursor = nxt
661
+ finally:
662
+ flush_log_tail()
663
+ if incomplete:
664
+ warn_logs_incomplete()
665
+ except exc.FabricError as e:
666
+ raise _handle_fabric_error(e)
667
+ finally:
668
+ client.close()
669
+
670
+
671
+ # --------------------------------------------------------------------------
672
+ # status
673
+ # --------------------------------------------------------------------------
674
+
675
+
676
+ @app.command()
677
+ def status(run_id: str = typer.Argument(..., help="Run id.")):
678
+ """Show a run's state; if pending, its position in the scheduler queue."""
679
+ from ._render import console, style_state, duration_str
680
+ from . import exceptions as exc
681
+
682
+ client, _ = _make_client()
683
+ try:
684
+ run = client.get_run(run_id)
685
+ except exc.FabricError as e:
686
+ raise _handle_fabric_error(e)
687
+ try:
688
+ state = run.get("state")
689
+ console.print(f"run : {run_id}")
690
+ console.print("state : ", style_state(state))
691
+ if run.get("node_id"):
692
+ console.print(f"node : {run['node_id']}")
693
+ res = run.get("resources") or {}
694
+ console.print(f"gpu : {res.get('gpu_type', '-')}")
695
+ console.print(
696
+ f"duration: {duration_str(run.get('started_at'), run.get('finished_at'))}"
697
+ )
698
+ console.print(
699
+ f"calls : {run.get('calls_finished', 0)}/{run.get('calls_total', 0)}"
700
+ )
701
+ if state == "pending":
702
+ pos = _queue_position(client, run)
703
+ if pos is not None:
704
+ console.print(f"queue : position {pos} (waiting for capacity)")
705
+ err = run.get("error")
706
+ if err:
707
+ console.print(
708
+ f"error : [red]{err.get('code')}[/red]: {err.get('message')}"
709
+ )
710
+ finally:
711
+ client.close()
712
+
713
+
714
+ def _queue_position(client, run: dict) -> Optional[int]:
715
+ """1-based position of this pending run in the FIFO queue.
716
+
717
+ The scheduler queue is FIFO by created_at (schema.sql runs_scheduler_queue);
718
+ position = number of pending runs created at-or-before this one, including
719
+ itself. ``GET /v1/runs?state=pending`` returns runs most-recent-first; we
720
+ page through and count those with created_at <= ours.
721
+ """
722
+ target_id = run.get("run_id")
723
+ target_created = run.get("created_at")
724
+ if not target_created:
725
+ return None
726
+ count = 0
727
+ cursor = None
728
+ seen_self = False
729
+ while True:
730
+ page = client.list_runs(state=["pending"], limit=200, cursor=cursor)
731
+ for r in page.get("items", []):
732
+ if r.get("run_id") == target_id:
733
+ seen_self = True
734
+ # Count pending runs ahead (created earlier) plus self.
735
+ if (r.get("created_at") or "") <= target_created:
736
+ count += 1
737
+ cursor = page.get("next_cursor")
738
+ if cursor is None:
739
+ break
740
+ if not seen_self:
741
+ # Run already left the pending queue between get_run and the list.
742
+ return None
743
+ return count
744
+
745
+
746
+ # --------------------------------------------------------------------------
747
+ # runs (list)
748
+ # --------------------------------------------------------------------------
749
+
750
+
751
+ @app.command()
752
+ def runs(
753
+ state: Optional[list[str]] = typer.Option(
754
+ None, "--state", help="Filter by state (repeatable)."
755
+ ),
756
+ limit: int = typer.Option(50, "--limit", min=1, max=200),
757
+ ):
758
+ """List runs (most recent first)."""
759
+ from ._render import console, runs_table
760
+ from . import exceptions as exc
761
+
762
+ client, _ = _make_client()
763
+ try:
764
+ page = client.list_runs(state=state, limit=limit)
765
+ except exc.FabricError as e:
766
+ raise _handle_fabric_error(e)
767
+ try:
768
+ items = page.get("items", [])
769
+ if not items:
770
+ console.print("[dim]no runs[/dim]")
771
+ return
772
+ console.print(runs_table(items))
773
+ finally:
774
+ client.close()
775
+
776
+
777
+ # --------------------------------------------------------------------------
778
+ # cancel
779
+ # --------------------------------------------------------------------------
780
+
781
+
782
+ @app.command()
783
+ def cancel(run_id: str = typer.Argument(..., help="Run id.")):
784
+ """Cancel a run."""
785
+ from ._render import console, err_console
786
+ from . import exceptions as exc
787
+
788
+ client, _ = _make_client()
789
+ try:
790
+ run = client.cancel_run(run_id)
791
+ except exc.FabricError as e:
792
+ raise _handle_fabric_error(e)
793
+ try:
794
+ state = run.get("state")
795
+ if state == "cancelling" and _node_unreachable(client, run.get("node_id")):
796
+ err_console.print(
797
+ f"[yellow]cancellation pending on node[/yellow] "
798
+ f"(node {run.get('node_id')} unreachable)"
799
+ )
800
+ else:
801
+ console.print(f"[magenta]cancel[/magenta] {run_id}: {state}")
802
+ finally:
803
+ client.close()
804
+
805
+
806
+ # --------------------------------------------------------------------------
807
+ # admin (PAAS-24): contain-user / force-stop
808
+ # --------------------------------------------------------------------------
809
+
810
+
811
+ _FORCE_STOP_KINDS = ("app", "deployment", "run", "sandbox", "map")
812
+
813
+
814
+ @admin_app.command("contain-user")
815
+ def admin_contain_user(
816
+ user_id: str = typer.Argument(..., help="User id (user_…)."),
817
+ reason: str = typer.Option(
818
+ "", "--reason", "-r", help="Operator note, recorded in the audit trail."
819
+ ),
820
+ ):
821
+ """Stop every workload of an account (graceful, then hard): runs, apps,
822
+ sandboxes and maps. Idempotent; nothing is restarted later.
823
+
824
+ This does NOT suspend the account. Suspension is iam's and is done in the
825
+ backoffice (the account page's Suspend action), which also calls this;
826
+ AI Fabric contains every account iam lists as suspended on its own within one
827
+ tick anyway. Use this command to stop an account's workloads now."""
828
+ from ._render import console
829
+ from . import exceptions as exc
830
+
831
+ client, _ = _make_client()
832
+ try:
833
+ out = client.admin_contain_user(user_id, reason)
834
+ except exc.FabricError as e:
835
+ raise _handle_fabric_error(e)
836
+ finally:
837
+ client.close()
838
+ c = out.get("contained", {})
839
+ console.print(
840
+ f"[magenta]contained[/magenta] {user_id}: "
841
+ f"{c.get('runs_stopped', 0)} run(s), {c.get('apps_stopped', 0)} app(s), "
842
+ f"{c.get('sandboxes_stopped', 0)} sandbox(es) stopped, "
843
+ f"{c.get('maps_cancelled', 0)} map(s) cancelled"
844
+ )
845
+
846
+
847
+ @admin_app.command("force-stop")
848
+ def admin_force_stop(
849
+ kind: str = typer.Argument(..., help="One of: app, deployment, run, sandbox, map."),
850
+ ref: str = typer.Argument(..., help="Id (or app name) of the workload."),
851
+ ):
852
+ """Force-stop ANY tenant's workload regardless of ownership. Audited."""
853
+ from ._render import console, err_console
854
+ from . import exceptions as exc
855
+
856
+ if kind not in _FORCE_STOP_KINDS:
857
+ err_console.print(
858
+ f"[red]unknown kind[/red] {kind!r}; expected one of {', '.join(_FORCE_STOP_KINDS)}"
859
+ )
860
+ raise typer.Exit(2)
861
+ client, _ = _make_client()
862
+ try:
863
+ out = client.admin_force_stop(kind, ref)
864
+ except exc.FabricError as e:
865
+ raise _handle_fabric_error(e)
866
+ finally:
867
+ client.close()
868
+ console.print(f"[magenta]force-stop[/magenta] {kind} {ref}: {out.get('state')}")
869
+
870
+
871
+ # --------------------------------------------------------------------------
872
+ # nodes
873
+ # --------------------------------------------------------------------------
874
+
875
+
876
+ @app.command()
877
+ def nodes(
878
+ state: Optional[list[str]] = typer.Option(
879
+ None, "--state", help="Filter by node state (repeatable)."
880
+ ),
881
+ ):
882
+ """Fleet view: GPUs free/busy per node (requires a staff session token).
883
+
884
+ STAFF ONLY. `GET /v1/nodes` is the whole physical fleet with no tenant
885
+ predicate — it names every node and, per GPU, the run id holding it, which
886
+ belongs to whichever tenant is on that GPU. It is gated on the `ops` or
887
+ `support` staff persona (contracts/admin-route-policy.json); an ordinary
888
+ user token gets `403 PERMISSION_DENIED`, and a staff `fg_` key
889
+ `403 INSUFFICIENT_SCOPE` (it carries no compute:admin, PAAS-630).
890
+
891
+ Kept as a top-level command rather than moved under `blade admin` (whose
892
+ help already says "requires a staff SESSION token"): `blade admin` is the
893
+ containment surface — contain-user, force-stop — and renaming a shipped
894
+ command is a breaking change for the operators who run this one. The help
895
+ text carries the requirement instead.
896
+ """
897
+ from ._render import console, nodes_table
898
+ from . import exceptions as exc
899
+
900
+ client, _ = _make_client()
901
+ try:
902
+ page = client._request(
903
+ "GET",
904
+ "/v1/nodes",
905
+ params={"limit": 200, **({"state": state} if state else {})},
906
+ )
907
+ except exc.FabricError as e:
908
+ raise _handle_fabric_error(e)
909
+ try:
910
+ items = page.get("items", [])
911
+ if not items:
912
+ console.print("[dim]no nodes registered[/dim]")
913
+ return
914
+ console.print(nodes_table(items))
915
+ finally:
916
+ client.close()
917
+
918
+
919
+ # --------------------------------------------------------------------------
920
+ # serve (dev loop)
921
+ # --------------------------------------------------------------------------
922
+
923
+
924
+ @app.command()
925
+ def serve(script: Path = typer.Argument(..., help="Python script defining the app.")):
926
+ """Dev loop: run the entrypoint, watch files, hot-reload the bundle (< 5s).
927
+
928
+ The run is created with serve=true so its container never drains on idle.
929
+ Edit a file → the bundle is re-pushed (deduplicated by sha256) and the
930
+ container reloads the code while keeping the loaded model hot. Ctrl-C stops.
931
+ """
932
+ from ._loader import load_script, select_app_and_entrypoint
933
+ from ._serve import serve as run_serve
934
+ from . import exceptions as exc
935
+
936
+ client, cfg = _make_client()
937
+ try:
938
+ module, module_name, bundle_root = load_script(script)
939
+ forge_app, entrypoint = select_app_and_entrypoint(module)
940
+ except exc.FabricError as e:
941
+ client.close()
942
+ raise _handle_fabric_error(e)
943
+
944
+ forge_app._bind_client(client)
945
+ forge_app._set_bundle_context(bundle_root, module_name)
946
+ try:
947
+ code = run_serve(client, forge_app, entrypoint, bundle_root)
948
+ finally:
949
+ client.close()
950
+ raise typer.Exit(code=code)
951
+
952
+
953
+ # --------------------------------------------------------------------------
954
+ # deploy / app
955
+ # --------------------------------------------------------------------------
956
+
957
+
958
+ def _parse_labels(labels: Optional[list[str]]) -> dict[str, str]:
959
+ parsed: dict[str, str] = {}
960
+ for item in labels or []:
961
+ if "=" not in item:
962
+ from ._render import err_console
963
+
964
+ err_console.print(f"[red]bad label[/red] {item!r} (expected KEY=VALUE)")
965
+ raise typer.Exit(code=2)
966
+ k, v = item.split("=", 1)
967
+ if not k:
968
+ from ._render import err_console
969
+
970
+ err_console.print(f"[red]bad label[/red] {item!r} (empty key)")
971
+ raise typer.Exit(code=2)
972
+ parsed[k] = v
973
+ return parsed
974
+
975
+
976
+ def _select_single_app(module):
977
+ from ._loader import find_apps
978
+ from . import exceptions as exc
979
+
980
+ apps = find_apps(module)
981
+ if not apps:
982
+ raise exc.FabricError("no bladekit.App found in script")
983
+ if len(apps) > 1:
984
+ raise exc.FabricError(
985
+ f"multiple bladekit.App objects found ({len(apps)}); deploy expects one"
986
+ )
987
+ return apps[0]
988
+
989
+
990
+ @app.command()
991
+ def deploy(
992
+ script: Path = typer.Argument(..., help="Python script defining the app."),
993
+ label: Optional[list[str]] = typer.Option(
994
+ None, "--label", help="Deployment label KEY=VALUE (repeatable)."
995
+ ),
996
+ verbose: bool = typer.Option(
997
+ False, "--verbose", "-v",
998
+ help="Verbose: raw build steps instead of the clean phase view.",
999
+ ),
1000
+ ):
1001
+ """Create a versioned deployment from @app.web_endpoint()/@app.asgi_app().
1002
+
1003
+ Clean mode renders one status line per phase (registering, building,
1004
+ deployed), a gold-arrow line per endpoint route, and an indented dim
1005
+ summary (autoscaling bounds, best-effort cold start). ``--verbose`` keeps
1006
+ the raw build-step stream.
1007
+ """
1008
+ from ._loader import load_script
1009
+ from ._render import phase, print_deployed, result_line, summary_line, set_verbose
1010
+ from rich.text import Text
1011
+ from . import exceptions as exc
1012
+
1013
+ if verbose:
1014
+ set_verbose(True)
1015
+
1016
+ client, _ = _make_client()
1017
+ try:
1018
+ module, module_name, bundle_root = load_script(script)
1019
+ forge_app = _select_single_app(module)
1020
+ forge_app._bind_client(client)
1021
+ forge_app._set_bundle_context(bundle_root, module_name)
1022
+
1023
+ labels = _parse_labels(label)
1024
+ # Resolve the manifest first — this validates endpoints/crons and (if the
1025
+ # image is a bladekit.Image DSL build) emits the `building` phase — before
1026
+ # any app is registered, so a validation error never leaves an orphan app.
1027
+ manifest = forge_app._deployment_manifest(
1028
+ resolve=True,
1029
+ require_endpoints=True,
1030
+ labels=labels or None,
1031
+ )
1032
+
1033
+ gpu = _manifest_gpu(manifest)
1034
+ reg_meta = forge_app.name if not gpu else f"{forge_app.name} ({gpu})"
1035
+ with phase("registering", reg_meta):
1036
+ app_resp = client.create_app(forge_app.name)
1037
+ app_obj = app_resp["app"]
1038
+
1039
+ url_text = Text(app_obj["url"], style="cyan") if app_obj.get("url") else None
1040
+ with phase("deployed", url_text) as ph:
1041
+ deployment = client.create_deployment(
1042
+ app_obj["app_id"], manifest.to_deployment_request()
1043
+ )
1044
+ ph.update(meta=url_text)
1045
+
1046
+ endpoints = deployment.get("endpoints") or []
1047
+ want_coldstart = any(
1048
+ ((e.get("autoscaling") or {}).get("min_replicas") or 0) > 0
1049
+ for e in endpoints
1050
+ )
1051
+ coldstart = (
1052
+ _best_effort_coldstart(client, app_obj["app_id"]) if want_coldstart else None
1053
+ )
1054
+ for endpoint in endpoints:
1055
+ methods = ",".join(endpoint.get("methods") or [])
1056
+ kind = endpoint.get("kind")
1057
+ route = f"{methods} {endpoint.get('path')}".strip()
1058
+ result_line(f"{route} {kind}" if kind else route)
1059
+ summary = _endpoint_summary(endpoint, coldstart)
1060
+ if summary:
1061
+ summary_line(summary)
1062
+ print_deployed(forge_app.name, deployment.get("version"))
1063
+ except exc.FabricError as e:
1064
+ raise _handle_fabric_error(e)
1065
+ finally:
1066
+ client.close()
1067
+
1068
+
1069
+ def _manifest_gpu(manifest) -> Optional[str]:
1070
+ """The single GPU type shared by every deployment endpoint, or None (mixed
1071
+ or CPU-only) — used for the `registering` phase metadata."""
1072
+ gpus = set()
1073
+ for ep in getattr(manifest, "endpoints", None) or []:
1074
+ gt = (ep.get("resources") or {}).get("gpu_type")
1075
+ if gt and gt != "none":
1076
+ gpus.add(gt)
1077
+ return next(iter(gpus)) if len(gpus) == 1 else None
1078
+
1079
+
1080
+ def _endpoint_summary(endpoint: dict, coldstart: Optional[str]) -> str:
1081
+ """Indented dim summary for one endpoint: autoscaling bounds + optional
1082
+ cold start. Only fields actually present are shown (middot-separated)."""
1083
+ a = endpoint.get("autoscaling") or {}
1084
+ parts: list[str] = []
1085
+ lo, hi = a.get("min_replicas"), a.get("max_replicas")
1086
+ if lo is not None and hi is not None:
1087
+ parts.append(f"min {lo} → max {hi}")
1088
+ if a.get("reserved_replicas"):
1089
+ parts.append(f"reserved {a['reserved_replicas']}")
1090
+ if a.get("max_concurrent_inputs"):
1091
+ parts.append(f"concurrency {a['max_concurrent_inputs']}")
1092
+ if coldstart:
1093
+ parts.append(f"cold-start {coldstart}")
1094
+ return " · ".join(parts)
1095
+
1096
+
1097
+ def _best_effort_coldstart(client, app_id: str) -> Optional[str]:
1098
+ """Newest cold-start total for an app as e.g. ``0.41s``; None when there is
1099
+ no ready replica yet (the common case right after deploy) or on any error —
1100
+ the cold-start line is optional, never blocking."""
1101
+ try:
1102
+ resp = client.get_app_coldstarts(app_id, limit=1)
1103
+ except Exception: # noqa: BLE001 — best-effort, optional summary field
1104
+ return None
1105
+ items = (resp or {}).get("items") or []
1106
+ if not items:
1107
+ return None
1108
+ total = ((items[0] or {}).get("phases") or {}).get("total_s")
1109
+ if not isinstance(total, (int, float)):
1110
+ return None
1111
+ return f"{total:.2f}s"
1112
+
1113
+
1114
+ def _resolve_app_id(client, name_or_id: str) -> str:
1115
+ from . import exceptions as exc
1116
+
1117
+ if name_or_id.startswith("app_"):
1118
+ return name_or_id
1119
+ page = client.list_apps(name=name_or_id, limit=1)
1120
+ items = page.get("items", [])
1121
+ if not items:
1122
+ raise exc.NotFoundError("APP_NOT_FOUND", f"no app named {name_or_id!r}")
1123
+ return items[0]["app_id"]
1124
+
1125
+
1126
+ @app_app.command("list")
1127
+ def app_list(
1128
+ state: Optional[str] = typer.Option(None, "--state", help="Filter by state."),
1129
+ limit: int = typer.Option(50, "--limit", min=1, max=200),
1130
+ ):
1131
+ """List deployed apps."""
1132
+ from rich.table import Table
1133
+ from ._render import console, clock_from_ts
1134
+ from . import exceptions as exc
1135
+
1136
+ client, _ = _make_client()
1137
+ try:
1138
+ page = client.list_apps(state=state, limit=limit)
1139
+ items = page.get("items", [])
1140
+ if not items:
1141
+ console.print("[dim]no apps[/dim]")
1142
+ return
1143
+ table = Table(title="apps", header_style="bold")
1144
+ table.add_column("app_id", overflow="fold")
1145
+ table.add_column("name")
1146
+ table.add_column("state")
1147
+ table.add_column("active_deployment")
1148
+ table.add_column("created")
1149
+ for item in items:
1150
+ table.add_row(
1151
+ item.get("app_id", "-"),
1152
+ item.get("name", "-"),
1153
+ item.get("state", "-"),
1154
+ item.get("active_deployment_id") or "-",
1155
+ clock_from_ts(item.get("created_at")),
1156
+ )
1157
+ console.print(table)
1158
+ except exc.FabricError as e:
1159
+ raise _handle_fabric_error(e)
1160
+ finally:
1161
+ client.close()
1162
+
1163
+
1164
+ @app_app.command("history")
1165
+ def app_history(
1166
+ app_name_or_id: str = typer.Argument(..., help="App name or app_ id."),
1167
+ limit: int = typer.Option(20, "--limit", min=1, max=200),
1168
+ ):
1169
+ """Show deployment history for an app."""
1170
+ from rich.table import Table
1171
+ from ._render import console, clock_from_ts
1172
+ from . import exceptions as exc
1173
+
1174
+ client, _ = _make_client()
1175
+ try:
1176
+ app_id = _resolve_app_id(client, app_name_or_id)
1177
+ page = client.list_deployments(app_id, limit=limit)
1178
+ items = page.get("items", [])
1179
+ if not items:
1180
+ console.print("[dim]no deployments[/dim]")
1181
+ return
1182
+ table = Table(title="deployments", header_style="bold")
1183
+ table.add_column("deployment_id", overflow="fold")
1184
+ table.add_column("version")
1185
+ table.add_column("state")
1186
+ table.add_column("endpoints")
1187
+ table.add_column("created")
1188
+ for dep in items:
1189
+ table.add_row(
1190
+ dep.get("deployment_id", "-"),
1191
+ str(dep.get("version", "-")),
1192
+ dep.get("state", "-"),
1193
+ str(len(dep.get("endpoints") or [])),
1194
+ clock_from_ts(dep.get("created_at")),
1195
+ )
1196
+ console.print(table)
1197
+ except exc.FabricError as e:
1198
+ raise _handle_fabric_error(e)
1199
+ finally:
1200
+ client.close()
1201
+
1202
+
1203
+ @app_app.command("rollback")
1204
+ def app_rollback(
1205
+ app_name_or_id: str = typer.Argument(..., help="App name or app_ id."),
1206
+ deployment_id: Optional[str] = typer.Option(None, "--deployment-id"),
1207
+ version: Optional[int] = typer.Option(None, "--version", min=1),
1208
+ ):
1209
+ """Rollback an app to a previous deployment id or version."""
1210
+ from ._render import console, err_console
1211
+ from . import exceptions as exc
1212
+
1213
+ if (deployment_id is None) == (version is None):
1214
+ err_console.print("[red]choose exactly one[/red]: --deployment-id or --version")
1215
+ raise typer.Exit(code=2)
1216
+ client, _ = _make_client()
1217
+ try:
1218
+ app_id = _resolve_app_id(client, app_name_or_id)
1219
+ app_obj = client.rollback_app(
1220
+ app_id, deployment_id=deployment_id, version=version
1221
+ )
1222
+ console.print(
1223
+ f"[green]rolled back[/green] {app_obj.get('name', app_id)} -> "
1224
+ f"{app_obj.get('active_deployment_id') or '-'}"
1225
+ )
1226
+ except exc.FabricError as e:
1227
+ raise _handle_fabric_error(e)
1228
+ finally:
1229
+ client.close()
1230
+
1231
+
1232
+ @app_app.command("stop")
1233
+ def app_stop(app_name_or_id: str = typer.Argument(..., help="App name or app_ id.")):
1234
+ """Stop an app and drain its active deployment replicas."""
1235
+ from ._render import console
1236
+ from . import exceptions as exc
1237
+
1238
+ client, _ = _make_client()
1239
+ try:
1240
+ app_id = _resolve_app_id(client, app_name_or_id)
1241
+ app_obj = client.stop_app(app_id)
1242
+ console.print(f"[magenta]stopped[/magenta] {app_obj.get('name', app_id)}")
1243
+ except exc.FabricError as e:
1244
+ raise _handle_fabric_error(e)
1245
+ finally:
1246
+ client.close()
1247
+
1248
+
1249
+ @app_app.command("scale")
1250
+ def app_scale(
1251
+ app_name_or_id: str = typer.Argument(..., help="App name or app_ id."),
1252
+ endpoint: Optional[str] = typer.Option(
1253
+ None, "--endpoint", "-e",
1254
+ help="Endpoint name to scale. Defaults to the only endpoint if there is one.",
1255
+ ),
1256
+ min_replicas: Optional[int] = typer.Option(None, "--min", help="Minimum replicas (0-50)."),
1257
+ max_replicas: Optional[int] = typer.Option(None, "--max", help="Maximum replicas (1-50)."),
1258
+ concurrency: Optional[int] = typer.Option(
1259
+ None, "--concurrency", help="Max concurrent inputs per replica (1-100)."
1260
+ ),
1261
+ idle: Optional[int] = typer.Option(
1262
+ None, "--idle", help="Scale-down idle seconds (30-3600)."
1263
+ ),
1264
+ reserved: Optional[int] = typer.Option(
1265
+ None, "--reserved",
1266
+ help="Hybrid: replicas kept on reserved (always-on); the rest burst on the endpoint's class.",
1267
+ ),
1268
+ target: Optional[int] = typer.Option(
1269
+ None, "--target",
1270
+ help="Autoscaling v2: per-replica concurrency the autoscaler AIMS for (1-100, <= concurrency).",
1271
+ ),
1272
+ buffer: Optional[int] = typer.Option(
1273
+ None, "--buffer",
1274
+ help="Autoscaling v2: warm spare replicas kept while active (0-10; 0 removes the buffer).",
1275
+ ),
1276
+ vllm_waiting: Optional[int] = typer.Option(
1277
+ None, "--vllm-waiting",
1278
+ help="vLLM preset: target requests waiting in vLLM's queue per replica (1-1000; 0 removes the preset; web_server only).",
1279
+ ),
1280
+ clear_policy: bool = typer.Option(
1281
+ False, "--clear-policy",
1282
+ help="Clear the scaling policy entirely (back to default concurrency autoscaling).",
1283
+ ),
1284
+ ):
1285
+ """Update a deployed endpoint's autoscaling bounds live (no redeploy).
1286
+
1287
+ Only the flags you pass change; the autoscaler applies the new bounds within
1288
+ seconds. Example: blade app scale my-api --min 2 --max 8 --reserved 2
1289
+ (2 always-on reserved replicas, bursting up to 6 more on spot.)
1290
+
1291
+ The v2 policy flags (--target/--buffer/--vllm-waiting) are merged into the
1292
+ endpoint's current policy client-side, then sent as a full replacement.
1293
+ Like every scale edit, they last until the next deploy (code wins).
1294
+ """
1295
+ from ._render import console
1296
+ from . import exceptions as exc
1297
+
1298
+ if (min_replicas is None and max_replicas is None and concurrency is None
1299
+ and idle is None and reserved is None
1300
+ and target is None and buffer is None and vllm_waiting is None
1301
+ and not clear_policy):
1302
+ console.print(
1303
+ "[red]Nothing to change.[/red] Pass at least one of "
1304
+ "--min/--max/--concurrency/--idle/--reserved/--target/--buffer/--vllm-waiting/--clear-policy."
1305
+ )
1306
+ raise typer.Exit(2)
1307
+ if clear_policy and (target is not None or buffer is not None or vllm_waiting is not None):
1308
+ console.print("[red]--clear-policy cannot be combined with --target/--buffer/--vllm-waiting.[/red]")
1309
+ raise typer.Exit(2)
1310
+
1311
+ client, _ = _make_client()
1312
+ try:
1313
+ app_id = _resolve_app_id(client, app_name_or_id)
1314
+ app_obj = client.get_app(app_id)
1315
+ endpoints = (app_obj.get("active_deployment") or {}).get("endpoints") or []
1316
+ if not endpoints:
1317
+ console.print("[red]No active deployment / endpoints to scale.[/red]")
1318
+ raise typer.Exit(1)
1319
+ target_ep = endpoint
1320
+ if target_ep is None:
1321
+ if len(endpoints) == 1:
1322
+ target_ep = endpoints[0]["name"]
1323
+ else:
1324
+ names = ", ".join(e["name"] for e in endpoints)
1325
+ console.print(f"[red]Multiple endpoints[/red] — pass --endpoint <name>: {names}")
1326
+ raise typer.Exit(2)
1327
+
1328
+ # v2 policy: read-modify-write. The server replaces the policy wholesale
1329
+ # (never merges), so the CLI merges the flags into the CURRENT policy.
1330
+ policy: Optional[dict] = None
1331
+ if target is not None or buffer is not None or vllm_waiting is not None:
1332
+ cur = next((e for e in endpoints if e["name"] == target_ep), None)
1333
+ policy = dict(((cur or {}).get("autoscaling") or {}).get("scaling_policy") or {})
1334
+ policy["v"] = 1
1335
+ if target is not None:
1336
+ policy["target_inputs"] = target
1337
+ if buffer is not None:
1338
+ if buffer > 0:
1339
+ policy["buffer_replicas"] = buffer
1340
+ else:
1341
+ policy.pop("buffer_replicas", None)
1342
+ if vllm_waiting is not None:
1343
+ if vllm_waiting > 0:
1344
+ policy["vllm"] = {"target_waiting": vllm_waiting}
1345
+ else:
1346
+ policy.pop("vllm", None)
1347
+ if set(policy.keys()) == {"v"}:
1348
+ # Everything removed → clear rather than send an empty policy.
1349
+ policy = None
1350
+ clear_policy = True
1351
+
1352
+ updated = client.scale_endpoint(
1353
+ app_id,
1354
+ target_ep,
1355
+ min_replicas=min_replicas,
1356
+ max_replicas=max_replicas,
1357
+ max_concurrent_inputs=concurrency,
1358
+ scale_down_idle_seconds=idle,
1359
+ reserved_replicas=reserved,
1360
+ scaling_policy=policy,
1361
+ clear_scaling_policy=clear_policy,
1362
+ )
1363
+ ep = next(
1364
+ (e for e in (updated.get("active_deployment") or {}).get("endpoints", []) if e["name"] == target_ep),
1365
+ None,
1366
+ )
1367
+ a = ep["autoscaling"] if ep else {}
1368
+ rsv = a.get("reserved_replicas") or 0
1369
+ hybrid = f", reserved {rsv}" if rsv else ""
1370
+ pol = a.get("scaling_policy") or {}
1371
+ extras = ""
1372
+ if pol.get("target_inputs"):
1373
+ extras += f", target {pol['target_inputs']}"
1374
+ if pol.get("buffer_replicas"):
1375
+ extras += f", buffer {pol['buffer_replicas']}"
1376
+ if (pol.get("vllm") or {}).get("target_waiting"):
1377
+ extras += f", vllm-waiting {pol['vllm']['target_waiting']}"
1378
+ console.print(
1379
+ f"[green]scaled[/green] {app_obj.get('name', app_id)} / {target_ep} → "
1380
+ f"replicas {a.get('min_replicas')}-{a.get('max_replicas')}{hybrid}, "
1381
+ f"concurrency {a.get('max_concurrent_inputs')}, idle {a.get('scale_down_idle_seconds')}s{extras}"
1382
+ )
1383
+ except exc.FabricError as e:
1384
+ raise _handle_fabric_error(e)
1385
+ finally:
1386
+ client.close()
1387
+
1388
+
1389
+ # --------------------------------------------------------------------------
1390
+ # volume
1391
+ # --------------------------------------------------------------------------
1392
+
1393
+
1394
+ @volume_app.command("list")
1395
+ def volume_list():
1396
+ """List volumes (generation, size, file count)."""
1397
+ from ._render import console, volumes_table
1398
+ from . import exceptions as exc
1399
+
1400
+ client, _ = _make_client()
1401
+ try:
1402
+ page = client.list_volumes(limit=200)
1403
+ except exc.FabricError as e:
1404
+ raise _handle_fabric_error(e)
1405
+ try:
1406
+ items = page.get("items", [])
1407
+ if not items:
1408
+ console.print("[dim]no volumes[/dim]")
1409
+ return
1410
+ console.print(volumes_table(items))
1411
+ finally:
1412
+ client.close()
1413
+
1414
+
1415
+ def _resolve_volume_id(client, name_or_id: str) -> str:
1416
+ """Accept a volume name or a vol_ id; return the id (404 if unknown name)."""
1417
+ from . import exceptions as exc
1418
+
1419
+ if name_or_id.startswith("vol_"):
1420
+ return name_or_id
1421
+ page = client.list_volumes(name=name_or_id, limit=1)
1422
+ items = page.get("items", [])
1423
+ if not items:
1424
+ raise exc.NotFoundError("VOLUME_NOT_FOUND", f"no volume named {name_or_id!r}")
1425
+ return items[0]["volume_id"]
1426
+
1427
+
1428
+ @volume_app.command("put")
1429
+ def volume_put(
1430
+ volume: str = typer.Argument(..., help="Volume name or id."),
1431
+ local_path: Path = typer.Argument(..., help="Local file to upload."),
1432
+ remote_path: str = typer.Argument(..., help="Destination path inside the volume."),
1433
+ ):
1434
+ """Upload a single file into the volume as a new generation (presigned)."""
1435
+ import hashlib
1436
+
1437
+ from ._render import console
1438
+ from . import exceptions as exc
1439
+
1440
+ client, _ = _make_client()
1441
+ try:
1442
+ if not local_path.is_file():
1443
+ console.print(f"[red]not a file[/red]: {local_path}")
1444
+ raise typer.Exit(code=1)
1445
+ data = local_path.read_bytes()
1446
+ sha = hashlib.sha256(data).hexdigest()
1447
+ rel = remote_path.lstrip("/")
1448
+ if ".." in Path(rel).parts:
1449
+ console.print(
1450
+ f"[red]invalid path[/red]: {remote_path!r} — '..' components are not allowed"
1451
+ )
1452
+ raise typer.Exit(code=2)
1453
+ volume_id = _resolve_volume_id(client, volume)
1454
+ body = {
1455
+ "entries": [
1456
+ {"path": rel, "type": "file", "size_bytes": len(data), "sha256": sha}
1457
+ ]
1458
+ }
1459
+ resp = client.create_volume_commit(volume_id, body)
1460
+ commit = resp["commit"]
1461
+ for up in resp.get("missing_uploads", []):
1462
+ if up["sha256"] == sha:
1463
+ client.upload_to_presigned(
1464
+ up["put_url"], data, "application/octet-stream"
1465
+ )
1466
+ final = client.finalize_volume_commit(volume_id, commit["commit_id"])
1467
+ gen = final.get("generation")
1468
+ console.print(
1469
+ f"[green]put[/green] {rel} → generation {gen} "
1470
+ f"[dim]({len(data)} bytes)[/dim]"
1471
+ )
1472
+ except exc.FabricError as e:
1473
+ raise _handle_fabric_error(e)
1474
+ finally:
1475
+ client.close()
1476
+
1477
+
1478
+ @volume_app.command("get")
1479
+ def volume_get(
1480
+ volume: str = typer.Argument(..., help="Volume name or id."),
1481
+ remote_path: str = typer.Argument(..., help="Path inside the volume."),
1482
+ out: Path = typer.Argument(..., help="Local destination file."),
1483
+ generation: Optional[int] = typer.Option(None, "--generation"),
1484
+ ):
1485
+ """Download a single file from the volume (presigned)."""
1486
+ from ._render import console
1487
+ from . import exceptions as exc
1488
+
1489
+ client, _ = _make_client()
1490
+ try:
1491
+ rel = remote_path.lstrip("/")
1492
+ if ".." in Path(rel).parts:
1493
+ from ._render import err_console as _ec
1494
+
1495
+ _ec.print(
1496
+ f"[red]invalid path[/red]: {remote_path!r} — '..' components are not allowed"
1497
+ )
1498
+ raise typer.Exit(code=2)
1499
+ volume_id = _resolve_volume_id(client, volume)
1500
+ info = client.get_volume_file_content(
1501
+ volume_id, rel, generation=generation
1502
+ )
1503
+ data = client.download_presigned(info["get_url"])
1504
+ out.write_bytes(data)
1505
+ console.print(
1506
+ f"[green]got[/green] {remote_path} → {out} [dim]({len(data)} bytes)[/dim]"
1507
+ )
1508
+ except exc.FabricError as e:
1509
+ raise _handle_fabric_error(e)
1510
+ finally:
1511
+ client.close()
1512
+
1513
+
1514
+ @volume_app.command("ls")
1515
+ def volume_ls(
1516
+ volume: str = typer.Argument(..., help="Volume name or id."),
1517
+ path: str = typer.Option("/", "--path", help="Directory to list."),
1518
+ generation: Optional[int] = typer.Option(None, "--generation"),
1519
+ ):
1520
+ """List entries under a path in the volume (from the manifest)."""
1521
+ from ._render import console, volume_files_table
1522
+ from . import exceptions as exc
1523
+
1524
+ client, _ = _make_client()
1525
+ try:
1526
+ volume_id = _resolve_volume_id(client, volume)
1527
+ page = client.list_volume_files(
1528
+ volume_id, path=path, generation=generation, limit=200
1529
+ )
1530
+ except exc.FabricError as e:
1531
+ raise _handle_fabric_error(e)
1532
+ try:
1533
+ items = page.get("items", [])
1534
+ console.print(f"[dim]generation {page.get('generation')}[/dim]")
1535
+ if not items:
1536
+ console.print("[dim](empty)[/dim]")
1537
+ return
1538
+ console.print(volume_files_table(items))
1539
+ finally:
1540
+ client.close()
1541
+
1542
+
1543
+ @volume_app.command("rm")
1544
+ def volume_rm(volume: str = typer.Argument(..., help="Volume name or id.")):
1545
+ """Delete a volume (refused if mounted by an active run: VOLUME_IN_USE)."""
1546
+ from ._render import console
1547
+ from . import exceptions as exc
1548
+
1549
+ client, _ = _make_client()
1550
+ try:
1551
+ volume_id = _resolve_volume_id(client, volume)
1552
+ client.delete_volume(volume_id)
1553
+ console.print(f"[magenta]deleted[/magenta] volume {volume}")
1554
+ except exc.FabricError as e:
1555
+ raise _handle_fabric_error(e)
1556
+ finally:
1557
+ client.close()
1558
+
1559
+
1560
+ # --------------------------------------------------------------------------
1561
+ # secret
1562
+ # --------------------------------------------------------------------------
1563
+
1564
+
1565
+ @secret_app.command("set")
1566
+ def secret_set(
1567
+ name: str = typer.Argument(..., help="Secret name."),
1568
+ from_env: Optional[list[str]] = typer.Option(
1569
+ None,
1570
+ "--from-env",
1571
+ help="Read KEY from the environment (repeatable: --from-env HF_TOKEN).",
1572
+ ),
1573
+ ):
1574
+ """Create or replace a secret. Values come from --from-env or stdin — NEVER argv.
1575
+
1576
+ Reading values from a positional argument would leak them into ``ps``/shell
1577
+ history, so it is intentionally impossible. With no --from-env, the secret is
1578
+ read from stdin as ``KEY=VALUE`` lines (one per line).
1579
+ """
1580
+ import os
1581
+
1582
+ from ._render import console
1583
+ from . import exceptions as exc
1584
+
1585
+ values: dict[str, str] = {}
1586
+ if from_env:
1587
+ for key in from_env:
1588
+ if key not in os.environ:
1589
+ console.print(f"[red]env var not set[/red]: {key}")
1590
+ raise typer.Exit(code=2)
1591
+ values[key] = os.environ[key]
1592
+ else:
1593
+ # Read KEY=VALUE lines from stdin (so the value never appears in argv).
1594
+ if sys.stdin.isatty():
1595
+ console.print(
1596
+ "[dim]reading KEY=VALUE lines from stdin (Ctrl-D to finish)…[/dim]"
1597
+ )
1598
+ for line in sys.stdin:
1599
+ line = line.rstrip("\n")
1600
+ if not line or line.startswith("#"):
1601
+ continue
1602
+ if "=" not in line:
1603
+ console.print(f"[red]bad line[/red] (expected KEY=VALUE): {line!r}")
1604
+ raise typer.Exit(code=2)
1605
+ k, v = line.split("=", 1)
1606
+ values[k.strip()] = v
1607
+ if not values:
1608
+ console.print("[red]no values provided[/red]")
1609
+ raise typer.Exit(code=2)
1610
+
1611
+ client, _ = _make_client()
1612
+ try:
1613
+ secret = client.set_secret(name, values)
1614
+ console.print(
1615
+ f"[green]set[/green] secret {secret['name']} "
1616
+ f"[dim](keys: {', '.join(secret.get('env_keys', []))})[/dim]"
1617
+ )
1618
+ except exc.FabricError as e:
1619
+ raise _handle_fabric_error(e)
1620
+ finally:
1621
+ client.close()
1622
+
1623
+
1624
+ @secret_app.command("list")
1625
+ def secret_list():
1626
+ """List secrets — names and env key names only (never values)."""
1627
+ from ._render import console, secrets_table
1628
+ from . import exceptions as exc
1629
+
1630
+ client, _ = _make_client()
1631
+ try:
1632
+ page = client.list_secrets(limit=200)
1633
+ except exc.FabricError as e:
1634
+ raise _handle_fabric_error(e)
1635
+ try:
1636
+ items = page.get("items", [])
1637
+ if not items:
1638
+ console.print("[dim]no secrets[/dim]")
1639
+ return
1640
+ console.print(secrets_table(items))
1641
+ finally:
1642
+ client.close()
1643
+
1644
+
1645
+ @secret_app.command("rm")
1646
+ def secret_rm(name: str = typer.Argument(..., help="Secret name or id.")):
1647
+ """Delete a secret (never blocked; pending runs referencing it will fail)."""
1648
+ from ._render import console
1649
+ from . import exceptions as exc
1650
+
1651
+ client, _ = _make_client()
1652
+ try:
1653
+ secret_id = name
1654
+ if not name.startswith("scr_"):
1655
+ page = client.list_secrets(name=name, limit=1)
1656
+ items = page.get("items", [])
1657
+ if not items:
1658
+ raise exc.NotFoundError(
1659
+ "SECRET_NOT_FOUND", f"no secret named {name!r}"
1660
+ )
1661
+ secret_id = items[0]["secret_id"]
1662
+ client.delete_secret(secret_id)
1663
+ console.print(f"[magenta]deleted[/magenta] secret {name}")
1664
+ except exc.FabricError as e:
1665
+ raise _handle_fabric_error(e)
1666
+ finally:
1667
+ client.close()
1668
+
1669
+
1670
+ # --------------------------------------------------------------------------
1671
+ # image
1672
+ # --------------------------------------------------------------------------
1673
+
1674
+
1675
+ def _load_image_from_script(module, script: Path):
1676
+ """Return the module-level ``image`` (a ``bladekit.Image``) that ``blade image
1677
+ build`` needs, or a clear :class:`bladekit.FabricError` otherwise.
1678
+
1679
+ Mirrors ``_select_single_app``'s "clear error, not a raw AttributeError/
1680
+ TypeError" pattern so a missing/wrong-type ``image`` hits the standard
1681
+ ``_handle_fabric_error`` rendering like every other script-loading error.
1682
+ """
1683
+ from ._image import Image
1684
+ from . import exceptions as exc
1685
+
1686
+ image_obj = getattr(module, "image", None)
1687
+ if image_obj is None:
1688
+ raise exc.FabricError(
1689
+ f"no module-level `image` found in {script} "
1690
+ "(expected `image = bladekit.Image(...)`)"
1691
+ )
1692
+ if not isinstance(image_obj, Image):
1693
+ raise exc.FabricError(
1694
+ f"`image` in {script} must be a bladekit.Image, got "
1695
+ f"{type(image_obj).__name__}"
1696
+ )
1697
+ return image_obj
1698
+
1699
+
1700
+ @image_app.command("build")
1701
+ def image_build(
1702
+ script: Path = typer.Argument(
1703
+ ..., help="Python script with a module-level `image = bladekit.Image(...)`."
1704
+ ),
1705
+ force: bool = typer.Option(
1706
+ False,
1707
+ "--force",
1708
+ help="Force a fresh build even if a succeeded build for this definition "
1709
+ "hash is already cached. Needed after a catalog base re-pin: the "
1710
+ "definition hash covers the alias, not the resolved digest, so an "
1711
+ "unchanged definition would otherwise dedup to the pre-re-pin build.",
1712
+ ),
1713
+ force_compression: bool = typer.Option(
1714
+ False,
1715
+ "--force-compression",
1716
+ help="Ask the control plane to force zstd recompression of inherited "
1717
+ "layers for this build (admin-authed callers only). Always set by the "
1718
+ "catalog:build:* CI jobs.",
1719
+ ),
1720
+ json_output: bool = typer.Option(
1721
+ False,
1722
+ "--json",
1723
+ help="Print a single JSON object to stdout "
1724
+ "({build_id, state, image_ref, definition_hash}) and nothing else — "
1725
+ "for CI to parse. Build logs still stream to stderr.",
1726
+ ),
1727
+ ):
1728
+ """Build a standalone bladekit.Image, outside of run/deploy.
1729
+
1730
+ Imports <script.py> as a module and requires a module-level `image`
1731
+ attribute (a bladekit.Image). Computes its definition + definition_hash the
1732
+ same way the implicit build path (run/deploy) does, then drives it
1733
+ through POST /v1/images/builds to a terminal state — its own command so
1734
+ CI can build (and promote) catalog base images
1735
+ (docs/specs/phase2-dogfood-catalog-build.md §4b).
1736
+ """
1737
+ from ._build import run_standalone_build
1738
+ from ._loader import load_script
1739
+ from ._render import console, result_line
1740
+ from . import exceptions as exc
1741
+
1742
+ try:
1743
+ module, _module_name, _bundle_root = load_script(script)
1744
+ image_obj = _load_image_from_script(module, script)
1745
+ definition = image_obj.to_definition()
1746
+ definition_hash = image_obj.definition_hash()
1747
+ except exc.FabricError as e:
1748
+ raise _handle_fabric_error(e)
1749
+
1750
+ client, _ = _make_client()
1751
+ try:
1752
+ build = run_standalone_build(
1753
+ client,
1754
+ definition,
1755
+ definition_hash,
1756
+ force=force,
1757
+ force_compression=force_compression,
1758
+ context_data=getattr(image_obj, "_context_data", None),
1759
+ )
1760
+ except exc.FabricError as e:
1761
+ raise _handle_fabric_error(e)
1762
+ finally:
1763
+ client.close()
1764
+
1765
+ image_ref = build.get("image_ref")
1766
+ if json_output:
1767
+ # Plain print, not console.print: a rich Console can soft-wrap long
1768
+ # lines to the terminal width, which would corrupt this as JSON once
1769
+ # piped/redirected (the CI case) — stdout must be EXACTLY one
1770
+ # parseable line, nothing else.
1771
+ print(
1772
+ json.dumps(
1773
+ {
1774
+ "build_id": build.get("build_id"),
1775
+ "state": build.get("state"),
1776
+ "image_ref": image_ref,
1777
+ "definition_hash": build.get("definition_hash", definition_hash),
1778
+ }
1779
+ )
1780
+ )
1781
+ return
1782
+
1783
+ console.print(f"[green]image built[/green] [dim]{definition_hash[:12]}…[/dim]")
1784
+ if image_ref:
1785
+ result_line(image_ref)
1786
+
1787
+
1788
+ @image_app.command("list")
1789
+ def image_list():
1790
+ """List the image catalogue aliases (maintained images)."""
1791
+ from ._render import console, images_table
1792
+ from . import exceptions as exc
1793
+
1794
+ client, _ = _make_client()
1795
+ try:
1796
+ page = client.list_images(limit=200)
1797
+ except exc.FabricError as e:
1798
+ raise _handle_fabric_error(e)
1799
+ try:
1800
+ items = page.get("items", [])
1801
+ if not items:
1802
+ console.print("[dim]no image aliases[/dim]")
1803
+ return
1804
+ console.print(images_table(items))
1805
+ finally:
1806
+ client.close()
1807
+
1808
+
1809
+ @image_app.command("builds")
1810
+ def image_builds(limit: int = typer.Option(50, "--limit", min=1, max=200)):
1811
+ """List recent DSL image builds (state, hash)."""
1812
+ from ._render import console, builds_table
1813
+ from . import exceptions as exc
1814
+
1815
+ client, _ = _make_client()
1816
+ try:
1817
+ page = client.list_image_builds(limit=limit)
1818
+ except exc.FabricError as e:
1819
+ raise _handle_fabric_error(e)
1820
+ try:
1821
+ items = page.get("items", [])
1822
+ if not items:
1823
+ console.print("[dim]no builds[/dim]")
1824
+ return
1825
+ console.print(builds_table(items))
1826
+ finally:
1827
+ client.close()
1828
+
1829
+
1830
+ @image_app.command("logs")
1831
+ def image_logs(
1832
+ build_id: str = typer.Argument(..., help="Image build id (ibld_…)."),
1833
+ follow: bool = typer.Option(False, "-f", "--follow", help="Stream live."),
1834
+ ):
1835
+ """Show an image build's logs (the failed-build hint points here)."""
1836
+ from ._render import print_log_chunk, flush_log_tail
1837
+ from ._build import _stream_build_logs
1838
+ from . import exceptions as exc
1839
+
1840
+ client, _ = _make_client()
1841
+ try:
1842
+ if follow:
1843
+ _stream_build_logs(client, build_id)
1844
+ else:
1845
+ cursor = None
1846
+ try:
1847
+ while True:
1848
+ page = client.get_build_logs(build_id, cursor=cursor)
1849
+ for chunk in page.get("items", []):
1850
+ print_log_chunk(chunk)
1851
+ nxt = page.get("next_cursor")
1852
+ if nxt is None:
1853
+ break
1854
+ cursor = nxt
1855
+ finally:
1856
+ flush_log_tail()
1857
+ except exc.FabricError as e:
1858
+ raise _handle_fabric_error(e)
1859
+ finally:
1860
+ client.close()
1861
+
1862
+
1863
+ # ---------------------------------------------------------------------------
1864
+ # map commands
1865
+ # ---------------------------------------------------------------------------
1866
+
1867
+
1868
+ @map_app.command("status")
1869
+ def map_status(
1870
+ map_id: str = typer.Argument(..., help="Map invocation ID (map_…)."),
1871
+ ):
1872
+ """Show the status of a map batch invocation."""
1873
+ from ._render import console
1874
+ from . import exceptions as exc
1875
+
1876
+ client, _ = _make_client()
1877
+ try:
1878
+ m = client.get_map(map_id)
1879
+ state = m.get("state", "unknown")
1880
+ input_count = m.get("input_count", 0)
1881
+ result_count = m.get("result_count", 0)
1882
+ console.print(f"[bold]{map_id}[/bold] state=[cyan]{state}[/cyan]")
1883
+ console.print(f" inputs: {input_count} completed: {result_count}")
1884
+ if m.get("finalized_at"):
1885
+ console.print(f" finalized_at: {m['finalized_at']}")
1886
+ if m.get("completed_at"):
1887
+ console.print(f" completed_at: {m['completed_at']}")
1888
+ if m.get("error"):
1889
+ err = m["error"]
1890
+ console.print(
1891
+ f" [red]error[/red]: [{err.get('code')}] {err.get('message')}"
1892
+ )
1893
+ except exc.FabricError as e:
1894
+ raise _handle_fabric_error(e)
1895
+ finally:
1896
+ client.close()
1897
+
1898
+
1899
+ @map_app.command("results")
1900
+ def map_results(
1901
+ map_id: str = typer.Argument(..., help="Map invocation ID (map_…)."),
1902
+ cursor: Optional[str] = typer.Option(None, "--cursor", help="Resume from cursor."),
1903
+ order: str = typer.Option("index", "--order", help="index or arrival."),
1904
+ return_exceptions: bool = typer.Option(
1905
+ False, "--return-exceptions", help="Yield exceptions instead of raising."
1906
+ ),
1907
+ limit: int = typer.Option(200, "--limit", help="Results per page."),
1908
+ ):
1909
+ """Stream results from a map batch (or resume from a cursor)."""
1910
+ from ._render import console
1911
+ from ._map import _stream_results
1912
+ from . import exceptions as exc
1913
+
1914
+ client, _ = _make_client()
1915
+ try:
1916
+ for i, result in enumerate(_stream_results(
1917
+ client,
1918
+ map_id,
1919
+ order=order,
1920
+ return_exceptions=return_exceptions,
1921
+ cursor=cursor,
1922
+ )):
1923
+ if isinstance(result, BaseException):
1924
+ console.print(f"[{i}] [red]exception[/red]: {result}")
1925
+ else:
1926
+ console.print(f"[{i}] {result!r}")
1927
+ except exc.FabricError as e:
1928
+ raise _handle_fabric_error(e)
1929
+ finally:
1930
+ client.close()
1931
+
1932
+
1933
+ # ---------------------------------------------------------------------------
1934
+ # cron commands
1935
+ # ---------------------------------------------------------------------------
1936
+
1937
+
1938
+ @cron_app.command("list")
1939
+ def cron_list(
1940
+ app_id: str = typer.Argument(..., help="App ID (app_…)."),
1941
+ ):
1942
+ """List cron schedules for an app."""
1943
+ from ._render import console
1944
+ from . import exceptions as exc
1945
+
1946
+ client, _ = _make_client()
1947
+ try:
1948
+ resp = client.list_crons(app_id)
1949
+ items = resp.get("items", [])
1950
+ if not items:
1951
+ console.print("[dim]no crons[/dim]")
1952
+ return
1953
+ for c in items:
1954
+ state = c.get("state", "unknown")
1955
+ expr = c.get("cron_expr", "?")
1956
+ cid = c.get("cron_id", "?")
1957
+ console.print(
1958
+ f"[bold]{cid}[/bold] expr=[cyan]{expr}[/cyan] state={state}"
1959
+ )
1960
+ except exc.FabricError as e:
1961
+ raise _handle_fabric_error(e)
1962
+ finally:
1963
+ client.close()
1964
+
1965
+
1966
+ @cron_app.command("history")
1967
+ def cron_history(
1968
+ cron_id: str = typer.Argument(..., help="Cron ID (cron_…)."),
1969
+ limit: int = typer.Option(20, "--limit", help="Max executions to show."),
1970
+ ):
1971
+ """Show recent cron executions."""
1972
+ from ._render import console
1973
+ from . import exceptions as exc
1974
+
1975
+ client, _ = _make_client()
1976
+ try:
1977
+ resp = client.list_cron_executions(cron_id, limit=limit)
1978
+ items = resp.get("items", [])
1979
+ if not items:
1980
+ console.print("[dim]no executions[/dim]")
1981
+ return
1982
+ for ex in items:
1983
+ eid = ex.get("execution_id", "?")
1984
+ state = ex.get("state", "?")
1985
+ sched = ex.get("scheduled_at", "?")
1986
+ console.print(
1987
+ f"[bold]{eid}[/bold] scheduled={sched} state=[cyan]{state}[/cyan]"
1988
+ )
1989
+ except exc.FabricError as e:
1990
+ raise _handle_fabric_error(e)
1991
+ finally:
1992
+ client.close()
1993
+
1994
+
1995
+ @cron_app.command("trigger")
1996
+ def cron_trigger(
1997
+ cron_id: str = typer.Argument(..., help="Cron ID (cron_…)."),
1998
+ ):
1999
+ """Trigger a one-off cron execution immediately."""
2000
+ from ._render import console
2001
+ from . import exceptions as exc
2002
+
2003
+ client, _ = _make_client()
2004
+ try:
2005
+ ex = client.trigger_cron(cron_id)
2006
+ eid = ex.get("execution_id", "?")
2007
+ console.print(
2008
+ f"[green]triggered[/green] execution [bold]{eid}[/bold]"
2009
+ )
2010
+ except exc.FabricError as e:
2011
+ raise _handle_fabric_error(e)
2012
+ finally:
2013
+ client.close()
2014
+
2015
+
2016
+ # ---------------------------------------------------------------------------
2017
+ # queue commands
2018
+ # ---------------------------------------------------------------------------
2019
+
2020
+
2021
+ @queue_app.command("len")
2022
+ def queue_len(
2023
+ name: str = typer.Argument(..., help="Queue name."),
2024
+ ):
2025
+ """Show the number of items in a queue."""
2026
+ from ._render import console
2027
+ from . import exceptions as exc
2028
+
2029
+ client, _ = _make_client()
2030
+ try:
2031
+ resp = client.get_queue(name)
2032
+ console.print(f"[cyan]{name}[/cyan]: {resp.get('len', 0)} items")
2033
+ except exc.FabricError as e:
2034
+ raise _handle_fabric_error(e)
2035
+ finally:
2036
+ client.close()
2037
+
2038
+
2039
+ # ---------------------------------------------------------------------------
2040
+ # dict commands
2041
+ # ---------------------------------------------------------------------------
2042
+
2043
+
2044
+ @dict_app.command("get")
2045
+ def dict_get_cmd(
2046
+ name: str = typer.Argument(..., help="Dict name."),
2047
+ key: str = typer.Argument(..., help="Key to retrieve."),
2048
+ ):
2049
+ """Get a value from a distributed dict."""
2050
+ from ._render import console
2051
+ from ._dict import Dict as DictClass
2052
+ from . import exceptions as exc
2053
+
2054
+ client, _ = _make_client()
2055
+ try:
2056
+ d = DictClass._from_client(name, client)
2057
+ value = d.get(key)
2058
+ if value is None:
2059
+ console.print(f"[dim]key {key!r} not found[/dim]")
2060
+ else:
2061
+ console.print(repr(value))
2062
+ except exc.FabricError as e:
2063
+ raise _handle_fabric_error(e)
2064
+ finally:
2065
+ client.close()
2066
+
2067
+
2068
+ @dict_app.command("set")
2069
+ def dict_set_cmd(
2070
+ name: str = typer.Argument(..., help="Dict name."),
2071
+ key: str = typer.Argument(..., help="Key to set."),
2072
+ value: str = typer.Argument(..., help="JSON-encoded value."),
2073
+ ):
2074
+ """Set a value in a distributed dict (value must be JSON)."""
2075
+ from ._render import console
2076
+ from ._dict import Dict as DictClass
2077
+ from . import exceptions as exc
2078
+
2079
+ try:
2080
+ import json as json_mod
2081
+ parsed = json_mod.loads(value)
2082
+ except json.JSONDecodeError as e:
2083
+ from ._render import err_console
2084
+ err_console.print(f"[red]error[/red]: value must be valid JSON: {e}")
2085
+ raise typer.Exit(code=1)
2086
+
2087
+ client, _ = _make_client()
2088
+ try:
2089
+ d = DictClass._from_client(name, client)
2090
+ d.put(key, parsed)
2091
+ console.print(f"[green]ok[/green] set [bold]{key}[/bold]")
2092
+ except exc.FabricError as e:
2093
+ raise _handle_fabric_error(e)
2094
+ finally:
2095
+ client.close()
2096
+
2097
+
2098
+ # ---------------------------------------------------------------------------
2099
+ # sandbox commands
2100
+ # ---------------------------------------------------------------------------
2101
+
2102
+
2103
+ def _fmt_sandbox_row(sb: dict) -> str:
2104
+ """One-line summary for a sandbox object."""
2105
+ sid = sb.get("sandbox_id", "?")
2106
+ state = sb.get("state", "?")
2107
+ rt = sb.get("runtime", "?")
2108
+ image = sb.get("image_alias") or sb.get("image_ref", "?")
2109
+ created = sb.get("created_at", "")[:19].replace("T", " ")
2110
+ # A sealed sandbox looks identical to any other otherwise, and "why can't this
2111
+ # thing resolve anything" is the first question it raises.
2112
+ net = " no-network" if sb.get("block_network") else ""
2113
+ return f"{sid} {state:<12} {rt:<6} {image:<30} {created}{net}"
2114
+
2115
+
2116
+ @sandbox_app.command("list")
2117
+ def sandbox_list(
2118
+ state: Optional[str] = typer.Option(
2119
+ None, "--state", help="Filter by state (creating/running/terminating/terminated/failed)."
2120
+ ),
2121
+ limit: int = typer.Option(50, "--limit", min=1, max=200),
2122
+ ):
2123
+ """List sandboxes (most recent first)."""
2124
+ from ._render import console
2125
+ from . import exceptions as exc
2126
+
2127
+ client, _ = _make_client()
2128
+ try:
2129
+ states = [state] if state else None
2130
+ page = client.list_sandboxes(state=states, limit=limit)
2131
+ items = page.get("items", [])
2132
+ if not items:
2133
+ console.print("[dim]no sandboxes[/dim]")
2134
+ return
2135
+ console.print(f"{'SANDBOX ID':<32} {'STATE':<12} {'RUNTIME':<6} {'IMAGE':<30} CREATED")
2136
+ console.print("-" * 100)
2137
+ for sb in items:
2138
+ console.print(_fmt_sandbox_row(sb))
2139
+ except exc.FabricError as e:
2140
+ raise _handle_fabric_error(e)
2141
+ finally:
2142
+ client.close()
2143
+
2144
+
2145
+ @sandbox_app.command("terminate")
2146
+ def sandbox_terminate(
2147
+ sandbox_id: str = typer.Argument(..., help="Sandbox ID (sb_…)."),
2148
+ ):
2149
+ """Terminate a running sandbox (SIGTERM -> grace -> SIGKILL)."""
2150
+ from ._render import console
2151
+ from . import exceptions as exc
2152
+
2153
+ client, _ = _make_client()
2154
+ try:
2155
+ client.sandbox_terminate(sandbox_id)
2156
+ console.print(f"[magenta]terminating[/magenta] sandbox {sandbox_id}")
2157
+ except exc.NotFoundError:
2158
+ console.print(f"[yellow]not found[/yellow]: sandbox {sandbox_id} (already gone?)")
2159
+ except exc.FabricError as e:
2160
+ raise _handle_fabric_error(e)
2161
+ finally:
2162
+ client.close()
2163
+
2164
+
2165
+ # ---------------------------------------------------------------------------
2166
+ # blade shell — interactive shell inside a sandbox
2167
+ # ---------------------------------------------------------------------------
2168
+
2169
+
2170
+ @app.command("shell")
2171
+ def shell(
2172
+ image: str = typer.Option("fabric/base", "--image", "-i", help="Image alias or ref."),
2173
+ gpu: Optional[str] = typer.Option(None, "--gpu", "-g", help="GPU type (e.g. RTX-A4500)."),
2174
+ runtime: str = typer.Option("runsc", "--runtime", help="Runtime: runsc (default) or runc."),
2175
+ timeout: int = typer.Option(3600, "--timeout", help="Sandbox lifetime in seconds."),
2176
+ block_network: bool = typer.Option(
2177
+ False,
2178
+ "--block-network",
2179
+ help="No network at all: loopback only, no DNS, no Internet (image/volumes still work).",
2180
+ ),
2181
+ cmd: Optional[list[str]] = typer.Argument(None, help="Command to run (default: /bin/bash)."),
2182
+ ):
2183
+ """Open an interactive shell inside a GPU sandbox.
2184
+
2185
+ Equivalent to: Sandbox.create(...) then exec /bin/bash (or the given cmd).
2186
+ Streams stdin/stdout/stderr interactively via the sandbox exec WebSocket.
2187
+
2188
+ \\b
2189
+ blade shell # /bin/bash in a CPU sandbox (runsc)
2190
+ blade shell --gpu RTX-A4500 # GPU sandbox
2191
+ blade shell --runtime runc # trusted-only debug (runc)
2192
+ blade shell --block-network # sealed: no Internet from inside
2193
+ blade shell -- python3 script.py # run a command and stream output
2194
+ """
2195
+ from ._render import err_console
2196
+ from ._sandbox import Sandbox
2197
+ from . import exceptions as exc
2198
+
2199
+ argv = list(cmd) if cmd else ["/bin/bash"]
2200
+
2201
+ client, _ = _make_client()
2202
+
2203
+ try:
2204
+ net = ", network=blocked" if block_network else ""
2205
+ err_console.print(
2206
+ f"[dim]Creating sandbox (image={image!r}, runtime={runtime!r}{net})...[/dim]"
2207
+ )
2208
+ sb = Sandbox.create(
2209
+ image,
2210
+ gpu=gpu,
2211
+ timeout=timeout,
2212
+ runtime=runtime,
2213
+ block_network=block_network,
2214
+ client=client,
2215
+ )
2216
+ err_console.print(f"[dim]Sandbox {sb.sandbox_id} created. Waiting for running...[/dim]")
2217
+ try:
2218
+ sb.wait_for_running(timeout=120.0)
2219
+ except (TimeoutError, exc.SandboxFailedError) as e:
2220
+ err_console.print(f"[red]sandbox failed to start[/red]: {e}")
2221
+ raise typer.Exit(code=1)
2222
+
2223
+ err_console.print(f"[green]Connected[/green] ({sb.sandbox_id}). Type 'exit' to quit.")
2224
+
2225
+ # Use streaming exec for interactive shell
2226
+ proc = sb.exec(*argv, stream=True)
2227
+
2228
+ # Interactive I/O pump: forward local stdin to the WebSocket and print
2229
+ # remote stdout/stderr. The websockets library is required for streaming.
2230
+ # For simplicity in CI/headless environments, fall back to one-shot mode.
2231
+ if proc.stdin is not None:
2232
+ # Streaming mode: drive stdin from the local terminal in a thread.
2233
+ import threading as _threading
2234
+
2235
+ def _feed_stdin() -> None:
2236
+ import sys as _sys
2237
+ stdin = proc.stdin
2238
+ assert stdin is not None # guarded by `if proc.stdin is not None`
2239
+ try:
2240
+ while True:
2241
+ line = _sys.stdin.readline()
2242
+ if not line:
2243
+ stdin.close()
2244
+ break
2245
+ stdin.write(line.encode())
2246
+ except Exception: # noqa: BLE001
2247
+ try:
2248
+ stdin.close()
2249
+ except Exception: # noqa: BLE001
2250
+ pass
2251
+
2252
+ feeder = _threading.Thread(target=_feed_stdin, daemon=True)
2253
+ feeder.start()
2254
+
2255
+ # Print stdout/stderr as they arrive.
2256
+ import sys as _sys
2257
+ for chunk in proc.stdout:
2258
+ _sys.stdout.buffer.write(chunk)
2259
+ _sys.stdout.buffer.flush()
2260
+ for chunk in proc.stderr:
2261
+ _sys.stderr.buffer.write(chunk)
2262
+ _sys.stderr.buffer.flush()
2263
+
2264
+ rc = proc.wait(timeout=float(timeout))
2265
+ raise typer.Exit(code=rc)
2266
+
2267
+ except exc.FabricError as e:
2268
+ raise _handle_fabric_error(e)
2269
+ finally:
2270
+ try:
2271
+ sb.terminate() # type: ignore[possibly-undefined]
2272
+ except Exception: # noqa: BLE001
2273
+ pass
2274
+ client.close()
2275
+
2276
+
2277
+ def main() -> None: # console_script entry indirection if ever needed
2278
+ app()
2279
+
2280
+
2281
+ if __name__ == "__main__": # pragma: no cover
2282
+ main()