bladekit 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bladekit/__init__.py +219 -0
- bladekit/_app.py +3241 -0
- bladekit/_banner.py +640 -0
- bladekit/_batch.py +150 -0
- bladekit/_bucket.py +211 -0
- bladekit/_build.py +337 -0
- bladekit/_bundle.py +459 -0
- bladekit/_client.py +1152 -0
- bladekit/_cls.py +574 -0
- bladekit/_constants.py +121 -0
- bladekit/_cron.py +88 -0
- bladekit/_dict.py +174 -0
- bladekit/_env.py +98 -0
- bladekit/_image.py +577 -0
- bladekit/_loader.py +79 -0
- bladekit/_map.py +555 -0
- bladekit/_preempt.py +94 -0
- bladekit/_queue.py +137 -0
- bladekit/_render.py +683 -0
- bladekit/_runner_api.py +148 -0
- bladekit/_sandbox.py +1051 -0
- bladekit/_secret.py +52 -0
- bladekit/_serialize.py +88 -0
- bladekit/_serve.py +188 -0
- bladekit/_volume.py +232 -0
- bladekit/_webhook.py +90 -0
- bladekit/cli.py +2282 -0
- bladekit/config.py +181 -0
- bladekit/exceptions.py +544 -0
- bladekit-4.5.0.dist-info/METADATA +373 -0
- bladekit-4.5.0.dist-info/RECORD +33 -0
- bladekit-4.5.0.dist-info/WHEEL +4 -0
- bladekit-4.5.0.dist-info/entry_points.txt +3 -0
bladekit/_render.py
ADDED
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
"""Terminal rendering helpers (rich) for the CLI.
|
|
2
|
+
|
|
3
|
+
Kept separate from CLI command wiring so the formatting logic is testable and the
|
|
4
|
+
``rich`` import stays out of ``import bladekit``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from typing import TYPE_CHECKING, Optional, Union
|
|
15
|
+
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
from rich.table import Table
|
|
18
|
+
from rich.text import Text
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING: # rich ships no py.typed, so this is Any at runtime-lint time
|
|
21
|
+
from rich.status import Status
|
|
22
|
+
|
|
23
|
+
console = Console()
|
|
24
|
+
err_console = Console(stderr=True)
|
|
25
|
+
|
|
26
|
+
# --------------------------------------------------------------------------
|
|
27
|
+
# verbosity
|
|
28
|
+
# --------------------------------------------------------------------------
|
|
29
|
+
#
|
|
30
|
+
# A single module-level flag toggled once from the CLI callback (and the
|
|
31
|
+
# ``--verbose``/``-v`` flag on ``run``/``deploy``, honouring ``FORGE_VERBOSE``).
|
|
32
|
+
# Every renderer branches on it via :func:`is_verbose` rather than threading a
|
|
33
|
+
# parameter through every call site. In verbose mode the CLI keeps the raw,
|
|
34
|
+
# chatty output (streamed build steps, per-transition state lines); in the
|
|
35
|
+
# default clean mode it collapses each step to a single phase line.
|
|
36
|
+
VERBOSE = False
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def set_verbose(value: bool) -> None:
|
|
40
|
+
"""Set the global clean-vs-verbose rendering mode (CLI callback only)."""
|
|
41
|
+
global VERBOSE
|
|
42
|
+
VERBOSE = bool(value)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def is_verbose() -> bool:
|
|
46
|
+
return VERBOSE
|
|
47
|
+
|
|
48
|
+
# --------------------------------------------------------------------------
|
|
49
|
+
# banners — Blade AI Fabric brand (PAAS-771)
|
|
50
|
+
# --------------------------------------------------------------------------
|
|
51
|
+
#
|
|
52
|
+
# The art itself lives in _banner.py, generated by the design system and
|
|
53
|
+
# vendored verbatim; these wrappers only decide WHEN to show which banner.
|
|
54
|
+
# _banner.print_banner already prints nothing when stdout is not a terminal and
|
|
55
|
+
# handles NO_COLOR / TERM=dumb, narrow terminals, legacy Windows consoles and
|
|
56
|
+
# non-UTF-8 streams.
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _stdout_is_tty() -> bool:
|
|
60
|
+
stream = sys.stdout
|
|
61
|
+
try:
|
|
62
|
+
return bool(stream.isatty())
|
|
63
|
+
except (AttributeError, ValueError):
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def print_intro() -> None:
|
|
68
|
+
"""The big Blade AI Fabric cartouche (``blade setup``); nothing when not a TTY."""
|
|
69
|
+
from . import __version__
|
|
70
|
+
from . import _banner
|
|
71
|
+
|
|
72
|
+
_banner.print_banner("intro", version=__version__)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def print_version(label: str) -> None:
|
|
76
|
+
"""``blade --version``: compact banner with ``label`` on a TTY, else plain ``label``."""
|
|
77
|
+
if _stdout_is_tty():
|
|
78
|
+
from . import _banner
|
|
79
|
+
|
|
80
|
+
_banner.print_banner("compact", text=label)
|
|
81
|
+
else:
|
|
82
|
+
sys.stdout.write(label + "\n")
|
|
83
|
+
sys.stdout.flush()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def print_deployed(app_name: str, version: Optional[int] = None) -> None:
|
|
87
|
+
"""Closing wink after a successful ``blade deploy`` (TTY only, else nothing)."""
|
|
88
|
+
from . import _banner
|
|
89
|
+
|
|
90
|
+
text = f"{app_name} is live" + (f" · v{version}" if version is not None else "")
|
|
91
|
+
_banner.print_banner("compact", text=text, expression="wink")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# Colors per run state for a quick visual scan.
|
|
95
|
+
_STATE_STYLE = {
|
|
96
|
+
"pending": "yellow",
|
|
97
|
+
"scheduled": "cyan",
|
|
98
|
+
"starting": "cyan",
|
|
99
|
+
"running": "blue",
|
|
100
|
+
"cancelling": "magenta",
|
|
101
|
+
"succeeded": "green",
|
|
102
|
+
"failed": "red",
|
|
103
|
+
"cancelled": "magenta",
|
|
104
|
+
"lost": "red",
|
|
105
|
+
"online": "green",
|
|
106
|
+
"unreachable": "yellow",
|
|
107
|
+
"decommissioned": "dim",
|
|
108
|
+
# image build states
|
|
109
|
+
"queued": "yellow",
|
|
110
|
+
"building": "blue",
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def style_state(state: Optional[str]) -> Text:
|
|
115
|
+
s = state or "unknown"
|
|
116
|
+
return Text(s, style=_STATE_STYLE.get(s, "white"))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _parse_ts(ts: Optional[str]) -> Optional[datetime]:
|
|
120
|
+
if not ts:
|
|
121
|
+
return None
|
|
122
|
+
try:
|
|
123
|
+
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
124
|
+
except ValueError:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def now_clock() -> str:
|
|
129
|
+
return datetime.now(timezone.utc).strftime("%H:%M:%S")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def clock_from_ts(ts: Optional[str]) -> str:
|
|
133
|
+
dt = _parse_ts(ts)
|
|
134
|
+
if dt is None:
|
|
135
|
+
return now_clock()
|
|
136
|
+
return dt.astimezone(timezone.utc).strftime("%H:%M:%S")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def duration_str(start: Optional[str], end: Optional[str]) -> str:
|
|
140
|
+
s = _parse_ts(start)
|
|
141
|
+
e = _parse_ts(end)
|
|
142
|
+
if s is None:
|
|
143
|
+
return "-"
|
|
144
|
+
if e is None:
|
|
145
|
+
e = datetime.now(timezone.utc)
|
|
146
|
+
secs = max(0.0, (e - s).total_seconds())
|
|
147
|
+
if secs < 60:
|
|
148
|
+
return f"{secs:.1f}s"
|
|
149
|
+
mins, rem = divmod(int(secs), 60)
|
|
150
|
+
if mins < 60:
|
|
151
|
+
return f"{mins}m{rem:02d}s"
|
|
152
|
+
hours, mins = divmod(mins, 60)
|
|
153
|
+
return f"{hours}h{mins:02d}m"
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def warn_logs_incomplete() -> None:
|
|
157
|
+
"""One stderr line: the log ingester was unreadable, so the logs shown are partial (PAAS-701)."""
|
|
158
|
+
err_console.print("[yellow]logs incomplete[/yellow]: the log store is unavailable, retry later")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def print_state_transition(state: str, ts: Optional[str] = None) -> None:
|
|
162
|
+
"""One line per run state transition with a timestamp (progress feed)."""
|
|
163
|
+
clock = clock_from_ts(ts)
|
|
164
|
+
arrow = Text("•", style="dim")
|
|
165
|
+
err_console.print(
|
|
166
|
+
Text.assemble(
|
|
167
|
+
(f"[{clock}] ", "dim"),
|
|
168
|
+
arrow,
|
|
169
|
+
" ",
|
|
170
|
+
style_state(state),
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# --------------------------------------------------------------------------
|
|
176
|
+
# phase rendering — fal-style status lines (Blade palette)
|
|
177
|
+
# --------------------------------------------------------------------------
|
|
178
|
+
#
|
|
179
|
+
# A "phase" is one step of a longer flow (registering, building, deploying …).
|
|
180
|
+
# On a TTY in clean mode it shows a live braille spinner while the step runs,
|
|
181
|
+
# then rewrites the line in place to a green check + verb + final metadata +
|
|
182
|
+
# duration; on failure it rewrites to a red cross and re-raises. When NOT a
|
|
183
|
+
# terminal (CI, pipes) or in verbose mode it never uses cursor rewriting —
|
|
184
|
+
# it prints a plain start line and a plain done/fail line — so redirected
|
|
185
|
+
# output stays clean and deterministic.
|
|
186
|
+
|
|
187
|
+
_CHECK = "✓" # ✓ success
|
|
188
|
+
_CROSS = "✗" # ✗ failure
|
|
189
|
+
_ARROW = "→" # → result / endpoint
|
|
190
|
+
_DOT = "·" # · dim start marker (non-TTY)
|
|
191
|
+
_SPINNER = "dots" # rich's braille spinner (⠋⠙⠹…, includes U+280B)
|
|
192
|
+
_GOLD = "#f2b92e" # Blade gold (design-system --b-gold, the banner chevrons)
|
|
193
|
+
|
|
194
|
+
# Colour of the resolved verb per terminal run/build state, for the final line.
|
|
195
|
+
_RESOLVED_STYLE = {
|
|
196
|
+
"succeeded": ("bold green", _CHECK),
|
|
197
|
+
"failed": ("bold red", _CROSS),
|
|
198
|
+
"lost": ("bold red", _CROSS),
|
|
199
|
+
"cancelled": ("bold magenta", _CROSS),
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
Meta = Union[str, Text, None]
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _is_runner_noise(line: str) -> bool:
|
|
206
|
+
"""True for the runner's own internal log lines (prefixed ``[forge-runner]``:
|
|
207
|
+
drain notices, mount/setup chatter). Hidden in clean mode — it is never the
|
|
208
|
+
user program's output — and kept under ``--verbose``."""
|
|
209
|
+
return line.lstrip().startswith("[forge-runner]")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _append_meta(t: Text, meta: Meta) -> None:
|
|
213
|
+
"""Append metadata to a line: a plain str is dimmed; a Text is kept as-is
|
|
214
|
+
(so callers can colour a URL cyan, embed a gold arrow, etc.)."""
|
|
215
|
+
if meta is None:
|
|
216
|
+
return
|
|
217
|
+
t.append(" ")
|
|
218
|
+
if isinstance(meta, Text):
|
|
219
|
+
t.append_text(meta)
|
|
220
|
+
else:
|
|
221
|
+
t.append(str(meta), style="dim")
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _phase_status_text(verb: str, meta: Meta) -> Text:
|
|
225
|
+
"""The renderable shown AFTER the spinner while a phase is running."""
|
|
226
|
+
t = Text()
|
|
227
|
+
t.append(verb, style="bold")
|
|
228
|
+
_append_meta(t, meta)
|
|
229
|
+
return t
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _resolved_line(
|
|
233
|
+
glyph: str, glyph_style: str, verb: str, meta: Meta = None, dur: Optional[str] = None
|
|
234
|
+
) -> Text:
|
|
235
|
+
"""A completed/failed/start phase line: glyph + bold verb + meta + (dur)."""
|
|
236
|
+
t = Text()
|
|
237
|
+
t.append(glyph + " ", style=glyph_style)
|
|
238
|
+
t.append(verb, style="bold")
|
|
239
|
+
_append_meta(t, meta)
|
|
240
|
+
if dur:
|
|
241
|
+
t.append(" ")
|
|
242
|
+
t.append(f"({dur})", style="dim")
|
|
243
|
+
return t
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class _PhaseHandle:
|
|
247
|
+
"""Handle yielded by :func:`phase`; lets the body update the metadata (live
|
|
248
|
+
on a TTY) and pin an explicit duration string (e.g. from build timestamps
|
|
249
|
+
rather than wall-clock)."""
|
|
250
|
+
|
|
251
|
+
def __init__(self, verb: str, meta: Meta, status: Optional["Status"] = None) -> None:
|
|
252
|
+
self.verb = verb
|
|
253
|
+
self.meta = meta
|
|
254
|
+
self.duration: Optional[str] = None
|
|
255
|
+
self._status = status
|
|
256
|
+
|
|
257
|
+
def update(self, *, meta: Meta = None, duration: Optional[str] = None) -> None:
|
|
258
|
+
if meta is not None:
|
|
259
|
+
self.meta = meta
|
|
260
|
+
if duration is not None:
|
|
261
|
+
self.duration = duration
|
|
262
|
+
if self._status is not None:
|
|
263
|
+
self._status.update(_phase_status_text(self.verb, self.meta))
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@contextmanager
|
|
267
|
+
def phase(verb: str, meta: Meta = None, *, stream: Optional[Console] = None):
|
|
268
|
+
"""Render one phase of a flow as a single status line.
|
|
269
|
+
|
|
270
|
+
TTY + clean mode: a live braille spinner while the block runs, rewritten in
|
|
271
|
+
place to a green check + final meta + duration on success, or a red cross on
|
|
272
|
+
failure (then re-raised).
|
|
273
|
+
|
|
274
|
+
Non-TTY or verbose: no cursor rewriting — a plain dim start line up front and
|
|
275
|
+
a plain done/fail line afterwards, so CI logs and pipes stay readable.
|
|
276
|
+
|
|
277
|
+
``stream`` overrides the target Console (default: the shared stdout
|
|
278
|
+
``console``) — used by ``blade image build``, whose stdout is reserved for
|
|
279
|
+
the machine-parsed ``--json`` result, so its phase decoration must land on
|
|
280
|
+
``err_console`` instead.
|
|
281
|
+
"""
|
|
282
|
+
target = stream or console
|
|
283
|
+
handle = _PhaseHandle(verb, meta)
|
|
284
|
+
start = time.monotonic()
|
|
285
|
+
live = target.is_terminal and not VERBOSE
|
|
286
|
+
status = None
|
|
287
|
+
if live:
|
|
288
|
+
status = target.status(
|
|
289
|
+
_phase_status_text(verb, meta), spinner=_SPINNER, spinner_style="cyan"
|
|
290
|
+
)
|
|
291
|
+
handle._status = status
|
|
292
|
+
status.start()
|
|
293
|
+
else:
|
|
294
|
+
target.print(_resolved_line(_DOT, "dim", verb, meta))
|
|
295
|
+
try:
|
|
296
|
+
yield handle
|
|
297
|
+
except BaseException:
|
|
298
|
+
if status is not None:
|
|
299
|
+
status.stop()
|
|
300
|
+
target.print(_resolved_line(_CROSS, "bold red", verb))
|
|
301
|
+
raise
|
|
302
|
+
if status is not None:
|
|
303
|
+
status.stop()
|
|
304
|
+
dur = handle.duration or f"{time.monotonic() - start:.1f}s"
|
|
305
|
+
target.print(_resolved_line(_CHECK, "bold green", verb, handle.meta, dur))
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def result_line(text: str, *, style: str = "cyan") -> None:
|
|
309
|
+
"""An indented result line prefixed with the Blade gold arrow →
|
|
310
|
+
(endpoint URLs, routes)."""
|
|
311
|
+
t = Text(" ")
|
|
312
|
+
t.append(_ARROW + " ", style=_GOLD)
|
|
313
|
+
t.append(text, style=style)
|
|
314
|
+
console.print(t)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def summary_line(text: str) -> None:
|
|
318
|
+
"""A further-indented dim summary line under a phase (autoscaling bounds,
|
|
319
|
+
cold-start …)."""
|
|
320
|
+
console.print(Text(" " + text, style="dim"))
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
# Runs created over a queue-mode GPU quota (namespace or org tier): their
|
|
324
|
+
# creation response carried a *_GPU_QUOTA_REACHED warning, which is never
|
|
325
|
+
# persisted server-side — so the creator records the run id here and
|
|
326
|
+
# RunProgress labels the pending wait "waiting for quota" instead of the
|
|
327
|
+
# misleading "waiting for capacity". Process-local, best-effort.
|
|
328
|
+
_QUOTA_QUEUED_RUNS: set = set()
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def mark_quota_queued(run_id: str) -> None:
|
|
332
|
+
"""Record that run_id was created at/over a queue-mode GPU quota."""
|
|
333
|
+
if run_id:
|
|
334
|
+
_QUOTA_QUEUED_RUNS.add(run_id)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
class RunProgress:
|
|
338
|
+
"""Clean-mode progress for ``blade run``: a braille spinner while the run is
|
|
339
|
+
pending/scheduled/starting, which yields to the user program's stdout the
|
|
340
|
+
instant execution begins, then a resolved check/cross summary line at the
|
|
341
|
+
terminal state.
|
|
342
|
+
|
|
343
|
+
The user program's stdout/stderr is NEVER suppressed — the spinner only
|
|
344
|
+
covers the wait BEFORE output starts and is retired before the first log
|
|
345
|
+
line so it never fights the streamed output. On a non-TTY (or verbose mode
|
|
346
|
+
handled by the caller) there is no spinner: a plain start line up front and
|
|
347
|
+
a plain resolved line at the end.
|
|
348
|
+
"""
|
|
349
|
+
|
|
350
|
+
_WAIT_STATES = frozenset({"pending", "scheduled", "starting"})
|
|
351
|
+
|
|
352
|
+
# The verb shown while the run is in each pre-execution state — so the
|
|
353
|
+
# spinner names the actual phase (fal-style) instead of a misleading
|
|
354
|
+
# "running" for the whole wait. `starting` is when the node pulls the image
|
|
355
|
+
# and creates the container, which dominates the wait, hence "pulling
|
|
356
|
+
# image".
|
|
357
|
+
_WAIT_VERB = {
|
|
358
|
+
"pending": "waiting for capacity",
|
|
359
|
+
"scheduled": "scheduling",
|
|
360
|
+
"starting": "pulling image",
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
def __init__(self, meta: Meta = None, run_id: Optional[str] = None) -> None:
|
|
364
|
+
self._meta = meta
|
|
365
|
+
self._start = time.monotonic()
|
|
366
|
+
self._status: Optional["Status"] = None
|
|
367
|
+
self._live = console.is_terminal and not VERBOSE
|
|
368
|
+
self._stopped = False
|
|
369
|
+
self._started = False
|
|
370
|
+
# Current pre-execution verb; refined by on_state as the run advances.
|
|
371
|
+
self._verb = "starting run"
|
|
372
|
+
self._printed_verb: Optional[str] = None # non-TTY line de-dup
|
|
373
|
+
# Per-instance wait verbs: a quota-queued run (mark_quota_queued) waits
|
|
374
|
+
# on its quota, not on fleet capacity — name the actual phase.
|
|
375
|
+
self._wait_verbs = dict(self._WAIT_VERB)
|
|
376
|
+
if run_id is not None and run_id in _QUOTA_QUEUED_RUNS:
|
|
377
|
+
self._wait_verbs["pending"] = "waiting for quota"
|
|
378
|
+
|
|
379
|
+
def start(self) -> None:
|
|
380
|
+
self._started = True
|
|
381
|
+
if self._live:
|
|
382
|
+
self._status = console.status(
|
|
383
|
+
_phase_status_text(self._verb, self._meta),
|
|
384
|
+
spinner=_SPINNER,
|
|
385
|
+
spinner_style="cyan",
|
|
386
|
+
)
|
|
387
|
+
self._status.start()
|
|
388
|
+
# Non-TTY: nothing is printed up front — the first state transition
|
|
389
|
+
# emits the first `· <phase>` line (avoids a redundant generic line
|
|
390
|
+
# before the real phase is known).
|
|
391
|
+
|
|
392
|
+
def _print_wait_line(self, verb: str) -> None:
|
|
393
|
+
# Non-TTY: one dim `· <verb>` line per distinct phase (no cursor
|
|
394
|
+
# rewriting), so CI/pipe output shows the pull progression as plain
|
|
395
|
+
# lines without spam.
|
|
396
|
+
if verb == self._printed_verb:
|
|
397
|
+
return
|
|
398
|
+
self._printed_verb = verb
|
|
399
|
+
console.print(_resolved_line(_DOT, "dim", verb, self._meta))
|
|
400
|
+
|
|
401
|
+
def _stop_spinner(self) -> None:
|
|
402
|
+
if self._status is not None and not self._stopped:
|
|
403
|
+
self._status.stop()
|
|
404
|
+
self._stopped = True
|
|
405
|
+
|
|
406
|
+
def on_state(self, state: Optional[str]) -> None:
|
|
407
|
+
if state is None:
|
|
408
|
+
return
|
|
409
|
+
if state in self._WAIT_STATES:
|
|
410
|
+
# Refine the phase label (spinner text on a TTY, a new dim line
|
|
411
|
+
# otherwise) so the user sees "pulling image" during the pull
|
|
412
|
+
# rather than a static verb.
|
|
413
|
+
verb = self._wait_verbs.get(state, self._verb)
|
|
414
|
+
if verb == self._verb:
|
|
415
|
+
return
|
|
416
|
+
self._verb = verb
|
|
417
|
+
if self._live:
|
|
418
|
+
if self._status is not None:
|
|
419
|
+
self._status.update(_phase_status_text(verb, self._meta))
|
|
420
|
+
else:
|
|
421
|
+
self._print_wait_line(verb)
|
|
422
|
+
else:
|
|
423
|
+
# Left the waiting states — retire the spinner so the user
|
|
424
|
+
# program's stdout starts on a clean line, never under a spinner.
|
|
425
|
+
self._stop_spinner()
|
|
426
|
+
|
|
427
|
+
def before_log(self) -> None:
|
|
428
|
+
# First byte of user output: retire the spinner unconditionally.
|
|
429
|
+
self._stop_spinner()
|
|
430
|
+
|
|
431
|
+
def finish(self, final_state: Optional[str]) -> None:
|
|
432
|
+
from ._constants import RUN_TERMINAL_STATES
|
|
433
|
+
|
|
434
|
+
self._stop_spinner()
|
|
435
|
+
if not self._started or final_state not in RUN_TERMINAL_STATES:
|
|
436
|
+
return
|
|
437
|
+
style, glyph = _RESOLVED_STYLE.get(final_state, ("bold red", _CROSS))
|
|
438
|
+
dur = f"{time.monotonic() - self._start:.1f}s"
|
|
439
|
+
console.print(_resolved_line(glyph, style, final_state, None, dur))
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
class LogRenderer:
|
|
443
|
+
"""Stateful renderer for a sequence of LogChunks from one stream.
|
|
444
|
+
|
|
445
|
+
LogChunks are flushed by forge-node purely by byte volume/time (every
|
|
446
|
+
100ms or at 32/64 KiB — see forge-node/internal/agent/logbuffer.go),
|
|
447
|
+
NOT aligned to ``\\n``/``\\r`` boundaries. A single logical line (e.g. a
|
|
448
|
+
tqdm/HF/whisper download bar) can straddle two chunks: the trailing
|
|
449
|
+
partial segment of chunk N has no following ``\\r``/``\\n`` *within that
|
|
450
|
+
chunk*, so treating each chunk independently prints it as if it were a
|
|
451
|
+
complete line, then prints the continuation from chunk N+1 as a second,
|
|
452
|
+
different-looking line for the SAME logical progress bar.
|
|
453
|
+
|
|
454
|
+
To render correctly we keep a per-stream (stdout/stderr) carry buffer of
|
|
455
|
+
the not-yet-terminated tail across calls, and only emit a segment once a
|
|
456
|
+
``\\n`` confirms it is complete. In-place carriage-return progress (tqdm)
|
|
457
|
+
is still collapsed to the final state of each logical line, now applied
|
|
458
|
+
across carry+chunk instead of within a single chunk.
|
|
459
|
+
"""
|
|
460
|
+
|
|
461
|
+
def __init__(self) -> None:
|
|
462
|
+
self._carry = {"stdout": "", "stderr": ""}
|
|
463
|
+
|
|
464
|
+
def feed(self, chunk: dict) -> None:
|
|
465
|
+
data = base64.b64decode(chunk.get("data_b64", ""))
|
|
466
|
+
text = data.decode("utf-8", "replace")
|
|
467
|
+
stream = "stderr" if chunk.get("stream", "stdout") == "stderr" else "stdout"
|
|
468
|
+
is_err = stream == "stderr"
|
|
469
|
+
target = err_console if is_err else console
|
|
470
|
+
style = "dim" if is_err else ""
|
|
471
|
+
|
|
472
|
+
buf = self._carry[stream] + text
|
|
473
|
+
# Split on newlines only (NOT splitlines, which also breaks on \r and
|
|
474
|
+
# turns one progress line into many). The last element has no
|
|
475
|
+
# trailing '\n' yet — it may be a complete-but-unterminated line or a
|
|
476
|
+
# mid-line fragment continued in the next chunk, so it is carried
|
|
477
|
+
# over rather than printed now.
|
|
478
|
+
parts = buf.split("\n")
|
|
479
|
+
self._carry[stream] = parts[-1]
|
|
480
|
+
for raw in parts[:-1]:
|
|
481
|
+
line = raw.rsplit("\r", 1)[-1].rstrip()
|
|
482
|
+
if not line:
|
|
483
|
+
continue
|
|
484
|
+
if is_err and not VERBOSE and _is_runner_noise(line):
|
|
485
|
+
continue
|
|
486
|
+
target.print(Text(line, style=style), highlight=False)
|
|
487
|
+
|
|
488
|
+
if chunk.get("truncated"):
|
|
489
|
+
self.flush(stream)
|
|
490
|
+
err_console.print(
|
|
491
|
+
Text(" … [log truncated by agent — output rate too high]", "yellow")
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
def flush(self, stream: Optional[str] = None) -> None:
|
|
495
|
+
"""Emit any residual partial line(s) — call at eof / stream end."""
|
|
496
|
+
streams = [stream] if stream else ["stdout", "stderr"]
|
|
497
|
+
for s in streams:
|
|
498
|
+
raw = self._carry.get(s, "")
|
|
499
|
+
self._carry[s] = ""
|
|
500
|
+
line = raw.rsplit("\r", 1)[-1].rstrip()
|
|
501
|
+
if not line:
|
|
502
|
+
continue
|
|
503
|
+
if s == "stderr" and not VERBOSE and _is_runner_noise(line):
|
|
504
|
+
continue
|
|
505
|
+
target = err_console if s == "stderr" else console
|
|
506
|
+
style = "dim" if s == "stderr" else ""
|
|
507
|
+
target.print(Text(line, style=style), highlight=False)
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
# Module-level default renderer for callers that render a single chunk at a
|
|
511
|
+
# time without managing their own LogRenderer (keeps print_log_chunk's
|
|
512
|
+
# existing call signature working, now stateful across calls).
|
|
513
|
+
_default_renderer = LogRenderer()
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def print_log_chunk(chunk: dict) -> None:
|
|
517
|
+
"""Render a single LogChunk (bytes are base64; decode UTF-8 with replacement,
|
|
518
|
+
conventions §8).
|
|
519
|
+
|
|
520
|
+
No stream prefix: stderr is NOT an error — tqdm bars, logging and
|
|
521
|
+
transformers/whisper download meters all write there — so we print it dimmed
|
|
522
|
+
(never red) and stdout at normal weight, letting brightness tell them apart
|
|
523
|
+
without an ``err|``/``out|`` label cluttering every line. In-place
|
|
524
|
+
carriage-return progress (tqdm) is collapsed to the final state of each
|
|
525
|
+
logical line, exactly as a real terminal renders ``\\r``, so a download bar
|
|
526
|
+
shows as one clean line instead of a wall of fragments + blank lines.
|
|
527
|
+
|
|
528
|
+
Stateful across calls (see :class:`LogRenderer`): a logical line split
|
|
529
|
+
across two LogChunks (chunk boundaries are byte/time-flush based, not
|
|
530
|
+
newline-aligned) is carried over and only emitted once its terminating
|
|
531
|
+
``\\n`` arrives, instead of being rendered as two separate lines. Callers
|
|
532
|
+
that stream a bounded sequence of chunks to completion should call
|
|
533
|
+
:func:`flush_log_tail` afterwards to emit any unterminated final line.
|
|
534
|
+
"""
|
|
535
|
+
_default_renderer.feed(chunk)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def flush_log_tail() -> None:
|
|
539
|
+
"""Flush the default renderer's residual partial line (call at stream end)."""
|
|
540
|
+
_default_renderer.flush()
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def runs_table(items: list) -> Table:
|
|
544
|
+
table = Table(title="runs", header_style="bold")
|
|
545
|
+
table.add_column("run_id", overflow="fold")
|
|
546
|
+
table.add_column("state")
|
|
547
|
+
table.add_column("app")
|
|
548
|
+
table.add_column("gpu")
|
|
549
|
+
table.add_column("created")
|
|
550
|
+
table.add_column("duration")
|
|
551
|
+
for r in items:
|
|
552
|
+
res = r.get("resources") or {}
|
|
553
|
+
table.add_row(
|
|
554
|
+
r.get("run_id", "-"),
|
|
555
|
+
style_state(r.get("state")),
|
|
556
|
+
r.get("app_name") or "-",
|
|
557
|
+
res.get("gpu_type", "-"),
|
|
558
|
+
clock_from_ts(r.get("created_at")),
|
|
559
|
+
duration_str(r.get("started_at"), r.get("finished_at")),
|
|
560
|
+
)
|
|
561
|
+
return table
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _human_bytes(n: Optional[int]) -> str:
|
|
565
|
+
if n is None:
|
|
566
|
+
return "-"
|
|
567
|
+
f = float(n)
|
|
568
|
+
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
|
|
569
|
+
if f < 1024 or unit == "TiB":
|
|
570
|
+
return f"{f:.0f}{unit}" if unit == "B" else f"{f:.1f}{unit}"
|
|
571
|
+
f /= 1024
|
|
572
|
+
return f"{f:.1f}TiB"
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def volumes_table(items: list) -> Table:
|
|
576
|
+
table = Table(title="volumes", header_style="bold")
|
|
577
|
+
table.add_column("volume_id", overflow="fold")
|
|
578
|
+
table.add_column("name")
|
|
579
|
+
table.add_column("generation")
|
|
580
|
+
table.add_column("size")
|
|
581
|
+
table.add_column("files")
|
|
582
|
+
table.add_column("last_committed")
|
|
583
|
+
for v in items:
|
|
584
|
+
table.add_row(
|
|
585
|
+
v.get("volume_id", "-"),
|
|
586
|
+
v.get("name", "-"),
|
|
587
|
+
str(v.get("current_generation", 0)),
|
|
588
|
+
_human_bytes(v.get("size_bytes")),
|
|
589
|
+
str(v.get("file_count")) if v.get("file_count") is not None else "-",
|
|
590
|
+
clock_from_ts(v.get("last_committed_at")) if v.get("last_committed_at") else "-",
|
|
591
|
+
)
|
|
592
|
+
return table
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def volume_files_table(items: list) -> Table:
|
|
596
|
+
table = Table(title="files", header_style="bold")
|
|
597
|
+
table.add_column("path", overflow="fold")
|
|
598
|
+
table.add_column("type")
|
|
599
|
+
table.add_column("size")
|
|
600
|
+
table.add_column("mode")
|
|
601
|
+
for f in items:
|
|
602
|
+
mode = f.get("mode")
|
|
603
|
+
table.add_row(
|
|
604
|
+
f.get("path", "-"),
|
|
605
|
+
f.get("type", "-"),
|
|
606
|
+
_human_bytes(f.get("size_bytes")) if f.get("type") == "file" else "-",
|
|
607
|
+
oct(mode)[2:] if isinstance(mode, int) else "-",
|
|
608
|
+
)
|
|
609
|
+
return table
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def secrets_table(items: list) -> Table:
|
|
613
|
+
table = Table(title="secrets", header_style="bold")
|
|
614
|
+
table.add_column("secret_id", overflow="fold")
|
|
615
|
+
table.add_column("name")
|
|
616
|
+
table.add_column("env_keys")
|
|
617
|
+
table.add_column("updated")
|
|
618
|
+
for s in items:
|
|
619
|
+
table.add_row(
|
|
620
|
+
s.get("secret_id", "-"),
|
|
621
|
+
s.get("name", "-"),
|
|
622
|
+
", ".join(s.get("env_keys", [])) or "-",
|
|
623
|
+
clock_from_ts(s.get("updated_at") or s.get("created_at")),
|
|
624
|
+
)
|
|
625
|
+
return table
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def images_table(items: list) -> Table:
|
|
629
|
+
table = Table(title="image catalogue", header_style="bold")
|
|
630
|
+
table.add_column("alias")
|
|
631
|
+
table.add_column("registry_ref", overflow="fold")
|
|
632
|
+
table.add_column("enabled")
|
|
633
|
+
from ._image import to_display_image
|
|
634
|
+
|
|
635
|
+
for i in items:
|
|
636
|
+
table.add_row(
|
|
637
|
+
# The server lists forge/<name>; users write fabric/<name> (PAAS-771).
|
|
638
|
+
to_display_image(i.get("alias", "-")),
|
|
639
|
+
i.get("registry_ref", "-"),
|
|
640
|
+
"yes" if i.get("enabled") else "no",
|
|
641
|
+
)
|
|
642
|
+
return table
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def builds_table(items: list) -> Table:
|
|
646
|
+
table = Table(title="image builds", header_style="bold")
|
|
647
|
+
table.add_column("build_id", overflow="fold")
|
|
648
|
+
table.add_column("state")
|
|
649
|
+
table.add_column("hash")
|
|
650
|
+
table.add_column("created")
|
|
651
|
+
for b in items:
|
|
652
|
+
h = b.get("definition_hash", "")
|
|
653
|
+
table.add_row(
|
|
654
|
+
b.get("build_id", "-"),
|
|
655
|
+
style_state(b.get("state")),
|
|
656
|
+
(h[:16] + "…") if len(h) > 16 else h,
|
|
657
|
+
clock_from_ts(b.get("created_at")),
|
|
658
|
+
)
|
|
659
|
+
return table
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def nodes_table(items: list) -> Table:
|
|
663
|
+
table = Table(title="fleet", header_style="bold")
|
|
664
|
+
table.add_column("node_id", overflow="fold")
|
|
665
|
+
table.add_column("hostname")
|
|
666
|
+
table.add_column("state")
|
|
667
|
+
table.add_column("gpu_type")
|
|
668
|
+
table.add_column("free/total")
|
|
669
|
+
table.add_column("fake")
|
|
670
|
+
for n in items:
|
|
671
|
+
gpus = n.get("gpus") or []
|
|
672
|
+
total = len(gpus)
|
|
673
|
+
free = sum(1 for g in gpus if not g.get("reserved_by_run_id"))
|
|
674
|
+
gpu_types = sorted({g.get("gpu_type", "?") for g in gpus}) or ["-"]
|
|
675
|
+
table.add_row(
|
|
676
|
+
n.get("node_id", "-"),
|
|
677
|
+
n.get("hostname", "-"),
|
|
678
|
+
style_state(n.get("state")),
|
|
679
|
+
",".join(gpu_types),
|
|
680
|
+
f"{free}/{total}",
|
|
681
|
+
"yes" if n.get("fake_gpu") else "no",
|
|
682
|
+
)
|
|
683
|
+
return table
|