cortex-training 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,646 @@
1
+ # Copyright 2025 Snowflake Inc.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """Read-only Textual TUI for Cortex Training job logs and scheduling events.
17
+
18
+ Flow: open → a list of jobs with their status (via the SDK's list_jobs) →
19
+ pick one → its log sources (zone server / job orchestration / Ray head /
20
+ scheduling events) → live tail. Pass a job_id on the command line to jump
21
+ straight to that job's logs and skip the picker.
22
+
23
+ Imports ``textual`` at module load. The interface is strictly read-only and
24
+ never mutates job state.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import os
30
+ import time
31
+ import traceback
32
+
33
+ from rich.markup import escape
34
+ from textual import work
35
+ from textual.app import App
36
+ from textual.app import ComposeResult
37
+ from textual.containers import Horizontal
38
+ from textual.screen import Screen
39
+ from textual.widgets import Footer
40
+ from textual.widgets import Header
41
+ from textual.widgets import Input
42
+ from textual.widgets import Label
43
+ from textual.widgets import ListItem
44
+ from textual.widgets import ListView
45
+ from textual.widgets import Log
46
+ from textual.widgets import Static
47
+ from textual.worker import get_current_worker
48
+
49
+ from cortex_training.tui.format import entry_at_level
50
+ from cortex_training.tui.format import entry_matches
51
+ from cortex_training.tui.format import format_job_config
52
+ from cortex_training.tui.format import format_job_row
53
+ from cortex_training.tui.format import format_log_entry
54
+ from cortex_training.tui.format import is_active_status
55
+ from cortex_training.tui.format import job_matches
56
+ from cortex_training.tui.format import job_status_color
57
+ from cortex_training.tui.format import sort_jobs
58
+ from cortex_training.tui.format import source_label
59
+ from cortex_training.tui.format import wrap_log_line
60
+ from cortex_training.tui.log_cache import LogCache
61
+ from cortex_training.tui.log_cache import cached_log_pages
62
+
63
+ _ERROR_LOG = "~/.cortex-training-errors.log"
64
+
65
+
66
+ class JobListScreen(Screen):
67
+ """Pick a job. Lists jobs + status via the SDK's ``list_jobs``."""
68
+
69
+ BINDINGS = [
70
+ ("/", "filter", "Filter"),
71
+ ("escape", "clear_filter", "Clear filter"),
72
+ ("r", "refresh", "Refresh"),
73
+ ("q", "quit", "Quit"),
74
+ ]
75
+ _REFRESH_SECONDS = 20
76
+
77
+ def __init__(self, client, *, sub_job_id=None, poll_interval=1.0):
78
+ super().__init__()
79
+ self._client = client
80
+ self._sub_job_id = sub_job_id
81
+ self._poll_interval = poll_interval
82
+ self._job_by_item: dict[str, str] = {}
83
+ self._status_by_item: dict[str, str] = {}
84
+ self._all_jobs: list = []
85
+ self._filter = ""
86
+ self._last_refreshed = None
87
+ self._refreshing = False
88
+
89
+ def compose(self) -> ComposeResult:
90
+ yield Header()
91
+ yield ListView(id="jobs")
92
+ yield Input(placeholder="filter jobs… (/ focus · Esc clear)", id="jobfilter")
93
+ yield Footer()
94
+
95
+ def on_mount(self) -> None:
96
+ self._refresh_jobs()
97
+ # Gentle auto-refresh so status changes appear without pressing r.
98
+ self.set_interval(self._REFRESH_SECONDS, self._auto_refresh)
99
+
100
+ def on_screen_resume(self) -> None:
101
+ # LogScreen owns the same app-level subtitle while it is active. Restore
102
+ # the picker status when the user returns from a job.
103
+ self._update_subtitle()
104
+
105
+ def _update_subtitle(self) -> None:
106
+ if self._refreshing:
107
+ refresh_status = "refreshing…"
108
+ if self._last_refreshed:
109
+ refresh_status += f" (last refreshed {self._last_refreshed})"
110
+ else:
111
+ refresh_status = f"last refreshed {self._last_refreshed or '—'}"
112
+ self.app.sub_title = (
113
+ "select a job (↑/↓, Enter · / filter · r refresh)"
114
+ f" · {refresh_status}"
115
+ )
116
+
117
+ def _refresh_jobs(self) -> None:
118
+ self._refreshing = True
119
+ self._update_subtitle()
120
+ self._load_jobs()
121
+
122
+ def _auto_refresh(self) -> None:
123
+ # Only refetch while the picker is the active screen — don't poll GS in
124
+ # the background while the user is down in the log view.
125
+ if self.app.screen is self:
126
+ self._refresh_jobs()
127
+
128
+ def action_refresh(self) -> None:
129
+ self._refresh_jobs()
130
+
131
+ def action_filter(self) -> None:
132
+ self.set_focus(self.query_one("#jobfilter", Input))
133
+
134
+ def action_clear_filter(self) -> None:
135
+ inp = self.query_one("#jobfilter", Input)
136
+ if self.focused is inp or self._filter:
137
+ inp.value = "" # fires on_input_changed → clears the filter
138
+ self.set_focus(self.query_one("#jobs", ListView))
139
+
140
+ async def on_input_changed(self, event: Input.Changed) -> None:
141
+ if event.input.id == "jobfilter":
142
+ self._filter = event.value.strip()
143
+ await self._render_jobs()
144
+
145
+ def on_input_submitted(self, event: Input.Submitted) -> None:
146
+ if event.input.id == "jobfilter":
147
+ self.set_focus(self.query_one("#jobs", ListView))
148
+
149
+ @work(thread=True, exclusive=True, group="jobs")
150
+ def _load_jobs(self) -> None:
151
+ worker = get_current_worker()
152
+ try:
153
+ jobs = self._client.list_jobs()
154
+ except Exception as exc: # noqa: BLE001 - surfaced in the UI
155
+ self.app.call_from_thread(self._on_jobs, None, f"list_jobs failed: {type(exc).__name__}: {exc}")
156
+ return
157
+ if worker.is_cancelled:
158
+ return
159
+ self.app.call_from_thread(self._on_jobs, jobs, None)
160
+
161
+ async def _on_jobs(self, jobs, error) -> None:
162
+ self._refreshing = False
163
+ if error:
164
+ self._update_subtitle()
165
+ lv = self.query_one("#jobs", ListView)
166
+ await lv.clear()
167
+ self._all_jobs = []
168
+ self._job_by_item = {}
169
+ self._status_by_item = {}
170
+ lv.append(ListItem(Label(f"[error] {error}")))
171
+ return
172
+ self._all_jobs = sort_jobs(jobs)
173
+ self._last_refreshed = time.strftime("%H:%M:%S")
174
+ self._update_subtitle()
175
+ await self._render_jobs()
176
+
177
+ async def _render_jobs(self) -> None:
178
+ lv = self.query_one("#jobs", ListView)
179
+ prev = lv.index
180
+ # await clear() so old items (job-0, …) are gone before re-appending
181
+ # ones with the same IDs.
182
+ await lv.clear()
183
+ self._job_by_item = {}
184
+ self._status_by_item = {}
185
+ visible = [j for j in self._all_jobs if job_matches(j, self._filter)]
186
+ if not visible:
187
+ lv.append(ListItem(Label("(no matching jobs)" if self._filter else "(no jobs found)")))
188
+ return
189
+ for i, job in enumerate(visible):
190
+ jid = job.get("job_id") or job.get("id")
191
+ if not jid:
192
+ continue
193
+ item_id = f"job-{i}"
194
+ self._job_by_item[item_id] = jid
195
+ self._status_by_item[item_id] = job.get("status")
196
+ # Tint only the leading status word (subtle); the rest stays default.
197
+ plain = format_job_row(job)
198
+ status_disp = (str(job.get("status", "")).strip() or "?").removeprefix("JOB_STATE_")
199
+ color = job_status_color(job.get("status"))
200
+ if color and plain.startswith(status_disp):
201
+ text = f"[{color}]{escape(status_disp)}[/]{escape(plain[len(status_disp):])}"
202
+ else:
203
+ text = escape(plain)
204
+ lv.append(ListItem(Label(text), id=item_id))
205
+ # Keep the cursor roughly where it was so auto-refresh doesn't jump it.
206
+ if prev is not None and self._job_by_item:
207
+ lv.index = min(prev, len(self._job_by_item) - 1)
208
+
209
+ def on_list_view_selected(self, event: ListView.Selected) -> None:
210
+ item_id = event.item.id or ""
211
+ jid = self._job_by_item.get(item_id)
212
+ if jid:
213
+ self.app.push_screen(
214
+ LogScreen(
215
+ self._client,
216
+ jid,
217
+ sub_job_id=self._sub_job_id,
218
+ poll_interval=self._poll_interval,
219
+ job_status=self._status_by_item.get(item_id),
220
+ )
221
+ )
222
+
223
+
224
+ class LogScreen(Screen):
225
+ """Source list + live tail for one job. Read-only."""
226
+
227
+ BINDINGS = [
228
+ ("b", "app.pop_screen", "Back"),
229
+ ("escape", "dismiss_or_back", "Back/clear"),
230
+ ("/", "filter", "Filter"),
231
+ ("L", "cycle_level", "Level"),
232
+ ("p", "toggle_pause", "Pause"),
233
+ ("s", "save_log", "Save"),
234
+ ("y", "copy_log", "Copy all"),
235
+ ("c", "copy_text", "Copy sel"),
236
+ ("r", "refresh_sources", "Refresh"),
237
+ ("[", "shrink_sources", "Narrower"),
238
+ ("]", "grow_sources", "Wider"),
239
+ ("q", "quit", "Quit"),
240
+ ]
241
+
242
+ _SOURCES_MIN = 24
243
+ _SOURCES_MAX = 140
244
+ _SOURCES_STEP = 8
245
+ _LEVEL_CYCLE = [None, "INFO", "WARNING", "ERROR"]
246
+
247
+ def __init__(self, client, job_id, *, sub_job_id=None, poll_interval=1.0, job_status=None):
248
+ super().__init__()
249
+ self._client = client
250
+ self._job_id = job_id
251
+ self._sub_job_id = sub_job_id
252
+ self._poll_interval = poll_interval
253
+ self._job_status = job_status
254
+ self._job_reason = None # failure/cancel reason, when terminal
255
+ self._sources_width = 48
256
+ self._cache = LogCache(job_id)
257
+ self._source_by_item: dict[str, str] = {}
258
+ self._logview: Log | None = None
259
+ self._filter = "" # grep substring for the log pane
260
+ self._min_level = None # None | INFO | WARNING | ERROR
261
+ self._paused = False # auto-scroll paused?
262
+ self._current_source = None # source currently being tailed
263
+ self._shown_lines: list[str] = [] # unwrapped lines on screen (for re-wrap)
264
+ self._wrap_width = -1 # content width the buffer is wrapped to
265
+ self._last_update = None # local HH:MM:SS of the last live line
266
+
267
+ def compose(self) -> ComposeResult:
268
+ yield Header()
269
+ yield Static("", id="summary")
270
+ with Horizontal():
271
+ yield ListView(id="sources")
272
+ # Log (not RichLog): it supports native text selection + copy. It
273
+ # does not soft-wrap, so we pre-wrap each line to the pane width.
274
+ yield Log(id="logview", highlight=False, max_lines=20000)
275
+ yield Input(placeholder="filter logs… (/ focus · Esc clear)", id="logfilter")
276
+ yield Footer()
277
+
278
+ def on_mount(self) -> None:
279
+ self._logview = self.query_one("#logview", Log)
280
+ # We drive scrolling ourselves (scroll to the tail on every write unless
281
+ # paused), so disable the widget's blanket auto-scroll.
282
+ self._logview.auto_scroll = False
283
+ self._apply_sources_width()
284
+ self._update_subtitle()
285
+ self._load_sources()
286
+
287
+ def on_unmount(self) -> None:
288
+ # Stop the tail worker when leaving this job (it polls follow=True).
289
+ self.workers.cancel_all()
290
+
291
+ def _apply_sources_width(self) -> None:
292
+ try:
293
+ self.query_one("#sources", ListView).styles.width = self._sources_width
294
+ except Exception: # noqa: BLE001 - not mounted yet
295
+ pass
296
+
297
+ def action_grow_sources(self) -> None:
298
+ self._sources_width = min(self._SOURCES_MAX, self._sources_width + self._SOURCES_STEP)
299
+ self._apply_sources_width()
300
+ self.call_after_refresh(self._rewrap) # log pane width changed
301
+
302
+ def action_shrink_sources(self) -> None:
303
+ self._sources_width = max(self._SOURCES_MIN, self._sources_width - self._SOURCES_STEP)
304
+ self._apply_sources_width()
305
+ self.call_after_refresh(self._rewrap) # log pane width changed
306
+
307
+ def on_resize(self, event) -> None:
308
+ # Terminal resize changes the log pane width → re-wrap the buffer.
309
+ self.call_after_refresh(self._rewrap)
310
+
311
+ def action_refresh_sources(self) -> None:
312
+ self._load_sources()
313
+
314
+ def _update_subtitle(self) -> None:
315
+ # Dynamic status line: everything that changes over the session lives
316
+ # here, leaving the #summary line for static config only.
317
+ bits = [self._job_id]
318
+ st = (self._job_status or "").strip()
319
+ if st:
320
+ bits.append(st.removeprefix("JOB_STATE_"))
321
+ if self._job_reason:
322
+ bits.append(f"reason: {self._job_reason}")
323
+ if self._filter:
324
+ bits.append(f"filter:{self._filter}")
325
+ if self._min_level:
326
+ bits.append(f"≥{self._min_level}")
327
+ if self._paused:
328
+ bits.append("PAUSED")
329
+ if self._last_update:
330
+ bits.append(f"updated {self._last_update}")
331
+ self.app.sub_title = " · ".join(bits)
332
+
333
+ def _set_summary(self, text: str) -> None:
334
+ try:
335
+ self.query_one("#summary", Static).update(text)
336
+ except Exception: # noqa: BLE001 - not mounted yet
337
+ pass
338
+
339
+ # ─── filter / level / pause ──────────────────────────────────────────
340
+ def action_filter(self) -> None:
341
+ self.set_focus(self.query_one("#logfilter", Input))
342
+
343
+ def action_dismiss_or_back(self) -> None:
344
+ # Esc clears an active filter (and unfocuses the input); otherwise backs out.
345
+ inp = self.query_one("#logfilter", Input)
346
+ if self.focused is inp or self._filter:
347
+ inp.value = "" # fires on_input_changed → clears the filter
348
+ self.set_focus(self.query_one("#sources", ListView))
349
+ else:
350
+ self.app.pop_screen()
351
+
352
+ def on_input_changed(self, event: Input.Changed) -> None:
353
+ if event.input.id == "logfilter":
354
+ self._filter = event.value.strip()
355
+ self._update_subtitle()
356
+ self._refilter()
357
+
358
+ def on_input_submitted(self, event: Input.Submitted) -> None:
359
+ if event.input.id == "logfilter":
360
+ self.set_focus(self.query_one("#sources", ListView))
361
+
362
+ def action_cycle_level(self) -> None:
363
+ i = self._LEVEL_CYCLE.index(self._min_level) if self._min_level in self._LEVEL_CYCLE else 0
364
+ self._min_level = self._LEVEL_CYCLE[(i + 1) % len(self._LEVEL_CYCLE)]
365
+ self._update_subtitle()
366
+ self._refilter()
367
+
368
+ def action_toggle_pause(self) -> None:
369
+ self._paused = not self._paused
370
+ # Resuming jumps to the end and follows again; pausing just stops
371
+ # following (the user can scroll freely).
372
+ if not self._paused and self._logview is not None:
373
+ self._logview.scroll_end(animate=False)
374
+ self._update_subtitle()
375
+
376
+ def _refilter(self) -> None:
377
+ # Re-run the tail so the filter/level apply to the replayed cache too.
378
+ if self._current_source:
379
+ self._start_tail(self._current_source)
380
+
381
+ # ─── export / copy ───────────────────────────────────────────────────
382
+ @work(thread=True, exclusive=True, group="export")
383
+ def action_save_log(self) -> None:
384
+ src = self._current_source
385
+ if not src:
386
+ return
387
+ entries = self._cache.load_entries(src)
388
+ text = "\n".join(format_log_entry(e) for e in entries)
389
+ safe = src.replace("/", "_").replace(":", "-")
390
+ path = os.path.expanduser(f"~/cortex-training-{self._job_id[:8]}-{safe}.log")
391
+ try:
392
+ with open(path, "w", encoding="utf-8") as f:
393
+ f.write(text + ("\n" if text else ""))
394
+ msg = f"[saved] {len(entries)} line(s) → {path}"
395
+ except OSError as exc: # noqa: BLE001
396
+ msg = f"[error] save failed: {exc}"
397
+ self.app.call_from_thread(self._write_line, msg)
398
+
399
+ @work(thread=True, exclusive=True, group="copy")
400
+ def action_copy_log(self) -> None:
401
+ src = self._current_source
402
+ if not src:
403
+ return
404
+ entries = self._cache.load_entries(src)
405
+ text = "\n".join(format_log_entry(e) for e in entries)
406
+ self.app.call_from_thread(self.app.copy_to_clipboard, text)
407
+ self.app.call_from_thread(self._write_line, f"[copied] {len(entries)} line(s) to clipboard")
408
+
409
+ @work(thread=True, exclusive=True, group="sources")
410
+ def _load_sources(self) -> None:
411
+ worker = get_current_worker()
412
+ # Fetch the job once (GS metadata — survives after the zone is gone) for
413
+ # the summary header, status (drives follow/cache), and the sub-job list.
414
+ try:
415
+ job = self._client.get_job(self._job_id) or {}
416
+ except Exception: # noqa: BLE001 - best-effort
417
+ job = {}
418
+ if job.get("status"):
419
+ self._job_status = job.get("status")
420
+ if job:
421
+ self._job_reason = job.get("reason") # clears a stale reason on refresh
422
+ # The summary line is reserved for static config; status/reason go
423
+ # on the subtitle line instead.
424
+ self._post(self._set_summary, format_job_config(job))
425
+ self._post(self._update_subtitle)
426
+
427
+ # Each sub-job is a tailable source (its zone-manager / Ray-head pod,
428
+ # whose stdout already includes worker output via Ray log_to_driver).
429
+ sources = []
430
+ for sj in job.get("sub_jobs") or []:
431
+ if isinstance(sj, dict):
432
+ sid = sj.get("sub_job_id") or sj.get("id")
433
+ if sid:
434
+ sources.append({"sub_job_id": sid, "job_type": sj.get("job_type") or sj.get("type")})
435
+ if not sources:
436
+ # get_job failed/empty (e.g. offline): fall back to cached sub-jobs.
437
+ sources = [{"sub_job_id": sid} for sid in self._cache.cached_sources()]
438
+ if not sources:
439
+ self._post(
440
+ self._write_line,
441
+ "[no logs] this job has no sub-jobs and no cached logs are available",
442
+ )
443
+ if worker.is_cancelled:
444
+ return
445
+ self._post(self._populate_sources, sources)
446
+
447
+ async def _populate_sources(self, sources) -> None:
448
+ lv = self.query_one("#sources", ListView)
449
+ await lv.clear() # await removal before re-appending src-* ids (see _populate)
450
+ self._source_by_item = {}
451
+ items = [s for s in sources if s.get("sub_job_id")]
452
+ for i, s in enumerate(items):
453
+ sid = s["sub_job_id"]
454
+ item_id = f"src-{i}"
455
+ self._source_by_item[item_id] = sid
456
+ lv.append(ListItem(Label(source_label(s)), id=item_id))
457
+ if items:
458
+ lv.index = 0
459
+ self._start_tail(items[0]["sub_job_id"])
460
+
461
+ def on_list_view_selected(self, event: ListView.Selected) -> None:
462
+ sid = self._source_by_item.get(event.item.id or "")
463
+ # Re-clicking the source you're already viewing shouldn't reload/reset
464
+ # the pane (that's what caused the jump-to-bottom on every click).
465
+ if sid and sid != self._current_source:
466
+ self._start_tail(sid)
467
+
468
+ def _post(self, fn, *args, **kwargs) -> bool:
469
+ """Run ``fn`` on the UI thread from a worker; return ``False`` instead of
470
+ raising once the app/event loop is gone (teardown), so background workers
471
+ stop cleanly rather than surfacing a NoActiveAppError as a WorkerError.
472
+
473
+ Only teardown-class errors (``RuntimeError`` — NoActiveAppError and
474
+ "event loop is closed" both derive from it) are swallowed; a real bug in
475
+ ``fn`` (Attribute/Type/Key/…) still propagates to ``_tail``'s handler so
476
+ it gets logged rather than silently stopping the tail."""
477
+ try:
478
+ self.app.call_from_thread(fn, *args, **kwargs)
479
+ return True
480
+ except RuntimeError:
481
+ return False
482
+
483
+ def _start_tail(self, sub_job_id: str) -> None:
484
+ self._current_source = sub_job_id
485
+ self._shown_lines = []
486
+ # A fresh tail context (new source, or a re-tail after a filter change)
487
+ # starts following the tail — otherwise a pause left over from a prior
488
+ # source would strand this one at the top, the very bug we're fixing.
489
+ self._paused = False
490
+ self._update_subtitle()
491
+ if self._logview is not None:
492
+ self._logview.clear()
493
+ self._write_line(f"— tailing {sub_job_id} —")
494
+ self._tail(sub_job_id)
495
+
496
+ @work(thread=True, exclusive=True, group="tail")
497
+ def _tail(self, sub_job_id: str) -> None:
498
+ worker = get_current_worker()
499
+ # A terminal job's zone is gone, so the operation API would error: serve
500
+ # cache only (no live fetch). A running/unknown job follows live.
501
+ active = is_active_status(self._job_status)
502
+ try:
503
+ # Replay the local cache instantly, then (only for a live job) resume
504
+ # the tail from the saved cursor so only new lines are fetched. Each
505
+ # yield is a whole page (or the cache batch); render it in one shot
506
+ # so high-velocity logs don't saturate the UI thread per line.
507
+ saw_live = False
508
+ for kind, entries in cached_log_pages(
509
+ self._cache,
510
+ self._client,
511
+ self._job_id,
512
+ sub_job_id,
513
+ follow=active,
514
+ live=active,
515
+ poll_interval=self._poll_interval,
516
+ replay_limit=2000,
517
+ is_cancelled=lambda: worker.is_cancelled,
518
+ ):
519
+ if worker.is_cancelled:
520
+ return
521
+ # Apply the grep filter + min-level to the whole batch.
522
+ lines = [
523
+ format_log_entry(e)
524
+ for e in entries
525
+ if entry_matches(e, self._filter) and entry_at_level(e, self._min_level)
526
+ ]
527
+ if not lines:
528
+ continue
529
+ if kind == "live" and not saw_live:
530
+ saw_live = True
531
+ lines = ["— live —"] + lines
532
+ # Every batch sticks to the tail (unless paused), so the view
533
+ # opens on the newest cached lines and follows live from there.
534
+ # _post fails closed if the app is tearing down, so the worker
535
+ # stops cleanly.
536
+ if not self._post(self._write_lines, lines):
537
+ return
538
+ if kind == "live":
539
+ # Stamp the freshness indicator on each live batch.
540
+ self._last_update = time.strftime("%H:%M:%S")
541
+ if not self._post(self._update_subtitle):
542
+ return
543
+ except Exception as exc: # noqa: BLE001 - surfaced in the UI
544
+ if not worker.is_cancelled:
545
+ try:
546
+ with open(os.path.expanduser(_ERROR_LOG), "a", encoding="utf-8") as f:
547
+ f.write(f"tail {sub_job_id} failed:\n{traceback.format_exc()}\n")
548
+ except Exception: # noqa: BLE001
549
+ pass
550
+ self._post(
551
+ self._write_line,
552
+ f"[error] tail {sub_job_id} failed: {type(exc).__name__}: {exc} (full traceback: {_ERROR_LOG})",
553
+ )
554
+
555
+ def _content_width(self) -> int:
556
+ """Usable text columns of the log pane (accounts for padding/scrollbar)."""
557
+ lv = self._logview
558
+ if lv is None:
559
+ return 0
560
+ try:
561
+ w = lv.scrollable_content_region.width
562
+ except Exception: # noqa: BLE001 - not laid out yet
563
+ w = 0
564
+ if w <= 0:
565
+ w = getattr(lv.size, "width", 0)
566
+ return max(0, w)
567
+
568
+ def _write_line(self, line: str) -> None:
569
+ self._write_lines([line])
570
+
571
+ def _write_lines(self, lines: list[str]) -> None:
572
+ lv = self._logview
573
+ if lv is None or not lines:
574
+ return
575
+ self._shown_lines.extend(lines)
576
+ # Bound the re-wrap buffer to the widget's own max_lines budget.
577
+ if len(self._shown_lines) > 20000:
578
+ del self._shown_lines[: len(self._shown_lines) - 20000]
579
+ # Always stick to the tail so every viewer sees the newest lines on
580
+ # open and as they stream — cache replay and live alike. Pause is the
581
+ # one escape hatch: it stops following so the user can scroll back.
582
+ follow = not self._paused
583
+ w = self._content_width()
584
+ wrapped: list[str] = []
585
+ for line in lines:
586
+ wrapped.extend(wrap_log_line(line, w))
587
+ # One write_lines call per page keeps the UI responsive under
588
+ # high-velocity logs (vs. a cross-thread hop + write per line).
589
+ lv.write_lines(wrapped, scroll_end=follow)
590
+
591
+ def _rewrap(self) -> None:
592
+ """Re-wrap the on-screen buffer after the pane width changes."""
593
+ lv = self._logview
594
+ if lv is None:
595
+ return
596
+ w = self._content_width()
597
+ if w <= 0 or w == self._wrap_width:
598
+ return
599
+ self._wrap_width = w
600
+ lv.clear()
601
+ for line in self._shown_lines:
602
+ for piece in wrap_log_line(line, w):
603
+ lv.write_line(piece)
604
+ # Re-wrapping clears + repaints (resetting scroll to the top), so snap
605
+ # back to the tail unless the user has paused to read history.
606
+ if not self._paused:
607
+ lv.scroll_end(animate=False)
608
+
609
+
610
+ class CortexTrainingLogTUI(App):
611
+ """Entry app: show the job picker, or jump straight to a job's logs when a
612
+ job_id is supplied."""
613
+
614
+ CSS = """
615
+ #jobs { padding: 0 1; }
616
+ #sources { width: 48; border-right: solid $panel; }
617
+ #logview { padding: 0 1; }
618
+ #summary { color: $text-muted; padding: 0 1; }
619
+ #logfilter { border: tall $panel; }
620
+ #jobfilter { border: tall $panel; }
621
+ /* Text-selection highlight: reuse the picker's subtle "blurred cursor"
622
+ shade so it's visible but not a heavy saturated block. */
623
+ LogScreen > .screen--selection {
624
+ background: $block-cursor-blurred-background;
625
+ color: $block-cursor-blurred-foreground;
626
+ }
627
+ """
628
+ BINDINGS = [("q", "quit", "Quit")]
629
+
630
+ def __init__(self, client, job_id=None, *, sub_job_id=None, poll_interval=1.0):
631
+ super().__init__()
632
+ self._client = client
633
+ self._job_id = job_id
634
+ self._sub_job_id = sub_job_id
635
+ self._poll_interval = poll_interval
636
+ self.title = "cortex training logs"
637
+
638
+ def on_mount(self) -> None:
639
+ if self._job_id:
640
+ self.push_screen(
641
+ LogScreen(self._client, self._job_id, sub_job_id=self._sub_job_id, poll_interval=self._poll_interval)
642
+ )
643
+ else:
644
+ self.push_screen(
645
+ JobListScreen(self._client, sub_job_id=self._sub_job_id, poll_interval=self._poll_interval)
646
+ )