cortex-training 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_training/__init__.py +49 -0
- cortex_training/__main__.py +20 -0
- cortex_training/_cli.py +1050 -0
- cortex_training/cli.py +103 -0
- cortex_training/client.py +2866 -0
- cortex_training/peft.py +112 -0
- cortex_training/telemetry.py +526 -0
- cortex_training/tui/__init__.py +23 -0
- cortex_training/tui/__main__.py +109 -0
- cortex_training/tui/app.py +646 -0
- cortex_training/tui/format.py +389 -0
- cortex_training/tui/log_cache.py +415 -0
- cortex_training/wire.py +414 -0
- cortex_training-0.0.3.dist-info/METADATA +143 -0
- cortex_training-0.0.3.dist-info/RECORD +19 -0
- cortex_training-0.0.3.dist-info/WHEEL +5 -0
- cortex_training-0.0.3.dist-info/entry_points.txt +3 -0
- cortex_training-0.0.3.dist-info/licenses/LICENSE +177 -0
- cortex_training-0.0.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,646 @@
|
|
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
"""Read-only Textual TUI for Cortex Training job logs and scheduling events.
|
|
17
|
+
|
|
18
|
+
Flow: open → a list of jobs with their status (via the SDK's list_jobs) →
|
|
19
|
+
pick one → its log sources (zone server / job orchestration / Ray head /
|
|
20
|
+
scheduling events) → live tail. Pass a job_id on the command line to jump
|
|
21
|
+
straight to that job's logs and skip the picker.
|
|
22
|
+
|
|
23
|
+
Imports ``textual`` at module load. The interface is strictly read-only and
|
|
24
|
+
never mutates job state.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import os
|
|
30
|
+
import time
|
|
31
|
+
import traceback
|
|
32
|
+
|
|
33
|
+
from rich.markup import escape
|
|
34
|
+
from textual import work
|
|
35
|
+
from textual.app import App
|
|
36
|
+
from textual.app import ComposeResult
|
|
37
|
+
from textual.containers import Horizontal
|
|
38
|
+
from textual.screen import Screen
|
|
39
|
+
from textual.widgets import Footer
|
|
40
|
+
from textual.widgets import Header
|
|
41
|
+
from textual.widgets import Input
|
|
42
|
+
from textual.widgets import Label
|
|
43
|
+
from textual.widgets import ListItem
|
|
44
|
+
from textual.widgets import ListView
|
|
45
|
+
from textual.widgets import Log
|
|
46
|
+
from textual.widgets import Static
|
|
47
|
+
from textual.worker import get_current_worker
|
|
48
|
+
|
|
49
|
+
from cortex_training.tui.format import entry_at_level
|
|
50
|
+
from cortex_training.tui.format import entry_matches
|
|
51
|
+
from cortex_training.tui.format import format_job_config
|
|
52
|
+
from cortex_training.tui.format import format_job_row
|
|
53
|
+
from cortex_training.tui.format import format_log_entry
|
|
54
|
+
from cortex_training.tui.format import is_active_status
|
|
55
|
+
from cortex_training.tui.format import job_matches
|
|
56
|
+
from cortex_training.tui.format import job_status_color
|
|
57
|
+
from cortex_training.tui.format import sort_jobs
|
|
58
|
+
from cortex_training.tui.format import source_label
|
|
59
|
+
from cortex_training.tui.format import wrap_log_line
|
|
60
|
+
from cortex_training.tui.log_cache import LogCache
|
|
61
|
+
from cortex_training.tui.log_cache import cached_log_pages
|
|
62
|
+
|
|
63
|
+
_ERROR_LOG = "~/.cortex-training-errors.log"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class JobListScreen(Screen):
|
|
67
|
+
"""Pick a job. Lists jobs + status via the SDK's ``list_jobs``."""
|
|
68
|
+
|
|
69
|
+
BINDINGS = [
|
|
70
|
+
("/", "filter", "Filter"),
|
|
71
|
+
("escape", "clear_filter", "Clear filter"),
|
|
72
|
+
("r", "refresh", "Refresh"),
|
|
73
|
+
("q", "quit", "Quit"),
|
|
74
|
+
]
|
|
75
|
+
_REFRESH_SECONDS = 20
|
|
76
|
+
|
|
77
|
+
def __init__(self, client, *, sub_job_id=None, poll_interval=1.0):
|
|
78
|
+
super().__init__()
|
|
79
|
+
self._client = client
|
|
80
|
+
self._sub_job_id = sub_job_id
|
|
81
|
+
self._poll_interval = poll_interval
|
|
82
|
+
self._job_by_item: dict[str, str] = {}
|
|
83
|
+
self._status_by_item: dict[str, str] = {}
|
|
84
|
+
self._all_jobs: list = []
|
|
85
|
+
self._filter = ""
|
|
86
|
+
self._last_refreshed = None
|
|
87
|
+
self._refreshing = False
|
|
88
|
+
|
|
89
|
+
def compose(self) -> ComposeResult:
|
|
90
|
+
yield Header()
|
|
91
|
+
yield ListView(id="jobs")
|
|
92
|
+
yield Input(placeholder="filter jobs… (/ focus · Esc clear)", id="jobfilter")
|
|
93
|
+
yield Footer()
|
|
94
|
+
|
|
95
|
+
def on_mount(self) -> None:
|
|
96
|
+
self._refresh_jobs()
|
|
97
|
+
# Gentle auto-refresh so status changes appear without pressing r.
|
|
98
|
+
self.set_interval(self._REFRESH_SECONDS, self._auto_refresh)
|
|
99
|
+
|
|
100
|
+
def on_screen_resume(self) -> None:
|
|
101
|
+
# LogScreen owns the same app-level subtitle while it is active. Restore
|
|
102
|
+
# the picker status when the user returns from a job.
|
|
103
|
+
self._update_subtitle()
|
|
104
|
+
|
|
105
|
+
def _update_subtitle(self) -> None:
|
|
106
|
+
if self._refreshing:
|
|
107
|
+
refresh_status = "refreshing…"
|
|
108
|
+
if self._last_refreshed:
|
|
109
|
+
refresh_status += f" (last refreshed {self._last_refreshed})"
|
|
110
|
+
else:
|
|
111
|
+
refresh_status = f"last refreshed {self._last_refreshed or '—'}"
|
|
112
|
+
self.app.sub_title = (
|
|
113
|
+
"select a job (↑/↓, Enter · / filter · r refresh)"
|
|
114
|
+
f" · {refresh_status}"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def _refresh_jobs(self) -> None:
|
|
118
|
+
self._refreshing = True
|
|
119
|
+
self._update_subtitle()
|
|
120
|
+
self._load_jobs()
|
|
121
|
+
|
|
122
|
+
def _auto_refresh(self) -> None:
|
|
123
|
+
# Only refetch while the picker is the active screen — don't poll GS in
|
|
124
|
+
# the background while the user is down in the log view.
|
|
125
|
+
if self.app.screen is self:
|
|
126
|
+
self._refresh_jobs()
|
|
127
|
+
|
|
128
|
+
def action_refresh(self) -> None:
|
|
129
|
+
self._refresh_jobs()
|
|
130
|
+
|
|
131
|
+
def action_filter(self) -> None:
|
|
132
|
+
self.set_focus(self.query_one("#jobfilter", Input))
|
|
133
|
+
|
|
134
|
+
def action_clear_filter(self) -> None:
|
|
135
|
+
inp = self.query_one("#jobfilter", Input)
|
|
136
|
+
if self.focused is inp or self._filter:
|
|
137
|
+
inp.value = "" # fires on_input_changed → clears the filter
|
|
138
|
+
self.set_focus(self.query_one("#jobs", ListView))
|
|
139
|
+
|
|
140
|
+
async def on_input_changed(self, event: Input.Changed) -> None:
|
|
141
|
+
if event.input.id == "jobfilter":
|
|
142
|
+
self._filter = event.value.strip()
|
|
143
|
+
await self._render_jobs()
|
|
144
|
+
|
|
145
|
+
def on_input_submitted(self, event: Input.Submitted) -> None:
|
|
146
|
+
if event.input.id == "jobfilter":
|
|
147
|
+
self.set_focus(self.query_one("#jobs", ListView))
|
|
148
|
+
|
|
149
|
+
@work(thread=True, exclusive=True, group="jobs")
|
|
150
|
+
def _load_jobs(self) -> None:
|
|
151
|
+
worker = get_current_worker()
|
|
152
|
+
try:
|
|
153
|
+
jobs = self._client.list_jobs()
|
|
154
|
+
except Exception as exc: # noqa: BLE001 - surfaced in the UI
|
|
155
|
+
self.app.call_from_thread(self._on_jobs, None, f"list_jobs failed: {type(exc).__name__}: {exc}")
|
|
156
|
+
return
|
|
157
|
+
if worker.is_cancelled:
|
|
158
|
+
return
|
|
159
|
+
self.app.call_from_thread(self._on_jobs, jobs, None)
|
|
160
|
+
|
|
161
|
+
async def _on_jobs(self, jobs, error) -> None:
|
|
162
|
+
self._refreshing = False
|
|
163
|
+
if error:
|
|
164
|
+
self._update_subtitle()
|
|
165
|
+
lv = self.query_one("#jobs", ListView)
|
|
166
|
+
await lv.clear()
|
|
167
|
+
self._all_jobs = []
|
|
168
|
+
self._job_by_item = {}
|
|
169
|
+
self._status_by_item = {}
|
|
170
|
+
lv.append(ListItem(Label(f"[error] {error}")))
|
|
171
|
+
return
|
|
172
|
+
self._all_jobs = sort_jobs(jobs)
|
|
173
|
+
self._last_refreshed = time.strftime("%H:%M:%S")
|
|
174
|
+
self._update_subtitle()
|
|
175
|
+
await self._render_jobs()
|
|
176
|
+
|
|
177
|
+
async def _render_jobs(self) -> None:
|
|
178
|
+
lv = self.query_one("#jobs", ListView)
|
|
179
|
+
prev = lv.index
|
|
180
|
+
# await clear() so old items (job-0, …) are gone before re-appending
|
|
181
|
+
# ones with the same IDs.
|
|
182
|
+
await lv.clear()
|
|
183
|
+
self._job_by_item = {}
|
|
184
|
+
self._status_by_item = {}
|
|
185
|
+
visible = [j for j in self._all_jobs if job_matches(j, self._filter)]
|
|
186
|
+
if not visible:
|
|
187
|
+
lv.append(ListItem(Label("(no matching jobs)" if self._filter else "(no jobs found)")))
|
|
188
|
+
return
|
|
189
|
+
for i, job in enumerate(visible):
|
|
190
|
+
jid = job.get("job_id") or job.get("id")
|
|
191
|
+
if not jid:
|
|
192
|
+
continue
|
|
193
|
+
item_id = f"job-{i}"
|
|
194
|
+
self._job_by_item[item_id] = jid
|
|
195
|
+
self._status_by_item[item_id] = job.get("status")
|
|
196
|
+
# Tint only the leading status word (subtle); the rest stays default.
|
|
197
|
+
plain = format_job_row(job)
|
|
198
|
+
status_disp = (str(job.get("status", "")).strip() or "?").removeprefix("JOB_STATE_")
|
|
199
|
+
color = job_status_color(job.get("status"))
|
|
200
|
+
if color and plain.startswith(status_disp):
|
|
201
|
+
text = f"[{color}]{escape(status_disp)}[/]{escape(plain[len(status_disp):])}"
|
|
202
|
+
else:
|
|
203
|
+
text = escape(plain)
|
|
204
|
+
lv.append(ListItem(Label(text), id=item_id))
|
|
205
|
+
# Keep the cursor roughly where it was so auto-refresh doesn't jump it.
|
|
206
|
+
if prev is not None and self._job_by_item:
|
|
207
|
+
lv.index = min(prev, len(self._job_by_item) - 1)
|
|
208
|
+
|
|
209
|
+
def on_list_view_selected(self, event: ListView.Selected) -> None:
|
|
210
|
+
item_id = event.item.id or ""
|
|
211
|
+
jid = self._job_by_item.get(item_id)
|
|
212
|
+
if jid:
|
|
213
|
+
self.app.push_screen(
|
|
214
|
+
LogScreen(
|
|
215
|
+
self._client,
|
|
216
|
+
jid,
|
|
217
|
+
sub_job_id=self._sub_job_id,
|
|
218
|
+
poll_interval=self._poll_interval,
|
|
219
|
+
job_status=self._status_by_item.get(item_id),
|
|
220
|
+
)
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class LogScreen(Screen):
|
|
225
|
+
"""Source list + live tail for one job. Read-only."""
|
|
226
|
+
|
|
227
|
+
BINDINGS = [
|
|
228
|
+
("b", "app.pop_screen", "Back"),
|
|
229
|
+
("escape", "dismiss_or_back", "Back/clear"),
|
|
230
|
+
("/", "filter", "Filter"),
|
|
231
|
+
("L", "cycle_level", "Level"),
|
|
232
|
+
("p", "toggle_pause", "Pause"),
|
|
233
|
+
("s", "save_log", "Save"),
|
|
234
|
+
("y", "copy_log", "Copy all"),
|
|
235
|
+
("c", "copy_text", "Copy sel"),
|
|
236
|
+
("r", "refresh_sources", "Refresh"),
|
|
237
|
+
("[", "shrink_sources", "Narrower"),
|
|
238
|
+
("]", "grow_sources", "Wider"),
|
|
239
|
+
("q", "quit", "Quit"),
|
|
240
|
+
]
|
|
241
|
+
|
|
242
|
+
_SOURCES_MIN = 24
|
|
243
|
+
_SOURCES_MAX = 140
|
|
244
|
+
_SOURCES_STEP = 8
|
|
245
|
+
_LEVEL_CYCLE = [None, "INFO", "WARNING", "ERROR"]
|
|
246
|
+
|
|
247
|
+
def __init__(self, client, job_id, *, sub_job_id=None, poll_interval=1.0, job_status=None):
|
|
248
|
+
super().__init__()
|
|
249
|
+
self._client = client
|
|
250
|
+
self._job_id = job_id
|
|
251
|
+
self._sub_job_id = sub_job_id
|
|
252
|
+
self._poll_interval = poll_interval
|
|
253
|
+
self._job_status = job_status
|
|
254
|
+
self._job_reason = None # failure/cancel reason, when terminal
|
|
255
|
+
self._sources_width = 48
|
|
256
|
+
self._cache = LogCache(job_id)
|
|
257
|
+
self._source_by_item: dict[str, str] = {}
|
|
258
|
+
self._logview: Log | None = None
|
|
259
|
+
self._filter = "" # grep substring for the log pane
|
|
260
|
+
self._min_level = None # None | INFO | WARNING | ERROR
|
|
261
|
+
self._paused = False # auto-scroll paused?
|
|
262
|
+
self._current_source = None # source currently being tailed
|
|
263
|
+
self._shown_lines: list[str] = [] # unwrapped lines on screen (for re-wrap)
|
|
264
|
+
self._wrap_width = -1 # content width the buffer is wrapped to
|
|
265
|
+
self._last_update = None # local HH:MM:SS of the last live line
|
|
266
|
+
|
|
267
|
+
def compose(self) -> ComposeResult:
|
|
268
|
+
yield Header()
|
|
269
|
+
yield Static("", id="summary")
|
|
270
|
+
with Horizontal():
|
|
271
|
+
yield ListView(id="sources")
|
|
272
|
+
# Log (not RichLog): it supports native text selection + copy. It
|
|
273
|
+
# does not soft-wrap, so we pre-wrap each line to the pane width.
|
|
274
|
+
yield Log(id="logview", highlight=False, max_lines=20000)
|
|
275
|
+
yield Input(placeholder="filter logs… (/ focus · Esc clear)", id="logfilter")
|
|
276
|
+
yield Footer()
|
|
277
|
+
|
|
278
|
+
def on_mount(self) -> None:
|
|
279
|
+
self._logview = self.query_one("#logview", Log)
|
|
280
|
+
# We drive scrolling ourselves (scroll to the tail on every write unless
|
|
281
|
+
# paused), so disable the widget's blanket auto-scroll.
|
|
282
|
+
self._logview.auto_scroll = False
|
|
283
|
+
self._apply_sources_width()
|
|
284
|
+
self._update_subtitle()
|
|
285
|
+
self._load_sources()
|
|
286
|
+
|
|
287
|
+
def on_unmount(self) -> None:
|
|
288
|
+
# Stop the tail worker when leaving this job (it polls follow=True).
|
|
289
|
+
self.workers.cancel_all()
|
|
290
|
+
|
|
291
|
+
def _apply_sources_width(self) -> None:
|
|
292
|
+
try:
|
|
293
|
+
self.query_one("#sources", ListView).styles.width = self._sources_width
|
|
294
|
+
except Exception: # noqa: BLE001 - not mounted yet
|
|
295
|
+
pass
|
|
296
|
+
|
|
297
|
+
def action_grow_sources(self) -> None:
|
|
298
|
+
self._sources_width = min(self._SOURCES_MAX, self._sources_width + self._SOURCES_STEP)
|
|
299
|
+
self._apply_sources_width()
|
|
300
|
+
self.call_after_refresh(self._rewrap) # log pane width changed
|
|
301
|
+
|
|
302
|
+
def action_shrink_sources(self) -> None:
|
|
303
|
+
self._sources_width = max(self._SOURCES_MIN, self._sources_width - self._SOURCES_STEP)
|
|
304
|
+
self._apply_sources_width()
|
|
305
|
+
self.call_after_refresh(self._rewrap) # log pane width changed
|
|
306
|
+
|
|
307
|
+
def on_resize(self, event) -> None:
|
|
308
|
+
# Terminal resize changes the log pane width → re-wrap the buffer.
|
|
309
|
+
self.call_after_refresh(self._rewrap)
|
|
310
|
+
|
|
311
|
+
def action_refresh_sources(self) -> None:
|
|
312
|
+
self._load_sources()
|
|
313
|
+
|
|
314
|
+
def _update_subtitle(self) -> None:
|
|
315
|
+
# Dynamic status line: everything that changes over the session lives
|
|
316
|
+
# here, leaving the #summary line for static config only.
|
|
317
|
+
bits = [self._job_id]
|
|
318
|
+
st = (self._job_status or "").strip()
|
|
319
|
+
if st:
|
|
320
|
+
bits.append(st.removeprefix("JOB_STATE_"))
|
|
321
|
+
if self._job_reason:
|
|
322
|
+
bits.append(f"reason: {self._job_reason}")
|
|
323
|
+
if self._filter:
|
|
324
|
+
bits.append(f"filter:{self._filter}")
|
|
325
|
+
if self._min_level:
|
|
326
|
+
bits.append(f"≥{self._min_level}")
|
|
327
|
+
if self._paused:
|
|
328
|
+
bits.append("PAUSED")
|
|
329
|
+
if self._last_update:
|
|
330
|
+
bits.append(f"updated {self._last_update}")
|
|
331
|
+
self.app.sub_title = " · ".join(bits)
|
|
332
|
+
|
|
333
|
+
def _set_summary(self, text: str) -> None:
|
|
334
|
+
try:
|
|
335
|
+
self.query_one("#summary", Static).update(text)
|
|
336
|
+
except Exception: # noqa: BLE001 - not mounted yet
|
|
337
|
+
pass
|
|
338
|
+
|
|
339
|
+
# ─── filter / level / pause ──────────────────────────────────────────
|
|
340
|
+
def action_filter(self) -> None:
|
|
341
|
+
self.set_focus(self.query_one("#logfilter", Input))
|
|
342
|
+
|
|
343
|
+
def action_dismiss_or_back(self) -> None:
|
|
344
|
+
# Esc clears an active filter (and unfocuses the input); otherwise backs out.
|
|
345
|
+
inp = self.query_one("#logfilter", Input)
|
|
346
|
+
if self.focused is inp or self._filter:
|
|
347
|
+
inp.value = "" # fires on_input_changed → clears the filter
|
|
348
|
+
self.set_focus(self.query_one("#sources", ListView))
|
|
349
|
+
else:
|
|
350
|
+
self.app.pop_screen()
|
|
351
|
+
|
|
352
|
+
def on_input_changed(self, event: Input.Changed) -> None:
|
|
353
|
+
if event.input.id == "logfilter":
|
|
354
|
+
self._filter = event.value.strip()
|
|
355
|
+
self._update_subtitle()
|
|
356
|
+
self._refilter()
|
|
357
|
+
|
|
358
|
+
def on_input_submitted(self, event: Input.Submitted) -> None:
|
|
359
|
+
if event.input.id == "logfilter":
|
|
360
|
+
self.set_focus(self.query_one("#sources", ListView))
|
|
361
|
+
|
|
362
|
+
def action_cycle_level(self) -> None:
|
|
363
|
+
i = self._LEVEL_CYCLE.index(self._min_level) if self._min_level in self._LEVEL_CYCLE else 0
|
|
364
|
+
self._min_level = self._LEVEL_CYCLE[(i + 1) % len(self._LEVEL_CYCLE)]
|
|
365
|
+
self._update_subtitle()
|
|
366
|
+
self._refilter()
|
|
367
|
+
|
|
368
|
+
def action_toggle_pause(self) -> None:
|
|
369
|
+
self._paused = not self._paused
|
|
370
|
+
# Resuming jumps to the end and follows again; pausing just stops
|
|
371
|
+
# following (the user can scroll freely).
|
|
372
|
+
if not self._paused and self._logview is not None:
|
|
373
|
+
self._logview.scroll_end(animate=False)
|
|
374
|
+
self._update_subtitle()
|
|
375
|
+
|
|
376
|
+
def _refilter(self) -> None:
|
|
377
|
+
# Re-run the tail so the filter/level apply to the replayed cache too.
|
|
378
|
+
if self._current_source:
|
|
379
|
+
self._start_tail(self._current_source)
|
|
380
|
+
|
|
381
|
+
# ─── export / copy ───────────────────────────────────────────────────
|
|
382
|
+
@work(thread=True, exclusive=True, group="export")
|
|
383
|
+
def action_save_log(self) -> None:
|
|
384
|
+
src = self._current_source
|
|
385
|
+
if not src:
|
|
386
|
+
return
|
|
387
|
+
entries = self._cache.load_entries(src)
|
|
388
|
+
text = "\n".join(format_log_entry(e) for e in entries)
|
|
389
|
+
safe = src.replace("/", "_").replace(":", "-")
|
|
390
|
+
path = os.path.expanduser(f"~/cortex-training-{self._job_id[:8]}-{safe}.log")
|
|
391
|
+
try:
|
|
392
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
393
|
+
f.write(text + ("\n" if text else ""))
|
|
394
|
+
msg = f"[saved] {len(entries)} line(s) → {path}"
|
|
395
|
+
except OSError as exc: # noqa: BLE001
|
|
396
|
+
msg = f"[error] save failed: {exc}"
|
|
397
|
+
self.app.call_from_thread(self._write_line, msg)
|
|
398
|
+
|
|
399
|
+
@work(thread=True, exclusive=True, group="copy")
|
|
400
|
+
def action_copy_log(self) -> None:
|
|
401
|
+
src = self._current_source
|
|
402
|
+
if not src:
|
|
403
|
+
return
|
|
404
|
+
entries = self._cache.load_entries(src)
|
|
405
|
+
text = "\n".join(format_log_entry(e) for e in entries)
|
|
406
|
+
self.app.call_from_thread(self.app.copy_to_clipboard, text)
|
|
407
|
+
self.app.call_from_thread(self._write_line, f"[copied] {len(entries)} line(s) to clipboard")
|
|
408
|
+
|
|
409
|
+
@work(thread=True, exclusive=True, group="sources")
|
|
410
|
+
def _load_sources(self) -> None:
|
|
411
|
+
worker = get_current_worker()
|
|
412
|
+
# Fetch the job once (GS metadata — survives after the zone is gone) for
|
|
413
|
+
# the summary header, status (drives follow/cache), and the sub-job list.
|
|
414
|
+
try:
|
|
415
|
+
job = self._client.get_job(self._job_id) or {}
|
|
416
|
+
except Exception: # noqa: BLE001 - best-effort
|
|
417
|
+
job = {}
|
|
418
|
+
if job.get("status"):
|
|
419
|
+
self._job_status = job.get("status")
|
|
420
|
+
if job:
|
|
421
|
+
self._job_reason = job.get("reason") # clears a stale reason on refresh
|
|
422
|
+
# The summary line is reserved for static config; status/reason go
|
|
423
|
+
# on the subtitle line instead.
|
|
424
|
+
self._post(self._set_summary, format_job_config(job))
|
|
425
|
+
self._post(self._update_subtitle)
|
|
426
|
+
|
|
427
|
+
# Each sub-job is a tailable source (its zone-manager / Ray-head pod,
|
|
428
|
+
# whose stdout already includes worker output via Ray log_to_driver).
|
|
429
|
+
sources = []
|
|
430
|
+
for sj in job.get("sub_jobs") or []:
|
|
431
|
+
if isinstance(sj, dict):
|
|
432
|
+
sid = sj.get("sub_job_id") or sj.get("id")
|
|
433
|
+
if sid:
|
|
434
|
+
sources.append({"sub_job_id": sid, "job_type": sj.get("job_type") or sj.get("type")})
|
|
435
|
+
if not sources:
|
|
436
|
+
# get_job failed/empty (e.g. offline): fall back to cached sub-jobs.
|
|
437
|
+
sources = [{"sub_job_id": sid} for sid in self._cache.cached_sources()]
|
|
438
|
+
if not sources:
|
|
439
|
+
self._post(
|
|
440
|
+
self._write_line,
|
|
441
|
+
"[no logs] this job has no sub-jobs and no cached logs are available",
|
|
442
|
+
)
|
|
443
|
+
if worker.is_cancelled:
|
|
444
|
+
return
|
|
445
|
+
self._post(self._populate_sources, sources)
|
|
446
|
+
|
|
447
|
+
async def _populate_sources(self, sources) -> None:
|
|
448
|
+
lv = self.query_one("#sources", ListView)
|
|
449
|
+
await lv.clear() # await removal before re-appending src-* ids (see _populate)
|
|
450
|
+
self._source_by_item = {}
|
|
451
|
+
items = [s for s in sources if s.get("sub_job_id")]
|
|
452
|
+
for i, s in enumerate(items):
|
|
453
|
+
sid = s["sub_job_id"]
|
|
454
|
+
item_id = f"src-{i}"
|
|
455
|
+
self._source_by_item[item_id] = sid
|
|
456
|
+
lv.append(ListItem(Label(source_label(s)), id=item_id))
|
|
457
|
+
if items:
|
|
458
|
+
lv.index = 0
|
|
459
|
+
self._start_tail(items[0]["sub_job_id"])
|
|
460
|
+
|
|
461
|
+
def on_list_view_selected(self, event: ListView.Selected) -> None:
|
|
462
|
+
sid = self._source_by_item.get(event.item.id or "")
|
|
463
|
+
# Re-clicking the source you're already viewing shouldn't reload/reset
|
|
464
|
+
# the pane (that's what caused the jump-to-bottom on every click).
|
|
465
|
+
if sid and sid != self._current_source:
|
|
466
|
+
self._start_tail(sid)
|
|
467
|
+
|
|
468
|
+
def _post(self, fn, *args, **kwargs) -> bool:
|
|
469
|
+
"""Run ``fn`` on the UI thread from a worker; return ``False`` instead of
|
|
470
|
+
raising once the app/event loop is gone (teardown), so background workers
|
|
471
|
+
stop cleanly rather than surfacing a NoActiveAppError as a WorkerError.
|
|
472
|
+
|
|
473
|
+
Only teardown-class errors (``RuntimeError`` — NoActiveAppError and
|
|
474
|
+
"event loop is closed" both derive from it) are swallowed; a real bug in
|
|
475
|
+
``fn`` (Attribute/Type/Key/…) still propagates to ``_tail``'s handler so
|
|
476
|
+
it gets logged rather than silently stopping the tail."""
|
|
477
|
+
try:
|
|
478
|
+
self.app.call_from_thread(fn, *args, **kwargs)
|
|
479
|
+
return True
|
|
480
|
+
except RuntimeError:
|
|
481
|
+
return False
|
|
482
|
+
|
|
483
|
+
def _start_tail(self, sub_job_id: str) -> None:
|
|
484
|
+
self._current_source = sub_job_id
|
|
485
|
+
self._shown_lines = []
|
|
486
|
+
# A fresh tail context (new source, or a re-tail after a filter change)
|
|
487
|
+
# starts following the tail — otherwise a pause left over from a prior
|
|
488
|
+
# source would strand this one at the top, the very bug we're fixing.
|
|
489
|
+
self._paused = False
|
|
490
|
+
self._update_subtitle()
|
|
491
|
+
if self._logview is not None:
|
|
492
|
+
self._logview.clear()
|
|
493
|
+
self._write_line(f"— tailing {sub_job_id} —")
|
|
494
|
+
self._tail(sub_job_id)
|
|
495
|
+
|
|
496
|
+
@work(thread=True, exclusive=True, group="tail")
|
|
497
|
+
def _tail(self, sub_job_id: str) -> None:
|
|
498
|
+
worker = get_current_worker()
|
|
499
|
+
# A terminal job's zone is gone, so the operation API would error: serve
|
|
500
|
+
# cache only (no live fetch). A running/unknown job follows live.
|
|
501
|
+
active = is_active_status(self._job_status)
|
|
502
|
+
try:
|
|
503
|
+
# Replay the local cache instantly, then (only for a live job) resume
|
|
504
|
+
# the tail from the saved cursor so only new lines are fetched. Each
|
|
505
|
+
# yield is a whole page (or the cache batch); render it in one shot
|
|
506
|
+
# so high-velocity logs don't saturate the UI thread per line.
|
|
507
|
+
saw_live = False
|
|
508
|
+
for kind, entries in cached_log_pages(
|
|
509
|
+
self._cache,
|
|
510
|
+
self._client,
|
|
511
|
+
self._job_id,
|
|
512
|
+
sub_job_id,
|
|
513
|
+
follow=active,
|
|
514
|
+
live=active,
|
|
515
|
+
poll_interval=self._poll_interval,
|
|
516
|
+
replay_limit=2000,
|
|
517
|
+
is_cancelled=lambda: worker.is_cancelled,
|
|
518
|
+
):
|
|
519
|
+
if worker.is_cancelled:
|
|
520
|
+
return
|
|
521
|
+
# Apply the grep filter + min-level to the whole batch.
|
|
522
|
+
lines = [
|
|
523
|
+
format_log_entry(e)
|
|
524
|
+
for e in entries
|
|
525
|
+
if entry_matches(e, self._filter) and entry_at_level(e, self._min_level)
|
|
526
|
+
]
|
|
527
|
+
if not lines:
|
|
528
|
+
continue
|
|
529
|
+
if kind == "live" and not saw_live:
|
|
530
|
+
saw_live = True
|
|
531
|
+
lines = ["— live —"] + lines
|
|
532
|
+
# Every batch sticks to the tail (unless paused), so the view
|
|
533
|
+
# opens on the newest cached lines and follows live from there.
|
|
534
|
+
# _post fails closed if the app is tearing down, so the worker
|
|
535
|
+
# stops cleanly.
|
|
536
|
+
if not self._post(self._write_lines, lines):
|
|
537
|
+
return
|
|
538
|
+
if kind == "live":
|
|
539
|
+
# Stamp the freshness indicator on each live batch.
|
|
540
|
+
self._last_update = time.strftime("%H:%M:%S")
|
|
541
|
+
if not self._post(self._update_subtitle):
|
|
542
|
+
return
|
|
543
|
+
except Exception as exc: # noqa: BLE001 - surfaced in the UI
|
|
544
|
+
if not worker.is_cancelled:
|
|
545
|
+
try:
|
|
546
|
+
with open(os.path.expanduser(_ERROR_LOG), "a", encoding="utf-8") as f:
|
|
547
|
+
f.write(f"tail {sub_job_id} failed:\n{traceback.format_exc()}\n")
|
|
548
|
+
except Exception: # noqa: BLE001
|
|
549
|
+
pass
|
|
550
|
+
self._post(
|
|
551
|
+
self._write_line,
|
|
552
|
+
f"[error] tail {sub_job_id} failed: {type(exc).__name__}: {exc} (full traceback: {_ERROR_LOG})",
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
def _content_width(self) -> int:
|
|
556
|
+
"""Usable text columns of the log pane (accounts for padding/scrollbar)."""
|
|
557
|
+
lv = self._logview
|
|
558
|
+
if lv is None:
|
|
559
|
+
return 0
|
|
560
|
+
try:
|
|
561
|
+
w = lv.scrollable_content_region.width
|
|
562
|
+
except Exception: # noqa: BLE001 - not laid out yet
|
|
563
|
+
w = 0
|
|
564
|
+
if w <= 0:
|
|
565
|
+
w = getattr(lv.size, "width", 0)
|
|
566
|
+
return max(0, w)
|
|
567
|
+
|
|
568
|
+
def _write_line(self, line: str) -> None:
|
|
569
|
+
self._write_lines([line])
|
|
570
|
+
|
|
571
|
+
def _write_lines(self, lines: list[str]) -> None:
|
|
572
|
+
lv = self._logview
|
|
573
|
+
if lv is None or not lines:
|
|
574
|
+
return
|
|
575
|
+
self._shown_lines.extend(lines)
|
|
576
|
+
# Bound the re-wrap buffer to the widget's own max_lines budget.
|
|
577
|
+
if len(self._shown_lines) > 20000:
|
|
578
|
+
del self._shown_lines[: len(self._shown_lines) - 20000]
|
|
579
|
+
# Always stick to the tail so every viewer sees the newest lines on
|
|
580
|
+
# open and as they stream — cache replay and live alike. Pause is the
|
|
581
|
+
# one escape hatch: it stops following so the user can scroll back.
|
|
582
|
+
follow = not self._paused
|
|
583
|
+
w = self._content_width()
|
|
584
|
+
wrapped: list[str] = []
|
|
585
|
+
for line in lines:
|
|
586
|
+
wrapped.extend(wrap_log_line(line, w))
|
|
587
|
+
# One write_lines call per page keeps the UI responsive under
|
|
588
|
+
# high-velocity logs (vs. a cross-thread hop + write per line).
|
|
589
|
+
lv.write_lines(wrapped, scroll_end=follow)
|
|
590
|
+
|
|
591
|
+
def _rewrap(self) -> None:
|
|
592
|
+
"""Re-wrap the on-screen buffer after the pane width changes."""
|
|
593
|
+
lv = self._logview
|
|
594
|
+
if lv is None:
|
|
595
|
+
return
|
|
596
|
+
w = self._content_width()
|
|
597
|
+
if w <= 0 or w == self._wrap_width:
|
|
598
|
+
return
|
|
599
|
+
self._wrap_width = w
|
|
600
|
+
lv.clear()
|
|
601
|
+
for line in self._shown_lines:
|
|
602
|
+
for piece in wrap_log_line(line, w):
|
|
603
|
+
lv.write_line(piece)
|
|
604
|
+
# Re-wrapping clears + repaints (resetting scroll to the top), so snap
|
|
605
|
+
# back to the tail unless the user has paused to read history.
|
|
606
|
+
if not self._paused:
|
|
607
|
+
lv.scroll_end(animate=False)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
class CortexTrainingLogTUI(App):
|
|
611
|
+
"""Entry app: show the job picker, or jump straight to a job's logs when a
|
|
612
|
+
job_id is supplied."""
|
|
613
|
+
|
|
614
|
+
CSS = """
|
|
615
|
+
#jobs { padding: 0 1; }
|
|
616
|
+
#sources { width: 48; border-right: solid $panel; }
|
|
617
|
+
#logview { padding: 0 1; }
|
|
618
|
+
#summary { color: $text-muted; padding: 0 1; }
|
|
619
|
+
#logfilter { border: tall $panel; }
|
|
620
|
+
#jobfilter { border: tall $panel; }
|
|
621
|
+
/* Text-selection highlight: reuse the picker's subtle "blurred cursor"
|
|
622
|
+
shade so it's visible but not a heavy saturated block. */
|
|
623
|
+
LogScreen > .screen--selection {
|
|
624
|
+
background: $block-cursor-blurred-background;
|
|
625
|
+
color: $block-cursor-blurred-foreground;
|
|
626
|
+
}
|
|
627
|
+
"""
|
|
628
|
+
BINDINGS = [("q", "quit", "Quit")]
|
|
629
|
+
|
|
630
|
+
def __init__(self, client, job_id=None, *, sub_job_id=None, poll_interval=1.0):
|
|
631
|
+
super().__init__()
|
|
632
|
+
self._client = client
|
|
633
|
+
self._job_id = job_id
|
|
634
|
+
self._sub_job_id = sub_job_id
|
|
635
|
+
self._poll_interval = poll_interval
|
|
636
|
+
self.title = "cortex training logs"
|
|
637
|
+
|
|
638
|
+
def on_mount(self) -> None:
|
|
639
|
+
if self._job_id:
|
|
640
|
+
self.push_screen(
|
|
641
|
+
LogScreen(self._client, self._job_id, sub_job_id=self._sub_job_id, poll_interval=self._poll_interval)
|
|
642
|
+
)
|
|
643
|
+
else:
|
|
644
|
+
self.push_screen(
|
|
645
|
+
JobListScreen(self._client, sub_job_id=self._sub_job_id, poll_interval=self._poll_interval)
|
|
646
|
+
)
|