experimaestro 2.0.0a8__py3-none-any.whl → 2.0.0b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of experimaestro might be problematic. Click here for more details.
- experimaestro/__init__.py +10 -11
- experimaestro/annotations.py +167 -206
- experimaestro/cli/__init__.py +130 -5
- experimaestro/cli/filter.py +42 -74
- experimaestro/cli/jobs.py +157 -106
- experimaestro/cli/refactor.py +249 -0
- experimaestro/click.py +0 -1
- experimaestro/commandline.py +19 -3
- experimaestro/connectors/__init__.py +20 -1
- experimaestro/connectors/local.py +12 -0
- experimaestro/core/arguments.py +182 -46
- experimaestro/core/identifier.py +107 -6
- experimaestro/core/objects/__init__.py +6 -0
- experimaestro/core/objects/config.py +542 -25
- experimaestro/core/objects/config_walk.py +20 -0
- experimaestro/core/serialization.py +91 -34
- experimaestro/core/subparameters.py +164 -0
- experimaestro/core/types.py +175 -38
- experimaestro/exceptions.py +26 -0
- experimaestro/experiments/cli.py +107 -25
- experimaestro/generators.py +50 -9
- experimaestro/huggingface.py +3 -1
- experimaestro/launcherfinder/parser.py +29 -0
- experimaestro/launchers/__init__.py +26 -1
- experimaestro/launchers/direct.py +12 -0
- experimaestro/launchers/slurm/base.py +154 -2
- experimaestro/mkdocs/metaloader.py +0 -1
- experimaestro/mypy.py +452 -7
- experimaestro/notifications.py +63 -13
- experimaestro/progress.py +0 -2
- experimaestro/rpyc.py +0 -1
- experimaestro/run.py +19 -6
- experimaestro/scheduler/base.py +489 -125
- experimaestro/scheduler/dependencies.py +43 -28
- experimaestro/scheduler/dynamic_outputs.py +259 -130
- experimaestro/scheduler/experiment.py +225 -30
- experimaestro/scheduler/interfaces.py +474 -0
- experimaestro/scheduler/jobs.py +216 -206
- experimaestro/scheduler/services.py +186 -12
- experimaestro/scheduler/state_db.py +388 -0
- experimaestro/scheduler/state_provider.py +2345 -0
- experimaestro/scheduler/state_sync.py +834 -0
- experimaestro/scheduler/workspace.py +52 -10
- experimaestro/scriptbuilder.py +7 -0
- experimaestro/server/__init__.py +147 -57
- experimaestro/server/data/index.css +0 -125
- experimaestro/server/data/index.css.map +1 -1
- experimaestro/server/data/index.js +194 -58
- experimaestro/server/data/index.js.map +1 -1
- experimaestro/settings.py +44 -5
- experimaestro/sphinx/__init__.py +3 -3
- experimaestro/taskglobals.py +20 -0
- experimaestro/tests/conftest.py +80 -0
- experimaestro/tests/core/test_generics.py +2 -2
- experimaestro/tests/identifier_stability.json +45 -0
- experimaestro/tests/launchers/bin/sacct +6 -2
- experimaestro/tests/launchers/bin/sbatch +4 -2
- experimaestro/tests/launchers/test_slurm.py +80 -0
- experimaestro/tests/tasks/test_dynamic.py +231 -0
- experimaestro/tests/test_cli_jobs.py +615 -0
- experimaestro/tests/test_deprecated.py +630 -0
- experimaestro/tests/test_environment.py +200 -0
- experimaestro/tests/test_file_progress_integration.py +1 -1
- experimaestro/tests/test_forward.py +3 -3
- experimaestro/tests/test_identifier.py +372 -41
- experimaestro/tests/test_identifier_stability.py +458 -0
- experimaestro/tests/test_instance.py +3 -3
- experimaestro/tests/test_multitoken.py +442 -0
- experimaestro/tests/test_mypy.py +433 -0
- experimaestro/tests/test_objects.py +312 -5
- experimaestro/tests/test_outputs.py +2 -2
- experimaestro/tests/test_param.py +8 -12
- experimaestro/tests/test_partial_paths.py +231 -0
- experimaestro/tests/test_progress.py +0 -48
- experimaestro/tests/test_resumable_task.py +480 -0
- experimaestro/tests/test_serializers.py +141 -1
- experimaestro/tests/test_state_db.py +434 -0
- experimaestro/tests/test_subparameters.py +160 -0
- experimaestro/tests/test_tags.py +136 -0
- experimaestro/tests/test_tasks.py +107 -121
- experimaestro/tests/test_token_locking.py +252 -0
- experimaestro/tests/test_tokens.py +17 -13
- experimaestro/tests/test_types.py +123 -1
- experimaestro/tests/test_workspace_triggers.py +158 -0
- experimaestro/tests/token_reschedule.py +4 -2
- experimaestro/tests/utils.py +2 -2
- experimaestro/tokens.py +154 -57
- experimaestro/tools/diff.py +1 -1
- experimaestro/tui/__init__.py +8 -0
- experimaestro/tui/app.py +2303 -0
- experimaestro/tui/app.tcss +353 -0
- experimaestro/tui/log_viewer.py +228 -0
- experimaestro/utils/__init__.py +23 -0
- experimaestro/utils/environment.py +148 -0
- experimaestro/utils/git.py +129 -0
- experimaestro/utils/resources.py +1 -1
- experimaestro/version.py +34 -0
- {experimaestro-2.0.0a8.dist-info → experimaestro-2.0.0b4.dist-info}/METADATA +68 -38
- experimaestro-2.0.0b4.dist-info/RECORD +181 -0
- {experimaestro-2.0.0a8.dist-info → experimaestro-2.0.0b4.dist-info}/WHEEL +1 -1
- experimaestro-2.0.0b4.dist-info/entry_points.txt +16 -0
- experimaestro/compat.py +0 -6
- experimaestro/core/objects.pyi +0 -221
- experimaestro/server/data/0c35d18bf06992036b69.woff2 +0 -0
- experimaestro/server/data/219aa9140e099e6c72ed.woff2 +0 -0
- experimaestro/server/data/3a4004a46a653d4b2166.woff +0 -0
- experimaestro/server/data/3baa5b8f3469222b822d.woff +0 -0
- experimaestro/server/data/4d73cb90e394b34b7670.woff +0 -0
- experimaestro/server/data/4ef4218c522f1eb6b5b1.woff2 +0 -0
- experimaestro/server/data/5d681e2edae8c60630db.woff +0 -0
- experimaestro/server/data/6f420cf17cc0d7676fad.woff2 +0 -0
- experimaestro/server/data/c380809fd3677d7d6903.woff2 +0 -0
- experimaestro/server/data/f882956fd323fd322f31.woff +0 -0
- experimaestro-2.0.0a8.dist-info/RECORD +0 -166
- experimaestro-2.0.0a8.dist-info/entry_points.txt +0 -17
- {experimaestro-2.0.0a8.dist-info → experimaestro-2.0.0b4.dist-info}/licenses/LICENSE +0 -0
experimaestro/tui/app.py
ADDED
|
@@ -0,0 +1,2303 @@
|
|
|
1
|
+
"""Main Textual TUI application for experiment monitoring"""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Optional
|
|
6
|
+
from textual.app import App, ComposeResult
|
|
7
|
+
from textual.containers import Container, Horizontal, Vertical
|
|
8
|
+
from textual.widgets import (
|
|
9
|
+
Header,
|
|
10
|
+
Footer,
|
|
11
|
+
DataTable,
|
|
12
|
+
Label,
|
|
13
|
+
TabbedContent,
|
|
14
|
+
TabPane,
|
|
15
|
+
RichLog,
|
|
16
|
+
Button,
|
|
17
|
+
Static,
|
|
18
|
+
Input,
|
|
19
|
+
)
|
|
20
|
+
from textual.widget import Widget
|
|
21
|
+
from textual.reactive import reactive
|
|
22
|
+
from textual.binding import Binding
|
|
23
|
+
from textual.message import Message
|
|
24
|
+
from textual.screen import ModalScreen, Screen
|
|
25
|
+
from textual import events
|
|
26
|
+
from rich.text import Text
|
|
27
|
+
from experimaestro.scheduler.state_provider import (
|
|
28
|
+
WorkspaceStateProvider,
|
|
29
|
+
StateEvent,
|
|
30
|
+
StateEventType,
|
|
31
|
+
)
|
|
32
|
+
from experimaestro.tui.log_viewer import LogViewerScreen
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def format_duration(seconds: float) -> str:
|
|
36
|
+
"""Format duration in seconds to human-readable string"""
|
|
37
|
+
if seconds < 0:
|
|
38
|
+
return "-"
|
|
39
|
+
seconds = int(seconds)
|
|
40
|
+
if seconds < 60:
|
|
41
|
+
return f"{seconds}s"
|
|
42
|
+
elif seconds < 3600:
|
|
43
|
+
return f"{seconds // 60}m {seconds % 60}s"
|
|
44
|
+
elif seconds < 86400:
|
|
45
|
+
return f"{seconds // 3600}h {(seconds % 3600) // 60}m"
|
|
46
|
+
else:
|
|
47
|
+
return f"{seconds // 86400}d {(seconds % 86400) // 3600}h"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class QuitConfirmScreen(ModalScreen[bool]):
|
|
51
|
+
"""Modal screen for quit confirmation"""
|
|
52
|
+
|
|
53
|
+
def __init__(self, has_active_experiment: bool = False):
|
|
54
|
+
super().__init__()
|
|
55
|
+
self.has_active_experiment = has_active_experiment
|
|
56
|
+
|
|
57
|
+
def compose(self) -> ComposeResult:
|
|
58
|
+
with Vertical(id="quit-dialog"):
|
|
59
|
+
yield Static("Quit Experimaestro?", id="quit-title")
|
|
60
|
+
|
|
61
|
+
if self.has_active_experiment:
|
|
62
|
+
yield Static(
|
|
63
|
+
"⚠️ The experiment is still in progress.\n"
|
|
64
|
+
"Quitting will prevent new jobs from being launched.",
|
|
65
|
+
id="quit-warning",
|
|
66
|
+
)
|
|
67
|
+
else:
|
|
68
|
+
yield Static("Are you sure you want to quit?", id="quit-message")
|
|
69
|
+
|
|
70
|
+
with Horizontal(id="quit-buttons"):
|
|
71
|
+
yield Button("Quit", variant="error", id="quit-yes")
|
|
72
|
+
yield Button("Cancel", variant="primary", id="quit-no")
|
|
73
|
+
|
|
74
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
75
|
+
if event.button.id == "quit-yes":
|
|
76
|
+
self.dismiss(True)
|
|
77
|
+
else:
|
|
78
|
+
self.dismiss(False)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class DeleteConfirmScreen(ModalScreen[bool]):
|
|
82
|
+
"""Modal screen for delete confirmation"""
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self, item_type: str, item_name: str, warning: Optional[str] = None
|
|
86
|
+
) -> None:
|
|
87
|
+
super().__init__()
|
|
88
|
+
self.item_type = item_type
|
|
89
|
+
self.item_name = item_name
|
|
90
|
+
self.warning = warning
|
|
91
|
+
|
|
92
|
+
def compose(self) -> ComposeResult:
|
|
93
|
+
with Vertical(id="delete-dialog"):
|
|
94
|
+
yield Static(f"Delete {self.item_type}?", id="delete-title")
|
|
95
|
+
yield Static(
|
|
96
|
+
f"This will permanently delete: {self.item_name}", id="delete-message"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
if self.warning:
|
|
100
|
+
yield Static(f"Warning: {self.warning}", id="delete-warning")
|
|
101
|
+
|
|
102
|
+
with Horizontal(id="delete-buttons"):
|
|
103
|
+
yield Button("Delete", variant="error", id="delete-yes")
|
|
104
|
+
yield Button("Cancel", variant="primary", id="delete-no")
|
|
105
|
+
|
|
106
|
+
def on_mount(self) -> None:
|
|
107
|
+
"""Focus cancel button by default"""
|
|
108
|
+
self.query_one("#delete-no", Button).focus()
|
|
109
|
+
|
|
110
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
111
|
+
if event.button.id == "delete-yes":
|
|
112
|
+
self.dismiss(True)
|
|
113
|
+
else:
|
|
114
|
+
self.dismiss(False)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class KillConfirmScreen(ModalScreen[bool]):
|
|
118
|
+
"""Modal screen for kill confirmation"""
|
|
119
|
+
|
|
120
|
+
def __init__(self, item_type: str, item_name: str) -> None:
|
|
121
|
+
super().__init__()
|
|
122
|
+
self.item_type = item_type
|
|
123
|
+
self.item_name = item_name
|
|
124
|
+
|
|
125
|
+
def compose(self) -> ComposeResult:
|
|
126
|
+
with Vertical(id="kill-dialog"):
|
|
127
|
+
yield Static(f"Kill {self.item_type}?", id="kill-title")
|
|
128
|
+
yield Static(f"This will terminate: {self.item_name}", id="kill-message")
|
|
129
|
+
|
|
130
|
+
with Horizontal(id="kill-buttons"):
|
|
131
|
+
yield Button("Kill", variant="warning", id="kill-yes")
|
|
132
|
+
yield Button("Cancel", variant="primary", id="kill-no")
|
|
133
|
+
|
|
134
|
+
def on_mount(self) -> None:
|
|
135
|
+
"""Focus cancel button by default"""
|
|
136
|
+
self.query_one("#kill-no", Button).focus()
|
|
137
|
+
|
|
138
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
139
|
+
if event.button.id == "kill-yes":
|
|
140
|
+
self.dismiss(True)
|
|
141
|
+
else:
|
|
142
|
+
self.dismiss(False)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def get_status_icon(status: str, failure_reason=None):
|
|
146
|
+
"""Get status icon for a job state.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
status: Job state name (e.g., "done", "error", "running")
|
|
150
|
+
failure_reason: Optional JobFailureStatus enum for error states
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
Status icon string
|
|
154
|
+
"""
|
|
155
|
+
if status == "done":
|
|
156
|
+
return "✓"
|
|
157
|
+
elif status == "error":
|
|
158
|
+
# Show different icons for different failure types
|
|
159
|
+
if failure_reason is not None:
|
|
160
|
+
from experimaestro.scheduler.interfaces import JobFailureStatus
|
|
161
|
+
|
|
162
|
+
if failure_reason == JobFailureStatus.DEPENDENCY:
|
|
163
|
+
return "🔗" # Dependency failed
|
|
164
|
+
elif failure_reason == JobFailureStatus.TIMEOUT:
|
|
165
|
+
return "⏱" # Timeout
|
|
166
|
+
elif failure_reason == JobFailureStatus.MEMORY:
|
|
167
|
+
return "💾" # Memory issue
|
|
168
|
+
# FAILED or unknown - use default error icon
|
|
169
|
+
return "❌"
|
|
170
|
+
elif status == "running":
|
|
171
|
+
return "▶"
|
|
172
|
+
elif status == "waiting":
|
|
173
|
+
return "⌛" # Waiting for dependencies
|
|
174
|
+
else:
|
|
175
|
+
# phantom, unscheduled or unknown
|
|
176
|
+
return "👻"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class CaptureLog(RichLog):
|
|
180
|
+
"""Custom RichLog widget that captures print statements with log highlighting"""
|
|
181
|
+
|
|
182
|
+
def on_mount(self) -> None:
|
|
183
|
+
"""Enable print capturing when widget is mounted"""
|
|
184
|
+
self.begin_capture_print()
|
|
185
|
+
|
|
186
|
+
def on_unmount(self) -> None:
|
|
187
|
+
"""Stop print capturing when widget is unmounted"""
|
|
188
|
+
self.end_capture_print()
|
|
189
|
+
|
|
190
|
+
def _format_log_line(self, text: str) -> Text:
|
|
191
|
+
"""Format a log line with appropriate styling based on log level"""
|
|
192
|
+
result = Text()
|
|
193
|
+
|
|
194
|
+
# Check for common log level patterns
|
|
195
|
+
if text.startswith("ERROR:") or ":ERROR:" in text:
|
|
196
|
+
result.append(text, style="bold red")
|
|
197
|
+
elif text.startswith("WARNING:") or ":WARNING:" in text:
|
|
198
|
+
result.append(text, style="yellow")
|
|
199
|
+
elif text.startswith("INFO:") or ":INFO:" in text:
|
|
200
|
+
result.append(text, style="green")
|
|
201
|
+
elif text.startswith("DEBUG:") or ":DEBUG:" in text:
|
|
202
|
+
result.append(text, style="dim")
|
|
203
|
+
elif text.startswith("CRITICAL:") or ":CRITICAL:" in text:
|
|
204
|
+
result.append(text, style="bold white on red")
|
|
205
|
+
else:
|
|
206
|
+
result.append(text)
|
|
207
|
+
|
|
208
|
+
return result
|
|
209
|
+
|
|
210
|
+
def on_print(self, event: events.Print) -> None:
|
|
211
|
+
"""Handle print events from captured stdout/stderr"""
|
|
212
|
+
if text := event.text.strip():
|
|
213
|
+
self.write(self._format_log_line(text))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class ExperimentsList(Widget):
|
|
217
|
+
"""Widget displaying list of experiments"""
|
|
218
|
+
|
|
219
|
+
BINDINGS = [
|
|
220
|
+
Binding("d", "delete_experiment", "Delete", show=False),
|
|
221
|
+
Binding("k", "kill_experiment", "Kill", show=False),
|
|
222
|
+
]
|
|
223
|
+
|
|
224
|
+
current_experiment: reactive[Optional[str]] = reactive(None)
|
|
225
|
+
collapsed: reactive[bool] = reactive(False)
|
|
226
|
+
|
|
227
|
+
def __init__(self, state_provider: WorkspaceStateProvider) -> None:
|
|
228
|
+
super().__init__()
|
|
229
|
+
self.state_provider = state_provider
|
|
230
|
+
self.experiments = []
|
|
231
|
+
|
|
232
|
+
def _get_selected_experiment_id(self) -> Optional[str]:
|
|
233
|
+
"""Get the experiment ID from the currently selected row"""
|
|
234
|
+
table = self.query_one("#experiments-table", DataTable)
|
|
235
|
+
if table.cursor_row is None:
|
|
236
|
+
return None
|
|
237
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
238
|
+
if row_key:
|
|
239
|
+
return str(row_key.value)
|
|
240
|
+
return None
|
|
241
|
+
|
|
242
|
+
def action_delete_experiment(self) -> None:
|
|
243
|
+
"""Request to delete the selected experiment"""
|
|
244
|
+
exp_id = self._get_selected_experiment_id()
|
|
245
|
+
if exp_id:
|
|
246
|
+
self.post_message(DeleteExperimentRequest(exp_id))
|
|
247
|
+
|
|
248
|
+
def action_kill_experiment(self) -> None:
|
|
249
|
+
"""Request to kill all running jobs in the selected experiment"""
|
|
250
|
+
exp_id = self._get_selected_experiment_id()
|
|
251
|
+
if exp_id:
|
|
252
|
+
self.post_message(KillExperimentRequest(exp_id))
|
|
253
|
+
|
|
254
|
+
def compose(self) -> ComposeResult:
|
|
255
|
+
# Collapsed header (hidden initially)
|
|
256
|
+
with Horizontal(id="collapsed-header", classes="hidden"):
|
|
257
|
+
yield Label("", id="collapsed-experiment-info")
|
|
258
|
+
|
|
259
|
+
# Full experiments table
|
|
260
|
+
with Container(id="experiments-table-container"):
|
|
261
|
+
yield Label("Experiments", classes="section-title")
|
|
262
|
+
yield DataTable(id="experiments-table", cursor_type="row")
|
|
263
|
+
|
|
264
|
+
def on_mount(self) -> None:
|
|
265
|
+
"""Initialize the experiments table"""
|
|
266
|
+
table = self.query_one("#experiments-table", DataTable)
|
|
267
|
+
table.add_column("ID", key="id")
|
|
268
|
+
table.add_column("Jobs", key="jobs")
|
|
269
|
+
table.add_column("Status", key="status")
|
|
270
|
+
table.add_column("Started", key="started")
|
|
271
|
+
table.add_column("Duration", key="duration")
|
|
272
|
+
self.refresh_experiments()
|
|
273
|
+
|
|
274
|
+
# If there's only one experiment, automatically select it
|
|
275
|
+
if len(self.experiments) == 1:
|
|
276
|
+
exp_id = self.experiments[0].experiment_id
|
|
277
|
+
self.current_experiment = exp_id
|
|
278
|
+
self.collapse_to_experiment(exp_id)
|
|
279
|
+
self.post_message(ExperimentSelected(exp_id))
|
|
280
|
+
|
|
281
|
+
def refresh_experiments(self) -> None:
|
|
282
|
+
"""Refresh the experiments list from state provider"""
|
|
283
|
+
table = self.query_one("#experiments-table", DataTable)
|
|
284
|
+
|
|
285
|
+
try:
|
|
286
|
+
self.experiments = self.state_provider.get_experiments()
|
|
287
|
+
self.log.debug(
|
|
288
|
+
f"Refreshing experiments: found {len(self.experiments)} experiments"
|
|
289
|
+
)
|
|
290
|
+
except Exception as e:
|
|
291
|
+
self.log.error(f"ERROR refreshing experiments: {e}")
|
|
292
|
+
import traceback
|
|
293
|
+
|
|
294
|
+
self.log.error(traceback.format_exc())
|
|
295
|
+
self.experiments = []
|
|
296
|
+
return
|
|
297
|
+
|
|
298
|
+
# Get existing row keys
|
|
299
|
+
existing_keys = set(table.rows.keys())
|
|
300
|
+
current_exp_ids = set()
|
|
301
|
+
|
|
302
|
+
from datetime import datetime
|
|
303
|
+
import time as time_module
|
|
304
|
+
|
|
305
|
+
for exp in self.experiments:
|
|
306
|
+
exp_id = exp.experiment_id
|
|
307
|
+
current_exp_ids.add(exp_id)
|
|
308
|
+
total = exp.total_jobs
|
|
309
|
+
finished = exp.finished_jobs
|
|
310
|
+
failed = exp.failed_jobs
|
|
311
|
+
|
|
312
|
+
# Determine status
|
|
313
|
+
if failed > 0:
|
|
314
|
+
status = f"❌ {failed} failed"
|
|
315
|
+
elif finished == total and total > 0:
|
|
316
|
+
status = "✓ Done"
|
|
317
|
+
elif finished < total:
|
|
318
|
+
status = f"▶ {finished}/{total}"
|
|
319
|
+
else:
|
|
320
|
+
status = "Empty"
|
|
321
|
+
|
|
322
|
+
jobs_text = f"{finished}/{total}"
|
|
323
|
+
|
|
324
|
+
# Format started time
|
|
325
|
+
if exp.started_at:
|
|
326
|
+
started = datetime.fromtimestamp(exp.started_at).strftime(
|
|
327
|
+
"%Y-%m-%d %H:%M"
|
|
328
|
+
)
|
|
329
|
+
else:
|
|
330
|
+
started = "-"
|
|
331
|
+
|
|
332
|
+
# Calculate duration
|
|
333
|
+
duration = "-"
|
|
334
|
+
if exp.started_at:
|
|
335
|
+
if exp.ended_at:
|
|
336
|
+
elapsed = exp.ended_at - exp.started_at
|
|
337
|
+
else:
|
|
338
|
+
# Still running - show elapsed time
|
|
339
|
+
elapsed = time_module.time() - exp.started_at
|
|
340
|
+
# Format duration
|
|
341
|
+
duration = format_duration(elapsed)
|
|
342
|
+
|
|
343
|
+
# Update existing row or add new one
|
|
344
|
+
if exp_id in existing_keys:
|
|
345
|
+
table.update_cell(exp_id, "id", exp_id, update_width=True)
|
|
346
|
+
table.update_cell(exp_id, "jobs", jobs_text, update_width=True)
|
|
347
|
+
table.update_cell(exp_id, "status", status, update_width=True)
|
|
348
|
+
table.update_cell(exp_id, "started", started, update_width=True)
|
|
349
|
+
table.update_cell(exp_id, "duration", duration, update_width=True)
|
|
350
|
+
else:
|
|
351
|
+
table.add_row(exp_id, jobs_text, status, started, duration, key=exp_id)
|
|
352
|
+
|
|
353
|
+
# Remove rows for experiments that no longer exist
|
|
354
|
+
for old_exp_id in existing_keys - current_exp_ids:
|
|
355
|
+
table.remove_row(old_exp_id)
|
|
356
|
+
|
|
357
|
+
# Update collapsed header if viewing an experiment
|
|
358
|
+
if self.collapsed and self.current_experiment:
|
|
359
|
+
self._update_collapsed_header(self.current_experiment)
|
|
360
|
+
|
|
361
|
+
def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
|
|
362
|
+
"""Handle experiment selection"""
|
|
363
|
+
if event.row_key:
|
|
364
|
+
self.current_experiment = str(event.row_key.value)
|
|
365
|
+
self.collapse_to_experiment(self.current_experiment)
|
|
366
|
+
self.post_message(ExperimentSelected(str(event.row_key.value)))
|
|
367
|
+
|
|
368
|
+
def _update_collapsed_header(self, experiment_id: str) -> None:
|
|
369
|
+
"""Update the collapsed experiment header with current stats"""
|
|
370
|
+
exp_info = next(
|
|
371
|
+
(exp for exp in self.experiments if exp.experiment_id == experiment_id),
|
|
372
|
+
None,
|
|
373
|
+
)
|
|
374
|
+
if not exp_info:
|
|
375
|
+
return
|
|
376
|
+
|
|
377
|
+
total = exp_info.total_jobs
|
|
378
|
+
finished = exp_info.finished_jobs
|
|
379
|
+
failed = exp_info.failed_jobs
|
|
380
|
+
|
|
381
|
+
if failed > 0:
|
|
382
|
+
status = f"❌ {failed} failed"
|
|
383
|
+
elif finished == total and total > 0:
|
|
384
|
+
status = "✓ Done"
|
|
385
|
+
elif finished < total:
|
|
386
|
+
status = f"▶ {finished}/{total}"
|
|
387
|
+
else:
|
|
388
|
+
status = "Empty"
|
|
389
|
+
|
|
390
|
+
collapsed_label = self.query_one("#collapsed-experiment-info", Label)
|
|
391
|
+
collapsed_label.update(f"📊 {experiment_id} - {status} (click to go back)")
|
|
392
|
+
|
|
393
|
+
def collapse_to_experiment(self, experiment_id: str) -> None:
|
|
394
|
+
"""Collapse the experiments list to show only the selected experiment"""
|
|
395
|
+
self._update_collapsed_header(experiment_id)
|
|
396
|
+
|
|
397
|
+
# Hide table, show collapsed header
|
|
398
|
+
self.query_one("#experiments-table-container").add_class("hidden")
|
|
399
|
+
self.query_one("#collapsed-header").remove_class("hidden")
|
|
400
|
+
self.collapsed = True
|
|
401
|
+
|
|
402
|
+
def expand_experiments(self) -> None:
|
|
403
|
+
"""Expand back to full experiments list"""
|
|
404
|
+
# Show table, hide collapsed header
|
|
405
|
+
self.query_one("#collapsed-header").add_class("hidden")
|
|
406
|
+
self.query_one("#experiments-table-container").remove_class("hidden")
|
|
407
|
+
self.collapsed = False
|
|
408
|
+
self.current_experiment = None
|
|
409
|
+
|
|
410
|
+
# Focus the experiments table
|
|
411
|
+
table = self.query_one("#experiments-table", DataTable)
|
|
412
|
+
table.focus()
|
|
413
|
+
|
|
414
|
+
def on_click(self) -> None:
|
|
415
|
+
"""Handle clicks on the widget"""
|
|
416
|
+
if self.collapsed:
|
|
417
|
+
self.expand_experiments()
|
|
418
|
+
self.post_message(ExperimentDeselected())
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
class ExperimentSelected(Message):
|
|
422
|
+
"""Message sent when an experiment is selected"""
|
|
423
|
+
|
|
424
|
+
def __init__(self, experiment_id: str) -> None:
|
|
425
|
+
super().__init__()
|
|
426
|
+
self.experiment_id = experiment_id
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
class ExperimentDeselected(Message):
|
|
430
|
+
"""Message sent when an experiment is deselected"""
|
|
431
|
+
|
|
432
|
+
pass
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
class JobSelected(Message):
|
|
436
|
+
"""Message sent when a job is selected"""
|
|
437
|
+
|
|
438
|
+
def __init__(self, job_id: str, experiment_id: str) -> None:
|
|
439
|
+
super().__init__()
|
|
440
|
+
self.job_id = job_id
|
|
441
|
+
self.experiment_id = experiment_id
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
class JobDeselected(Message):
|
|
445
|
+
"""Message sent when returning from job detail view"""
|
|
446
|
+
|
|
447
|
+
pass
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
class ViewJobLogs(Message):
|
|
451
|
+
"""Message sent when user wants to view job logs"""
|
|
452
|
+
|
|
453
|
+
def __init__(self, job_path: str, task_id: str) -> None:
|
|
454
|
+
super().__init__()
|
|
455
|
+
self.job_path = job_path
|
|
456
|
+
self.task_id = task_id
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
class ViewJobLogsRequest(Message):
|
|
460
|
+
"""Message sent when user requests to view logs from jobs table"""
|
|
461
|
+
|
|
462
|
+
def __init__(self, job_id: str, experiment_id: str) -> None:
|
|
463
|
+
super().__init__()
|
|
464
|
+
self.job_id = job_id
|
|
465
|
+
self.experiment_id = experiment_id
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
class DeleteJobRequest(Message):
|
|
469
|
+
"""Message sent when user requests to delete a job"""
|
|
470
|
+
|
|
471
|
+
def __init__(self, job_id: str, experiment_id: str) -> None:
|
|
472
|
+
super().__init__()
|
|
473
|
+
self.job_id = job_id
|
|
474
|
+
self.experiment_id = experiment_id
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
class DeleteExperimentRequest(Message):
|
|
478
|
+
"""Message sent when user requests to delete an experiment"""
|
|
479
|
+
|
|
480
|
+
def __init__(self, experiment_id: str) -> None:
|
|
481
|
+
super().__init__()
|
|
482
|
+
self.experiment_id = experiment_id
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
class KillJobRequest(Message):
|
|
486
|
+
"""Message sent when user requests to kill a running job"""
|
|
487
|
+
|
|
488
|
+
def __init__(self, job_id: str, experiment_id: str) -> None:
|
|
489
|
+
super().__init__()
|
|
490
|
+
self.job_id = job_id
|
|
491
|
+
self.experiment_id = experiment_id
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
class KillExperimentRequest(Message):
|
|
495
|
+
"""Message sent when user requests to kill all running jobs in an experiment"""
|
|
496
|
+
|
|
497
|
+
def __init__(self, experiment_id: str) -> None:
|
|
498
|
+
super().__init__()
|
|
499
|
+
self.experiment_id = experiment_id
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
class FilterChanged(Message):
|
|
503
|
+
"""Message sent when search filter changes"""
|
|
504
|
+
|
|
505
|
+
def __init__(self, filter_fn) -> None:
|
|
506
|
+
super().__init__()
|
|
507
|
+
self.filter_fn = filter_fn
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
class ServicesList(Vertical):
|
|
511
|
+
"""Widget displaying services for selected experiment
|
|
512
|
+
|
|
513
|
+
Services are retrieved from WorkspaceStateProvider.get_services() which
|
|
514
|
+
abstracts away whether services are live (from scheduler) or recreated
|
|
515
|
+
from database state_dict. The UI treats all services uniformly.
|
|
516
|
+
"""
|
|
517
|
+
|
|
518
|
+
BINDINGS = [
|
|
519
|
+
Binding("s", "start_service", "Start"),
|
|
520
|
+
Binding("x", "stop_service", "Stop"),
|
|
521
|
+
Binding("u", "copy_url", "Copy URL", show=False),
|
|
522
|
+
]
|
|
523
|
+
|
|
524
|
+
# State icons for display
|
|
525
|
+
STATE_ICONS = {
|
|
526
|
+
"STOPPED": "⏹",
|
|
527
|
+
"STARTING": "⏳",
|
|
528
|
+
"RUNNING": "▶",
|
|
529
|
+
"STOPPING": "⏳",
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
def __init__(self, state_provider: WorkspaceStateProvider) -> None:
|
|
533
|
+
super().__init__()
|
|
534
|
+
self.state_provider = state_provider
|
|
535
|
+
self.current_experiment: Optional[str] = None
|
|
536
|
+
self._services: dict = {} # service_id -> Service object
|
|
537
|
+
|
|
538
|
+
def compose(self) -> ComposeResult:
|
|
539
|
+
yield DataTable(id="services-table", cursor_type="row")
|
|
540
|
+
|
|
541
|
+
def on_mount(self) -> None:
|
|
542
|
+
"""Set up the services table"""
|
|
543
|
+
table = self.query_one("#services-table", DataTable)
|
|
544
|
+
table.add_columns("ID", "Description", "State", "URL")
|
|
545
|
+
table.cursor_type = "row"
|
|
546
|
+
|
|
547
|
+
def set_experiment(self, experiment_id: Optional[str]) -> None:
|
|
548
|
+
"""Set the current experiment and refresh services"""
|
|
549
|
+
self.current_experiment = experiment_id
|
|
550
|
+
self.refresh_services()
|
|
551
|
+
|
|
552
|
+
def refresh_services(self) -> None:
|
|
553
|
+
"""Refresh the services list from state provider"""
|
|
554
|
+
table = self.query_one("#services-table", DataTable)
|
|
555
|
+
table.clear()
|
|
556
|
+
self._services = {}
|
|
557
|
+
|
|
558
|
+
if not self.current_experiment:
|
|
559
|
+
return
|
|
560
|
+
|
|
561
|
+
# Get services from state provider (handles live vs DB automatically)
|
|
562
|
+
services = self.state_provider.get_services(self.current_experiment)
|
|
563
|
+
|
|
564
|
+
for service in services:
|
|
565
|
+
service_id = service.id
|
|
566
|
+
self._services[service_id] = service
|
|
567
|
+
|
|
568
|
+
state_name = service.state.name if hasattr(service, "state") else "UNKNOWN"
|
|
569
|
+
state_icon = self.STATE_ICONS.get(state_name, "?")
|
|
570
|
+
url = getattr(service, "url", None) or "-"
|
|
571
|
+
description = (
|
|
572
|
+
service.description() if hasattr(service, "description") else ""
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
table.add_row(
|
|
576
|
+
service_id,
|
|
577
|
+
description,
|
|
578
|
+
f"{state_icon} {state_name}",
|
|
579
|
+
url,
|
|
580
|
+
key=service_id,
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
def _get_selected_service(self):
|
|
584
|
+
"""Get the currently selected Service object"""
|
|
585
|
+
table = self.query_one("#services-table", DataTable)
|
|
586
|
+
if table.cursor_row is not None and table.row_count > 0:
|
|
587
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
588
|
+
if row_key:
|
|
589
|
+
service_id = str(row_key.value)
|
|
590
|
+
return self._services.get(service_id)
|
|
591
|
+
return None
|
|
592
|
+
|
|
593
|
+
def action_start_service(self) -> None:
|
|
594
|
+
"""Start the selected service"""
|
|
595
|
+
service = self._get_selected_service()
|
|
596
|
+
if not service:
|
|
597
|
+
return
|
|
598
|
+
|
|
599
|
+
try:
|
|
600
|
+
if hasattr(service, "get_url"):
|
|
601
|
+
url = service.get_url()
|
|
602
|
+
self.notify(f"Service started: {url}", severity="information")
|
|
603
|
+
else:
|
|
604
|
+
self.notify("Service does not support starting", severity="warning")
|
|
605
|
+
self.refresh_services()
|
|
606
|
+
except Exception as e:
|
|
607
|
+
self.notify(f"Failed to start service: {e}", severity="error")
|
|
608
|
+
|
|
609
|
+
def action_stop_service(self) -> None:
|
|
610
|
+
"""Stop the selected service"""
|
|
611
|
+
service = self._get_selected_service()
|
|
612
|
+
if not service:
|
|
613
|
+
return
|
|
614
|
+
|
|
615
|
+
from experimaestro.scheduler.services import ServiceState
|
|
616
|
+
|
|
617
|
+
if service.state == ServiceState.STOPPED:
|
|
618
|
+
self.notify("Service is not running", severity="warning")
|
|
619
|
+
return
|
|
620
|
+
|
|
621
|
+
try:
|
|
622
|
+
if hasattr(service, "stop"):
|
|
623
|
+
service.stop()
|
|
624
|
+
self.notify(f"Service stopped: {service.id}", severity="information")
|
|
625
|
+
else:
|
|
626
|
+
self.notify("Service does not support stopping", severity="warning")
|
|
627
|
+
self.refresh_services()
|
|
628
|
+
except Exception as e:
|
|
629
|
+
self.notify(f"Failed to stop service: {e}", severity="error")
|
|
630
|
+
|
|
631
|
+
def action_copy_url(self) -> None:
|
|
632
|
+
"""Copy the service URL to clipboard"""
|
|
633
|
+
service = self._get_selected_service()
|
|
634
|
+
if not service:
|
|
635
|
+
return
|
|
636
|
+
|
|
637
|
+
url = getattr(service, "url", None)
|
|
638
|
+
if url:
|
|
639
|
+
try:
|
|
640
|
+
import pyperclip
|
|
641
|
+
|
|
642
|
+
pyperclip.copy(url)
|
|
643
|
+
self.notify(f"URL copied: {url}", severity="information")
|
|
644
|
+
except Exception as e:
|
|
645
|
+
self.notify(f"Failed to copy: {e}", severity="error")
|
|
646
|
+
else:
|
|
647
|
+
self.notify("Start the service first to get URL", severity="warning")
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
class JobDetailView(Widget):
|
|
651
|
+
"""Widget displaying detailed job information"""
|
|
652
|
+
|
|
653
|
+
BINDINGS = [
|
|
654
|
+
Binding("l", "view_logs", "View Logs", priority=True),
|
|
655
|
+
]
|
|
656
|
+
|
|
657
|
+
def __init__(self, state_provider: WorkspaceStateProvider) -> None:
|
|
658
|
+
super().__init__()
|
|
659
|
+
self.state_provider = state_provider
|
|
660
|
+
self.current_job_id: Optional[str] = None
|
|
661
|
+
self.current_experiment_id: Optional[str] = None
|
|
662
|
+
self.job_data: Optional[dict] = None
|
|
663
|
+
|
|
664
|
+
def compose(self) -> ComposeResult:
|
|
665
|
+
yield Label("Job Details", classes="section-title")
|
|
666
|
+
with Vertical(id="job-detail-content"):
|
|
667
|
+
yield Label("", id="job-id-label")
|
|
668
|
+
yield Label("", id="job-task-label")
|
|
669
|
+
yield Label("", id="job-status-label")
|
|
670
|
+
yield Label("", id="job-path-label")
|
|
671
|
+
yield Label("", id="job-times-label")
|
|
672
|
+
yield Label("Tags:", classes="subsection-title")
|
|
673
|
+
yield Label("", id="job-tags-label")
|
|
674
|
+
yield Label("Progress:", classes="subsection-title")
|
|
675
|
+
yield Label("", id="job-progress-label")
|
|
676
|
+
yield Label("", id="job-logs-hint")
|
|
677
|
+
|
|
678
|
+
def action_view_logs(self) -> None:
|
|
679
|
+
"""View job logs with toolong"""
|
|
680
|
+
if self.job_data and self.job_data.path and self.job_data.task_id:
|
|
681
|
+
self.post_message(
|
|
682
|
+
ViewJobLogs(str(self.job_data.path), self.job_data.task_id)
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
def set_job(self, job_id: str, experiment_id: str) -> None:
|
|
686
|
+
"""Set the job to display"""
|
|
687
|
+
self.current_job_id = job_id
|
|
688
|
+
self.current_experiment_id = experiment_id
|
|
689
|
+
self.refresh_job_detail()
|
|
690
|
+
|
|
691
|
+
def refresh_job_detail(self) -> None:
|
|
692
|
+
"""Refresh job details from state provider"""
|
|
693
|
+
if not self.current_job_id or not self.current_experiment_id:
|
|
694
|
+
return
|
|
695
|
+
|
|
696
|
+
job = self.state_provider.get_job(
|
|
697
|
+
self.current_job_id, self.current_experiment_id
|
|
698
|
+
)
|
|
699
|
+
if not job:
|
|
700
|
+
self.log(f"Job not found: {self.current_job_id}")
|
|
701
|
+
return
|
|
702
|
+
|
|
703
|
+
self.job_data = job
|
|
704
|
+
|
|
705
|
+
# Update labels
|
|
706
|
+
self.query_one("#job-id-label", Label).update(f"Job ID: {job.identifier}")
|
|
707
|
+
self.query_one("#job-task-label", Label).update(f"Task: {job.task_id}")
|
|
708
|
+
|
|
709
|
+
# Format status with icon and name
|
|
710
|
+
status_name = job.state.name if job.state else "unknown"
|
|
711
|
+
failure_reason = getattr(job, "failure_reason", None)
|
|
712
|
+
status_icon = get_status_icon(status_name, failure_reason)
|
|
713
|
+
status_text = f"{status_icon} {status_name}"
|
|
714
|
+
if failure_reason:
|
|
715
|
+
status_text += f" ({failure_reason.name})"
|
|
716
|
+
|
|
717
|
+
self.query_one("#job-status-label", Label).update(f"Status: {status_text}")
|
|
718
|
+
|
|
719
|
+
# Path (from locator)
|
|
720
|
+
locator = job.locator or "-"
|
|
721
|
+
self.query_one("#job-path-label", Label).update(f"Locator: {locator}")
|
|
722
|
+
|
|
723
|
+
# Times - format timestamps
|
|
724
|
+
from datetime import datetime
|
|
725
|
+
import time as time_module
|
|
726
|
+
|
|
727
|
+
def format_time(ts):
|
|
728
|
+
if ts:
|
|
729
|
+
return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
|
|
730
|
+
return "-"
|
|
731
|
+
|
|
732
|
+
submitted = format_time(job.submittime)
|
|
733
|
+
start = format_time(job.starttime)
|
|
734
|
+
end = format_time(job.endtime)
|
|
735
|
+
|
|
736
|
+
# Calculate duration
|
|
737
|
+
duration = "-"
|
|
738
|
+
if job.starttime:
|
|
739
|
+
if job.endtime:
|
|
740
|
+
duration = format_duration(job.endtime - job.starttime)
|
|
741
|
+
else:
|
|
742
|
+
duration = (
|
|
743
|
+
format_duration(time_module.time() - job.starttime) + " (running)"
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
times_text = f"Submitted: {submitted} | Start: {start} | End: {end} | Duration: {duration}"
|
|
747
|
+
self.query_one("#job-times-label", Label).update(times_text)
|
|
748
|
+
|
|
749
|
+
# Tags - job.tags is now a dict
|
|
750
|
+
tags = job.tags
|
|
751
|
+
if tags:
|
|
752
|
+
tags_text = ", ".join(f"{k}={v}" for k, v in tags.items())
|
|
753
|
+
else:
|
|
754
|
+
tags_text = "(no tags)"
|
|
755
|
+
self.query_one("#job-tags-label", Label).update(tags_text)
|
|
756
|
+
|
|
757
|
+
# Progress
|
|
758
|
+
progress_list = job.progress or []
|
|
759
|
+
if progress_list:
|
|
760
|
+
progress_lines = []
|
|
761
|
+
for p in progress_list:
|
|
762
|
+
level = p.get("level", 0)
|
|
763
|
+
pct = p.get("progress", 0) * 100
|
|
764
|
+
desc = p.get("desc", "")
|
|
765
|
+
indent = " " * level
|
|
766
|
+
progress_lines.append(f"{indent}{pct:.1f}% {desc}")
|
|
767
|
+
progress_text = "\n".join(progress_lines) if progress_lines else "-"
|
|
768
|
+
else:
|
|
769
|
+
progress_text = "-"
|
|
770
|
+
self.query_one("#job-progress-label", Label).update(progress_text)
|
|
771
|
+
|
|
772
|
+
# Log files hint - log files are named after the last part of the task ID
|
|
773
|
+
job_path = job.path
|
|
774
|
+
task_id = job.task_id
|
|
775
|
+
if job_path and task_id:
|
|
776
|
+
# Extract the last component of the task ID (e.g., "evaluate" from "mnist_xp.learn.evaluate")
|
|
777
|
+
task_name = task_id.split(".")[-1]
|
|
778
|
+
stdout_path = job_path / f"{task_name}.out"
|
|
779
|
+
stderr_path = job_path / f"{task_name}.err"
|
|
780
|
+
logs_exist = stdout_path.exists() or stderr_path.exists()
|
|
781
|
+
if logs_exist:
|
|
782
|
+
self.query_one("#job-logs-hint", Label).update(
|
|
783
|
+
"[bold cyan]Press 'l' to view logs[/bold cyan]"
|
|
784
|
+
)
|
|
785
|
+
else:
|
|
786
|
+
self.query_one("#job-logs-hint", Label).update("(no log files found)")
|
|
787
|
+
else:
|
|
788
|
+
self.query_one("#job-logs-hint", Label).update("")
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
class SearchBar(Widget):
|
|
792
|
+
"""Search bar widget with filter hints for filtering jobs"""
|
|
793
|
+
|
|
794
|
+
visible: reactive[bool] = reactive(False)
|
|
795
|
+
_keep_filter: bool = False # Flag to keep filter when hiding
|
|
796
|
+
_query_valid: bool = False # Track if current query is valid
|
|
797
|
+
|
|
798
|
+
def __init__(self) -> None:
|
|
799
|
+
super().__init__()
|
|
800
|
+
self.filter_fn = None
|
|
801
|
+
self.active_query = "" # Store the active query text
|
|
802
|
+
|
|
803
|
+
def compose(self) -> ComposeResult:
|
|
804
|
+
# Active filter indicator (shown when filter active but bar hidden)
|
|
805
|
+
yield Static("", id="active-filter")
|
|
806
|
+
# Search input container
|
|
807
|
+
with Vertical(id="search-container"):
|
|
808
|
+
yield Input(
|
|
809
|
+
placeholder="Filter: @state = 'done', @name ~ 'pattern', tag = 'value'",
|
|
810
|
+
id="search-input",
|
|
811
|
+
)
|
|
812
|
+
yield Static(
|
|
813
|
+
"Syntax: @state = 'done' | @name ~ 'regex' | tag = 'value' | and/or",
|
|
814
|
+
id="search-hints",
|
|
815
|
+
)
|
|
816
|
+
yield Static("", id="search-error")
|
|
817
|
+
|
|
818
|
+
def on_mount(self) -> None:
|
|
819
|
+
"""Initialize visibility state"""
|
|
820
|
+
# Start with everything hidden
|
|
821
|
+
self.display = False
|
|
822
|
+
self.query_one("#search-container").display = False
|
|
823
|
+
self.query_one("#active-filter").display = False
|
|
824
|
+
self.query_one("#search-error").display = False
|
|
825
|
+
|
|
826
|
+
def watch_visible(self, visible: bool) -> None:
|
|
827
|
+
"""Show/hide search bar"""
|
|
828
|
+
search_container = self.query_one("#search-container")
|
|
829
|
+
active_filter = self.query_one("#active-filter")
|
|
830
|
+
error_widget = self.query_one("#search-error")
|
|
831
|
+
|
|
832
|
+
if visible:
|
|
833
|
+
self.display = True
|
|
834
|
+
search_container.display = True
|
|
835
|
+
active_filter.display = False
|
|
836
|
+
self.query_one("#search-input", Input).focus()
|
|
837
|
+
else:
|
|
838
|
+
if not self._keep_filter:
|
|
839
|
+
self.query_one("#search-input", Input).value = ""
|
|
840
|
+
self.filter_fn = None
|
|
841
|
+
self.active_query = ""
|
|
842
|
+
self._query_valid = False
|
|
843
|
+
self._keep_filter = False
|
|
844
|
+
|
|
845
|
+
# Show/hide based on whether filter is active
|
|
846
|
+
if self.filter_fn is not None:
|
|
847
|
+
# Filter active - show indicator, hide input
|
|
848
|
+
self.display = True
|
|
849
|
+
search_container.display = False
|
|
850
|
+
error_widget.display = False
|
|
851
|
+
active_filter.update(
|
|
852
|
+
f"Filter: {self.active_query} (/ to edit, c to clear)"
|
|
853
|
+
)
|
|
854
|
+
active_filter.display = True
|
|
855
|
+
else:
|
|
856
|
+
# No filter - hide everything including this widget
|
|
857
|
+
self.display = False
|
|
858
|
+
search_container.display = False
|
|
859
|
+
active_filter.display = False
|
|
860
|
+
error_widget.display = False
|
|
861
|
+
|
|
862
|
+
def on_input_changed(self, event: Input.Changed) -> None:
|
|
863
|
+
"""Parse filter expression when input changes"""
|
|
864
|
+
query = event.value.strip()
|
|
865
|
+
input_widget = self.query_one("#search-input", Input)
|
|
866
|
+
error_widget = self.query_one("#search-error", Static)
|
|
867
|
+
|
|
868
|
+
if not query:
|
|
869
|
+
self.filter_fn = None
|
|
870
|
+
self._query_valid = False
|
|
871
|
+
self.post_message(FilterChanged(None))
|
|
872
|
+
input_widget.remove_class("error")
|
|
873
|
+
input_widget.remove_class("valid")
|
|
874
|
+
error_widget.display = False
|
|
875
|
+
return
|
|
876
|
+
|
|
877
|
+
try:
|
|
878
|
+
from experimaestro.cli.filter import createFilter
|
|
879
|
+
|
|
880
|
+
self.filter_fn = createFilter(query)
|
|
881
|
+
self._query_valid = True
|
|
882
|
+
self.active_query = query
|
|
883
|
+
self.post_message(FilterChanged(self.filter_fn))
|
|
884
|
+
input_widget.remove_class("error")
|
|
885
|
+
input_widget.add_class("valid")
|
|
886
|
+
error_widget.display = False
|
|
887
|
+
except Exception as e:
|
|
888
|
+
self.filter_fn = None
|
|
889
|
+
self._query_valid = False
|
|
890
|
+
self.post_message(FilterChanged(None))
|
|
891
|
+
input_widget.remove_class("valid")
|
|
892
|
+
input_widget.add_class("error")
|
|
893
|
+
error_widget.update(f"Invalid query: {str(e)[:50]}")
|
|
894
|
+
error_widget.display = True
|
|
895
|
+
|
|
896
|
+
def on_input_submitted(self, event: Input.Submitted) -> None:
|
|
897
|
+
"""Apply filter and hide search bar (only if query is valid)"""
|
|
898
|
+
if self._query_valid and self.filter_fn is not None:
|
|
899
|
+
# Set flag to keep filter when hiding
|
|
900
|
+
self._keep_filter = True
|
|
901
|
+
self.visible = False
|
|
902
|
+
# Post message to focus jobs table
|
|
903
|
+
self.post_message(SearchApplied())
|
|
904
|
+
# If invalid, do nothing (keep input focused for correction)
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
class SearchApplied(Message):
|
|
908
|
+
"""Message sent when search filter is applied via Enter"""
|
|
909
|
+
|
|
910
|
+
pass
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
class JobsTable(Vertical):
|
|
914
|
+
"""Widget displaying jobs for selected experiment"""
|
|
915
|
+
|
|
916
|
+
BINDINGS = [
|
|
917
|
+
Binding("d", "delete_job", "Delete", show=False),
|
|
918
|
+
Binding("k", "kill_job", "Kill", show=False),
|
|
919
|
+
Binding("l", "view_logs", "Logs"),
|
|
920
|
+
Binding("f", "copy_path", "Copy Path", show=False),
|
|
921
|
+
Binding("/", "toggle_search", "Search"),
|
|
922
|
+
Binding("c", "clear_filter", "Clear", show=False),
|
|
923
|
+
Binding("r", "refresh_live", "Refresh"),
|
|
924
|
+
Binding("S", "sort_by_status", "Sort ⚑", show=False),
|
|
925
|
+
Binding("T", "sort_by_task", "Sort Task", show=False),
|
|
926
|
+
Binding("D", "sort_by_submitted", "Sort Date", show=False),
|
|
927
|
+
Binding("escape", "clear_search", show=False, priority=True),
|
|
928
|
+
]
|
|
929
|
+
|
|
930
|
+
# Track current sort state
|
|
931
|
+
_sort_column: Optional[str] = None
|
|
932
|
+
_sort_reverse: bool = False
|
|
933
|
+
_needs_rebuild: bool = True # Start with rebuild needed
|
|
934
|
+
|
|
935
|
+
def __init__(self, state_provider: WorkspaceStateProvider) -> None:
|
|
936
|
+
super().__init__()
|
|
937
|
+
self.state_provider = state_provider
|
|
938
|
+
self.filter_fn = None
|
|
939
|
+
self.current_experiment: Optional[str] = None
|
|
940
|
+
|
|
941
|
+
def compose(self) -> ComposeResult:
|
|
942
|
+
yield SearchBar()
|
|
943
|
+
yield DataTable(id="jobs-table", cursor_type="row")
|
|
944
|
+
|
|
945
|
+
def action_toggle_search(self) -> None:
|
|
946
|
+
"""Toggle search bar visibility"""
|
|
947
|
+
search_bar = self.query_one(SearchBar)
|
|
948
|
+
search_bar.visible = not search_bar.visible
|
|
949
|
+
|
|
950
|
+
def action_clear_filter(self) -> None:
|
|
951
|
+
"""Clear the active filter"""
|
|
952
|
+
if self.filter_fn is not None:
|
|
953
|
+
search_bar = self.query_one(SearchBar)
|
|
954
|
+
search_bar.query_one("#search-input", Input).value = ""
|
|
955
|
+
search_bar.filter_fn = None
|
|
956
|
+
search_bar.active_query = ""
|
|
957
|
+
search_bar._query_valid = False
|
|
958
|
+
# Hide the SearchBar completely
|
|
959
|
+
search_bar.display = False
|
|
960
|
+
search_bar.query_one("#search-container").display = False
|
|
961
|
+
search_bar.query_one("#active-filter").display = False
|
|
962
|
+
search_bar.query_one("#search-error").display = False
|
|
963
|
+
self.filter_fn = None
|
|
964
|
+
self.refresh_jobs()
|
|
965
|
+
self.notify("Filter cleared", severity="information")
|
|
966
|
+
|
|
967
|
+
def action_sort_by_status(self) -> None:
|
|
968
|
+
"""Sort jobs by status"""
|
|
969
|
+
if self._sort_column == "status":
|
|
970
|
+
self._sort_reverse = not self._sort_reverse
|
|
971
|
+
else:
|
|
972
|
+
self._sort_column = "status"
|
|
973
|
+
self._sort_reverse = False
|
|
974
|
+
self._needs_rebuild = True
|
|
975
|
+
self._update_column_headers()
|
|
976
|
+
self.refresh_jobs()
|
|
977
|
+
order = "desc" if self._sort_reverse else "asc"
|
|
978
|
+
self.notify(f"Sorted by status ({order})", severity="information")
|
|
979
|
+
|
|
980
|
+
def action_sort_by_task(self) -> None:
|
|
981
|
+
"""Sort jobs by task"""
|
|
982
|
+
if self._sort_column == "task":
|
|
983
|
+
self._sort_reverse = not self._sort_reverse
|
|
984
|
+
else:
|
|
985
|
+
self._sort_column = "task"
|
|
986
|
+
self._sort_reverse = False
|
|
987
|
+
self._needs_rebuild = True
|
|
988
|
+
self._update_column_headers()
|
|
989
|
+
self.refresh_jobs()
|
|
990
|
+
order = "desc" if self._sort_reverse else "asc"
|
|
991
|
+
self.notify(f"Sorted by task ({order})", severity="information")
|
|
992
|
+
|
|
993
|
+
def action_sort_by_submitted(self) -> None:
|
|
994
|
+
"""Sort jobs by submission time"""
|
|
995
|
+
if self._sort_column == "submitted":
|
|
996
|
+
self._sort_reverse = not self._sort_reverse
|
|
997
|
+
else:
|
|
998
|
+
self._sort_column = "submitted"
|
|
999
|
+
self._sort_reverse = False
|
|
1000
|
+
self._needs_rebuild = True
|
|
1001
|
+
self._update_column_headers()
|
|
1002
|
+
self.refresh_jobs()
|
|
1003
|
+
order = "newest first" if self._sort_reverse else "oldest first"
|
|
1004
|
+
self.notify(f"Sorted by date ({order})", severity="information")
|
|
1005
|
+
|
|
1006
|
+
def action_clear_search(self) -> None:
|
|
1007
|
+
"""Handle escape: hide search bar if visible, or go back"""
|
|
1008
|
+
search_bar = self.query_one(SearchBar)
|
|
1009
|
+
if search_bar.visible:
|
|
1010
|
+
# Search bar visible - hide it and clear filter
|
|
1011
|
+
search_bar.visible = False
|
|
1012
|
+
self.filter_fn = None
|
|
1013
|
+
self.refresh_jobs()
|
|
1014
|
+
# Focus the jobs table
|
|
1015
|
+
self.query_one("#jobs-table", DataTable).focus()
|
|
1016
|
+
else:
|
|
1017
|
+
# Search bar hidden - go back (keep filter)
|
|
1018
|
+
self.app.action_go_back()
|
|
1019
|
+
|
|
1020
|
+
def action_refresh_live(self) -> None:
|
|
1021
|
+
"""Refresh the jobs table"""
|
|
1022
|
+
self.refresh_jobs()
|
|
1023
|
+
self.notify("Jobs refreshed", severity="information")
|
|
1024
|
+
|
|
1025
|
+
def on_filter_changed(self, message: FilterChanged) -> None:
|
|
1026
|
+
"""Apply new filter"""
|
|
1027
|
+
self.filter_fn = message.filter_fn
|
|
1028
|
+
self.refresh_jobs()
|
|
1029
|
+
|
|
1030
|
+
def on_search_applied(self, message: SearchApplied) -> None:
|
|
1031
|
+
"""Focus jobs table when search is applied"""
|
|
1032
|
+
self.query_one("#jobs-table", DataTable).focus()
|
|
1033
|
+
|
|
1034
|
+
def _get_selected_job_id(self) -> Optional[str]:
|
|
1035
|
+
"""Get the job ID from the currently selected row"""
|
|
1036
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1037
|
+
if table.cursor_row is None:
|
|
1038
|
+
return None
|
|
1039
|
+
row_key = table.get_row_at(table.cursor_row)
|
|
1040
|
+
if row_key:
|
|
1041
|
+
# The first column is job_id
|
|
1042
|
+
return str(table.get_row_at(table.cursor_row)[0])
|
|
1043
|
+
return None
|
|
1044
|
+
|
|
1045
|
+
def action_delete_job(self) -> None:
|
|
1046
|
+
"""Request to delete the selected job"""
|
|
1047
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1048
|
+
if table.cursor_row is None or not self.current_experiment:
|
|
1049
|
+
return
|
|
1050
|
+
|
|
1051
|
+
# Get job ID from the row key
|
|
1052
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
1053
|
+
if row_key:
|
|
1054
|
+
job_id = str(row_key.value)
|
|
1055
|
+
self.post_message(DeleteJobRequest(job_id, self.current_experiment))
|
|
1056
|
+
|
|
1057
|
+
def action_kill_job(self) -> None:
|
|
1058
|
+
"""Request to kill the selected job"""
|
|
1059
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1060
|
+
if table.cursor_row is None or not self.current_experiment:
|
|
1061
|
+
return
|
|
1062
|
+
|
|
1063
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
1064
|
+
if row_key:
|
|
1065
|
+
job_id = str(row_key.value)
|
|
1066
|
+
self.post_message(KillJobRequest(job_id, self.current_experiment))
|
|
1067
|
+
|
|
1068
|
+
def action_view_logs(self) -> None:
|
|
1069
|
+
"""Request to view logs for the selected job"""
|
|
1070
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1071
|
+
if table.cursor_row is None or not self.current_experiment:
|
|
1072
|
+
return
|
|
1073
|
+
|
|
1074
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
1075
|
+
if row_key:
|
|
1076
|
+
job_id = str(row_key.value)
|
|
1077
|
+
self.post_message(ViewJobLogsRequest(job_id, self.current_experiment))
|
|
1078
|
+
|
|
1079
|
+
def action_copy_path(self) -> None:
|
|
1080
|
+
"""Copy the job folder path to clipboard"""
|
|
1081
|
+
import pyperclip
|
|
1082
|
+
|
|
1083
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1084
|
+
if table.cursor_row is None or not self.current_experiment:
|
|
1085
|
+
return
|
|
1086
|
+
|
|
1087
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
1088
|
+
if row_key:
|
|
1089
|
+
job_id = str(row_key.value)
|
|
1090
|
+
job = self.state_provider.get_job(job_id, self.current_experiment)
|
|
1091
|
+
if job and job.path:
|
|
1092
|
+
try:
|
|
1093
|
+
pyperclip.copy(str(job.path))
|
|
1094
|
+
self.notify(f"Path copied: {job.path}", severity="information")
|
|
1095
|
+
except Exception as e:
|
|
1096
|
+
self.notify(f"Failed to copy: {e}", severity="error")
|
|
1097
|
+
else:
|
|
1098
|
+
self.notify("No path available for this job", severity="warning")
|
|
1099
|
+
|
|
1100
|
+
# Status sort order (for sorting by status)
|
|
1101
|
+
STATUS_ORDER = {
|
|
1102
|
+
"running": 0,
|
|
1103
|
+
"waiting": 1,
|
|
1104
|
+
"error": 2,
|
|
1105
|
+
"done": 3,
|
|
1106
|
+
"unscheduled": 4,
|
|
1107
|
+
"phantom": 5,
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
# Failure reason sort order (within error status)
|
|
1111
|
+
# More actionable failures first
|
|
1112
|
+
FAILURE_ORDER = {
|
|
1113
|
+
"TIMEOUT": 0, # Might just need retry
|
|
1114
|
+
"MEMORY": 1, # Might need resource adjustment
|
|
1115
|
+
"DEPENDENCY": 2, # Need to fix upstream job first
|
|
1116
|
+
"FAILED": 3, # Generic failure
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
@classmethod
|
|
1120
|
+
def _get_status_sort_key(cls, job):
|
|
1121
|
+
"""Get sort key for a job based on status and failure reason.
|
|
1122
|
+
|
|
1123
|
+
Returns tuple (status_order, failure_order) for proper sorting.
|
|
1124
|
+
"""
|
|
1125
|
+
state_name = job.state.name if job.state else "unknown"
|
|
1126
|
+
status_order = cls.STATUS_ORDER.get(state_name, 99)
|
|
1127
|
+
|
|
1128
|
+
# For error jobs, also sort by failure reason
|
|
1129
|
+
if state_name == "error":
|
|
1130
|
+
failure_reason = getattr(job, "failure_reason", None)
|
|
1131
|
+
if failure_reason:
|
|
1132
|
+
failure_order = cls.FAILURE_ORDER.get(failure_reason.name, 99)
|
|
1133
|
+
else:
|
|
1134
|
+
failure_order = 99 # Unknown failure at end
|
|
1135
|
+
else:
|
|
1136
|
+
failure_order = 0
|
|
1137
|
+
|
|
1138
|
+
return (status_order, failure_order)
|
|
1139
|
+
|
|
1140
|
+
# Column key to display name mapping
|
|
1141
|
+
COLUMN_LABELS = {
|
|
1142
|
+
"job_id": "ID",
|
|
1143
|
+
"task": "Task",
|
|
1144
|
+
"status": "⚑",
|
|
1145
|
+
"tags": "Tags",
|
|
1146
|
+
"submitted": "Submitted",
|
|
1147
|
+
"duration": "Duration",
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
# Columns that support sorting (column key -> sort column name)
|
|
1151
|
+
SORTABLE_COLUMNS = {
|
|
1152
|
+
"status": "status",
|
|
1153
|
+
"task": "task",
|
|
1154
|
+
"submitted": "submitted",
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
def on_mount(self) -> None:
|
|
1158
|
+
"""Initialize the jobs table"""
|
|
1159
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1160
|
+
table.add_column("ID", key="job_id")
|
|
1161
|
+
table.add_column("Task", key="task")
|
|
1162
|
+
table.add_column("⚑", key="status", width=6)
|
|
1163
|
+
table.add_column("Tags", key="tags")
|
|
1164
|
+
table.add_column("Submitted", key="submitted")
|
|
1165
|
+
table.add_column("Duration", key="duration")
|
|
1166
|
+
table.cursor_type = "row"
|
|
1167
|
+
table.zebra_stripes = True
|
|
1168
|
+
|
|
1169
|
+
def _update_column_headers(self) -> None:
|
|
1170
|
+
"""Update column headers with sort indicators"""
|
|
1171
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1172
|
+
for column in table.columns.values():
|
|
1173
|
+
col_key = str(column.key.value) if column.key else None
|
|
1174
|
+
if col_key and col_key in self.COLUMN_LABELS:
|
|
1175
|
+
label = self.COLUMN_LABELS[col_key]
|
|
1176
|
+
sort_col = self.SORTABLE_COLUMNS.get(col_key)
|
|
1177
|
+
if sort_col and self._sort_column == sort_col:
|
|
1178
|
+
# Add sort indicator
|
|
1179
|
+
indicator = "▼" if self._sort_reverse else "▲"
|
|
1180
|
+
new_label = f"{label} {indicator}"
|
|
1181
|
+
else:
|
|
1182
|
+
new_label = label
|
|
1183
|
+
column.label = new_label
|
|
1184
|
+
|
|
1185
|
+
def on_data_table_header_selected(self, event: DataTable.HeaderSelected) -> None:
|
|
1186
|
+
"""Handle column header click for sorting"""
|
|
1187
|
+
col_key = str(event.column_key.value) if event.column_key else None
|
|
1188
|
+
if col_key and col_key in self.SORTABLE_COLUMNS:
|
|
1189
|
+
sort_col = self.SORTABLE_COLUMNS[col_key]
|
|
1190
|
+
if self._sort_column == sort_col:
|
|
1191
|
+
self._sort_reverse = not self._sort_reverse
|
|
1192
|
+
else:
|
|
1193
|
+
self._sort_column = sort_col
|
|
1194
|
+
self._sort_reverse = False
|
|
1195
|
+
self._needs_rebuild = True
|
|
1196
|
+
self._update_column_headers()
|
|
1197
|
+
self.refresh_jobs()
|
|
1198
|
+
|
|
1199
|
+
def set_experiment(self, experiment_id: Optional[str]) -> None:
|
|
1200
|
+
"""Set the current experiment and refresh jobs"""
|
|
1201
|
+
self.current_experiment = experiment_id
|
|
1202
|
+
self.refresh_jobs()
|
|
1203
|
+
|
|
1204
|
+
def refresh_jobs(self) -> None: # noqa: C901
|
|
1205
|
+
"""Refresh the jobs list from state provider"""
|
|
1206
|
+
table = self.query_one("#jobs-table", DataTable)
|
|
1207
|
+
|
|
1208
|
+
if not self.current_experiment:
|
|
1209
|
+
return
|
|
1210
|
+
|
|
1211
|
+
jobs = self.state_provider.get_jobs(self.current_experiment)
|
|
1212
|
+
self.log.debug(
|
|
1213
|
+
f"Refreshing jobs for {self.current_experiment}: {len(jobs)} jobs"
|
|
1214
|
+
)
|
|
1215
|
+
|
|
1216
|
+
# Apply filter if set
|
|
1217
|
+
if self.filter_fn:
|
|
1218
|
+
jobs = [j for j in jobs if self.filter_fn(j)]
|
|
1219
|
+
self.log.debug(f"After filter: {len(jobs)} jobs")
|
|
1220
|
+
|
|
1221
|
+
# Sort jobs based on selected column
|
|
1222
|
+
if self._sort_column == "status":
|
|
1223
|
+
# Sort by status priority, then by failure reason for errors
|
|
1224
|
+
jobs.sort(
|
|
1225
|
+
key=self._get_status_sort_key,
|
|
1226
|
+
reverse=self._sort_reverse,
|
|
1227
|
+
)
|
|
1228
|
+
elif self._sort_column == "task":
|
|
1229
|
+
# Sort by task name
|
|
1230
|
+
jobs.sort(
|
|
1231
|
+
key=lambda j: j.task_id or "",
|
|
1232
|
+
reverse=self._sort_reverse,
|
|
1233
|
+
)
|
|
1234
|
+
else:
|
|
1235
|
+
# Default: sort by submission time (oldest first by default)
|
|
1236
|
+
# Jobs without submittime go to the end
|
|
1237
|
+
jobs.sort(
|
|
1238
|
+
key=lambda j: j.submittime or float("inf"),
|
|
1239
|
+
reverse=self._sort_reverse,
|
|
1240
|
+
)
|
|
1241
|
+
|
|
1242
|
+
# Check if we need to rebuild (new/removed jobs, or status changed when sorting by status)
|
|
1243
|
+
from datetime import datetime
|
|
1244
|
+
import time as time_module
|
|
1245
|
+
|
|
1246
|
+
existing_keys = {str(k.value) for k in table.rows.keys()}
|
|
1247
|
+
current_job_ids = {job.identifier for job in jobs}
|
|
1248
|
+
|
|
1249
|
+
# Check if job set changed
|
|
1250
|
+
jobs_changed = existing_keys != current_job_ids
|
|
1251
|
+
|
|
1252
|
+
# Check if status changed when sorting by status
|
|
1253
|
+
status_changed = False
|
|
1254
|
+
if self._sort_column == "status" and not jobs_changed:
|
|
1255
|
+
current_statuses = {
|
|
1256
|
+
job.identifier: (job.state.name if job.state else "unknown")
|
|
1257
|
+
for job in jobs
|
|
1258
|
+
}
|
|
1259
|
+
if (
|
|
1260
|
+
hasattr(self, "_last_statuses")
|
|
1261
|
+
and self._last_statuses != current_statuses
|
|
1262
|
+
):
|
|
1263
|
+
status_changed = True
|
|
1264
|
+
self._last_statuses = current_statuses
|
|
1265
|
+
|
|
1266
|
+
needs_rebuild = self._needs_rebuild or jobs_changed or status_changed
|
|
1267
|
+
self._needs_rebuild = False
|
|
1268
|
+
|
|
1269
|
+
# Build row data for all jobs
|
|
1270
|
+
rows_data = {}
|
|
1271
|
+
for job in jobs:
|
|
1272
|
+
job_id = job.identifier
|
|
1273
|
+
task_id = job.task_id
|
|
1274
|
+
status = job.state.name if job.state else "unknown"
|
|
1275
|
+
|
|
1276
|
+
# Format status with icon (and progress % if running)
|
|
1277
|
+
if status == "running":
|
|
1278
|
+
progress_list = job.progress or []
|
|
1279
|
+
if progress_list:
|
|
1280
|
+
last_progress = progress_list[-1]
|
|
1281
|
+
progress_pct = last_progress.get("progress", 0) * 100
|
|
1282
|
+
status_text = f"▶ {progress_pct:.0f}%"
|
|
1283
|
+
else:
|
|
1284
|
+
status_text = "▶"
|
|
1285
|
+
else:
|
|
1286
|
+
failure_reason = getattr(job, "failure_reason", None)
|
|
1287
|
+
status_text = get_status_icon(status, failure_reason)
|
|
1288
|
+
|
|
1289
|
+
# Format tags - show all tags on single line
|
|
1290
|
+
tags = job.tags
|
|
1291
|
+
if tags:
|
|
1292
|
+
tags_text = Text()
|
|
1293
|
+
for i, (k, v) in enumerate(tags.items()):
|
|
1294
|
+
if i > 0:
|
|
1295
|
+
tags_text.append(", ")
|
|
1296
|
+
tags_text.append(f"{k}", style="bold")
|
|
1297
|
+
tags_text.append(f"={v}")
|
|
1298
|
+
else:
|
|
1299
|
+
tags_text = Text("-")
|
|
1300
|
+
|
|
1301
|
+
submitted = "-"
|
|
1302
|
+
if job.submittime:
|
|
1303
|
+
submitted = datetime.fromtimestamp(job.submittime).strftime(
|
|
1304
|
+
"%Y-%m-%d %H:%M"
|
|
1305
|
+
)
|
|
1306
|
+
|
|
1307
|
+
# Calculate duration
|
|
1308
|
+
start = job.starttime
|
|
1309
|
+
end = job.endtime
|
|
1310
|
+
duration = "-"
|
|
1311
|
+
if start:
|
|
1312
|
+
if end:
|
|
1313
|
+
elapsed = end - start
|
|
1314
|
+
else:
|
|
1315
|
+
elapsed = time_module.time() - start
|
|
1316
|
+
duration = self._format_duration(elapsed)
|
|
1317
|
+
|
|
1318
|
+
job_id_short = job_id[:7]
|
|
1319
|
+
rows_data[job_id] = (
|
|
1320
|
+
job_id_short,
|
|
1321
|
+
task_id,
|
|
1322
|
+
status_text,
|
|
1323
|
+
tags_text,
|
|
1324
|
+
submitted,
|
|
1325
|
+
duration,
|
|
1326
|
+
)
|
|
1327
|
+
|
|
1328
|
+
if needs_rebuild:
|
|
1329
|
+
# Full rebuild needed - save selection, clear, rebuild
|
|
1330
|
+
selected_key = None
|
|
1331
|
+
if table.cursor_row is not None and table.row_count > 0:
|
|
1332
|
+
try:
|
|
1333
|
+
row_keys = list(table.rows.keys())
|
|
1334
|
+
if table.cursor_row < len(row_keys):
|
|
1335
|
+
selected_key = str(row_keys[table.cursor_row].value)
|
|
1336
|
+
except (IndexError, KeyError):
|
|
1337
|
+
pass
|
|
1338
|
+
|
|
1339
|
+
table.clear()
|
|
1340
|
+
new_cursor_row = None
|
|
1341
|
+
for idx, job in enumerate(jobs):
|
|
1342
|
+
job_id = job.identifier
|
|
1343
|
+
table.add_row(*rows_data[job_id], key=job_id)
|
|
1344
|
+
if selected_key == job_id:
|
|
1345
|
+
new_cursor_row = idx
|
|
1346
|
+
|
|
1347
|
+
if new_cursor_row is not None and table.row_count > 0:
|
|
1348
|
+
table.move_cursor(row=new_cursor_row)
|
|
1349
|
+
else:
|
|
1350
|
+
# Just update cells in place - no reordering needed
|
|
1351
|
+
for job_id, row_data in rows_data.items():
|
|
1352
|
+
(
|
|
1353
|
+
job_id_short,
|
|
1354
|
+
task_id,
|
|
1355
|
+
status_text,
|
|
1356
|
+
tags_text,
|
|
1357
|
+
submitted,
|
|
1358
|
+
duration,
|
|
1359
|
+
) = row_data
|
|
1360
|
+
table.update_cell(job_id, "job_id", job_id_short, update_width=True)
|
|
1361
|
+
table.update_cell(job_id, "task", task_id, update_width=True)
|
|
1362
|
+
table.update_cell(job_id, "status", status_text, update_width=True)
|
|
1363
|
+
table.update_cell(job_id, "tags", tags_text, update_width=True)
|
|
1364
|
+
table.update_cell(job_id, "submitted", submitted, update_width=True)
|
|
1365
|
+
table.update_cell(job_id, "duration", duration, update_width=True)
|
|
1366
|
+
|
|
1367
|
+
self.log.debug(
|
|
1368
|
+
f"Jobs table now has {table.row_count} rows (rebuild={needs_rebuild})"
|
|
1369
|
+
)
|
|
1370
|
+
|
|
1371
|
+
def _format_duration(self, seconds: float) -> str:
|
|
1372
|
+
"""Format duration in seconds to human-readable string"""
|
|
1373
|
+
if seconds < 0:
|
|
1374
|
+
return "-"
|
|
1375
|
+
|
|
1376
|
+
seconds = int(seconds)
|
|
1377
|
+
if seconds < 60:
|
|
1378
|
+
return f"{seconds}s"
|
|
1379
|
+
elif seconds < 3600:
|
|
1380
|
+
minutes = seconds // 60
|
|
1381
|
+
secs = seconds % 60
|
|
1382
|
+
return f"{minutes}m {secs}s"
|
|
1383
|
+
elif seconds < 86400:
|
|
1384
|
+
hours = seconds // 3600
|
|
1385
|
+
minutes = (seconds % 3600) // 60
|
|
1386
|
+
return f"{hours}h {minutes}m"
|
|
1387
|
+
else:
|
|
1388
|
+
days = seconds // 86400
|
|
1389
|
+
hours = (seconds % 86400) // 3600
|
|
1390
|
+
return f"{days}d {hours}h"
|
|
1391
|
+
|
|
1392
|
+
def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
|
|
1393
|
+
"""Handle job selection"""
|
|
1394
|
+
if event.row_key and self.current_experiment:
|
|
1395
|
+
job_id = str(event.row_key.value)
|
|
1396
|
+
self.post_message(JobSelected(job_id, self.current_experiment))
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
class SizeCalculated(Message):
|
|
1400
|
+
"""Message sent when a folder size has been calculated"""
|
|
1401
|
+
|
|
1402
|
+
def __init__(self, job_id: str, size: str, size_bytes: int) -> None:
|
|
1403
|
+
super().__init__()
|
|
1404
|
+
self.job_id = job_id
|
|
1405
|
+
self.size = size
|
|
1406
|
+
self.size_bytes = size_bytes
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
class OrphanJobsScreen(Screen):
|
|
1410
|
+
"""Screen for viewing and managing orphan jobs"""
|
|
1411
|
+
|
|
1412
|
+
BINDINGS = [
|
|
1413
|
+
Binding("d", "delete_selected", "Delete"),
|
|
1414
|
+
Binding("D", "delete_all", "Delete All", key_display="D"),
|
|
1415
|
+
Binding("escape", "go_back", "Back"),
|
|
1416
|
+
Binding("q", "go_back", "Quit"),
|
|
1417
|
+
Binding("r", "refresh", "Refresh"),
|
|
1418
|
+
Binding("f", "copy_path", "Copy Path", show=False),
|
|
1419
|
+
Binding("T", "sort_by_task", "Sort Task", show=False),
|
|
1420
|
+
Binding("Z", "sort_by_size", "Sort Size", show=False),
|
|
1421
|
+
]
|
|
1422
|
+
|
|
1423
|
+
_size_cache: dict = {} # Class-level cache (formatted strings)
|
|
1424
|
+
_size_bytes_cache: dict = {} # Class-level cache (raw bytes for sorting)
|
|
1425
|
+
|
|
1426
|
+
def __init__(self, state_provider: WorkspaceStateProvider) -> None:
|
|
1427
|
+
super().__init__()
|
|
1428
|
+
self.state_provider = state_provider
|
|
1429
|
+
self.orphan_jobs = []
|
|
1430
|
+
self._pending_jobs = [] # Jobs waiting for size calculation
|
|
1431
|
+
self._sort_column: Optional[str] = None
|
|
1432
|
+
self._sort_reverse: bool = False
|
|
1433
|
+
|
|
1434
|
+
def compose(self) -> ComposeResult:
|
|
1435
|
+
yield Header()
|
|
1436
|
+
with Vertical(id="orphan-container"):
|
|
1437
|
+
yield Static("Orphan Jobs", id="orphan-title")
|
|
1438
|
+
yield Static("", id="orphan-stats")
|
|
1439
|
+
yield DataTable(id="orphan-table", cursor_type="row")
|
|
1440
|
+
yield Static("", id="orphan-job-info")
|
|
1441
|
+
yield Footer()
|
|
1442
|
+
|
|
1443
|
+
def on_mount(self) -> None:
|
|
1444
|
+
"""Initialize the orphan jobs table"""
|
|
1445
|
+
table = self.query_one("#orphan-table", DataTable)
|
|
1446
|
+
table.add_column("⚑", key="status", width=3)
|
|
1447
|
+
table.add_column("Job ID", key="job_id", width=10)
|
|
1448
|
+
table.add_column("Task", key="task")
|
|
1449
|
+
table.add_column("Size", key="size", width=10)
|
|
1450
|
+
self.refresh_orphans()
|
|
1451
|
+
|
|
1452
|
+
def action_sort_by_task(self) -> None:
|
|
1453
|
+
"""Sort by task name"""
|
|
1454
|
+
if self._sort_column == "task":
|
|
1455
|
+
self._sort_reverse = not self._sort_reverse
|
|
1456
|
+
else:
|
|
1457
|
+
self._sort_column = "task"
|
|
1458
|
+
self._sort_reverse = False
|
|
1459
|
+
self._rebuild_table()
|
|
1460
|
+
order = "desc" if self._sort_reverse else "asc"
|
|
1461
|
+
self.notify(f"Sorted by task ({order})", severity="information")
|
|
1462
|
+
|
|
1463
|
+
def action_sort_by_size(self) -> None:
|
|
1464
|
+
"""Sort by size"""
|
|
1465
|
+
if self._sort_column == "size":
|
|
1466
|
+
self._sort_reverse = not self._sort_reverse
|
|
1467
|
+
else:
|
|
1468
|
+
self._sort_column = "size"
|
|
1469
|
+
self._sort_reverse = True # Default: largest first
|
|
1470
|
+
self._rebuild_table()
|
|
1471
|
+
order = "largest first" if self._sort_reverse else "smallest first"
|
|
1472
|
+
self.notify(f"Sorted by size ({order})", severity="information")
|
|
1473
|
+
|
|
1474
|
+
def _get_sorted_jobs(self):
|
|
1475
|
+
"""Return jobs sorted by current sort column"""
|
|
1476
|
+
jobs = self.orphan_jobs[:]
|
|
1477
|
+
if self._sort_column == "task":
|
|
1478
|
+
jobs.sort(key=lambda j: j.task_id or "", reverse=self._sort_reverse)
|
|
1479
|
+
elif self._sort_column == "size":
|
|
1480
|
+
# Sort by raw bytes, jobs not in cache go to end
|
|
1481
|
+
jobs.sort(
|
|
1482
|
+
key=lambda j: self._size_bytes_cache.get(j.identifier, -1),
|
|
1483
|
+
reverse=self._sort_reverse,
|
|
1484
|
+
)
|
|
1485
|
+
return jobs
|
|
1486
|
+
|
|
1487
|
+
def _rebuild_table(self) -> None:
|
|
1488
|
+
"""Rebuild the table with current sort order"""
|
|
1489
|
+
table = self.query_one("#orphan-table", DataTable)
|
|
1490
|
+
table.clear()
|
|
1491
|
+
|
|
1492
|
+
for job in self._get_sorted_jobs():
|
|
1493
|
+
failure_reason = getattr(job, "failure_reason", None)
|
|
1494
|
+
status_icon = get_status_icon(
|
|
1495
|
+
job.state.name if job.state else "unknown", failure_reason
|
|
1496
|
+
)
|
|
1497
|
+
if job.identifier in self._size_cache:
|
|
1498
|
+
size_text = self._size_cache[job.identifier]
|
|
1499
|
+
else:
|
|
1500
|
+
size_text = "waiting"
|
|
1501
|
+
table.add_row(
|
|
1502
|
+
status_icon,
|
|
1503
|
+
job.identifier[:7],
|
|
1504
|
+
job.task_id,
|
|
1505
|
+
size_text,
|
|
1506
|
+
key=job.identifier,
|
|
1507
|
+
)
|
|
1508
|
+
|
|
1509
|
+
def refresh_orphans(self) -> None:
|
|
1510
|
+
"""Refresh the orphan jobs list"""
|
|
1511
|
+
# Only include orphan jobs that have an existing folder
|
|
1512
|
+
all_orphans = self.state_provider.get_orphan_jobs()
|
|
1513
|
+
self.orphan_jobs = [j for j in all_orphans if j.path and j.path.exists()]
|
|
1514
|
+
|
|
1515
|
+
# Update stats
|
|
1516
|
+
stats = self.query_one("#orphan-stats", Static)
|
|
1517
|
+
stats.update(f"Found {len(self.orphan_jobs)} orphan jobs")
|
|
1518
|
+
|
|
1519
|
+
# Collect jobs needing size calculation
|
|
1520
|
+
self._pending_jobs = [
|
|
1521
|
+
j for j in self.orphan_jobs if j.identifier not in self._size_cache
|
|
1522
|
+
]
|
|
1523
|
+
|
|
1524
|
+
# Rebuild table
|
|
1525
|
+
self._rebuild_table()
|
|
1526
|
+
|
|
1527
|
+
# Start calculating sizes
|
|
1528
|
+
if self._pending_jobs:
|
|
1529
|
+
self._calculate_next_size()
|
|
1530
|
+
|
|
1531
|
+
def _calculate_next_size(self) -> None:
|
|
1532
|
+
"""Calculate size for the next pending job using a worker"""
|
|
1533
|
+
if not self._pending_jobs:
|
|
1534
|
+
return
|
|
1535
|
+
|
|
1536
|
+
job = self._pending_jobs.pop(0)
|
|
1537
|
+
# Update to "calc..."
|
|
1538
|
+
self._update_size_cell(job.identifier, "calc...")
|
|
1539
|
+
# Run calculation in worker thread
|
|
1540
|
+
self.run_worker(
|
|
1541
|
+
self._calc_size_worker(job.identifier, job.path),
|
|
1542
|
+
thread=True,
|
|
1543
|
+
)
|
|
1544
|
+
|
|
1545
|
+
async def _calc_size_worker(self, job_id: str, path):
|
|
1546
|
+
"""Worker to calculate folder size"""
|
|
1547
|
+
size_bytes = await self._get_folder_size_async(path)
|
|
1548
|
+
size_str = self._format_size(size_bytes)
|
|
1549
|
+
self._size_cache[job_id] = size_str
|
|
1550
|
+
self._size_bytes_cache[job_id] = size_bytes
|
|
1551
|
+
self.post_message(SizeCalculated(job_id, size_str, size_bytes))
|
|
1552
|
+
|
|
1553
|
+
def on_size_calculated(self, message: SizeCalculated) -> None:
|
|
1554
|
+
"""Handle size calculation completion"""
|
|
1555
|
+
self._size_bytes_cache[message.job_id] = message.size_bytes
|
|
1556
|
+
self._update_size_cell(message.job_id, message.size)
|
|
1557
|
+
# Calculate next one
|
|
1558
|
+
self._calculate_next_size()
|
|
1559
|
+
|
|
1560
|
+
@staticmethod
|
|
1561
|
+
async def _get_folder_size_async(path) -> int:
|
|
1562
|
+
"""Calculate total size of a folder using du command if available"""
|
|
1563
|
+
import asyncio
|
|
1564
|
+
import shutil
|
|
1565
|
+
import sys
|
|
1566
|
+
|
|
1567
|
+
# Try using du command for better performance
|
|
1568
|
+
if shutil.which("du"):
|
|
1569
|
+
try:
|
|
1570
|
+
if sys.platform == "darwin":
|
|
1571
|
+
# macOS: du -sk gives size in KB
|
|
1572
|
+
proc = await asyncio.create_subprocess_exec(
|
|
1573
|
+
"du",
|
|
1574
|
+
"-sk",
|
|
1575
|
+
str(path),
|
|
1576
|
+
stdout=asyncio.subprocess.PIPE,
|
|
1577
|
+
stderr=asyncio.subprocess.DEVNULL,
|
|
1578
|
+
)
|
|
1579
|
+
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=30)
|
|
1580
|
+
if proc.returncode == 0 and stdout:
|
|
1581
|
+
# Output format: "SIZE\tPATH"
|
|
1582
|
+
size_kb = int(stdout.decode().split()[0])
|
|
1583
|
+
return size_kb * 1024
|
|
1584
|
+
else:
|
|
1585
|
+
# Linux: du -sb gives size in bytes
|
|
1586
|
+
proc = await asyncio.create_subprocess_exec(
|
|
1587
|
+
"du",
|
|
1588
|
+
"-sb",
|
|
1589
|
+
str(path),
|
|
1590
|
+
stdout=asyncio.subprocess.PIPE,
|
|
1591
|
+
stderr=asyncio.subprocess.DEVNULL,
|
|
1592
|
+
)
|
|
1593
|
+
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=30)
|
|
1594
|
+
if proc.returncode == 0 and stdout:
|
|
1595
|
+
# Output format: "SIZE\tPATH"
|
|
1596
|
+
return int(stdout.decode().split()[0])
|
|
1597
|
+
except (asyncio.TimeoutError, ValueError, IndexError, OSError):
|
|
1598
|
+
pass # Fall back to Python implementation
|
|
1599
|
+
|
|
1600
|
+
# Fallback: Python implementation
|
|
1601
|
+
return OrphanJobsScreen._get_folder_size_sync(path)
|
|
1602
|
+
|
|
1603
|
+
@staticmethod
|
|
1604
|
+
def _get_folder_size_sync(path) -> int:
|
|
1605
|
+
"""Calculate total size of a folder using Python (fallback)"""
|
|
1606
|
+
total = 0
|
|
1607
|
+
try:
|
|
1608
|
+
for entry in path.rglob("*"):
|
|
1609
|
+
if entry.is_file():
|
|
1610
|
+
total += entry.stat().st_size
|
|
1611
|
+
except (OSError, PermissionError):
|
|
1612
|
+
pass
|
|
1613
|
+
return total
|
|
1614
|
+
|
|
1615
|
+
@staticmethod
|
|
1616
|
+
def _format_size(size: int) -> str:
|
|
1617
|
+
"""Format size in human-readable format"""
|
|
1618
|
+
for unit in ["B", "KB", "MB", "GB"]:
|
|
1619
|
+
if size < 1024:
|
|
1620
|
+
return f"{size:.1f}{unit}" if unit != "B" else f"{size}{unit}"
|
|
1621
|
+
size /= 1024
|
|
1622
|
+
return f"{size:.1f}TB"
|
|
1623
|
+
|
|
1624
|
+
def _update_size_cell(self, job_id: str, value: str = None) -> None:
|
|
1625
|
+
"""Update the size cell for a job"""
|
|
1626
|
+
try:
|
|
1627
|
+
table = self.query_one("#orphan-table", DataTable)
|
|
1628
|
+
size_text = (
|
|
1629
|
+
value if value is not None else self._size_cache.get(job_id, "-")
|
|
1630
|
+
)
|
|
1631
|
+
table.update_cell(job_id, "size", size_text)
|
|
1632
|
+
except Exception:
|
|
1633
|
+
pass # Table may have changed
|
|
1634
|
+
|
|
1635
|
+
def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
|
|
1636
|
+
"""Show job details when a row is selected"""
|
|
1637
|
+
self._update_job_info()
|
|
1638
|
+
|
|
1639
|
+
def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
|
|
1640
|
+
"""Show job details when cursor moves"""
|
|
1641
|
+
self._update_job_info()
|
|
1642
|
+
|
|
1643
|
+
def _update_job_info(self) -> None:
|
|
1644
|
+
"""Update the job info display"""
|
|
1645
|
+
table = self.query_one("#orphan-table", DataTable)
|
|
1646
|
+
info = self.query_one("#orphan-job-info", Static)
|
|
1647
|
+
|
|
1648
|
+
if table.cursor_row is None:
|
|
1649
|
+
info.update("")
|
|
1650
|
+
return
|
|
1651
|
+
|
|
1652
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
1653
|
+
if row_key:
|
|
1654
|
+
job_id = str(row_key.value)
|
|
1655
|
+
job = next((j for j in self.orphan_jobs if j.identifier == job_id), None)
|
|
1656
|
+
if job and job.path:
|
|
1657
|
+
size = self._size_cache.get(job.identifier, "calculating...")
|
|
1658
|
+
info.update(f"Path: {job.path} | Size: {size}")
|
|
1659
|
+
else:
|
|
1660
|
+
info.update("")
|
|
1661
|
+
|
|
1662
|
+
def action_copy_path(self) -> None:
|
|
1663
|
+
"""Copy the job folder path to clipboard"""
|
|
1664
|
+
import pyperclip
|
|
1665
|
+
|
|
1666
|
+
table = self.query_one("#orphan-table", DataTable)
|
|
1667
|
+
if table.cursor_row is None:
|
|
1668
|
+
return
|
|
1669
|
+
|
|
1670
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
1671
|
+
if row_key:
|
|
1672
|
+
job_id = str(row_key.value)
|
|
1673
|
+
job = next((j for j in self.orphan_jobs if j.identifier == job_id), None)
|
|
1674
|
+
if job and job.path:
|
|
1675
|
+
try:
|
|
1676
|
+
pyperclip.copy(str(job.path))
|
|
1677
|
+
self.notify("Path copied", severity="information")
|
|
1678
|
+
except Exception as e:
|
|
1679
|
+
self.notify(f"Failed to copy: {e}", severity="error")
|
|
1680
|
+
|
|
1681
|
+
def action_delete_selected(self) -> None:
|
|
1682
|
+
"""Delete the selected orphan job"""
|
|
1683
|
+
table = self.query_one("#orphan-table", DataTable)
|
|
1684
|
+
if table.cursor_row is None:
|
|
1685
|
+
return
|
|
1686
|
+
|
|
1687
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
1688
|
+
if row_key:
|
|
1689
|
+
job_id = str(row_key.value)
|
|
1690
|
+
job = next((j for j in self.orphan_jobs if j.identifier == job_id), None)
|
|
1691
|
+
if job:
|
|
1692
|
+
self._delete_job(job)
|
|
1693
|
+
|
|
1694
|
+
def _delete_job(self, job) -> None:
|
|
1695
|
+
"""Delete a single orphan job with confirmation"""
|
|
1696
|
+
|
|
1697
|
+
def handle_delete(confirmed: bool) -> None:
|
|
1698
|
+
if confirmed:
|
|
1699
|
+
success, msg = self.state_provider.delete_job_safely(job)
|
|
1700
|
+
if success:
|
|
1701
|
+
self.notify(msg, severity="information")
|
|
1702
|
+
self.refresh_orphans()
|
|
1703
|
+
else:
|
|
1704
|
+
self.notify(msg, severity="error")
|
|
1705
|
+
|
|
1706
|
+
self.app.push_screen(
|
|
1707
|
+
DeleteConfirmScreen("orphan job", job.identifier),
|
|
1708
|
+
handle_delete,
|
|
1709
|
+
)
|
|
1710
|
+
|
|
1711
|
+
def action_delete_all(self) -> None:
|
|
1712
|
+
"""Delete all orphan jobs"""
|
|
1713
|
+
if not self.orphan_jobs:
|
|
1714
|
+
self.notify("No orphan jobs to delete", severity="warning")
|
|
1715
|
+
return
|
|
1716
|
+
|
|
1717
|
+
# Filter out running jobs
|
|
1718
|
+
deletable_jobs = [j for j in self.orphan_jobs if not j.state.running()]
|
|
1719
|
+
|
|
1720
|
+
if not deletable_jobs:
|
|
1721
|
+
self.notify("All orphan jobs are running", severity="warning")
|
|
1722
|
+
return
|
|
1723
|
+
|
|
1724
|
+
def handle_delete_all(confirmed: bool) -> None:
|
|
1725
|
+
if confirmed:
|
|
1726
|
+
deleted = 0
|
|
1727
|
+
for job in deletable_jobs:
|
|
1728
|
+
success, _ = self.state_provider.delete_job_safely(
|
|
1729
|
+
job, cascade_orphans=False
|
|
1730
|
+
)
|
|
1731
|
+
if success:
|
|
1732
|
+
deleted += 1
|
|
1733
|
+
|
|
1734
|
+
# Clean up orphan partials once at the end
|
|
1735
|
+
self.state_provider.cleanup_orphan_partials(perform=True)
|
|
1736
|
+
|
|
1737
|
+
self.notify(f"Deleted {deleted} orphan jobs", severity="information")
|
|
1738
|
+
self.refresh_orphans()
|
|
1739
|
+
|
|
1740
|
+
self.app.push_screen(
|
|
1741
|
+
DeleteConfirmScreen(
|
|
1742
|
+
"all orphan jobs",
|
|
1743
|
+
f"{len(deletable_jobs)} jobs",
|
|
1744
|
+
"This action cannot be undone",
|
|
1745
|
+
),
|
|
1746
|
+
handle_delete_all,
|
|
1747
|
+
)
|
|
1748
|
+
|
|
1749
|
+
def action_refresh(self) -> None:
|
|
1750
|
+
"""Refresh the orphan jobs list"""
|
|
1751
|
+
self.refresh_orphans()
|
|
1752
|
+
|
|
1753
|
+
def action_go_back(self) -> None:
|
|
1754
|
+
"""Go back to main screen"""
|
|
1755
|
+
self.dismiss()
|
|
1756
|
+
|
|
1757
|
+
|
|
1758
|
+
class HelpScreen(ModalScreen[None]):
|
|
1759
|
+
"""Modal screen showing keyboard shortcuts"""
|
|
1760
|
+
|
|
1761
|
+
BINDINGS = [
|
|
1762
|
+
Binding("escape", "close", "Close"),
|
|
1763
|
+
Binding("?", "close", "Close"),
|
|
1764
|
+
]
|
|
1765
|
+
|
|
1766
|
+
def compose(self) -> ComposeResult:
|
|
1767
|
+
from textual.containers import VerticalScroll
|
|
1768
|
+
|
|
1769
|
+
help_text = """
|
|
1770
|
+
[bold]Keyboard Shortcuts[/bold]
|
|
1771
|
+
|
|
1772
|
+
[bold cyan]Navigation[/bold cyan]
|
|
1773
|
+
q Quit application
|
|
1774
|
+
Esc Go back / Close dialog
|
|
1775
|
+
r Refresh data
|
|
1776
|
+
? Show this help
|
|
1777
|
+
j Switch to Jobs tab
|
|
1778
|
+
s Switch to Services tab
|
|
1779
|
+
|
|
1780
|
+
[bold cyan]Experiments[/bold cyan]
|
|
1781
|
+
Enter Select experiment
|
|
1782
|
+
d Delete experiment
|
|
1783
|
+
k Kill all running jobs
|
|
1784
|
+
|
|
1785
|
+
[bold cyan]Jobs[/bold cyan]
|
|
1786
|
+
l View job logs
|
|
1787
|
+
d Delete job
|
|
1788
|
+
k Kill running job
|
|
1789
|
+
/ Open search filter
|
|
1790
|
+
c Clear search filter
|
|
1791
|
+
S Sort by status
|
|
1792
|
+
T Sort by task
|
|
1793
|
+
D Sort by date
|
|
1794
|
+
f Copy folder path
|
|
1795
|
+
|
|
1796
|
+
[bold cyan]Services[/bold cyan]
|
|
1797
|
+
s Start service
|
|
1798
|
+
x Stop service
|
|
1799
|
+
u Copy URL
|
|
1800
|
+
|
|
1801
|
+
[bold cyan]Search Filter[/bold cyan]
|
|
1802
|
+
Enter Apply filter
|
|
1803
|
+
Esc Close and clear filter
|
|
1804
|
+
|
|
1805
|
+
[bold cyan]Orphan Jobs[/bold cyan]
|
|
1806
|
+
o Show orphan jobs
|
|
1807
|
+
T Sort by task
|
|
1808
|
+
Z Sort by size
|
|
1809
|
+
d Delete selected
|
|
1810
|
+
D Delete all
|
|
1811
|
+
f Copy folder path
|
|
1812
|
+
"""
|
|
1813
|
+
with Vertical(id="help-dialog"):
|
|
1814
|
+
yield Static("Experimaestro Help", id="help-title")
|
|
1815
|
+
with VerticalScroll(id="help-scroll"):
|
|
1816
|
+
yield Static(help_text, id="help-content")
|
|
1817
|
+
yield Button("Close", id="help-close-btn")
|
|
1818
|
+
|
|
1819
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
1820
|
+
self.dismiss()
|
|
1821
|
+
|
|
1822
|
+
def action_close(self) -> None:
|
|
1823
|
+
self.dismiss()
|
|
1824
|
+
|
|
1825
|
+
|
|
1826
|
+
class ExperimaestroUI(App):
|
|
1827
|
+
"""Textual TUI for monitoring experiments"""
|
|
1828
|
+
|
|
1829
|
+
TITLE = "Experimaestro UI"
|
|
1830
|
+
CSS_PATH = "app.tcss"
|
|
1831
|
+
|
|
1832
|
+
BINDINGS = [
|
|
1833
|
+
Binding("q", "quit", "Quit"),
|
|
1834
|
+
Binding("?", "show_help", "Help"),
|
|
1835
|
+
Binding("escape", "go_back", "Back", show=False),
|
|
1836
|
+
Binding("l", "view_logs", "Logs", show=False),
|
|
1837
|
+
Binding("o", "show_orphans", "Orphans", show=False),
|
|
1838
|
+
Binding("j", "focus_jobs", "Jobs", show=False),
|
|
1839
|
+
Binding("s", "focus_services", "Services", show=False),
|
|
1840
|
+
]
|
|
1841
|
+
|
|
1842
|
+
def __init__(
|
|
1843
|
+
self,
|
|
1844
|
+
workdir: Optional[Path] = None,
|
|
1845
|
+
watch: bool = True,
|
|
1846
|
+
state_provider: Optional[WorkspaceStateProvider] = None,
|
|
1847
|
+
show_logs: bool = False,
|
|
1848
|
+
):
|
|
1849
|
+
"""Initialize the TUI
|
|
1850
|
+
|
|
1851
|
+
Args:
|
|
1852
|
+
workdir: Workspace directory (required if state_provider not provided)
|
|
1853
|
+
watch: Enable filesystem watching for workspace mode
|
|
1854
|
+
state_provider: Pre-initialized state provider (for active experiments)
|
|
1855
|
+
show_logs: Whether to show the logs tab (for active experiments)
|
|
1856
|
+
"""
|
|
1857
|
+
super().__init__()
|
|
1858
|
+
self.workdir = workdir
|
|
1859
|
+
self.watch = watch
|
|
1860
|
+
self.show_logs = show_logs
|
|
1861
|
+
self._listener_registered = False
|
|
1862
|
+
|
|
1863
|
+
# Initialize state provider before compose
|
|
1864
|
+
if state_provider:
|
|
1865
|
+
self.state_provider = state_provider
|
|
1866
|
+
self.owns_provider = False # Don't close external provider
|
|
1867
|
+
self._has_active_experiment = True # External provider = active experiment
|
|
1868
|
+
else:
|
|
1869
|
+
from experimaestro.scheduler.state_provider import WorkspaceStateProvider
|
|
1870
|
+
|
|
1871
|
+
# Get singleton provider instance for this workspace
|
|
1872
|
+
self.state_provider = WorkspaceStateProvider.get_instance(
|
|
1873
|
+
self.workdir,
|
|
1874
|
+
read_only=False,
|
|
1875
|
+
sync_on_start=True,
|
|
1876
|
+
sync_interval_minutes=5,
|
|
1877
|
+
)
|
|
1878
|
+
self.owns_provider = False # Provider is singleton, don't close
|
|
1879
|
+
self._has_active_experiment = False # Just viewing, no active experiment
|
|
1880
|
+
|
|
1881
|
+
def compose(self) -> ComposeResult:
|
|
1882
|
+
"""Compose the TUI layout"""
|
|
1883
|
+
yield Header()
|
|
1884
|
+
|
|
1885
|
+
if self.show_logs:
|
|
1886
|
+
# Tabbed layout with logs
|
|
1887
|
+
with TabbedContent(id="main-tabs"):
|
|
1888
|
+
with TabPane("Monitor", id="monitor-tab"):
|
|
1889
|
+
yield from self._compose_monitor_view()
|
|
1890
|
+
with TabPane("Logs", id="logs-tab"):
|
|
1891
|
+
yield CaptureLog(id="logs", auto_scroll=True)
|
|
1892
|
+
else:
|
|
1893
|
+
# Simple layout without logs
|
|
1894
|
+
with Vertical(id="main-container"):
|
|
1895
|
+
yield from self._compose_monitor_view()
|
|
1896
|
+
|
|
1897
|
+
yield Footer()
|
|
1898
|
+
|
|
1899
|
+
def _compose_monitor_view(self):
|
|
1900
|
+
"""Compose the monitor view with experiments, jobs/services tabs, and job details"""
|
|
1901
|
+
yield ExperimentsList(self.state_provider)
|
|
1902
|
+
# Tabbed view for jobs and services (hidden initially)
|
|
1903
|
+
with TabbedContent(id="experiment-tabs", classes="hidden"):
|
|
1904
|
+
with TabPane("Jobs", id="jobs-tab"):
|
|
1905
|
+
yield JobsTable(self.state_provider)
|
|
1906
|
+
with TabPane("Services", id="services-tab"):
|
|
1907
|
+
yield ServicesList(self.state_provider)
|
|
1908
|
+
# Job detail view (hidden initially)
|
|
1909
|
+
with Vertical(id="job-detail-container", classes="hidden"):
|
|
1910
|
+
yield JobDetailView(self.state_provider)
|
|
1911
|
+
|
|
1912
|
+
def on_mount(self) -> None:
|
|
1913
|
+
"""Initialize the application"""
|
|
1914
|
+
# Resets logging
|
|
1915
|
+
logging.basicConfig(level=logging.INFO, force=True)
|
|
1916
|
+
|
|
1917
|
+
# Get the widgets
|
|
1918
|
+
experiments_list = self.query_one(ExperimentsList)
|
|
1919
|
+
experiments_list.refresh_experiments()
|
|
1920
|
+
|
|
1921
|
+
# Register as listener for push notifications from state provider
|
|
1922
|
+
if self.state_provider:
|
|
1923
|
+
self.state_provider.add_listener(self._on_state_event)
|
|
1924
|
+
self._listener_registered = True
|
|
1925
|
+
self.log("Registered state listener for push notifications")
|
|
1926
|
+
|
|
1927
|
+
def _on_state_event(self, event: StateEvent) -> None:
|
|
1928
|
+
"""Handle state change events from the state provider
|
|
1929
|
+
|
|
1930
|
+
This is called from the state provider's thread, so we use
|
|
1931
|
+
call_from_thread to safely update the UI.
|
|
1932
|
+
"""
|
|
1933
|
+
self.call_from_thread(self._handle_state_event, event)
|
|
1934
|
+
|
|
1935
|
+
def _handle_state_event(self, event: StateEvent) -> None:
|
|
1936
|
+
"""Process state event on the main thread"""
|
|
1937
|
+
# Use query() instead of query_one() to avoid NoMatches exception
|
|
1938
|
+
# when widgets aren't visible yet
|
|
1939
|
+
jobs_tables = self.query(JobsTable)
|
|
1940
|
+
services_lists = self.query(ServicesList)
|
|
1941
|
+
|
|
1942
|
+
self.log.debug(
|
|
1943
|
+
f"State event {event.event_type.name}, "
|
|
1944
|
+
f"JobsTable found: {len(jobs_tables)}, ServicesList found: {len(services_lists)}"
|
|
1945
|
+
)
|
|
1946
|
+
|
|
1947
|
+
if event.event_type == StateEventType.EXPERIMENT_UPDATED:
|
|
1948
|
+
# Refresh experiments list
|
|
1949
|
+
for exp_list in self.query(ExperimentsList):
|
|
1950
|
+
exp_list.refresh_experiments()
|
|
1951
|
+
|
|
1952
|
+
elif event.event_type == StateEventType.JOB_UPDATED:
|
|
1953
|
+
event_exp_id = event.data.get("experimentId")
|
|
1954
|
+
|
|
1955
|
+
# Refresh jobs table if we're viewing the affected experiment
|
|
1956
|
+
for jobs_table in jobs_tables:
|
|
1957
|
+
if jobs_table.current_experiment == event_exp_id:
|
|
1958
|
+
jobs_table.refresh_jobs()
|
|
1959
|
+
|
|
1960
|
+
# Also refresh job detail if we're viewing the affected job
|
|
1961
|
+
for job_detail_container in self.query("#job-detail-container"):
|
|
1962
|
+
if not job_detail_container.has_class("hidden"):
|
|
1963
|
+
for job_detail_view in self.query(JobDetailView):
|
|
1964
|
+
event_job_id = event.data.get("jobId")
|
|
1965
|
+
if job_detail_view.current_job_id == event_job_id:
|
|
1966
|
+
job_detail_view.refresh_job_detail()
|
|
1967
|
+
|
|
1968
|
+
# Also update the experiment stats in the experiments list
|
|
1969
|
+
for exp_list in self.query(ExperimentsList):
|
|
1970
|
+
exp_list.refresh_experiments()
|
|
1971
|
+
|
|
1972
|
+
elif event.event_type == StateEventType.RUN_UPDATED:
|
|
1973
|
+
# Refresh experiments list to show updated run info
|
|
1974
|
+
for exp_list in self.query(ExperimentsList):
|
|
1975
|
+
exp_list.refresh_experiments()
|
|
1976
|
+
|
|
1977
|
+
elif event.event_type == StateEventType.SERVICE_UPDATED:
|
|
1978
|
+
event_exp_id = event.data.get("experimentId")
|
|
1979
|
+
|
|
1980
|
+
# Refresh services list if we're viewing the affected experiment
|
|
1981
|
+
for services_list in services_lists:
|
|
1982
|
+
if services_list.current_experiment == event_exp_id:
|
|
1983
|
+
services_list.refresh_services()
|
|
1984
|
+
|
|
1985
|
+
def on_experiment_selected(self, message: ExperimentSelected) -> None:
|
|
1986
|
+
"""Handle experiment selection - show jobs/services tabs"""
|
|
1987
|
+
self.log(f"Experiment selected: {message.experiment_id}")
|
|
1988
|
+
|
|
1989
|
+
# Set up services list
|
|
1990
|
+
services_list = self.query_one(ServicesList)
|
|
1991
|
+
services_list.set_experiment(message.experiment_id)
|
|
1992
|
+
|
|
1993
|
+
# Set up jobs table
|
|
1994
|
+
jobs_table_widget = self.query_one(JobsTable)
|
|
1995
|
+
jobs_table_widget.set_experiment(message.experiment_id)
|
|
1996
|
+
|
|
1997
|
+
# Show the tabbed content
|
|
1998
|
+
tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
1999
|
+
tabs.remove_class("hidden")
|
|
2000
|
+
|
|
2001
|
+
# Focus the jobs table
|
|
2002
|
+
jobs_table = self.query_one("#jobs-table", DataTable)
|
|
2003
|
+
jobs_table.focus()
|
|
2004
|
+
|
|
2005
|
+
def on_experiment_deselected(self, message: ExperimentDeselected) -> None:
|
|
2006
|
+
"""Handle experiment deselection - hide jobs/services tabs"""
|
|
2007
|
+
# Hide the tabbed content
|
|
2008
|
+
tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
2009
|
+
tabs.add_class("hidden")
|
|
2010
|
+
# Also hide job detail if visible
|
|
2011
|
+
job_detail_container = self.query_one("#job-detail-container")
|
|
2012
|
+
job_detail_container.add_class("hidden")
|
|
2013
|
+
|
|
2014
|
+
def on_job_selected(self, message: JobSelected) -> None:
|
|
2015
|
+
"""Handle job selection - show job detail view"""
|
|
2016
|
+
self.log(f"Job selected: {message.job_id} from {message.experiment_id}")
|
|
2017
|
+
|
|
2018
|
+
# Hide tabs, show job detail
|
|
2019
|
+
tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
2020
|
+
tabs.add_class("hidden")
|
|
2021
|
+
|
|
2022
|
+
job_detail_container = self.query_one("#job-detail-container")
|
|
2023
|
+
job_detail_container.remove_class("hidden")
|
|
2024
|
+
|
|
2025
|
+
# Set the job to display
|
|
2026
|
+
job_detail_view = self.query_one(JobDetailView)
|
|
2027
|
+
job_detail_view.set_job(message.job_id, message.experiment_id)
|
|
2028
|
+
|
|
2029
|
+
def on_job_deselected(self, message: JobDeselected) -> None:
|
|
2030
|
+
"""Handle job deselection - go back to jobs view"""
|
|
2031
|
+
# Hide job detail, show tabs
|
|
2032
|
+
job_detail_container = self.query_one("#job-detail-container")
|
|
2033
|
+
job_detail_container.add_class("hidden")
|
|
2034
|
+
|
|
2035
|
+
tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
2036
|
+
tabs.remove_class("hidden")
|
|
2037
|
+
|
|
2038
|
+
# Focus the jobs table
|
|
2039
|
+
jobs_table = self.query_one("#jobs-table", DataTable)
|
|
2040
|
+
jobs_table.focus()
|
|
2041
|
+
|
|
2042
|
+
def action_refresh(self) -> None:
|
|
2043
|
+
"""Manually refresh the data"""
|
|
2044
|
+
experiments_list = self.query_one(ExperimentsList)
|
|
2045
|
+
jobs_table = self.query_one(JobsTable)
|
|
2046
|
+
|
|
2047
|
+
experiments_list.refresh_experiments()
|
|
2048
|
+
jobs_table.refresh_jobs()
|
|
2049
|
+
|
|
2050
|
+
# Also refresh job detail if visible
|
|
2051
|
+
job_detail_container = self.query_one("#job-detail-container")
|
|
2052
|
+
if not job_detail_container.has_class("hidden"):
|
|
2053
|
+
job_detail_view = self.query_one(JobDetailView)
|
|
2054
|
+
job_detail_view.refresh_job_detail()
|
|
2055
|
+
|
|
2056
|
+
def action_go_back(self) -> None:
|
|
2057
|
+
"""Go back one level in the navigation hierarchy"""
|
|
2058
|
+
# Check if job detail is visible -> go back to jobs/services tabs
|
|
2059
|
+
job_detail_container = self.query_one("#job-detail-container")
|
|
2060
|
+
if not job_detail_container.has_class("hidden"):
|
|
2061
|
+
self.post_message(JobDeselected())
|
|
2062
|
+
return
|
|
2063
|
+
|
|
2064
|
+
# Check if experiment tabs visible -> go back to experiments list
|
|
2065
|
+
experiment_tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
2066
|
+
if not experiment_tabs.has_class("hidden"):
|
|
2067
|
+
experiments_list = self.query_one(ExperimentsList)
|
|
2068
|
+
if experiments_list.collapsed:
|
|
2069
|
+
experiments_list.expand_experiments()
|
|
2070
|
+
experiment_tabs.add_class("hidden")
|
|
2071
|
+
self.post_message(ExperimentDeselected())
|
|
2072
|
+
|
|
2073
|
+
def action_view_logs(self) -> None:
|
|
2074
|
+
"""View logs for the current job (if job detail is visible)"""
|
|
2075
|
+
job_detail_container = self.query_one("#job-detail-container")
|
|
2076
|
+
if not job_detail_container.has_class("hidden"):
|
|
2077
|
+
job_detail_view = self.query_one(JobDetailView)
|
|
2078
|
+
job_detail_view.action_view_logs()
|
|
2079
|
+
|
|
2080
|
+
def action_show_orphans(self) -> None:
|
|
2081
|
+
"""Show orphan jobs screen"""
|
|
2082
|
+
self.push_screen(OrphanJobsScreen(self.state_provider))
|
|
2083
|
+
|
|
2084
|
+
def on_view_job_logs(self, message: ViewJobLogs) -> None:
|
|
2085
|
+
"""Handle request to view job logs - push LogViewerScreen"""
|
|
2086
|
+
job_path = Path(message.job_path)
|
|
2087
|
+
# Log files are named after the last part of the task ID
|
|
2088
|
+
task_name = message.task_id.split(".")[-1]
|
|
2089
|
+
stdout_path = job_path / f"{task_name}.out"
|
|
2090
|
+
stderr_path = job_path / f"{task_name}.err"
|
|
2091
|
+
|
|
2092
|
+
# Collect existing log files
|
|
2093
|
+
log_files = []
|
|
2094
|
+
if stdout_path.exists():
|
|
2095
|
+
log_files.append(str(stdout_path))
|
|
2096
|
+
if stderr_path.exists():
|
|
2097
|
+
log_files.append(str(stderr_path))
|
|
2098
|
+
|
|
2099
|
+
if not log_files:
|
|
2100
|
+
self.notify(
|
|
2101
|
+
f"No log files found: {task_name}.out/.err in {job_path}",
|
|
2102
|
+
severity="warning",
|
|
2103
|
+
)
|
|
2104
|
+
return
|
|
2105
|
+
|
|
2106
|
+
# Push the log viewer screen
|
|
2107
|
+
job_id = job_path.name
|
|
2108
|
+
self.push_screen(LogViewerScreen(log_files, job_id))
|
|
2109
|
+
|
|
2110
|
+
def on_view_job_logs_request(self, message: ViewJobLogsRequest) -> None:
|
|
2111
|
+
"""Handle log viewing request from jobs table"""
|
|
2112
|
+
job = self.state_provider.get_job(message.job_id, message.experiment_id)
|
|
2113
|
+
if not job or not job.path or not job.task_id:
|
|
2114
|
+
self.notify("Cannot find job logs", severity="warning")
|
|
2115
|
+
return
|
|
2116
|
+
self.post_message(ViewJobLogs(str(job.path), job.task_id))
|
|
2117
|
+
|
|
2118
|
+
def on_delete_job_request(self, message: DeleteJobRequest) -> None:
|
|
2119
|
+
"""Handle job deletion request"""
|
|
2120
|
+
job = self.state_provider.get_job(message.job_id, message.experiment_id)
|
|
2121
|
+
if not job:
|
|
2122
|
+
self.notify("Job not found", severity="error")
|
|
2123
|
+
return
|
|
2124
|
+
|
|
2125
|
+
if job.state.running():
|
|
2126
|
+
self.notify("Cannot delete a running job", severity="warning")
|
|
2127
|
+
return
|
|
2128
|
+
|
|
2129
|
+
# Save cursor position to restore after delete
|
|
2130
|
+
jobs_table = self.query_one(JobsTable)
|
|
2131
|
+
table = jobs_table.query_one("#jobs-table", DataTable)
|
|
2132
|
+
cursor_row = table.cursor_row
|
|
2133
|
+
|
|
2134
|
+
def handle_delete_response(confirmed: bool) -> None:
|
|
2135
|
+
if confirmed:
|
|
2136
|
+
success, msg = self.state_provider.delete_job_safely(job)
|
|
2137
|
+
if success:
|
|
2138
|
+
self.notify(msg, severity="information")
|
|
2139
|
+
self.action_refresh()
|
|
2140
|
+
# Move cursor to previous row (or first if was at top)
|
|
2141
|
+
if cursor_row is not None and table.row_count > 0:
|
|
2142
|
+
new_row = min(cursor_row, table.row_count - 1)
|
|
2143
|
+
if new_row > 0 and cursor_row > 0:
|
|
2144
|
+
new_row = cursor_row - 1
|
|
2145
|
+
table.move_cursor(row=new_row)
|
|
2146
|
+
else:
|
|
2147
|
+
self.notify(msg, severity="error")
|
|
2148
|
+
|
|
2149
|
+
self.push_screen(
|
|
2150
|
+
DeleteConfirmScreen("job", job.identifier),
|
|
2151
|
+
handle_delete_response,
|
|
2152
|
+
)
|
|
2153
|
+
|
|
2154
|
+
def on_delete_experiment_request(self, message: DeleteExperimentRequest) -> None:
|
|
2155
|
+
"""Handle experiment deletion request"""
|
|
2156
|
+
jobs = self.state_provider.get_jobs(message.experiment_id)
|
|
2157
|
+
running_jobs = [j for j in jobs if j.state.running()]
|
|
2158
|
+
|
|
2159
|
+
if running_jobs:
|
|
2160
|
+
self.notify(
|
|
2161
|
+
f"Cannot delete: {len(running_jobs)} jobs are running",
|
|
2162
|
+
severity="warning",
|
|
2163
|
+
)
|
|
2164
|
+
return
|
|
2165
|
+
|
|
2166
|
+
warning = (
|
|
2167
|
+
f"{len(jobs)} jobs will remain (not deleted by default)" if jobs else None
|
|
2168
|
+
)
|
|
2169
|
+
|
|
2170
|
+
def handle_delete_response(confirmed: bool) -> None:
|
|
2171
|
+
if confirmed:
|
|
2172
|
+
success, msg = self.state_provider.delete_experiment(
|
|
2173
|
+
message.experiment_id, delete_jobs=False
|
|
2174
|
+
)
|
|
2175
|
+
if success:
|
|
2176
|
+
self.notify(msg, severity="information")
|
|
2177
|
+
# Go back to experiments list
|
|
2178
|
+
experiments_list = self.query_one(ExperimentsList)
|
|
2179
|
+
experiments_list.expand_experiments()
|
|
2180
|
+
self.post_message(ExperimentDeselected())
|
|
2181
|
+
self.action_refresh()
|
|
2182
|
+
else:
|
|
2183
|
+
self.notify(msg, severity="error")
|
|
2184
|
+
|
|
2185
|
+
self.push_screen(
|
|
2186
|
+
DeleteConfirmScreen("experiment", message.experiment_id, warning),
|
|
2187
|
+
handle_delete_response,
|
|
2188
|
+
)
|
|
2189
|
+
|
|
2190
|
+
def on_kill_job_request(self, message: KillJobRequest) -> None:
|
|
2191
|
+
"""Handle job kill request"""
|
|
2192
|
+
job = self.state_provider.get_job(message.job_id, message.experiment_id)
|
|
2193
|
+
if not job:
|
|
2194
|
+
self.notify("Job not found", severity="error")
|
|
2195
|
+
return
|
|
2196
|
+
|
|
2197
|
+
if not job.state.running():
|
|
2198
|
+
self.notify("Job is not running", severity="warning")
|
|
2199
|
+
return
|
|
2200
|
+
|
|
2201
|
+
def handle_kill_response(confirmed: bool) -> None:
|
|
2202
|
+
if confirmed:
|
|
2203
|
+
success = self.state_provider.kill_job(job, perform=True)
|
|
2204
|
+
if success:
|
|
2205
|
+
self.notify(f"Job {job.identifier} killed", severity="information")
|
|
2206
|
+
self.action_refresh()
|
|
2207
|
+
else:
|
|
2208
|
+
self.notify("Failed to kill job", severity="error")
|
|
2209
|
+
|
|
2210
|
+
self.push_screen(
|
|
2211
|
+
KillConfirmScreen("job", job.identifier),
|
|
2212
|
+
handle_kill_response,
|
|
2213
|
+
)
|
|
2214
|
+
|
|
2215
|
+
def on_kill_experiment_request(self, message: KillExperimentRequest) -> None:
|
|
2216
|
+
"""Handle experiment kill request (kill all running jobs)"""
|
|
2217
|
+
jobs = self.state_provider.get_jobs(message.experiment_id)
|
|
2218
|
+
running_jobs = [j for j in jobs if j.state.running()]
|
|
2219
|
+
|
|
2220
|
+
if not running_jobs:
|
|
2221
|
+
self.notify("No running jobs in experiment", severity="warning")
|
|
2222
|
+
return
|
|
2223
|
+
|
|
2224
|
+
def handle_kill_response(confirmed: bool) -> None:
|
|
2225
|
+
if confirmed:
|
|
2226
|
+
killed = 0
|
|
2227
|
+
for job in running_jobs:
|
|
2228
|
+
if self.state_provider.kill_job(job, perform=True):
|
|
2229
|
+
killed += 1
|
|
2230
|
+
self.notify(
|
|
2231
|
+
f"Killed {killed} of {len(running_jobs)} running jobs",
|
|
2232
|
+
severity="information",
|
|
2233
|
+
)
|
|
2234
|
+
self.action_refresh()
|
|
2235
|
+
|
|
2236
|
+
self.push_screen(
|
|
2237
|
+
KillConfirmScreen("experiment", f"{len(running_jobs)} running jobs"),
|
|
2238
|
+
handle_kill_response,
|
|
2239
|
+
)
|
|
2240
|
+
|
|
2241
|
+
def action_focus_jobs(self) -> None:
|
|
2242
|
+
"""Switch to the jobs tab"""
|
|
2243
|
+
tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
2244
|
+
if not tabs.has_class("hidden"):
|
|
2245
|
+
tabs.active = "jobs-tab"
|
|
2246
|
+
jobs_table = self.query_one("#jobs-table", DataTable)
|
|
2247
|
+
jobs_table.focus()
|
|
2248
|
+
else:
|
|
2249
|
+
self.notify("Select an experiment first", severity="warning")
|
|
2250
|
+
|
|
2251
|
+
def action_focus_services(self) -> None:
|
|
2252
|
+
"""Switch to the services tab"""
|
|
2253
|
+
tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
2254
|
+
if not tabs.has_class("hidden"):
|
|
2255
|
+
tabs.active = "services-tab"
|
|
2256
|
+
services_table = self.query_one("#services-table", DataTable)
|
|
2257
|
+
services_table.focus()
|
|
2258
|
+
else:
|
|
2259
|
+
self.notify("Select an experiment first", severity="warning")
|
|
2260
|
+
|
|
2261
|
+
def action_switch_focus(self) -> None:
|
|
2262
|
+
"""Switch focus between experiments table and current tab"""
|
|
2263
|
+
focused = self.focused
|
|
2264
|
+
if focused:
|
|
2265
|
+
experiments_table = self.query_one("#experiments-table", DataTable)
|
|
2266
|
+
tabs = self.query_one("#experiment-tabs", TabbedContent)
|
|
2267
|
+
|
|
2268
|
+
if focused == experiments_table and not tabs.has_class("hidden"):
|
|
2269
|
+
# Focus the active tab's table
|
|
2270
|
+
if tabs.active == "services-tab":
|
|
2271
|
+
self.query_one("#services-table", DataTable).focus()
|
|
2272
|
+
else:
|
|
2273
|
+
self.query_one("#jobs-table", DataTable).focus()
|
|
2274
|
+
else:
|
|
2275
|
+
experiments_table.focus()
|
|
2276
|
+
|
|
2277
|
+
def action_quit(self) -> None:
|
|
2278
|
+
"""Show quit confirmation dialog"""
|
|
2279
|
+
|
|
2280
|
+
def handle_quit_response(confirmed: bool) -> None:
|
|
2281
|
+
if confirmed:
|
|
2282
|
+
self.exit()
|
|
2283
|
+
|
|
2284
|
+
self.push_screen(
|
|
2285
|
+
QuitConfirmScreen(has_active_experiment=self._has_active_experiment),
|
|
2286
|
+
handle_quit_response,
|
|
2287
|
+
)
|
|
2288
|
+
|
|
2289
|
+
def action_show_help(self) -> None:
|
|
2290
|
+
"""Show help screen with keyboard shortcuts"""
|
|
2291
|
+
self.push_screen(HelpScreen())
|
|
2292
|
+
|
|
2293
|
+
def on_unmount(self) -> None:
|
|
2294
|
+
"""Clean up when closing"""
|
|
2295
|
+
# Unregister listener
|
|
2296
|
+
if self._listener_registered and self.state_provider:
|
|
2297
|
+
self.state_provider.remove_listener(self._on_state_event)
|
|
2298
|
+
self._listener_registered = False
|
|
2299
|
+
self.log("Unregistered state listener")
|
|
2300
|
+
|
|
2301
|
+
# Only close state provider if we own it (not external/active experiment)
|
|
2302
|
+
if self.state_provider and self.owns_provider:
|
|
2303
|
+
self.state_provider.close()
|