experimaestro 2.0.0b4__py3-none-any.whl → 2.0.0b17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of experimaestro might be problematic. Click here for more details.
- experimaestro/__init__.py +12 -5
- experimaestro/cli/__init__.py +393 -134
- experimaestro/cli/filter.py +48 -23
- experimaestro/cli/jobs.py +253 -71
- experimaestro/cli/refactor.py +1 -2
- experimaestro/commandline.py +7 -4
- experimaestro/connectors/__init__.py +9 -1
- experimaestro/connectors/local.py +43 -3
- experimaestro/core/arguments.py +18 -18
- experimaestro/core/identifier.py +11 -11
- experimaestro/core/objects/config.py +96 -39
- experimaestro/core/objects/config_walk.py +3 -3
- experimaestro/core/{subparameters.py → partial.py} +16 -16
- experimaestro/core/partial_lock.py +394 -0
- experimaestro/core/types.py +12 -15
- experimaestro/dynamic.py +290 -0
- experimaestro/experiments/__init__.py +6 -2
- experimaestro/experiments/cli.py +223 -52
- experimaestro/experiments/configuration.py +24 -0
- experimaestro/generators.py +5 -5
- experimaestro/ipc.py +118 -1
- experimaestro/launcherfinder/__init__.py +2 -2
- experimaestro/launcherfinder/registry.py +6 -7
- experimaestro/launcherfinder/specs.py +2 -9
- experimaestro/launchers/slurm/__init__.py +2 -2
- experimaestro/launchers/slurm/base.py +62 -0
- experimaestro/locking.py +957 -1
- experimaestro/notifications.py +89 -201
- experimaestro/progress.py +63 -366
- experimaestro/rpyc.py +0 -2
- experimaestro/run.py +29 -2
- experimaestro/scheduler/__init__.py +8 -1
- experimaestro/scheduler/base.py +650 -53
- experimaestro/scheduler/dependencies.py +20 -16
- experimaestro/scheduler/experiment.py +764 -169
- experimaestro/scheduler/interfaces.py +338 -96
- experimaestro/scheduler/jobs.py +58 -20
- experimaestro/scheduler/remote/__init__.py +31 -0
- experimaestro/scheduler/remote/adaptive_sync.py +265 -0
- experimaestro/scheduler/remote/client.py +928 -0
- experimaestro/scheduler/remote/protocol.py +282 -0
- experimaestro/scheduler/remote/server.py +447 -0
- experimaestro/scheduler/remote/sync.py +144 -0
- experimaestro/scheduler/services.py +186 -35
- experimaestro/scheduler/state_provider.py +811 -2157
- experimaestro/scheduler/state_status.py +1247 -0
- experimaestro/scheduler/transient.py +31 -0
- experimaestro/scheduler/workspace.py +1 -1
- experimaestro/scheduler/workspace_state_provider.py +1273 -0
- experimaestro/scriptbuilder.py +4 -4
- experimaestro/settings.py +36 -0
- experimaestro/tests/conftest.py +33 -5
- experimaestro/tests/connectors/bin/executable.py +1 -1
- experimaestro/tests/fixtures/pre_experiment/experiment_check_env.py +16 -0
- experimaestro/tests/fixtures/pre_experiment/experiment_check_mock.py +14 -0
- experimaestro/tests/fixtures/pre_experiment/experiment_simple.py +12 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_env.py +5 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_error.py +3 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_mock.py +8 -0
- experimaestro/tests/launchers/bin/test.py +1 -0
- experimaestro/tests/launchers/test_slurm.py +9 -9
- experimaestro/tests/partial_reschedule.py +46 -0
- experimaestro/tests/restart.py +3 -3
- experimaestro/tests/restart_main.py +1 -0
- experimaestro/tests/scripts/notifyandwait.py +1 -0
- experimaestro/tests/task_partial.py +38 -0
- experimaestro/tests/task_tokens.py +2 -2
- experimaestro/tests/tasks/test_dynamic.py +6 -6
- experimaestro/tests/test_dependencies.py +3 -3
- experimaestro/tests/test_deprecated.py +15 -15
- experimaestro/tests/test_dynamic_locking.py +317 -0
- experimaestro/tests/test_environment.py +24 -14
- experimaestro/tests/test_experiment.py +171 -36
- experimaestro/tests/test_identifier.py +25 -25
- experimaestro/tests/test_identifier_stability.py +3 -5
- experimaestro/tests/test_multitoken.py +2 -4
- experimaestro/tests/{test_subparameters.py → test_partial.py} +25 -25
- experimaestro/tests/test_partial_paths.py +81 -138
- experimaestro/tests/test_pre_experiment.py +219 -0
- experimaestro/tests/test_progress.py +2 -8
- experimaestro/tests/test_remote_state.py +1132 -0
- experimaestro/tests/test_stray_jobs.py +261 -0
- experimaestro/tests/test_tasks.py +1 -2
- experimaestro/tests/test_token_locking.py +52 -67
- experimaestro/tests/test_tokens.py +5 -6
- experimaestro/tests/test_transient.py +225 -0
- experimaestro/tests/test_workspace_state_provider.py +768 -0
- experimaestro/tests/token_reschedule.py +1 -3
- experimaestro/tests/utils.py +2 -7
- experimaestro/tokens.py +227 -372
- experimaestro/tools/diff.py +1 -0
- experimaestro/tools/documentation.py +4 -5
- experimaestro/tools/jobs.py +1 -2
- experimaestro/tui/app.py +459 -1895
- experimaestro/tui/app.tcss +162 -0
- experimaestro/tui/dialogs.py +172 -0
- experimaestro/tui/log_viewer.py +253 -3
- experimaestro/tui/messages.py +137 -0
- experimaestro/tui/utils.py +54 -0
- experimaestro/tui/widgets/__init__.py +23 -0
- experimaestro/tui/widgets/experiments.py +468 -0
- experimaestro/tui/widgets/global_services.py +238 -0
- experimaestro/tui/widgets/jobs.py +972 -0
- experimaestro/tui/widgets/log.py +156 -0
- experimaestro/tui/widgets/orphans.py +363 -0
- experimaestro/tui/widgets/runs.py +185 -0
- experimaestro/tui/widgets/services.py +314 -0
- experimaestro/tui/widgets/stray_jobs.py +528 -0
- experimaestro/utils/__init__.py +1 -1
- experimaestro/utils/environment.py +105 -22
- experimaestro/utils/fswatcher.py +124 -0
- experimaestro/utils/jobs.py +1 -2
- experimaestro/utils/jupyter.py +1 -2
- experimaestro/utils/logging.py +72 -0
- experimaestro/version.py +2 -2
- experimaestro/webui/__init__.py +9 -0
- experimaestro/webui/app.py +117 -0
- experimaestro/{server → webui}/data/index.css +66 -11
- experimaestro/webui/data/index.css.map +1 -0
- experimaestro/{server → webui}/data/index.js +82763 -87217
- experimaestro/webui/data/index.js.map +1 -0
- experimaestro/webui/routes/__init__.py +5 -0
- experimaestro/webui/routes/auth.py +53 -0
- experimaestro/webui/routes/proxy.py +117 -0
- experimaestro/webui/server.py +200 -0
- experimaestro/webui/state_bridge.py +152 -0
- experimaestro/webui/websocket.py +413 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/METADATA +8 -9
- experimaestro-2.0.0b17.dist-info/RECORD +219 -0
- experimaestro/cli/progress.py +0 -269
- experimaestro/scheduler/state.py +0 -75
- experimaestro/scheduler/state_db.py +0 -388
- experimaestro/scheduler/state_sync.py +0 -834
- experimaestro/server/__init__.py +0 -467
- experimaestro/server/data/index.css.map +0 -1
- experimaestro/server/data/index.js.map +0 -1
- experimaestro/tests/test_cli_jobs.py +0 -615
- experimaestro/tests/test_file_progress.py +0 -425
- experimaestro/tests/test_file_progress_integration.py +0 -477
- experimaestro/tests/test_state_db.py +0 -434
- experimaestro-2.0.0b4.dist-info/RECORD +0 -181
- /experimaestro/{server → webui}/data/1815e00441357e01619e.ttf +0 -0
- /experimaestro/{server → webui}/data/2463b90d9a316e4e5294.woff2 +0 -0
- /experimaestro/{server → webui}/data/2582b0e4bcf85eceead0.ttf +0 -0
- /experimaestro/{server → webui}/data/89999bdf5d835c012025.woff2 +0 -0
- /experimaestro/{server → webui}/data/914997e1bdfc990d0897.ttf +0 -0
- /experimaestro/{server → webui}/data/c210719e60948b211a12.woff2 +0 -0
- /experimaestro/{server → webui}/data/favicon.ico +0 -0
- /experimaestro/{server → webui}/data/index.html +0 -0
- /experimaestro/{server → webui}/data/login.html +0 -0
- /experimaestro/{server → webui}/data/manifest.json +0 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/WHEEL +0 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/entry_points.txt +0 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/licenses/LICENSE +0 -0
experimaestro/tui/app.tcss
CHANGED
|
@@ -10,6 +10,10 @@ ExperimentsList {
|
|
|
10
10
|
height: auto;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
ExperimentsList.expanded {
|
|
14
|
+
height: 1fr;
|
|
15
|
+
}
|
|
16
|
+
|
|
13
17
|
Monitor {
|
|
14
18
|
height: 100%;
|
|
15
19
|
}
|
|
@@ -92,6 +96,10 @@ JobsTable {
|
|
|
92
96
|
min-height: 5;
|
|
93
97
|
}
|
|
94
98
|
|
|
99
|
+
ExperimentsList.expanded #experiments-table {
|
|
100
|
+
height: 1fr;
|
|
101
|
+
}
|
|
102
|
+
|
|
95
103
|
TabbedContent {
|
|
96
104
|
height: 100%;
|
|
97
105
|
}
|
|
@@ -115,6 +123,8 @@ JobDetailView {
|
|
|
115
123
|
|
|
116
124
|
#job-detail-content {
|
|
117
125
|
padding: 1;
|
|
126
|
+
overflow-y: auto;
|
|
127
|
+
height: 1fr;
|
|
118
128
|
}
|
|
119
129
|
|
|
120
130
|
#job-detail-content Label {
|
|
@@ -351,3 +361,155 @@ HelpScreen {
|
|
|
351
361
|
width: 100%;
|
|
352
362
|
height: auto;
|
|
353
363
|
}
|
|
364
|
+
|
|
365
|
+
/* Past run banner */
|
|
366
|
+
#past-run-banner {
|
|
367
|
+
background: $warning-darken-2;
|
|
368
|
+
color: $text;
|
|
369
|
+
padding: 0 1;
|
|
370
|
+
text-align: center;
|
|
371
|
+
text-style: bold;
|
|
372
|
+
height: auto;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/* Runs List */
|
|
376
|
+
RunsList {
|
|
377
|
+
height: 1fr;
|
|
378
|
+
width: 100%;
|
|
379
|
+
display: none;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
RunsList.hidden {
|
|
383
|
+
display: none;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
#runs-container {
|
|
387
|
+
height: 100%;
|
|
388
|
+
width: 100%;
|
|
389
|
+
border: solid $primary;
|
|
390
|
+
padding: 0 1;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
#runs-title {
|
|
394
|
+
text-style: bold;
|
|
395
|
+
color: $primary;
|
|
396
|
+
height: auto;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
#runs-table {
|
|
400
|
+
height: 1fr;
|
|
401
|
+
min-height: 3;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/* Stray Jobs Tab */
|
|
405
|
+
StrayJobsTab {
|
|
406
|
+
height: 100%;
|
|
407
|
+
width: 100%;
|
|
408
|
+
padding: 1;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
#stray-title {
|
|
412
|
+
text-align: center;
|
|
413
|
+
text-style: bold;
|
|
414
|
+
margin-bottom: 1;
|
|
415
|
+
color: $warning;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
#stray-warning {
|
|
419
|
+
background: $warning-darken-2;
|
|
420
|
+
color: $text;
|
|
421
|
+
padding: 0 1;
|
|
422
|
+
margin-bottom: 1;
|
|
423
|
+
text-align: center;
|
|
424
|
+
height: auto;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
#stray-warning.hidden {
|
|
428
|
+
display: none;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
.warning-banner {
|
|
432
|
+
height: auto;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
#stray-stats {
|
|
436
|
+
margin-bottom: 1;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
#stray-controls {
|
|
440
|
+
height: auto;
|
|
441
|
+
margin-bottom: 1;
|
|
442
|
+
align: left middle;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
#stray-controls Button {
|
|
446
|
+
margin-right: 1;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
#stray-controls Switch {
|
|
450
|
+
margin-right: 0;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
#show-orphans-label {
|
|
454
|
+
margin-right: 2;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
.controls-bar {
|
|
458
|
+
height: auto;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
#stray-table {
|
|
462
|
+
height: 1fr;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
#stray-job-info {
|
|
466
|
+
height: auto;
|
|
467
|
+
margin-top: 1;
|
|
468
|
+
padding: 0 1;
|
|
469
|
+
color: $text-muted;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/* Orphan Jobs Tab */
|
|
473
|
+
OrphanJobsTab {
|
|
474
|
+
height: 1fr;
|
|
475
|
+
width: 100%;
|
|
476
|
+
padding: 1;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
OrphanJobsTab #orphan-warning {
|
|
480
|
+
background: $warning-darken-2;
|
|
481
|
+
color: $text;
|
|
482
|
+
padding: 0 1;
|
|
483
|
+
margin-bottom: 1;
|
|
484
|
+
text-align: center;
|
|
485
|
+
height: auto;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
OrphanJobsTab #orphan-warning.hidden {
|
|
489
|
+
display: none;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
OrphanJobsTab #orphan-controls {
|
|
493
|
+
height: auto;
|
|
494
|
+
margin-bottom: 1;
|
|
495
|
+
align: left middle;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
OrphanJobsTab #orphan-controls Button {
|
|
499
|
+
margin-right: 1;
|
|
500
|
+
min-width: 8;
|
|
501
|
+
height: auto;
|
|
502
|
+
padding: 0 1;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
OrphanJobsTab #orphan-table {
|
|
506
|
+
height: 1fr;
|
|
507
|
+
min-height: 10;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
OrphanJobsTab #orphan-job-info {
|
|
511
|
+
height: auto;
|
|
512
|
+
margin-top: 1;
|
|
513
|
+
padding: 0 1;
|
|
514
|
+
color: $text-muted;
|
|
515
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Modal dialog screens for the TUI"""
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
from textual.app import ComposeResult
|
|
5
|
+
from textual.containers import Horizontal, Vertical, VerticalScroll
|
|
6
|
+
from textual.widgets import Button, Static
|
|
7
|
+
from textual.screen import ModalScreen
|
|
8
|
+
from textual.binding import Binding
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class QuitConfirmScreen(ModalScreen[bool]):
|
|
12
|
+
"""Modal screen for quit confirmation"""
|
|
13
|
+
|
|
14
|
+
def __init__(self, has_active_experiment: bool = False):
|
|
15
|
+
super().__init__()
|
|
16
|
+
self.has_active_experiment = has_active_experiment
|
|
17
|
+
|
|
18
|
+
def compose(self) -> ComposeResult:
|
|
19
|
+
with Vertical(id="quit-dialog"):
|
|
20
|
+
yield Static("Quit Experimaestro?", id="quit-title")
|
|
21
|
+
|
|
22
|
+
if self.has_active_experiment:
|
|
23
|
+
yield Static(
|
|
24
|
+
"⚠️ The experiment is still in progress.\nQuitting will prevent new jobs from being launched.",
|
|
25
|
+
id="quit-warning",
|
|
26
|
+
)
|
|
27
|
+
else:
|
|
28
|
+
yield Static("Are you sure you want to quit?", id="quit-message")
|
|
29
|
+
|
|
30
|
+
with Horizontal(id="quit-buttons"):
|
|
31
|
+
yield Button("Quit", variant="error", id="quit-yes")
|
|
32
|
+
yield Button("Cancel", variant="primary", id="quit-no")
|
|
33
|
+
|
|
34
|
+
def on_mount(self) -> None:
|
|
35
|
+
"""Focus Cancel button by default"""
|
|
36
|
+
self.query_one("#quit-no", Button).focus()
|
|
37
|
+
|
|
38
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
39
|
+
if event.button.id == "quit-yes":
|
|
40
|
+
self.dismiss(True)
|
|
41
|
+
else:
|
|
42
|
+
self.dismiss(False)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class DeleteConfirmScreen(ModalScreen[bool]):
|
|
46
|
+
"""Modal screen for delete confirmation"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self, item_type: str, item_name: str, warning: Optional[str] = None
|
|
50
|
+
) -> None:
|
|
51
|
+
super().__init__()
|
|
52
|
+
self.item_type = item_type
|
|
53
|
+
self.item_name = item_name
|
|
54
|
+
self.warning = warning
|
|
55
|
+
|
|
56
|
+
def compose(self) -> ComposeResult:
|
|
57
|
+
with Vertical(id="delete-dialog"):
|
|
58
|
+
yield Static(f"Delete {self.item_type}?", id="delete-title")
|
|
59
|
+
yield Static(
|
|
60
|
+
f"This will permanently delete: {self.item_name}", id="delete-message"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
if self.warning:
|
|
64
|
+
yield Static(f"Warning: {self.warning}", id="delete-warning")
|
|
65
|
+
|
|
66
|
+
with Horizontal(id="delete-buttons"):
|
|
67
|
+
yield Button("Delete", variant="error", id="delete-yes")
|
|
68
|
+
yield Button("Cancel", variant="primary", id="delete-no")
|
|
69
|
+
|
|
70
|
+
def on_mount(self) -> None:
|
|
71
|
+
"""Focus cancel button by default"""
|
|
72
|
+
self.query_one("#delete-no", Button).focus()
|
|
73
|
+
|
|
74
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
75
|
+
if event.button.id == "delete-yes":
|
|
76
|
+
self.dismiss(True)
|
|
77
|
+
else:
|
|
78
|
+
self.dismiss(False)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class KillConfirmScreen(ModalScreen[bool]):
|
|
82
|
+
"""Modal screen for kill confirmation"""
|
|
83
|
+
|
|
84
|
+
def __init__(self, item_type: str, item_name: str) -> None:
|
|
85
|
+
super().__init__()
|
|
86
|
+
self.item_type = item_type
|
|
87
|
+
self.item_name = item_name
|
|
88
|
+
|
|
89
|
+
def compose(self) -> ComposeResult:
|
|
90
|
+
with Vertical(id="kill-dialog"):
|
|
91
|
+
yield Static(f"Kill {self.item_type}?", id="kill-title")
|
|
92
|
+
yield Static(f"This will terminate: {self.item_name}", id="kill-message")
|
|
93
|
+
|
|
94
|
+
with Horizontal(id="kill-buttons"):
|
|
95
|
+
yield Button("Kill", variant="warning", id="kill-yes")
|
|
96
|
+
yield Button("Cancel", variant="primary", id="kill-no")
|
|
97
|
+
|
|
98
|
+
def on_mount(self) -> None:
|
|
99
|
+
"""Focus cancel button by default"""
|
|
100
|
+
self.query_one("#kill-no", Button).focus()
|
|
101
|
+
|
|
102
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
103
|
+
if event.button.id == "kill-yes":
|
|
104
|
+
self.dismiss(True)
|
|
105
|
+
else:
|
|
106
|
+
self.dismiss(False)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class HelpScreen(ModalScreen[None]):
|
|
110
|
+
"""Modal screen showing keyboard shortcuts"""
|
|
111
|
+
|
|
112
|
+
BINDINGS = [
|
|
113
|
+
Binding("escape", "close", "Close"),
|
|
114
|
+
Binding("?", "close", "Close"),
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
def compose(self) -> ComposeResult:
|
|
118
|
+
help_text = """
|
|
119
|
+
[bold]Keyboard Shortcuts[/bold]
|
|
120
|
+
|
|
121
|
+
[bold cyan]Navigation[/bold cyan]
|
|
122
|
+
q Quit application
|
|
123
|
+
Esc Go back / Close dialog
|
|
124
|
+
r Refresh data
|
|
125
|
+
? Show this help
|
|
126
|
+
j Switch to Jobs tab
|
|
127
|
+
s Switch to Services tab
|
|
128
|
+
|
|
129
|
+
[bold cyan]Experiments[/bold cyan]
|
|
130
|
+
Enter Select experiment
|
|
131
|
+
d Delete experiment
|
|
132
|
+
k Kill all running jobs
|
|
133
|
+
|
|
134
|
+
[bold cyan]Jobs[/bold cyan]
|
|
135
|
+
l View job logs
|
|
136
|
+
d Delete job
|
|
137
|
+
k Kill running job
|
|
138
|
+
/ Open search filter
|
|
139
|
+
c Clear search filter
|
|
140
|
+
S Sort by status
|
|
141
|
+
T Sort by task
|
|
142
|
+
D Sort by date
|
|
143
|
+
f Copy folder path
|
|
144
|
+
|
|
145
|
+
[bold cyan]Services[/bold cyan]
|
|
146
|
+
s Start service
|
|
147
|
+
x Stop service
|
|
148
|
+
u Copy URL
|
|
149
|
+
|
|
150
|
+
[bold cyan]Search Filter[/bold cyan]
|
|
151
|
+
Enter Apply filter
|
|
152
|
+
Esc Close and clear filter
|
|
153
|
+
|
|
154
|
+
[bold cyan]Orphan Jobs[/bold cyan]
|
|
155
|
+
o Show orphan jobs
|
|
156
|
+
T Sort by task
|
|
157
|
+
Z Sort by size
|
|
158
|
+
d Delete selected
|
|
159
|
+
D Delete all
|
|
160
|
+
f Copy folder path
|
|
161
|
+
"""
|
|
162
|
+
with Vertical(id="help-dialog"):
|
|
163
|
+
yield Static("Experimaestro Help", id="help-title")
|
|
164
|
+
with VerticalScroll(id="help-scroll"):
|
|
165
|
+
yield Static(help_text, id="help-content")
|
|
166
|
+
yield Button("Close", id="help-close-btn")
|
|
167
|
+
|
|
168
|
+
def on_button_pressed(self, event: Button.Pressed) -> None:
|
|
169
|
+
self.dismiss()
|
|
170
|
+
|
|
171
|
+
def action_close(self) -> None:
|
|
172
|
+
self.dismiss()
|
experimaestro/tui/log_viewer.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Log viewer screen for viewing job logs efficiently"""
|
|
2
2
|
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
from typing import Callable, Optional
|
|
4
5
|
|
|
5
6
|
from textual.app import ComposeResult
|
|
6
7
|
from textual.binding import Binding
|
|
@@ -8,6 +9,8 @@ from textual.containers import Vertical
|
|
|
8
9
|
from textual.screen import Screen
|
|
9
10
|
from textual.widgets import Footer, Header, RichLog, Static, TabbedContent, TabPane
|
|
10
11
|
|
|
12
|
+
from experimaestro.scheduler.interfaces import JobState
|
|
13
|
+
|
|
11
14
|
|
|
12
15
|
# Default chunk size for reading file (64KB)
|
|
13
16
|
CHUNK_SIZE = 64 * 1024
|
|
@@ -129,7 +132,12 @@ class LogWidget(Vertical):
|
|
|
129
132
|
|
|
130
133
|
|
|
131
134
|
class LogViewerScreen(Screen, inherit_bindings=False):
|
|
132
|
-
"""Screen for viewing job logs efficiently
|
|
135
|
+
"""Screen for viewing job logs efficiently
|
|
136
|
+
|
|
137
|
+
Supports both local and remote log viewing:
|
|
138
|
+
- Local: reads log files directly
|
|
139
|
+
- Remote: uses adaptive sync to periodically rsync logs from remote
|
|
140
|
+
"""
|
|
133
141
|
|
|
134
142
|
CSS = """
|
|
135
143
|
LogViewerScreen {
|
|
@@ -144,6 +152,21 @@ class LogViewerScreen(Screen, inherit_bindings=False):
|
|
|
144
152
|
color: $text-muted;
|
|
145
153
|
}
|
|
146
154
|
|
|
155
|
+
.loading-message {
|
|
156
|
+
content-align: center middle;
|
|
157
|
+
height: 100%;
|
|
158
|
+
text-style: italic;
|
|
159
|
+
color: $text-muted;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
.sync-status {
|
|
163
|
+
background: $boost;
|
|
164
|
+
padding: 0 1;
|
|
165
|
+
height: auto;
|
|
166
|
+
text-style: italic;
|
|
167
|
+
color: $text-muted;
|
|
168
|
+
}
|
|
169
|
+
|
|
147
170
|
LogWidget {
|
|
148
171
|
height: 1fr;
|
|
149
172
|
}
|
|
@@ -158,21 +181,53 @@ class LogViewerScreen(Screen, inherit_bindings=False):
|
|
|
158
181
|
Binding("f", "toggle_follow", "Follow"),
|
|
159
182
|
Binding("g", "go_to_top", "Top"),
|
|
160
183
|
Binding("G", "go_to_bottom", "Bottom"),
|
|
184
|
+
Binding("r", "sync_now", "Sync", show=False),
|
|
161
185
|
Binding("escape", "close_viewer", "Back", priority=True),
|
|
162
186
|
Binding("q", "close_viewer", "Quit", priority=True),
|
|
163
187
|
]
|
|
164
188
|
|
|
165
|
-
def __init__(
|
|
189
|
+
def __init__(
|
|
190
|
+
self,
|
|
191
|
+
log_files: list[str],
|
|
192
|
+
job_id: str,
|
|
193
|
+
sync_func: Optional[Callable[[str], Optional[Path]]] = None,
|
|
194
|
+
remote_path: Optional[str] = None,
|
|
195
|
+
task_id: Optional[str] = None,
|
|
196
|
+
job_state: Optional[JobState] = None,
|
|
197
|
+
):
|
|
198
|
+
"""Initialize the log viewer
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
log_files: List of local log file paths
|
|
202
|
+
job_id: Job identifier
|
|
203
|
+
sync_func: Function to sync remote path (for remote monitoring)
|
|
204
|
+
remote_path: Remote job directory path (for remote monitoring)
|
|
205
|
+
task_id: Task ID for log file naming (for remote monitoring)
|
|
206
|
+
job_state: Current job state (for adaptive sync decisions)
|
|
207
|
+
"""
|
|
166
208
|
super().__init__()
|
|
167
209
|
self.log_files = log_files
|
|
168
210
|
self.job_id = job_id
|
|
169
211
|
self.following = True
|
|
170
212
|
self.log_widgets: list[LogWidget] = []
|
|
171
213
|
|
|
214
|
+
# Remote sync support
|
|
215
|
+
self.sync_func = sync_func
|
|
216
|
+
self.remote_path = remote_path
|
|
217
|
+
self.task_id = task_id
|
|
218
|
+
self.job_state = job_state
|
|
219
|
+
self._synchronizer = None
|
|
220
|
+
self._loading = bool(sync_func and not log_files)
|
|
221
|
+
|
|
172
222
|
def compose(self) -> ComposeResult:
|
|
173
223
|
yield Header()
|
|
174
224
|
|
|
175
|
-
if
|
|
225
|
+
if self._loading:
|
|
226
|
+
# Show loading message while waiting for first sync
|
|
227
|
+
yield Static(
|
|
228
|
+
"Syncing logs from remote...", id="loading", classes="loading-message"
|
|
229
|
+
)
|
|
230
|
+
elif len(self.log_files) == 1:
|
|
176
231
|
# Single file - simple view
|
|
177
232
|
widget = LogWidget(self.log_files[0], "log-0")
|
|
178
233
|
self.log_widgets.append(widget)
|
|
@@ -187,12 +242,183 @@ class LogViewerScreen(Screen, inherit_bindings=False):
|
|
|
187
242
|
self.log_widgets.append(widget)
|
|
188
243
|
yield widget
|
|
189
244
|
|
|
245
|
+
# Sync status for remote monitoring
|
|
246
|
+
if self.sync_func:
|
|
247
|
+
yield Static("", id="sync-status", classes="sync-status")
|
|
248
|
+
|
|
190
249
|
yield Footer()
|
|
191
250
|
|
|
192
251
|
def on_mount(self) -> None:
|
|
193
252
|
"""Start watching for changes"""
|
|
194
253
|
self.set_interval(0.5, self._refresh_logs)
|
|
195
254
|
|
|
255
|
+
# Start adaptive sync for remote monitoring
|
|
256
|
+
if self.sync_func and self.remote_path:
|
|
257
|
+
self._start_adaptive_sync()
|
|
258
|
+
|
|
259
|
+
def _start_adaptive_sync(self) -> None:
|
|
260
|
+
"""Start adaptive sync for remote log files"""
|
|
261
|
+
import logging
|
|
262
|
+
|
|
263
|
+
from experimaestro.scheduler.remote.adaptive_sync import AdaptiveSynchronizer
|
|
264
|
+
|
|
265
|
+
logger = logging.getLogger("xpm.tui.log_viewer")
|
|
266
|
+
|
|
267
|
+
# Only use adaptive sync for running jobs
|
|
268
|
+
is_running = self.job_state and self.job_state.running()
|
|
269
|
+
logger.info(
|
|
270
|
+
f"Starting sync: job_state={self.job_state}, "
|
|
271
|
+
f"is_running={is_running}, remote_path={self.remote_path}"
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
# Update loading message to show we're syncing
|
|
275
|
+
try:
|
|
276
|
+
loading = self.query_one("#loading", Static)
|
|
277
|
+
loading.update("Syncing logs from remote (this may take a moment)...")
|
|
278
|
+
except Exception:
|
|
279
|
+
pass
|
|
280
|
+
|
|
281
|
+
if is_running:
|
|
282
|
+
# Build a name for logging
|
|
283
|
+
sync_name = f"job:{self.task_id}" if self.task_id else f"job:{self.job_id}"
|
|
284
|
+
self._synchronizer = AdaptiveSynchronizer(
|
|
285
|
+
sync_func=self.sync_func,
|
|
286
|
+
remote_path=self.remote_path,
|
|
287
|
+
name=sync_name,
|
|
288
|
+
on_sync_start=lambda: self.app.call_from_thread(self._on_sync_start),
|
|
289
|
+
on_sync_complete=lambda p: self.app.call_from_thread(
|
|
290
|
+
self._on_sync_complete, p
|
|
291
|
+
),
|
|
292
|
+
on_sync_error=lambda e: self.app.call_from_thread(
|
|
293
|
+
self._on_sync_error, e
|
|
294
|
+
),
|
|
295
|
+
)
|
|
296
|
+
self._synchronizer.start()
|
|
297
|
+
else:
|
|
298
|
+
# For completed jobs, just sync once
|
|
299
|
+
self._do_single_sync()
|
|
300
|
+
|
|
301
|
+
def _do_single_sync(self) -> None:
|
|
302
|
+
"""Do a single sync for completed jobs"""
|
|
303
|
+
import threading
|
|
304
|
+
|
|
305
|
+
def sync_thread():
|
|
306
|
+
try:
|
|
307
|
+
local_path = self.sync_func(self.remote_path)
|
|
308
|
+
if local_path:
|
|
309
|
+
self.app.call_from_thread(self._on_sync_complete, local_path)
|
|
310
|
+
else:
|
|
311
|
+
self.app.call_from_thread(self._on_sync_error, "Sync failed")
|
|
312
|
+
except Exception as e:
|
|
313
|
+
self.app.call_from_thread(self._on_sync_error, str(e))
|
|
314
|
+
|
|
315
|
+
thread = threading.Thread(target=sync_thread, daemon=True)
|
|
316
|
+
thread.start()
|
|
317
|
+
|
|
318
|
+
def _on_sync_start(self) -> None:
|
|
319
|
+
"""Handle sync start - update status"""
|
|
320
|
+
try:
|
|
321
|
+
status = self.query_one("#sync-status", Static)
|
|
322
|
+
status.update("⟳ Syncing...")
|
|
323
|
+
except Exception:
|
|
324
|
+
pass
|
|
325
|
+
|
|
326
|
+
def _on_sync_complete(self, local_path) -> None:
|
|
327
|
+
"""Handle sync completion - update log widgets"""
|
|
328
|
+
import logging
|
|
329
|
+
|
|
330
|
+
logger = logging.getLogger("xpm.tui.log_viewer")
|
|
331
|
+
|
|
332
|
+
# Ensure local_path is a Path object
|
|
333
|
+
if not isinstance(local_path, Path):
|
|
334
|
+
local_path = Path(local_path)
|
|
335
|
+
|
|
336
|
+
logger.info(f"Sync complete: {local_path}, loading={self._loading}")
|
|
337
|
+
|
|
338
|
+
# Update sync status
|
|
339
|
+
try:
|
|
340
|
+
status = self.query_one("#sync-status", Static)
|
|
341
|
+
if self._synchronizer:
|
|
342
|
+
status.update(f"✓ Next sync: {self._synchronizer.interval:.0f}s")
|
|
343
|
+
else:
|
|
344
|
+
status.update("✓ Synced")
|
|
345
|
+
except Exception as e:
|
|
346
|
+
logger.warning(f"Failed to update sync status: {e}")
|
|
347
|
+
|
|
348
|
+
# If loading, find log files and create widgets
|
|
349
|
+
if self._loading:
|
|
350
|
+
self._loading = False
|
|
351
|
+
task_name = self.task_id.split(".")[-1] if self.task_id else "task"
|
|
352
|
+
stdout_path = local_path / f"{task_name}.out"
|
|
353
|
+
stderr_path = local_path / f"{task_name}.err"
|
|
354
|
+
|
|
355
|
+
logger.info(f"Looking for log files: {stdout_path}, {stderr_path}")
|
|
356
|
+
|
|
357
|
+
log_files = []
|
|
358
|
+
if stdout_path.exists():
|
|
359
|
+
log_files.append(str(stdout_path))
|
|
360
|
+
if stderr_path.exists():
|
|
361
|
+
log_files.append(str(stderr_path))
|
|
362
|
+
|
|
363
|
+
if not log_files:
|
|
364
|
+
try:
|
|
365
|
+
loading = self.query_one("#loading", Static)
|
|
366
|
+
loading.update(
|
|
367
|
+
f"No log files found: {task_name}.out/.err in {local_path}"
|
|
368
|
+
)
|
|
369
|
+
except Exception as e:
|
|
370
|
+
logger.warning(f"Failed to update loading: {e}")
|
|
371
|
+
return
|
|
372
|
+
|
|
373
|
+
logger.info(f"Found log files: {log_files}")
|
|
374
|
+
self.log_files = log_files
|
|
375
|
+
self._rebuild_log_widgets()
|
|
376
|
+
else:
|
|
377
|
+
# Just refresh existing widgets
|
|
378
|
+
for widget in self.log_widgets:
|
|
379
|
+
widget.refresh_content()
|
|
380
|
+
|
|
381
|
+
def _on_sync_error(self, error: str) -> None:
|
|
382
|
+
"""Handle sync error"""
|
|
383
|
+
try:
|
|
384
|
+
status = self.query_one("#sync-status", Static)
|
|
385
|
+
status.update(f"Sync error: {error}")
|
|
386
|
+
except Exception:
|
|
387
|
+
pass
|
|
388
|
+
|
|
389
|
+
if self._loading:
|
|
390
|
+
try:
|
|
391
|
+
loading = self.query_one("#loading", Static)
|
|
392
|
+
loading.update(f"Error: {error}")
|
|
393
|
+
except Exception:
|
|
394
|
+
pass
|
|
395
|
+
|
|
396
|
+
def _rebuild_log_widgets(self) -> None:
|
|
397
|
+
"""Rebuild log widgets after first sync"""
|
|
398
|
+
# Remove loading message
|
|
399
|
+
try:
|
|
400
|
+
loading = self.query_one("#loading", Static)
|
|
401
|
+
loading.remove()
|
|
402
|
+
except Exception:
|
|
403
|
+
pass
|
|
404
|
+
|
|
405
|
+
# Add log widgets
|
|
406
|
+
self.log_widgets = []
|
|
407
|
+
if len(self.log_files) == 1:
|
|
408
|
+
widget = LogWidget(self.log_files[0], "log-0")
|
|
409
|
+
self.log_widgets.append(widget)
|
|
410
|
+
self.mount(widget, before=self.query_one("#sync-status"))
|
|
411
|
+
else:
|
|
412
|
+
tabbed = TabbedContent()
|
|
413
|
+
self.mount(tabbed, before=self.query_one("#sync-status"))
|
|
414
|
+
for i, log_file in enumerate(self.log_files):
|
|
415
|
+
file_name = Path(log_file).name
|
|
416
|
+
pane = TabPane(file_name, id=f"tab-{i}")
|
|
417
|
+
widget = LogWidget(log_file, f"log-{i}")
|
|
418
|
+
self.log_widgets.append(widget)
|
|
419
|
+
pane.compose_add_child(widget)
|
|
420
|
+
tabbed.add_pane(pane)
|
|
421
|
+
|
|
196
422
|
def _refresh_logs(self) -> None:
|
|
197
423
|
"""Refresh all log widgets"""
|
|
198
424
|
if not self.following:
|
|
@@ -203,6 +429,21 @@ class LogViewerScreen(Screen, inherit_bindings=False):
|
|
|
203
429
|
|
|
204
430
|
def action_close_viewer(self) -> None:
|
|
205
431
|
"""Go back to the job detail view"""
|
|
432
|
+
import logging
|
|
433
|
+
|
|
434
|
+
logger = logging.getLogger("xpm.tui.log_viewer")
|
|
435
|
+
|
|
436
|
+
# Warn if closing during first sync
|
|
437
|
+
if self._loading:
|
|
438
|
+
self.notify("Sync in progress, please wait...", severity="warning")
|
|
439
|
+
logger.info("User tried to close during first sync, ignoring")
|
|
440
|
+
return
|
|
441
|
+
|
|
442
|
+
# Stop adaptive sync if running
|
|
443
|
+
if self._synchronizer:
|
|
444
|
+
logger.info("Closing log viewer, stopping sync")
|
|
445
|
+
self._synchronizer.stop()
|
|
446
|
+
|
|
206
447
|
self.dismiss()
|
|
207
448
|
|
|
208
449
|
def action_toggle_follow(self) -> None:
|
|
@@ -226,3 +467,12 @@ class LogViewerScreen(Screen, inherit_bindings=False):
|
|
|
226
467
|
for widget in self.log_widgets:
|
|
227
468
|
widget.following = True
|
|
228
469
|
widget.scroll_to_end()
|
|
470
|
+
|
|
471
|
+
def action_sync_now(self) -> None:
|
|
472
|
+
"""Trigger immediate sync (for remote monitoring)"""
|
|
473
|
+
if self._synchronizer:
|
|
474
|
+
self._synchronizer.sync_now()
|
|
475
|
+
self.notify("Syncing...")
|
|
476
|
+
elif self.sync_func and self.remote_path:
|
|
477
|
+
self._do_single_sync()
|
|
478
|
+
self.notify("Syncing...")
|