experimaestro 2.0.0b8__py3-none-any.whl → 2.0.0b17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of experimaestro might be problematic. Click here for more details.

Files changed (152) hide show
  1. experimaestro/__init__.py +12 -5
  2. experimaestro/cli/__init__.py +239 -126
  3. experimaestro/cli/filter.py +48 -23
  4. experimaestro/cli/jobs.py +253 -71
  5. experimaestro/cli/refactor.py +1 -2
  6. experimaestro/commandline.py +7 -4
  7. experimaestro/connectors/__init__.py +9 -1
  8. experimaestro/connectors/local.py +43 -3
  9. experimaestro/core/arguments.py +18 -18
  10. experimaestro/core/identifier.py +11 -11
  11. experimaestro/core/objects/config.py +96 -39
  12. experimaestro/core/objects/config_walk.py +3 -3
  13. experimaestro/core/{subparameters.py → partial.py} +16 -16
  14. experimaestro/core/partial_lock.py +394 -0
  15. experimaestro/core/types.py +12 -15
  16. experimaestro/dynamic.py +290 -0
  17. experimaestro/experiments/__init__.py +6 -2
  18. experimaestro/experiments/cli.py +217 -50
  19. experimaestro/experiments/configuration.py +24 -0
  20. experimaestro/generators.py +5 -5
  21. experimaestro/ipc.py +118 -1
  22. experimaestro/launcherfinder/__init__.py +2 -2
  23. experimaestro/launcherfinder/registry.py +6 -7
  24. experimaestro/launcherfinder/specs.py +2 -9
  25. experimaestro/launchers/slurm/__init__.py +2 -2
  26. experimaestro/launchers/slurm/base.py +62 -0
  27. experimaestro/locking.py +957 -1
  28. experimaestro/notifications.py +89 -201
  29. experimaestro/progress.py +63 -366
  30. experimaestro/rpyc.py +0 -2
  31. experimaestro/run.py +29 -2
  32. experimaestro/scheduler/__init__.py +8 -1
  33. experimaestro/scheduler/base.py +629 -53
  34. experimaestro/scheduler/dependencies.py +20 -16
  35. experimaestro/scheduler/experiment.py +732 -167
  36. experimaestro/scheduler/interfaces.py +316 -101
  37. experimaestro/scheduler/jobs.py +58 -20
  38. experimaestro/scheduler/remote/adaptive_sync.py +265 -0
  39. experimaestro/scheduler/remote/client.py +171 -117
  40. experimaestro/scheduler/remote/protocol.py +8 -193
  41. experimaestro/scheduler/remote/server.py +95 -71
  42. experimaestro/scheduler/services.py +53 -28
  43. experimaestro/scheduler/state_provider.py +663 -2430
  44. experimaestro/scheduler/state_status.py +1247 -0
  45. experimaestro/scheduler/transient.py +31 -0
  46. experimaestro/scheduler/workspace.py +1 -1
  47. experimaestro/scheduler/workspace_state_provider.py +1273 -0
  48. experimaestro/scriptbuilder.py +4 -4
  49. experimaestro/settings.py +36 -0
  50. experimaestro/tests/conftest.py +33 -5
  51. experimaestro/tests/connectors/bin/executable.py +1 -1
  52. experimaestro/tests/fixtures/pre_experiment/experiment_check_env.py +16 -0
  53. experimaestro/tests/fixtures/pre_experiment/experiment_check_mock.py +14 -0
  54. experimaestro/tests/fixtures/pre_experiment/experiment_simple.py +12 -0
  55. experimaestro/tests/fixtures/pre_experiment/pre_setup_env.py +5 -0
  56. experimaestro/tests/fixtures/pre_experiment/pre_setup_error.py +3 -0
  57. experimaestro/tests/fixtures/pre_experiment/pre_setup_mock.py +8 -0
  58. experimaestro/tests/launchers/bin/test.py +1 -0
  59. experimaestro/tests/launchers/test_slurm.py +9 -9
  60. experimaestro/tests/partial_reschedule.py +46 -0
  61. experimaestro/tests/restart.py +3 -3
  62. experimaestro/tests/restart_main.py +1 -0
  63. experimaestro/tests/scripts/notifyandwait.py +1 -0
  64. experimaestro/tests/task_partial.py +38 -0
  65. experimaestro/tests/task_tokens.py +2 -2
  66. experimaestro/tests/tasks/test_dynamic.py +6 -6
  67. experimaestro/tests/test_dependencies.py +3 -3
  68. experimaestro/tests/test_deprecated.py +15 -15
  69. experimaestro/tests/test_dynamic_locking.py +317 -0
  70. experimaestro/tests/test_environment.py +24 -14
  71. experimaestro/tests/test_experiment.py +171 -36
  72. experimaestro/tests/test_identifier.py +25 -25
  73. experimaestro/tests/test_identifier_stability.py +3 -5
  74. experimaestro/tests/test_multitoken.py +2 -4
  75. experimaestro/tests/{test_subparameters.py → test_partial.py} +25 -25
  76. experimaestro/tests/test_partial_paths.py +81 -138
  77. experimaestro/tests/test_pre_experiment.py +219 -0
  78. experimaestro/tests/test_progress.py +2 -8
  79. experimaestro/tests/test_remote_state.py +560 -99
  80. experimaestro/tests/test_stray_jobs.py +261 -0
  81. experimaestro/tests/test_tasks.py +1 -2
  82. experimaestro/tests/test_token_locking.py +52 -67
  83. experimaestro/tests/test_tokens.py +5 -6
  84. experimaestro/tests/test_transient.py +225 -0
  85. experimaestro/tests/test_workspace_state_provider.py +768 -0
  86. experimaestro/tests/token_reschedule.py +1 -3
  87. experimaestro/tests/utils.py +2 -7
  88. experimaestro/tokens.py +227 -372
  89. experimaestro/tools/diff.py +1 -0
  90. experimaestro/tools/documentation.py +4 -5
  91. experimaestro/tools/jobs.py +1 -2
  92. experimaestro/tui/app.py +438 -1966
  93. experimaestro/tui/app.tcss +162 -0
  94. experimaestro/tui/dialogs.py +172 -0
  95. experimaestro/tui/log_viewer.py +253 -3
  96. experimaestro/tui/messages.py +137 -0
  97. experimaestro/tui/utils.py +54 -0
  98. experimaestro/tui/widgets/__init__.py +23 -0
  99. experimaestro/tui/widgets/experiments.py +468 -0
  100. experimaestro/tui/widgets/global_services.py +238 -0
  101. experimaestro/tui/widgets/jobs.py +972 -0
  102. experimaestro/tui/widgets/log.py +156 -0
  103. experimaestro/tui/widgets/orphans.py +363 -0
  104. experimaestro/tui/widgets/runs.py +185 -0
  105. experimaestro/tui/widgets/services.py +314 -0
  106. experimaestro/tui/widgets/stray_jobs.py +528 -0
  107. experimaestro/utils/__init__.py +1 -1
  108. experimaestro/utils/environment.py +105 -22
  109. experimaestro/utils/fswatcher.py +124 -0
  110. experimaestro/utils/jobs.py +1 -2
  111. experimaestro/utils/jupyter.py +1 -2
  112. experimaestro/utils/logging.py +72 -0
  113. experimaestro/version.py +2 -2
  114. experimaestro/webui/__init__.py +9 -0
  115. experimaestro/webui/app.py +117 -0
  116. experimaestro/{server → webui}/data/index.css +66 -11
  117. experimaestro/webui/data/index.css.map +1 -0
  118. experimaestro/{server → webui}/data/index.js +82763 -87217
  119. experimaestro/webui/data/index.js.map +1 -0
  120. experimaestro/webui/routes/__init__.py +5 -0
  121. experimaestro/webui/routes/auth.py +53 -0
  122. experimaestro/webui/routes/proxy.py +117 -0
  123. experimaestro/webui/server.py +200 -0
  124. experimaestro/webui/state_bridge.py +152 -0
  125. experimaestro/webui/websocket.py +413 -0
  126. {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/METADATA +5 -6
  127. experimaestro-2.0.0b17.dist-info/RECORD +219 -0
  128. experimaestro/cli/progress.py +0 -269
  129. experimaestro/scheduler/state.py +0 -75
  130. experimaestro/scheduler/state_db.py +0 -437
  131. experimaestro/scheduler/state_sync.py +0 -891
  132. experimaestro/server/__init__.py +0 -467
  133. experimaestro/server/data/index.css.map +0 -1
  134. experimaestro/server/data/index.js.map +0 -1
  135. experimaestro/tests/test_cli_jobs.py +0 -615
  136. experimaestro/tests/test_file_progress.py +0 -425
  137. experimaestro/tests/test_file_progress_integration.py +0 -477
  138. experimaestro/tests/test_state_db.py +0 -434
  139. experimaestro-2.0.0b8.dist-info/RECORD +0 -187
  140. /experimaestro/{server → webui}/data/1815e00441357e01619e.ttf +0 -0
  141. /experimaestro/{server → webui}/data/2463b90d9a316e4e5294.woff2 +0 -0
  142. /experimaestro/{server → webui}/data/2582b0e4bcf85eceead0.ttf +0 -0
  143. /experimaestro/{server → webui}/data/89999bdf5d835c012025.woff2 +0 -0
  144. /experimaestro/{server → webui}/data/914997e1bdfc990d0897.ttf +0 -0
  145. /experimaestro/{server → webui}/data/c210719e60948b211a12.woff2 +0 -0
  146. /experimaestro/{server → webui}/data/favicon.ico +0 -0
  147. /experimaestro/{server → webui}/data/index.html +0 -0
  148. /experimaestro/{server → webui}/data/login.html +0 -0
  149. /experimaestro/{server → webui}/data/manifest.json +0 -0
  150. {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/WHEEL +0 -0
  151. {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/entry_points.txt +0 -0
  152. {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/licenses/LICENSE +0 -0
@@ -10,6 +10,10 @@ ExperimentsList {
10
10
  height: auto;
11
11
  }
12
12
 
13
+ ExperimentsList.expanded {
14
+ height: 1fr;
15
+ }
16
+
13
17
  Monitor {
14
18
  height: 100%;
15
19
  }
@@ -92,6 +96,10 @@ JobsTable {
92
96
  min-height: 5;
93
97
  }
94
98
 
99
+ ExperimentsList.expanded #experiments-table {
100
+ height: 1fr;
101
+ }
102
+
95
103
  TabbedContent {
96
104
  height: 100%;
97
105
  }
@@ -115,6 +123,8 @@ JobDetailView {
115
123
 
116
124
  #job-detail-content {
117
125
  padding: 1;
126
+ overflow-y: auto;
127
+ height: 1fr;
118
128
  }
119
129
 
120
130
  #job-detail-content Label {
@@ -351,3 +361,155 @@ HelpScreen {
351
361
  width: 100%;
352
362
  height: auto;
353
363
  }
364
+
365
+ /* Past run banner */
366
+ #past-run-banner {
367
+ background: $warning-darken-2;
368
+ color: $text;
369
+ padding: 0 1;
370
+ text-align: center;
371
+ text-style: bold;
372
+ height: auto;
373
+ }
374
+
375
+ /* Runs List */
376
+ RunsList {
377
+ height: 1fr;
378
+ width: 100%;
379
+ display: none;
380
+ }
381
+
382
+ RunsList.hidden {
383
+ display: none;
384
+ }
385
+
386
+ #runs-container {
387
+ height: 100%;
388
+ width: 100%;
389
+ border: solid $primary;
390
+ padding: 0 1;
391
+ }
392
+
393
+ #runs-title {
394
+ text-style: bold;
395
+ color: $primary;
396
+ height: auto;
397
+ }
398
+
399
+ #runs-table {
400
+ height: 1fr;
401
+ min-height: 3;
402
+ }
403
+
404
+ /* Stray Jobs Tab */
405
+ StrayJobsTab {
406
+ height: 100%;
407
+ width: 100%;
408
+ padding: 1;
409
+ }
410
+
411
+ #stray-title {
412
+ text-align: center;
413
+ text-style: bold;
414
+ margin-bottom: 1;
415
+ color: $warning;
416
+ }
417
+
418
+ #stray-warning {
419
+ background: $warning-darken-2;
420
+ color: $text;
421
+ padding: 0 1;
422
+ margin-bottom: 1;
423
+ text-align: center;
424
+ height: auto;
425
+ }
426
+
427
+ #stray-warning.hidden {
428
+ display: none;
429
+ }
430
+
431
+ .warning-banner {
432
+ height: auto;
433
+ }
434
+
435
+ #stray-stats {
436
+ margin-bottom: 1;
437
+ }
438
+
439
+ #stray-controls {
440
+ height: auto;
441
+ margin-bottom: 1;
442
+ align: left middle;
443
+ }
444
+
445
+ #stray-controls Button {
446
+ margin-right: 1;
447
+ }
448
+
449
+ #stray-controls Switch {
450
+ margin-right: 0;
451
+ }
452
+
453
+ #show-orphans-label {
454
+ margin-right: 2;
455
+ }
456
+
457
+ .controls-bar {
458
+ height: auto;
459
+ }
460
+
461
+ #stray-table {
462
+ height: 1fr;
463
+ }
464
+
465
+ #stray-job-info {
466
+ height: auto;
467
+ margin-top: 1;
468
+ padding: 0 1;
469
+ color: $text-muted;
470
+ }
471
+
472
+ /* Orphan Jobs Tab */
473
+ OrphanJobsTab {
474
+ height: 1fr;
475
+ width: 100%;
476
+ padding: 1;
477
+ }
478
+
479
+ OrphanJobsTab #orphan-warning {
480
+ background: $warning-darken-2;
481
+ color: $text;
482
+ padding: 0 1;
483
+ margin-bottom: 1;
484
+ text-align: center;
485
+ height: auto;
486
+ }
487
+
488
+ OrphanJobsTab #orphan-warning.hidden {
489
+ display: none;
490
+ }
491
+
492
+ OrphanJobsTab #orphan-controls {
493
+ height: auto;
494
+ margin-bottom: 1;
495
+ align: left middle;
496
+ }
497
+
498
+ OrphanJobsTab #orphan-controls Button {
499
+ margin-right: 1;
500
+ min-width: 8;
501
+ height: auto;
502
+ padding: 0 1;
503
+ }
504
+
505
+ OrphanJobsTab #orphan-table {
506
+ height: 1fr;
507
+ min-height: 10;
508
+ }
509
+
510
+ OrphanJobsTab #orphan-job-info {
511
+ height: auto;
512
+ margin-top: 1;
513
+ padding: 0 1;
514
+ color: $text-muted;
515
+ }
@@ -0,0 +1,172 @@
1
+ """Modal dialog screens for the TUI"""
2
+
3
+ from typing import Optional
4
+ from textual.app import ComposeResult
5
+ from textual.containers import Horizontal, Vertical, VerticalScroll
6
+ from textual.widgets import Button, Static
7
+ from textual.screen import ModalScreen
8
+ from textual.binding import Binding
9
+
10
+
11
+ class QuitConfirmScreen(ModalScreen[bool]):
12
+ """Modal screen for quit confirmation"""
13
+
14
+ def __init__(self, has_active_experiment: bool = False):
15
+ super().__init__()
16
+ self.has_active_experiment = has_active_experiment
17
+
18
+ def compose(self) -> ComposeResult:
19
+ with Vertical(id="quit-dialog"):
20
+ yield Static("Quit Experimaestro?", id="quit-title")
21
+
22
+ if self.has_active_experiment:
23
+ yield Static(
24
+ "⚠️ The experiment is still in progress.\nQuitting will prevent new jobs from being launched.",
25
+ id="quit-warning",
26
+ )
27
+ else:
28
+ yield Static("Are you sure you want to quit?", id="quit-message")
29
+
30
+ with Horizontal(id="quit-buttons"):
31
+ yield Button("Quit", variant="error", id="quit-yes")
32
+ yield Button("Cancel", variant="primary", id="quit-no")
33
+
34
+ def on_mount(self) -> None:
35
+ """Focus Cancel button by default"""
36
+ self.query_one("#quit-no", Button).focus()
37
+
38
+ def on_button_pressed(self, event: Button.Pressed) -> None:
39
+ if event.button.id == "quit-yes":
40
+ self.dismiss(True)
41
+ else:
42
+ self.dismiss(False)
43
+
44
+
45
+ class DeleteConfirmScreen(ModalScreen[bool]):
46
+ """Modal screen for delete confirmation"""
47
+
48
+ def __init__(
49
+ self, item_type: str, item_name: str, warning: Optional[str] = None
50
+ ) -> None:
51
+ super().__init__()
52
+ self.item_type = item_type
53
+ self.item_name = item_name
54
+ self.warning = warning
55
+
56
+ def compose(self) -> ComposeResult:
57
+ with Vertical(id="delete-dialog"):
58
+ yield Static(f"Delete {self.item_type}?", id="delete-title")
59
+ yield Static(
60
+ f"This will permanently delete: {self.item_name}", id="delete-message"
61
+ )
62
+
63
+ if self.warning:
64
+ yield Static(f"Warning: {self.warning}", id="delete-warning")
65
+
66
+ with Horizontal(id="delete-buttons"):
67
+ yield Button("Delete", variant="error", id="delete-yes")
68
+ yield Button("Cancel", variant="primary", id="delete-no")
69
+
70
+ def on_mount(self) -> None:
71
+ """Focus cancel button by default"""
72
+ self.query_one("#delete-no", Button).focus()
73
+
74
+ def on_button_pressed(self, event: Button.Pressed) -> None:
75
+ if event.button.id == "delete-yes":
76
+ self.dismiss(True)
77
+ else:
78
+ self.dismiss(False)
79
+
80
+
81
+ class KillConfirmScreen(ModalScreen[bool]):
82
+ """Modal screen for kill confirmation"""
83
+
84
+ def __init__(self, item_type: str, item_name: str) -> None:
85
+ super().__init__()
86
+ self.item_type = item_type
87
+ self.item_name = item_name
88
+
89
+ def compose(self) -> ComposeResult:
90
+ with Vertical(id="kill-dialog"):
91
+ yield Static(f"Kill {self.item_type}?", id="kill-title")
92
+ yield Static(f"This will terminate: {self.item_name}", id="kill-message")
93
+
94
+ with Horizontal(id="kill-buttons"):
95
+ yield Button("Kill", variant="warning", id="kill-yes")
96
+ yield Button("Cancel", variant="primary", id="kill-no")
97
+
98
+ def on_mount(self) -> None:
99
+ """Focus cancel button by default"""
100
+ self.query_one("#kill-no", Button).focus()
101
+
102
+ def on_button_pressed(self, event: Button.Pressed) -> None:
103
+ if event.button.id == "kill-yes":
104
+ self.dismiss(True)
105
+ else:
106
+ self.dismiss(False)
107
+
108
+
109
+ class HelpScreen(ModalScreen[None]):
110
+ """Modal screen showing keyboard shortcuts"""
111
+
112
+ BINDINGS = [
113
+ Binding("escape", "close", "Close"),
114
+ Binding("?", "close", "Close"),
115
+ ]
116
+
117
+ def compose(self) -> ComposeResult:
118
+ help_text = """
119
+ [bold]Keyboard Shortcuts[/bold]
120
+
121
+ [bold cyan]Navigation[/bold cyan]
122
+ q Quit application
123
+ Esc Go back / Close dialog
124
+ r Refresh data
125
+ ? Show this help
126
+ j Switch to Jobs tab
127
+ s Switch to Services tab
128
+
129
+ [bold cyan]Experiments[/bold cyan]
130
+ Enter Select experiment
131
+ d Delete experiment
132
+ k Kill all running jobs
133
+
134
+ [bold cyan]Jobs[/bold cyan]
135
+ l View job logs
136
+ d Delete job
137
+ k Kill running job
138
+ / Open search filter
139
+ c Clear search filter
140
+ S Sort by status
141
+ T Sort by task
142
+ D Sort by date
143
+ f Copy folder path
144
+
145
+ [bold cyan]Services[/bold cyan]
146
+ s Start service
147
+ x Stop service
148
+ u Copy URL
149
+
150
+ [bold cyan]Search Filter[/bold cyan]
151
+ Enter Apply filter
152
+ Esc Close and clear filter
153
+
154
+ [bold cyan]Orphan Jobs[/bold cyan]
155
+ o Show orphan jobs
156
+ T Sort by task
157
+ Z Sort by size
158
+ d Delete selected
159
+ D Delete all
160
+ f Copy folder path
161
+ """
162
+ with Vertical(id="help-dialog"):
163
+ yield Static("Experimaestro Help", id="help-title")
164
+ with VerticalScroll(id="help-scroll"):
165
+ yield Static(help_text, id="help-content")
166
+ yield Button("Close", id="help-close-btn")
167
+
168
+ def on_button_pressed(self, event: Button.Pressed) -> None:
169
+ self.dismiss()
170
+
171
+ def action_close(self) -> None:
172
+ self.dismiss()
@@ -1,6 +1,7 @@
1
1
  """Log viewer screen for viewing job logs efficiently"""
2
2
 
3
3
  from pathlib import Path
4
+ from typing import Callable, Optional
4
5
 
5
6
  from textual.app import ComposeResult
6
7
  from textual.binding import Binding
@@ -8,6 +9,8 @@ from textual.containers import Vertical
8
9
  from textual.screen import Screen
9
10
  from textual.widgets import Footer, Header, RichLog, Static, TabbedContent, TabPane
10
11
 
12
+ from experimaestro.scheduler.interfaces import JobState
13
+
11
14
 
12
15
  # Default chunk size for reading file (64KB)
13
16
  CHUNK_SIZE = 64 * 1024
@@ -129,7 +132,12 @@ class LogWidget(Vertical):
129
132
 
130
133
 
131
134
  class LogViewerScreen(Screen, inherit_bindings=False):
132
- """Screen for viewing job logs efficiently"""
135
+ """Screen for viewing job logs efficiently
136
+
137
+ Supports both local and remote log viewing:
138
+ - Local: reads log files directly
139
+ - Remote: uses adaptive sync to periodically rsync logs from remote
140
+ """
133
141
 
134
142
  CSS = """
135
143
  LogViewerScreen {
@@ -144,6 +152,21 @@ class LogViewerScreen(Screen, inherit_bindings=False):
144
152
  color: $text-muted;
145
153
  }
146
154
 
155
+ .loading-message {
156
+ content-align: center middle;
157
+ height: 100%;
158
+ text-style: italic;
159
+ color: $text-muted;
160
+ }
161
+
162
+ .sync-status {
163
+ background: $boost;
164
+ padding: 0 1;
165
+ height: auto;
166
+ text-style: italic;
167
+ color: $text-muted;
168
+ }
169
+
147
170
  LogWidget {
148
171
  height: 1fr;
149
172
  }
@@ -158,21 +181,53 @@ class LogViewerScreen(Screen, inherit_bindings=False):
158
181
  Binding("f", "toggle_follow", "Follow"),
159
182
  Binding("g", "go_to_top", "Top"),
160
183
  Binding("G", "go_to_bottom", "Bottom"),
184
+ Binding("r", "sync_now", "Sync", show=False),
161
185
  Binding("escape", "close_viewer", "Back", priority=True),
162
186
  Binding("q", "close_viewer", "Quit", priority=True),
163
187
  ]
164
188
 
165
- def __init__(self, log_files: list[str], job_id: str):
189
+ def __init__(
190
+ self,
191
+ log_files: list[str],
192
+ job_id: str,
193
+ sync_func: Optional[Callable[[str], Optional[Path]]] = None,
194
+ remote_path: Optional[str] = None,
195
+ task_id: Optional[str] = None,
196
+ job_state: Optional[JobState] = None,
197
+ ):
198
+ """Initialize the log viewer
199
+
200
+ Args:
201
+ log_files: List of local log file paths
202
+ job_id: Job identifier
203
+ sync_func: Function to sync remote path (for remote monitoring)
204
+ remote_path: Remote job directory path (for remote monitoring)
205
+ task_id: Task ID for log file naming (for remote monitoring)
206
+ job_state: Current job state (for adaptive sync decisions)
207
+ """
166
208
  super().__init__()
167
209
  self.log_files = log_files
168
210
  self.job_id = job_id
169
211
  self.following = True
170
212
  self.log_widgets: list[LogWidget] = []
171
213
 
214
+ # Remote sync support
215
+ self.sync_func = sync_func
216
+ self.remote_path = remote_path
217
+ self.task_id = task_id
218
+ self.job_state = job_state
219
+ self._synchronizer = None
220
+ self._loading = bool(sync_func and not log_files)
221
+
172
222
  def compose(self) -> ComposeResult:
173
223
  yield Header()
174
224
 
175
- if len(self.log_files) == 1:
225
+ if self._loading:
226
+ # Show loading message while waiting for first sync
227
+ yield Static(
228
+ "Syncing logs from remote...", id="loading", classes="loading-message"
229
+ )
230
+ elif len(self.log_files) == 1:
176
231
  # Single file - simple view
177
232
  widget = LogWidget(self.log_files[0], "log-0")
178
233
  self.log_widgets.append(widget)
@@ -187,12 +242,183 @@ class LogViewerScreen(Screen, inherit_bindings=False):
187
242
  self.log_widgets.append(widget)
188
243
  yield widget
189
244
 
245
+ # Sync status for remote monitoring
246
+ if self.sync_func:
247
+ yield Static("", id="sync-status", classes="sync-status")
248
+
190
249
  yield Footer()
191
250
 
192
251
  def on_mount(self) -> None:
193
252
  """Start watching for changes"""
194
253
  self.set_interval(0.5, self._refresh_logs)
195
254
 
255
+ # Start adaptive sync for remote monitoring
256
+ if self.sync_func and self.remote_path:
257
+ self._start_adaptive_sync()
258
+
259
+ def _start_adaptive_sync(self) -> None:
260
+ """Start adaptive sync for remote log files"""
261
+ import logging
262
+
263
+ from experimaestro.scheduler.remote.adaptive_sync import AdaptiveSynchronizer
264
+
265
+ logger = logging.getLogger("xpm.tui.log_viewer")
266
+
267
+ # Only use adaptive sync for running jobs
268
+ is_running = self.job_state and self.job_state.running()
269
+ logger.info(
270
+ f"Starting sync: job_state={self.job_state}, "
271
+ f"is_running={is_running}, remote_path={self.remote_path}"
272
+ )
273
+
274
+ # Update loading message to show we're syncing
275
+ try:
276
+ loading = self.query_one("#loading", Static)
277
+ loading.update("Syncing logs from remote (this may take a moment)...")
278
+ except Exception:
279
+ pass
280
+
281
+ if is_running:
282
+ # Build a name for logging
283
+ sync_name = f"job:{self.task_id}" if self.task_id else f"job:{self.job_id}"
284
+ self._synchronizer = AdaptiveSynchronizer(
285
+ sync_func=self.sync_func,
286
+ remote_path=self.remote_path,
287
+ name=sync_name,
288
+ on_sync_start=lambda: self.app.call_from_thread(self._on_sync_start),
289
+ on_sync_complete=lambda p: self.app.call_from_thread(
290
+ self._on_sync_complete, p
291
+ ),
292
+ on_sync_error=lambda e: self.app.call_from_thread(
293
+ self._on_sync_error, e
294
+ ),
295
+ )
296
+ self._synchronizer.start()
297
+ else:
298
+ # For completed jobs, just sync once
299
+ self._do_single_sync()
300
+
301
+ def _do_single_sync(self) -> None:
302
+ """Do a single sync for completed jobs"""
303
+ import threading
304
+
305
+ def sync_thread():
306
+ try:
307
+ local_path = self.sync_func(self.remote_path)
308
+ if local_path:
309
+ self.app.call_from_thread(self._on_sync_complete, local_path)
310
+ else:
311
+ self.app.call_from_thread(self._on_sync_error, "Sync failed")
312
+ except Exception as e:
313
+ self.app.call_from_thread(self._on_sync_error, str(e))
314
+
315
+ thread = threading.Thread(target=sync_thread, daemon=True)
316
+ thread.start()
317
+
318
+ def _on_sync_start(self) -> None:
319
+ """Handle sync start - update status"""
320
+ try:
321
+ status = self.query_one("#sync-status", Static)
322
+ status.update("⟳ Syncing...")
323
+ except Exception:
324
+ pass
325
+
326
+ def _on_sync_complete(self, local_path) -> None:
327
+ """Handle sync completion - update log widgets"""
328
+ import logging
329
+
330
+ logger = logging.getLogger("xpm.tui.log_viewer")
331
+
332
+ # Ensure local_path is a Path object
333
+ if not isinstance(local_path, Path):
334
+ local_path = Path(local_path)
335
+
336
+ logger.info(f"Sync complete: {local_path}, loading={self._loading}")
337
+
338
+ # Update sync status
339
+ try:
340
+ status = self.query_one("#sync-status", Static)
341
+ if self._synchronizer:
342
+ status.update(f"✓ Next sync: {self._synchronizer.interval:.0f}s")
343
+ else:
344
+ status.update("✓ Synced")
345
+ except Exception as e:
346
+ logger.warning(f"Failed to update sync status: {e}")
347
+
348
+ # If loading, find log files and create widgets
349
+ if self._loading:
350
+ self._loading = False
351
+ task_name = self.task_id.split(".")[-1] if self.task_id else "task"
352
+ stdout_path = local_path / f"{task_name}.out"
353
+ stderr_path = local_path / f"{task_name}.err"
354
+
355
+ logger.info(f"Looking for log files: {stdout_path}, {stderr_path}")
356
+
357
+ log_files = []
358
+ if stdout_path.exists():
359
+ log_files.append(str(stdout_path))
360
+ if stderr_path.exists():
361
+ log_files.append(str(stderr_path))
362
+
363
+ if not log_files:
364
+ try:
365
+ loading = self.query_one("#loading", Static)
366
+ loading.update(
367
+ f"No log files found: {task_name}.out/.err in {local_path}"
368
+ )
369
+ except Exception as e:
370
+ logger.warning(f"Failed to update loading: {e}")
371
+ return
372
+
373
+ logger.info(f"Found log files: {log_files}")
374
+ self.log_files = log_files
375
+ self._rebuild_log_widgets()
376
+ else:
377
+ # Just refresh existing widgets
378
+ for widget in self.log_widgets:
379
+ widget.refresh_content()
380
+
381
+ def _on_sync_error(self, error: str) -> None:
382
+ """Handle sync error"""
383
+ try:
384
+ status = self.query_one("#sync-status", Static)
385
+ status.update(f"Sync error: {error}")
386
+ except Exception:
387
+ pass
388
+
389
+ if self._loading:
390
+ try:
391
+ loading = self.query_one("#loading", Static)
392
+ loading.update(f"Error: {error}")
393
+ except Exception:
394
+ pass
395
+
396
+ def _rebuild_log_widgets(self) -> None:
397
+ """Rebuild log widgets after first sync"""
398
+ # Remove loading message
399
+ try:
400
+ loading = self.query_one("#loading", Static)
401
+ loading.remove()
402
+ except Exception:
403
+ pass
404
+
405
+ # Add log widgets
406
+ self.log_widgets = []
407
+ if len(self.log_files) == 1:
408
+ widget = LogWidget(self.log_files[0], "log-0")
409
+ self.log_widgets.append(widget)
410
+ self.mount(widget, before=self.query_one("#sync-status"))
411
+ else:
412
+ tabbed = TabbedContent()
413
+ self.mount(tabbed, before=self.query_one("#sync-status"))
414
+ for i, log_file in enumerate(self.log_files):
415
+ file_name = Path(log_file).name
416
+ pane = TabPane(file_name, id=f"tab-{i}")
417
+ widget = LogWidget(log_file, f"log-{i}")
418
+ self.log_widgets.append(widget)
419
+ pane.compose_add_child(widget)
420
+ tabbed.add_pane(pane)
421
+
196
422
  def _refresh_logs(self) -> None:
197
423
  """Refresh all log widgets"""
198
424
  if not self.following:
@@ -203,6 +429,21 @@ class LogViewerScreen(Screen, inherit_bindings=False):
203
429
 
204
430
  def action_close_viewer(self) -> None:
205
431
  """Go back to the job detail view"""
432
+ import logging
433
+
434
+ logger = logging.getLogger("xpm.tui.log_viewer")
435
+
436
+ # Warn if closing during first sync
437
+ if self._loading:
438
+ self.notify("Sync in progress, please wait...", severity="warning")
439
+ logger.info("User tried to close during first sync, ignoring")
440
+ return
441
+
442
+ # Stop adaptive sync if running
443
+ if self._synchronizer:
444
+ logger.info("Closing log viewer, stopping sync")
445
+ self._synchronizer.stop()
446
+
206
447
  self.dismiss()
207
448
 
208
449
  def action_toggle_follow(self) -> None:
@@ -226,3 +467,12 @@ class LogViewerScreen(Screen, inherit_bindings=False):
226
467
  for widget in self.log_widgets:
227
468
  widget.following = True
228
469
  widget.scroll_to_end()
470
+
471
+ def action_sync_now(self) -> None:
472
+ """Trigger immediate sync (for remote monitoring)"""
473
+ if self._synchronizer:
474
+ self._synchronizer.sync_now()
475
+ self.notify("Syncing...")
476
+ elif self.sync_func and self.remote_path:
477
+ self._do_single_sync()
478
+ self.notify("Syncing...")