experimaestro 2.0.0a8__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of experimaestro might be problematic. Click here for more details.

Files changed (116) hide show
  1. experimaestro/__init__.py +10 -11
  2. experimaestro/annotations.py +167 -206
  3. experimaestro/cli/__init__.py +130 -5
  4. experimaestro/cli/filter.py +42 -74
  5. experimaestro/cli/jobs.py +157 -106
  6. experimaestro/cli/refactor.py +249 -0
  7. experimaestro/click.py +0 -1
  8. experimaestro/commandline.py +19 -3
  9. experimaestro/connectors/__init__.py +20 -1
  10. experimaestro/connectors/local.py +12 -0
  11. experimaestro/core/arguments.py +182 -46
  12. experimaestro/core/identifier.py +107 -6
  13. experimaestro/core/objects/__init__.py +6 -0
  14. experimaestro/core/objects/config.py +542 -25
  15. experimaestro/core/objects/config_walk.py +20 -0
  16. experimaestro/core/serialization.py +91 -34
  17. experimaestro/core/subparameters.py +164 -0
  18. experimaestro/core/types.py +175 -38
  19. experimaestro/exceptions.py +26 -0
  20. experimaestro/experiments/cli.py +107 -25
  21. experimaestro/generators.py +50 -9
  22. experimaestro/huggingface.py +3 -1
  23. experimaestro/launcherfinder/parser.py +29 -0
  24. experimaestro/launchers/__init__.py +26 -1
  25. experimaestro/launchers/direct.py +12 -0
  26. experimaestro/launchers/slurm/base.py +154 -2
  27. experimaestro/mkdocs/metaloader.py +0 -1
  28. experimaestro/mypy.py +452 -7
  29. experimaestro/notifications.py +63 -13
  30. experimaestro/progress.py +0 -2
  31. experimaestro/rpyc.py +0 -1
  32. experimaestro/run.py +19 -6
  33. experimaestro/scheduler/base.py +489 -125
  34. experimaestro/scheduler/dependencies.py +43 -28
  35. experimaestro/scheduler/dynamic_outputs.py +259 -130
  36. experimaestro/scheduler/experiment.py +225 -30
  37. experimaestro/scheduler/interfaces.py +474 -0
  38. experimaestro/scheduler/jobs.py +216 -206
  39. experimaestro/scheduler/services.py +186 -12
  40. experimaestro/scheduler/state_db.py +388 -0
  41. experimaestro/scheduler/state_provider.py +2345 -0
  42. experimaestro/scheduler/state_sync.py +834 -0
  43. experimaestro/scheduler/workspace.py +52 -10
  44. experimaestro/scriptbuilder.py +7 -0
  45. experimaestro/server/__init__.py +147 -57
  46. experimaestro/server/data/index.css +0 -125
  47. experimaestro/server/data/index.css.map +1 -1
  48. experimaestro/server/data/index.js +194 -58
  49. experimaestro/server/data/index.js.map +1 -1
  50. experimaestro/settings.py +44 -5
  51. experimaestro/sphinx/__init__.py +3 -3
  52. experimaestro/taskglobals.py +20 -0
  53. experimaestro/tests/conftest.py +80 -0
  54. experimaestro/tests/core/test_generics.py +2 -2
  55. experimaestro/tests/identifier_stability.json +45 -0
  56. experimaestro/tests/launchers/bin/sacct +6 -2
  57. experimaestro/tests/launchers/bin/sbatch +4 -2
  58. experimaestro/tests/launchers/test_slurm.py +80 -0
  59. experimaestro/tests/tasks/test_dynamic.py +231 -0
  60. experimaestro/tests/test_cli_jobs.py +615 -0
  61. experimaestro/tests/test_deprecated.py +630 -0
  62. experimaestro/tests/test_environment.py +200 -0
  63. experimaestro/tests/test_file_progress_integration.py +1 -1
  64. experimaestro/tests/test_forward.py +3 -3
  65. experimaestro/tests/test_identifier.py +372 -41
  66. experimaestro/tests/test_identifier_stability.py +458 -0
  67. experimaestro/tests/test_instance.py +3 -3
  68. experimaestro/tests/test_multitoken.py +442 -0
  69. experimaestro/tests/test_mypy.py +433 -0
  70. experimaestro/tests/test_objects.py +312 -5
  71. experimaestro/tests/test_outputs.py +2 -2
  72. experimaestro/tests/test_param.py +8 -12
  73. experimaestro/tests/test_partial_paths.py +231 -0
  74. experimaestro/tests/test_progress.py +0 -48
  75. experimaestro/tests/test_resumable_task.py +480 -0
  76. experimaestro/tests/test_serializers.py +141 -1
  77. experimaestro/tests/test_state_db.py +434 -0
  78. experimaestro/tests/test_subparameters.py +160 -0
  79. experimaestro/tests/test_tags.py +136 -0
  80. experimaestro/tests/test_tasks.py +107 -121
  81. experimaestro/tests/test_token_locking.py +252 -0
  82. experimaestro/tests/test_tokens.py +17 -13
  83. experimaestro/tests/test_types.py +123 -1
  84. experimaestro/tests/test_workspace_triggers.py +158 -0
  85. experimaestro/tests/token_reschedule.py +4 -2
  86. experimaestro/tests/utils.py +2 -2
  87. experimaestro/tokens.py +154 -57
  88. experimaestro/tools/diff.py +1 -1
  89. experimaestro/tui/__init__.py +8 -0
  90. experimaestro/tui/app.py +2303 -0
  91. experimaestro/tui/app.tcss +353 -0
  92. experimaestro/tui/log_viewer.py +228 -0
  93. experimaestro/utils/__init__.py +23 -0
  94. experimaestro/utils/environment.py +148 -0
  95. experimaestro/utils/git.py +129 -0
  96. experimaestro/utils/resources.py +1 -1
  97. experimaestro/version.py +34 -0
  98. {experimaestro-2.0.0a8.dist-info → experimaestro-2.0.0b4.dist-info}/METADATA +68 -38
  99. experimaestro-2.0.0b4.dist-info/RECORD +181 -0
  100. {experimaestro-2.0.0a8.dist-info → experimaestro-2.0.0b4.dist-info}/WHEEL +1 -1
  101. experimaestro-2.0.0b4.dist-info/entry_points.txt +16 -0
  102. experimaestro/compat.py +0 -6
  103. experimaestro/core/objects.pyi +0 -221
  104. experimaestro/server/data/0c35d18bf06992036b69.woff2 +0 -0
  105. experimaestro/server/data/219aa9140e099e6c72ed.woff2 +0 -0
  106. experimaestro/server/data/3a4004a46a653d4b2166.woff +0 -0
  107. experimaestro/server/data/3baa5b8f3469222b822d.woff +0 -0
  108. experimaestro/server/data/4d73cb90e394b34b7670.woff +0 -0
  109. experimaestro/server/data/4ef4218c522f1eb6b5b1.woff2 +0 -0
  110. experimaestro/server/data/5d681e2edae8c60630db.woff +0 -0
  111. experimaestro/server/data/6f420cf17cc0d7676fad.woff2 +0 -0
  112. experimaestro/server/data/c380809fd3677d7d6903.woff2 +0 -0
  113. experimaestro/server/data/f882956fd323fd322f31.woff +0 -0
  114. experimaestro-2.0.0a8.dist-info/RECORD +0 -166
  115. experimaestro-2.0.0a8.dist-info/entry_points.txt +0 -17
  116. {experimaestro-2.0.0a8.dist-info → experimaestro-2.0.0b4.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,353 @@
1
+ /* Main Experimaestro UI styles */
2
+
3
+ #main-container {
4
+ width: 100%;
5
+ height: 100%;
6
+ }
7
+
8
+ ExperimentsList {
9
+ width: 100%;
10
+ height: auto;
11
+ }
12
+
13
+ Monitor {
14
+ height: 100%;
15
+ }
16
+
17
+ #logs-tab {
18
+ height: 1fr;
19
+ width: 100%;
20
+ }
21
+
22
+ CaptureLog {
23
+ width: 1fr;
24
+ height: 1fr;
25
+ min-height: 10;
26
+ border: solid cyan;
27
+ }
28
+
29
+ #experiments-table-container {
30
+ width: 100%;
31
+ height: auto;
32
+ }
33
+
34
+ #collapsed-header {
35
+ background: $boost;
36
+ padding: 0;
37
+ text-style: bold;
38
+ border: solid green;
39
+ height: auto;
40
+ width: 100%;
41
+ }
42
+
43
+ #collapsed-header:hover {
44
+ background: $primary;
45
+ }
46
+
47
+ #collapsed-experiment-info {
48
+ width: 100%;
49
+ height: auto;
50
+ }
51
+
52
+ #experiment-tabs {
53
+ width: 100%;
54
+ height: 1fr;
55
+ }
56
+
57
+ ServicesList {
58
+ width: 100%;
59
+ height: 1fr;
60
+ }
61
+
62
+ #services-table {
63
+ width: 100%;
64
+ height: 1fr;
65
+ min-height: 10;
66
+ }
67
+
68
+ JobsTable {
69
+ width: 100%;
70
+ height: 1fr;
71
+ }
72
+
73
+ #jobs-table {
74
+ width: 100%;
75
+ height: 1fr;
76
+ min-height: 10;
77
+ }
78
+
79
+ .hidden {
80
+ display: none;
81
+ }
82
+
83
+ .section-title {
84
+ background: $boost;
85
+ padding: 1;
86
+ text-style: bold;
87
+ height: auto;
88
+ }
89
+
90
+ #experiments-table {
91
+ height: auto;
92
+ min-height: 5;
93
+ }
94
+
95
+ TabbedContent {
96
+ height: 100%;
97
+ }
98
+
99
+ #logs {
100
+ height: 100%;
101
+ width: 100%;
102
+ }
103
+
104
+ #job-detail-container {
105
+ width: 100%;
106
+ height: 1fr;
107
+ border: solid magenta;
108
+ padding: 1;
109
+ }
110
+
111
+ JobDetailView {
112
+ width: 100%;
113
+ height: 100%;
114
+ }
115
+
116
+ #job-detail-content {
117
+ padding: 1;
118
+ }
119
+
120
+ #job-detail-content Label {
121
+ margin-bottom: 1;
122
+ }
123
+
124
+ .subsection-title {
125
+ background: $surface;
126
+ padding: 0 1;
127
+ text-style: italic;
128
+ margin-top: 1;
129
+ }
130
+
131
+ #job-logs-hint {
132
+ margin-top: 1;
133
+ }
134
+
135
+ /* SearchBar styles */
136
+ SearchBar {
137
+ height: auto;
138
+ }
139
+
140
+ #search-container {
141
+ width: 100%;
142
+ height: auto;
143
+ padding: 1;
144
+ background: $boost;
145
+ }
146
+
147
+ #search-input {
148
+ width: 100%;
149
+ }
150
+
151
+ #search-input.valid {
152
+ border: solid $success;
153
+ }
154
+
155
+ #search-input.error {
156
+ border: solid $error;
157
+ }
158
+
159
+ #search-hints {
160
+ color: $text-muted;
161
+ text-style: italic;
162
+ margin-top: 1;
163
+ }
164
+
165
+ #search-error {
166
+ color: $error;
167
+ margin-top: 1;
168
+ }
169
+
170
+ #active-filter {
171
+ background: $success-darken-2;
172
+ color: $text;
173
+ padding: 0 1;
174
+ }
175
+
176
+ /* Modal dialogs */
177
+ QuitConfirmScreen {
178
+ align: center middle;
179
+ }
180
+
181
+ #quit-dialog {
182
+ width: 60;
183
+ height: auto;
184
+ border: thick $primary;
185
+ background: $surface;
186
+ padding: 1 2;
187
+ }
188
+
189
+ #quit-title {
190
+ text-align: center;
191
+ text-style: bold;
192
+ margin-bottom: 1;
193
+ }
194
+
195
+ #quit-message {
196
+ margin-bottom: 1;
197
+ }
198
+
199
+ #quit-warning {
200
+ color: $warning;
201
+ text-style: bold;
202
+ margin-bottom: 1;
203
+ }
204
+
205
+ #quit-buttons {
206
+ width: 100%;
207
+ height: auto;
208
+ align: center middle;
209
+ }
210
+
211
+ #quit-buttons Button {
212
+ margin: 0 1;
213
+ }
214
+
215
+ DeleteConfirmScreen {
216
+ align: center middle;
217
+ }
218
+
219
+ #delete-dialog {
220
+ width: 70;
221
+ height: auto;
222
+ border: thick $error;
223
+ background: $surface;
224
+ padding: 1 2;
225
+ }
226
+
227
+ #delete-title {
228
+ text-align: center;
229
+ text-style: bold;
230
+ color: $error;
231
+ margin-bottom: 1;
232
+ }
233
+
234
+ #delete-message {
235
+ margin-bottom: 1;
236
+ }
237
+
238
+ #delete-warning {
239
+ color: $warning;
240
+ text-style: bold;
241
+ margin-bottom: 1;
242
+ }
243
+
244
+ #delete-buttons {
245
+ width: 100%;
246
+ height: auto;
247
+ align: center middle;
248
+ }
249
+
250
+ #delete-buttons Button {
251
+ margin: 0 1;
252
+ }
253
+
254
+ KillConfirmScreen {
255
+ align: center middle;
256
+ }
257
+
258
+ #kill-dialog {
259
+ width: 70;
260
+ height: auto;
261
+ border: thick $warning;
262
+ background: $surface;
263
+ padding: 1 2;
264
+ }
265
+
266
+ #kill-title {
267
+ text-align: center;
268
+ text-style: bold;
269
+ color: $warning;
270
+ margin-bottom: 1;
271
+ }
272
+
273
+ #kill-message {
274
+ margin-bottom: 1;
275
+ }
276
+
277
+ #kill-buttons {
278
+ width: 100%;
279
+ height: auto;
280
+ align: center middle;
281
+ }
282
+
283
+ #kill-buttons Button {
284
+ margin: 0 1;
285
+ }
286
+
287
+ /* Orphan Jobs Screen */
288
+ OrphanJobsScreen {
289
+ background: $surface;
290
+ }
291
+
292
+ #orphan-container {
293
+ height: 100%;
294
+ width: 100%;
295
+ padding: 1;
296
+ }
297
+
298
+ #orphan-title {
299
+ text-align: center;
300
+ text-style: bold;
301
+ margin-bottom: 1;
302
+ color: $warning;
303
+ }
304
+
305
+ #orphan-stats {
306
+ margin-bottom: 1;
307
+ }
308
+
309
+ #orphan-table {
310
+ height: 1fr;
311
+ }
312
+
313
+ #orphan-job-info {
314
+ height: auto;
315
+ margin-top: 1;
316
+ padding: 0 1;
317
+ color: $text-muted;
318
+ }
319
+
320
+ /* Help Screen */
321
+ HelpScreen {
322
+ align: center middle;
323
+ }
324
+
325
+ #help-dialog {
326
+ width: 65;
327
+ height: 80%;
328
+ max-height: 80%;
329
+ border: thick $primary;
330
+ background: $surface;
331
+ padding: 1 2;
332
+ }
333
+
334
+ #help-title {
335
+ text-align: center;
336
+ text-style: bold;
337
+ margin-bottom: 1;
338
+ height: auto;
339
+ }
340
+
341
+ #help-scroll {
342
+ height: 1fr;
343
+ margin-bottom: 1;
344
+ }
345
+
346
+ #help-content {
347
+ height: auto;
348
+ }
349
+
350
+ #help-close-btn {
351
+ width: 100%;
352
+ height: auto;
353
+ }
@@ -0,0 +1,228 @@
1
+ """Log viewer screen for viewing job logs efficiently"""
2
+
3
+ from pathlib import Path
4
+
5
+ from textual.app import ComposeResult
6
+ from textual.binding import Binding
7
+ from textual.containers import Vertical
8
+ from textual.screen import Screen
9
+ from textual.widgets import Footer, Header, RichLog, Static, TabbedContent, TabPane
10
+
11
+
12
+ # Default chunk size for reading file (64KB)
13
+ CHUNK_SIZE = 64 * 1024
14
+ # How many bytes to read from end of file initially
15
+ INITIAL_TAIL_SIZE = 256 * 1024 # 256KB
16
+
17
+
18
+ class LogFile:
19
+ """Efficient log file reader that tracks position and watches for changes"""
20
+
21
+ def __init__(self, path: str):
22
+ self.path = Path(path)
23
+ self.position = 0
24
+ self.size = 0
25
+ self._update_size()
26
+
27
+ def _update_size(self) -> None:
28
+ """Update the known file size"""
29
+ try:
30
+ self.size = self.path.stat().st_size
31
+ except OSError:
32
+ self.size = 0
33
+
34
+ def read_tail(self, max_bytes: int = INITIAL_TAIL_SIZE) -> str:
35
+ """Read the last N bytes of the file"""
36
+ if not self.path.exists():
37
+ return ""
38
+
39
+ self._update_size()
40
+ if self.size == 0:
41
+ return ""
42
+
43
+ try:
44
+ with open(self.path, "r", errors="replace") as f:
45
+ # Start from max_bytes before end, or beginning
46
+ start_pos = max(0, self.size - max_bytes)
47
+ f.seek(start_pos)
48
+
49
+ # If we're not at the start, skip to the next newline
50
+ if start_pos > 0:
51
+ f.readline() # Skip partial line
52
+
53
+ content = f.read()
54
+ self.position = f.tell()
55
+ return content
56
+ except Exception:
57
+ return ""
58
+
59
+ def read_new_content(self) -> str:
60
+ """Read any new content since last read"""
61
+ if not self.path.exists():
62
+ return ""
63
+
64
+ self._update_size()
65
+
66
+ # File was truncated or rotated
67
+ if self.size < self.position:
68
+ self.position = 0
69
+
70
+ if self.position >= self.size:
71
+ return ""
72
+
73
+ try:
74
+ with open(self.path, "r", errors="replace") as f:
75
+ f.seek(self.position)
76
+ content = f.read()
77
+ self.position = f.tell()
78
+ return content
79
+ except Exception:
80
+ return ""
81
+
82
+ def has_new_content(self) -> bool:
83
+ """Check if there's new content without reading it"""
84
+ self._update_size()
85
+ return self.size > self.position
86
+
87
+
88
+ class LogWidget(Vertical):
89
+ """Widget for displaying a single log file with efficient loading"""
90
+
91
+ def __init__(self, file_path: str, widget_id: str):
92
+ super().__init__(id=widget_id)
93
+ self.file_path = file_path
94
+ self.log_file = LogFile(file_path)
95
+ self.following = True
96
+
97
+ def compose(self) -> ComposeResult:
98
+ yield Static(f"📄 {self.file_path}", classes="log-file-path")
99
+ yield RichLog(id=f"{self.id}-content", wrap=True, highlight=True, markup=False)
100
+
101
+ def on_mount(self) -> None:
102
+ """Load initial content from tail of file"""
103
+ content = self.log_file.read_tail()
104
+ if content:
105
+ log_widget = self.query_one(f"#{self.id}-content", RichLog)
106
+ for line in content.splitlines():
107
+ log_widget.write(line)
108
+
109
+ def refresh_content(self) -> None:
110
+ """Check for and append new content"""
111
+ if not self.following:
112
+ return
113
+
114
+ new_content = self.log_file.read_new_content()
115
+ if new_content:
116
+ log_widget = self.query_one(f"#{self.id}-content", RichLog)
117
+ for line in new_content.splitlines():
118
+ log_widget.write(line)
119
+
120
+ def scroll_to_end(self) -> None:
121
+ """Scroll to end of log"""
122
+ log_widget = self.query_one(f"#{self.id}-content", RichLog)
123
+ log_widget.scroll_end()
124
+
125
+ def scroll_to_top(self) -> None:
126
+ """Scroll to top of log"""
127
+ log_widget = self.query_one(f"#{self.id}-content", RichLog)
128
+ log_widget.scroll_home()
129
+
130
+
131
+ class LogViewerScreen(Screen, inherit_bindings=False):
132
+ """Screen for viewing job logs efficiently"""
133
+
134
+ CSS = """
135
+ LogViewerScreen {
136
+ background: $surface;
137
+ }
138
+
139
+ .log-file-path {
140
+ background: $boost;
141
+ padding: 0 1;
142
+ height: auto;
143
+ text-style: bold;
144
+ color: $text-muted;
145
+ }
146
+
147
+ LogWidget {
148
+ height: 1fr;
149
+ }
150
+
151
+ LogWidget RichLog {
152
+ height: 1fr;
153
+ border: solid $primary;
154
+ }
155
+ """
156
+
157
+ BINDINGS = [
158
+ Binding("f", "toggle_follow", "Follow"),
159
+ Binding("g", "go_to_top", "Top"),
160
+ Binding("G", "go_to_bottom", "Bottom"),
161
+ Binding("escape", "close_viewer", "Back", priority=True),
162
+ Binding("q", "close_viewer", "Quit", priority=True),
163
+ ]
164
+
165
+ def __init__(self, log_files: list[str], job_id: str):
166
+ super().__init__()
167
+ self.log_files = log_files
168
+ self.job_id = job_id
169
+ self.following = True
170
+ self.log_widgets: list[LogWidget] = []
171
+
172
+ def compose(self) -> ComposeResult:
173
+ yield Header()
174
+
175
+ if len(self.log_files) == 1:
176
+ # Single file - simple view
177
+ widget = LogWidget(self.log_files[0], "log-0")
178
+ self.log_widgets.append(widget)
179
+ yield widget
180
+ else:
181
+ # Multiple files - tabbed view
182
+ with TabbedContent():
183
+ for i, log_file in enumerate(self.log_files):
184
+ file_name = Path(log_file).name
185
+ with TabPane(file_name, id=f"tab-{i}"):
186
+ widget = LogWidget(log_file, f"log-{i}")
187
+ self.log_widgets.append(widget)
188
+ yield widget
189
+
190
+ yield Footer()
191
+
192
+ def on_mount(self) -> None:
193
+ """Start watching for changes"""
194
+ self.set_interval(0.5, self._refresh_logs)
195
+
196
+ def _refresh_logs(self) -> None:
197
+ """Refresh all log widgets"""
198
+ if not self.following:
199
+ return
200
+
201
+ for widget in self.log_widgets:
202
+ widget.refresh_content()
203
+
204
+ def action_close_viewer(self) -> None:
205
+ """Go back to the job detail view"""
206
+ self.dismiss()
207
+
208
+ def action_toggle_follow(self) -> None:
209
+ """Toggle following mode"""
210
+ self.following = not self.following
211
+ for widget in self.log_widgets:
212
+ widget.following = self.following
213
+ status = "ON" if self.following else "OFF"
214
+ self.notify(f"Follow mode: {status}")
215
+
216
+ def action_go_to_top(self) -> None:
217
+ """Scroll to top"""
218
+ self.following = False
219
+ for widget in self.log_widgets:
220
+ widget.following = False
221
+ widget.scroll_to_top()
222
+
223
+ def action_go_to_bottom(self) -> None:
224
+ """Scroll to bottom and resume following"""
225
+ self.following = True
226
+ for widget in self.log_widgets:
227
+ widget.following = True
228
+ widget.scroll_to_end()
@@ -4,10 +4,33 @@ import threading
4
4
  from typing import Union
5
5
  import logging
6
6
  import shutil
7
+ import inspect
7
8
 
8
9
  logger = logging.getLogger("xpm")
9
10
 
10
11
 
12
+ def get_caller_location(skip_frames: int = 1) -> str:
13
+ """Get the source location of the caller
14
+
15
+ Args:
16
+ skip_frames: Number of frames to skip (1 = immediate caller)
17
+
18
+ Returns:
19
+ Source location string in format "path:line"
20
+ """
21
+ frame = inspect.currentframe()
22
+ try:
23
+ # Skip this function's frame plus requested frames
24
+ for _ in range(skip_frames + 1):
25
+ if frame is not None:
26
+ frame = frame.f_back
27
+ if frame is not None:
28
+ return f"{Path(frame.f_code.co_filename).absolute()}:{frame.f_lineno}"
29
+ finally:
30
+ del frame # Avoid reference cycles
31
+ return "<unknown>"
32
+
33
+
11
34
  def aspath(path: Union[str, Path]):
12
35
  if isinstance(path, Path):
13
36
  return path