experimaestro 2.0.0b4__py3-none-any.whl → 2.0.0b17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of experimaestro might be problematic. Click here for more details.

Files changed (154) hide show
  1. experimaestro/__init__.py +12 -5
  2. experimaestro/cli/__init__.py +393 -134
  3. experimaestro/cli/filter.py +48 -23
  4. experimaestro/cli/jobs.py +253 -71
  5. experimaestro/cli/refactor.py +1 -2
  6. experimaestro/commandline.py +7 -4
  7. experimaestro/connectors/__init__.py +9 -1
  8. experimaestro/connectors/local.py +43 -3
  9. experimaestro/core/arguments.py +18 -18
  10. experimaestro/core/identifier.py +11 -11
  11. experimaestro/core/objects/config.py +96 -39
  12. experimaestro/core/objects/config_walk.py +3 -3
  13. experimaestro/core/{subparameters.py → partial.py} +16 -16
  14. experimaestro/core/partial_lock.py +394 -0
  15. experimaestro/core/types.py +12 -15
  16. experimaestro/dynamic.py +290 -0
  17. experimaestro/experiments/__init__.py +6 -2
  18. experimaestro/experiments/cli.py +223 -52
  19. experimaestro/experiments/configuration.py +24 -0
  20. experimaestro/generators.py +5 -5
  21. experimaestro/ipc.py +118 -1
  22. experimaestro/launcherfinder/__init__.py +2 -2
  23. experimaestro/launcherfinder/registry.py +6 -7
  24. experimaestro/launcherfinder/specs.py +2 -9
  25. experimaestro/launchers/slurm/__init__.py +2 -2
  26. experimaestro/launchers/slurm/base.py +62 -0
  27. experimaestro/locking.py +957 -1
  28. experimaestro/notifications.py +89 -201
  29. experimaestro/progress.py +63 -366
  30. experimaestro/rpyc.py +0 -2
  31. experimaestro/run.py +29 -2
  32. experimaestro/scheduler/__init__.py +8 -1
  33. experimaestro/scheduler/base.py +650 -53
  34. experimaestro/scheduler/dependencies.py +20 -16
  35. experimaestro/scheduler/experiment.py +764 -169
  36. experimaestro/scheduler/interfaces.py +338 -96
  37. experimaestro/scheduler/jobs.py +58 -20
  38. experimaestro/scheduler/remote/__init__.py +31 -0
  39. experimaestro/scheduler/remote/adaptive_sync.py +265 -0
  40. experimaestro/scheduler/remote/client.py +928 -0
  41. experimaestro/scheduler/remote/protocol.py +282 -0
  42. experimaestro/scheduler/remote/server.py +447 -0
  43. experimaestro/scheduler/remote/sync.py +144 -0
  44. experimaestro/scheduler/services.py +186 -35
  45. experimaestro/scheduler/state_provider.py +811 -2157
  46. experimaestro/scheduler/state_status.py +1247 -0
  47. experimaestro/scheduler/transient.py +31 -0
  48. experimaestro/scheduler/workspace.py +1 -1
  49. experimaestro/scheduler/workspace_state_provider.py +1273 -0
  50. experimaestro/scriptbuilder.py +4 -4
  51. experimaestro/settings.py +36 -0
  52. experimaestro/tests/conftest.py +33 -5
  53. experimaestro/tests/connectors/bin/executable.py +1 -1
  54. experimaestro/tests/fixtures/pre_experiment/experiment_check_env.py +16 -0
  55. experimaestro/tests/fixtures/pre_experiment/experiment_check_mock.py +14 -0
  56. experimaestro/tests/fixtures/pre_experiment/experiment_simple.py +12 -0
  57. experimaestro/tests/fixtures/pre_experiment/pre_setup_env.py +5 -0
  58. experimaestro/tests/fixtures/pre_experiment/pre_setup_error.py +3 -0
  59. experimaestro/tests/fixtures/pre_experiment/pre_setup_mock.py +8 -0
  60. experimaestro/tests/launchers/bin/test.py +1 -0
  61. experimaestro/tests/launchers/test_slurm.py +9 -9
  62. experimaestro/tests/partial_reschedule.py +46 -0
  63. experimaestro/tests/restart.py +3 -3
  64. experimaestro/tests/restart_main.py +1 -0
  65. experimaestro/tests/scripts/notifyandwait.py +1 -0
  66. experimaestro/tests/task_partial.py +38 -0
  67. experimaestro/tests/task_tokens.py +2 -2
  68. experimaestro/tests/tasks/test_dynamic.py +6 -6
  69. experimaestro/tests/test_dependencies.py +3 -3
  70. experimaestro/tests/test_deprecated.py +15 -15
  71. experimaestro/tests/test_dynamic_locking.py +317 -0
  72. experimaestro/tests/test_environment.py +24 -14
  73. experimaestro/tests/test_experiment.py +171 -36
  74. experimaestro/tests/test_identifier.py +25 -25
  75. experimaestro/tests/test_identifier_stability.py +3 -5
  76. experimaestro/tests/test_multitoken.py +2 -4
  77. experimaestro/tests/{test_subparameters.py → test_partial.py} +25 -25
  78. experimaestro/tests/test_partial_paths.py +81 -138
  79. experimaestro/tests/test_pre_experiment.py +219 -0
  80. experimaestro/tests/test_progress.py +2 -8
  81. experimaestro/tests/test_remote_state.py +1132 -0
  82. experimaestro/tests/test_stray_jobs.py +261 -0
  83. experimaestro/tests/test_tasks.py +1 -2
  84. experimaestro/tests/test_token_locking.py +52 -67
  85. experimaestro/tests/test_tokens.py +5 -6
  86. experimaestro/tests/test_transient.py +225 -0
  87. experimaestro/tests/test_workspace_state_provider.py +768 -0
  88. experimaestro/tests/token_reschedule.py +1 -3
  89. experimaestro/tests/utils.py +2 -7
  90. experimaestro/tokens.py +227 -372
  91. experimaestro/tools/diff.py +1 -0
  92. experimaestro/tools/documentation.py +4 -5
  93. experimaestro/tools/jobs.py +1 -2
  94. experimaestro/tui/app.py +459 -1895
  95. experimaestro/tui/app.tcss +162 -0
  96. experimaestro/tui/dialogs.py +172 -0
  97. experimaestro/tui/log_viewer.py +253 -3
  98. experimaestro/tui/messages.py +137 -0
  99. experimaestro/tui/utils.py +54 -0
  100. experimaestro/tui/widgets/__init__.py +23 -0
  101. experimaestro/tui/widgets/experiments.py +468 -0
  102. experimaestro/tui/widgets/global_services.py +238 -0
  103. experimaestro/tui/widgets/jobs.py +972 -0
  104. experimaestro/tui/widgets/log.py +156 -0
  105. experimaestro/tui/widgets/orphans.py +363 -0
  106. experimaestro/tui/widgets/runs.py +185 -0
  107. experimaestro/tui/widgets/services.py +314 -0
  108. experimaestro/tui/widgets/stray_jobs.py +528 -0
  109. experimaestro/utils/__init__.py +1 -1
  110. experimaestro/utils/environment.py +105 -22
  111. experimaestro/utils/fswatcher.py +124 -0
  112. experimaestro/utils/jobs.py +1 -2
  113. experimaestro/utils/jupyter.py +1 -2
  114. experimaestro/utils/logging.py +72 -0
  115. experimaestro/version.py +2 -2
  116. experimaestro/webui/__init__.py +9 -0
  117. experimaestro/webui/app.py +117 -0
  118. experimaestro/{server → webui}/data/index.css +66 -11
  119. experimaestro/webui/data/index.css.map +1 -0
  120. experimaestro/{server → webui}/data/index.js +82763 -87217
  121. experimaestro/webui/data/index.js.map +1 -0
  122. experimaestro/webui/routes/__init__.py +5 -0
  123. experimaestro/webui/routes/auth.py +53 -0
  124. experimaestro/webui/routes/proxy.py +117 -0
  125. experimaestro/webui/server.py +200 -0
  126. experimaestro/webui/state_bridge.py +152 -0
  127. experimaestro/webui/websocket.py +413 -0
  128. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/METADATA +8 -9
  129. experimaestro-2.0.0b17.dist-info/RECORD +219 -0
  130. experimaestro/cli/progress.py +0 -269
  131. experimaestro/scheduler/state.py +0 -75
  132. experimaestro/scheduler/state_db.py +0 -388
  133. experimaestro/scheduler/state_sync.py +0 -834
  134. experimaestro/server/__init__.py +0 -467
  135. experimaestro/server/data/index.css.map +0 -1
  136. experimaestro/server/data/index.js.map +0 -1
  137. experimaestro/tests/test_cli_jobs.py +0 -615
  138. experimaestro/tests/test_file_progress.py +0 -425
  139. experimaestro/tests/test_file_progress_integration.py +0 -477
  140. experimaestro/tests/test_state_db.py +0 -434
  141. experimaestro-2.0.0b4.dist-info/RECORD +0 -181
  142. /experimaestro/{server → webui}/data/1815e00441357e01619e.ttf +0 -0
  143. /experimaestro/{server → webui}/data/2463b90d9a316e4e5294.woff2 +0 -0
  144. /experimaestro/{server → webui}/data/2582b0e4bcf85eceead0.ttf +0 -0
  145. /experimaestro/{server → webui}/data/89999bdf5d835c012025.woff2 +0 -0
  146. /experimaestro/{server → webui}/data/914997e1bdfc990d0897.ttf +0 -0
  147. /experimaestro/{server → webui}/data/c210719e60948b211a12.woff2 +0 -0
  148. /experimaestro/{server → webui}/data/favicon.ico +0 -0
  149. /experimaestro/{server → webui}/data/index.html +0 -0
  150. /experimaestro/{server → webui}/data/login.html +0 -0
  151. /experimaestro/{server → webui}/data/manifest.json +0 -0
  152. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/WHEEL +0 -0
  153. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/entry_points.txt +0 -0
  154. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,238 @@
1
+ """Global services widget - shows all running services across experiments"""
2
+
3
+ import logging
4
+ from pathlib import Path
5
+ from typing import Optional
6
+
7
+ from textual.app import ComposeResult
8
+ from textual.containers import Vertical
9
+ from textual.widgets import DataTable, Static
10
+
11
+ from experimaestro.scheduler.state_provider import StateProvider
12
+
13
+ logger = logging.getLogger("xpm.tui.global_services")
14
+
15
+
16
+ class GlobalServiceSyncs(Vertical):
17
+ """Widget displaying all running services across all experiments
18
+
19
+ Shows services from all experiments with their state and URL.
20
+ For remote monitoring, also tracks file synchronization status.
21
+ """
22
+
23
+ def __init__(self, state_provider: StateProvider) -> None:
24
+ super().__init__()
25
+ self.state_provider = state_provider
26
+ # service_key -> {synchronizer, ...} for remote file syncs
27
+ self._syncs: dict[str, dict] = {}
28
+
29
+ def compose(self) -> ComposeResult:
30
+ yield Static("Running Services", classes="section-title")
31
+ yield DataTable(id="global-services-table", cursor_type="row")
32
+
33
+ def on_mount(self) -> None:
34
+ """Set up the table"""
35
+ table = self.query_one("#global-services-table", DataTable)
36
+ table.add_columns("Experiment", "Service", "State", "Sync", "URL")
37
+ table.cursor_type = "row"
38
+ self.log.info(
39
+ f"GlobalServiceSyncs mounted, state_provider={type(self.state_provider).__name__}"
40
+ )
41
+ # Initial refresh
42
+ self.refresh_services()
43
+
44
+ def refresh_services(self) -> None:
45
+ """Refresh the services list from state provider"""
46
+
47
+ try:
48
+ table = self.query_one("#global-services-table", DataTable)
49
+ except Exception:
50
+ return
51
+
52
+ # Guard: ensure columns have been added
53
+ if len(table.columns) == 0:
54
+ return
55
+
56
+ table.clear()
57
+
58
+ try:
59
+ # Get all services from state provider
60
+ all_services = self.state_provider.get_services()
61
+ self.log.info(
62
+ f"GlobalServiceSyncs.refresh_services: got {len(all_services)} services"
63
+ )
64
+
65
+ for service in all_services:
66
+ service_id = service.id
67
+ state = service.state if hasattr(service, "state") else None
68
+ state_name = state.name if state else "UNKNOWN"
69
+ exp_id = getattr(service, "_experiment_id", None) or "-"
70
+ self.log.info(
71
+ f" Service: {service_id}, state={state_name}, exp={exp_id}"
72
+ )
73
+
74
+ # Get description
75
+ description = ""
76
+ if hasattr(service, "description"):
77
+ try:
78
+ description = service.description()
79
+ except Exception:
80
+ description = service_id
81
+
82
+ # Get URL
83
+ url = getattr(service, "url", None) or "-"
84
+
85
+ # Get sync status for remote monitoring
86
+ sync_status = "-"
87
+ service_key = f"{exp_id}:{service_id}"
88
+ if service_key in self._syncs:
89
+ sync_info = self._syncs[service_key]
90
+ synchronizer = sync_info.get("synchronizer")
91
+ if synchronizer:
92
+ if synchronizer.syncing:
93
+ sync_status = "⟳ Syncing"
94
+ else:
95
+ sync_status = f"✓ {synchronizer.interval:.0f}s"
96
+
97
+ # State icon
98
+ state_icons = {
99
+ "RUNNING": "▶",
100
+ "STOPPED": "⏹",
101
+ "STARTING": "⏳",
102
+ "STOPPING": "⏳",
103
+ }
104
+ state_icon = state_icons.get(state_name, "?")
105
+
106
+ table.add_row(
107
+ exp_id,
108
+ description or service_id,
109
+ f"{state_icon} {state_name}",
110
+ sync_status,
111
+ url,
112
+ key=service_key,
113
+ )
114
+
115
+ except Exception as e:
116
+ logger.warning(f"Failed to refresh global services: {e}")
117
+
118
+ # Update tab title
119
+ self._update_tab_title()
120
+
121
+ def add_service_sync(
122
+ self,
123
+ experiment_id: str,
124
+ service_id: str,
125
+ description: str,
126
+ remote_path: str,
127
+ url: Optional[str] = None,
128
+ ) -> None:
129
+ """Add a new service sync (called from ServicesList for remote monitoring)"""
130
+ from experimaestro.scheduler.remote.adaptive_sync import AdaptiveSynchronizer
131
+
132
+ service_key = f"{experiment_id}:{service_id}"
133
+
134
+ # Don't restart if already syncing
135
+ if service_key in self._syncs:
136
+ return
137
+
138
+ if not self.state_provider.is_remote:
139
+ return
140
+
141
+ sync_name = f"service:{description}"
142
+
143
+ synchronizer = AdaptiveSynchronizer(
144
+ sync_func=self.state_provider.sync_path,
145
+ remote_path=remote_path,
146
+ name=sync_name,
147
+ on_sync_start=lambda sk=service_key: self.app.call_from_thread(
148
+ self._on_sync_start, sk
149
+ ),
150
+ on_sync_complete=lambda p, sk=service_key: self.app.call_from_thread(
151
+ self._on_sync_complete, sk, p
152
+ ),
153
+ )
154
+
155
+ self._syncs[service_key] = {
156
+ "synchronizer": synchronizer,
157
+ "experiment_id": experiment_id,
158
+ "service_id": service_id,
159
+ "description": description,
160
+ "remote_path": remote_path,
161
+ "url": url or "-",
162
+ }
163
+
164
+ synchronizer.start()
165
+ logger.info(f"Started global sync for {service_key}: {remote_path}")
166
+
167
+ # Refresh to show sync status
168
+ self.refresh_services()
169
+
170
+ def stop_service_sync(self, experiment_id: str, service_id: str) -> None:
171
+ """Stop a service sync (called when service is STOPPED)"""
172
+ service_key = f"{experiment_id}:{service_id}"
173
+
174
+ if service_key in self._syncs:
175
+ self._syncs[service_key]["synchronizer"].stop()
176
+ del self._syncs[service_key]
177
+ logger.info(f"Stopped global sync for {service_key}")
178
+ self.refresh_services()
179
+
180
+ def _update_tab_title(self) -> None:
181
+ """Update the Services tab title with count"""
182
+ try:
183
+ self.app.update_services_tab_title()
184
+ except Exception:
185
+ pass
186
+
187
+ def has_sync(self, experiment_id: str, service_id: str) -> bool:
188
+ """Check if a sync exists for this service"""
189
+ return f"{experiment_id}:{service_id}" in self._syncs
190
+
191
+ def get_sync_status(self, experiment_id: str, service_id: str) -> Optional[str]:
192
+ """Get sync status string for display"""
193
+ service_key = f"{experiment_id}:{service_id}"
194
+ if service_key not in self._syncs:
195
+ return None
196
+
197
+ sync_info = self._syncs[service_key]
198
+ synchronizer = sync_info["synchronizer"]
199
+
200
+ if synchronizer.syncing:
201
+ return "⟳"
202
+ else:
203
+ return f"✓ {synchronizer.interval:.0f}s"
204
+
205
+ def _on_sync_start(self, service_key: str) -> None:
206
+ """Handle sync start"""
207
+ self.refresh_services()
208
+
209
+ def _on_sync_complete(self, service_key: str, local_path: Path) -> None:
210
+ """Handle sync complete"""
211
+ self.refresh_services()
212
+
213
+ def on_unmount(self) -> None:
214
+ """Stop all syncs when app closes"""
215
+ for service_key, info in list(self._syncs.items()):
216
+ if "synchronizer" in info:
217
+ info["synchronizer"].stop()
218
+ self._syncs.clear()
219
+
220
+ @property
221
+ def sync_count(self) -> int:
222
+ """Number of active syncs (for backward compatibility)"""
223
+ return len(self._syncs)
224
+
225
+ @property
226
+ def running_service_count(self) -> int:
227
+ """Number of running services"""
228
+ from experimaestro.scheduler.services import ServiceState
229
+
230
+ try:
231
+ all_services = self.state_provider.get_services()
232
+ return sum(
233
+ 1
234
+ for s in all_services
235
+ if hasattr(s, "state") and s.state == ServiceState.RUNNING
236
+ )
237
+ except Exception:
238
+ return 0