experimaestro 2.0.0b4__py3-none-any.whl → 2.0.0b17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of experimaestro might be problematic. Click here for more details.
- experimaestro/__init__.py +12 -5
- experimaestro/cli/__init__.py +393 -134
- experimaestro/cli/filter.py +48 -23
- experimaestro/cli/jobs.py +253 -71
- experimaestro/cli/refactor.py +1 -2
- experimaestro/commandline.py +7 -4
- experimaestro/connectors/__init__.py +9 -1
- experimaestro/connectors/local.py +43 -3
- experimaestro/core/arguments.py +18 -18
- experimaestro/core/identifier.py +11 -11
- experimaestro/core/objects/config.py +96 -39
- experimaestro/core/objects/config_walk.py +3 -3
- experimaestro/core/{subparameters.py → partial.py} +16 -16
- experimaestro/core/partial_lock.py +394 -0
- experimaestro/core/types.py +12 -15
- experimaestro/dynamic.py +290 -0
- experimaestro/experiments/__init__.py +6 -2
- experimaestro/experiments/cli.py +223 -52
- experimaestro/experiments/configuration.py +24 -0
- experimaestro/generators.py +5 -5
- experimaestro/ipc.py +118 -1
- experimaestro/launcherfinder/__init__.py +2 -2
- experimaestro/launcherfinder/registry.py +6 -7
- experimaestro/launcherfinder/specs.py +2 -9
- experimaestro/launchers/slurm/__init__.py +2 -2
- experimaestro/launchers/slurm/base.py +62 -0
- experimaestro/locking.py +957 -1
- experimaestro/notifications.py +89 -201
- experimaestro/progress.py +63 -366
- experimaestro/rpyc.py +0 -2
- experimaestro/run.py +29 -2
- experimaestro/scheduler/__init__.py +8 -1
- experimaestro/scheduler/base.py +650 -53
- experimaestro/scheduler/dependencies.py +20 -16
- experimaestro/scheduler/experiment.py +764 -169
- experimaestro/scheduler/interfaces.py +338 -96
- experimaestro/scheduler/jobs.py +58 -20
- experimaestro/scheduler/remote/__init__.py +31 -0
- experimaestro/scheduler/remote/adaptive_sync.py +265 -0
- experimaestro/scheduler/remote/client.py +928 -0
- experimaestro/scheduler/remote/protocol.py +282 -0
- experimaestro/scheduler/remote/server.py +447 -0
- experimaestro/scheduler/remote/sync.py +144 -0
- experimaestro/scheduler/services.py +186 -35
- experimaestro/scheduler/state_provider.py +811 -2157
- experimaestro/scheduler/state_status.py +1247 -0
- experimaestro/scheduler/transient.py +31 -0
- experimaestro/scheduler/workspace.py +1 -1
- experimaestro/scheduler/workspace_state_provider.py +1273 -0
- experimaestro/scriptbuilder.py +4 -4
- experimaestro/settings.py +36 -0
- experimaestro/tests/conftest.py +33 -5
- experimaestro/tests/connectors/bin/executable.py +1 -1
- experimaestro/tests/fixtures/pre_experiment/experiment_check_env.py +16 -0
- experimaestro/tests/fixtures/pre_experiment/experiment_check_mock.py +14 -0
- experimaestro/tests/fixtures/pre_experiment/experiment_simple.py +12 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_env.py +5 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_error.py +3 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_mock.py +8 -0
- experimaestro/tests/launchers/bin/test.py +1 -0
- experimaestro/tests/launchers/test_slurm.py +9 -9
- experimaestro/tests/partial_reschedule.py +46 -0
- experimaestro/tests/restart.py +3 -3
- experimaestro/tests/restart_main.py +1 -0
- experimaestro/tests/scripts/notifyandwait.py +1 -0
- experimaestro/tests/task_partial.py +38 -0
- experimaestro/tests/task_tokens.py +2 -2
- experimaestro/tests/tasks/test_dynamic.py +6 -6
- experimaestro/tests/test_dependencies.py +3 -3
- experimaestro/tests/test_deprecated.py +15 -15
- experimaestro/tests/test_dynamic_locking.py +317 -0
- experimaestro/tests/test_environment.py +24 -14
- experimaestro/tests/test_experiment.py +171 -36
- experimaestro/tests/test_identifier.py +25 -25
- experimaestro/tests/test_identifier_stability.py +3 -5
- experimaestro/tests/test_multitoken.py +2 -4
- experimaestro/tests/{test_subparameters.py → test_partial.py} +25 -25
- experimaestro/tests/test_partial_paths.py +81 -138
- experimaestro/tests/test_pre_experiment.py +219 -0
- experimaestro/tests/test_progress.py +2 -8
- experimaestro/tests/test_remote_state.py +1132 -0
- experimaestro/tests/test_stray_jobs.py +261 -0
- experimaestro/tests/test_tasks.py +1 -2
- experimaestro/tests/test_token_locking.py +52 -67
- experimaestro/tests/test_tokens.py +5 -6
- experimaestro/tests/test_transient.py +225 -0
- experimaestro/tests/test_workspace_state_provider.py +768 -0
- experimaestro/tests/token_reschedule.py +1 -3
- experimaestro/tests/utils.py +2 -7
- experimaestro/tokens.py +227 -372
- experimaestro/tools/diff.py +1 -0
- experimaestro/tools/documentation.py +4 -5
- experimaestro/tools/jobs.py +1 -2
- experimaestro/tui/app.py +459 -1895
- experimaestro/tui/app.tcss +162 -0
- experimaestro/tui/dialogs.py +172 -0
- experimaestro/tui/log_viewer.py +253 -3
- experimaestro/tui/messages.py +137 -0
- experimaestro/tui/utils.py +54 -0
- experimaestro/tui/widgets/__init__.py +23 -0
- experimaestro/tui/widgets/experiments.py +468 -0
- experimaestro/tui/widgets/global_services.py +238 -0
- experimaestro/tui/widgets/jobs.py +972 -0
- experimaestro/tui/widgets/log.py +156 -0
- experimaestro/tui/widgets/orphans.py +363 -0
- experimaestro/tui/widgets/runs.py +185 -0
- experimaestro/tui/widgets/services.py +314 -0
- experimaestro/tui/widgets/stray_jobs.py +528 -0
- experimaestro/utils/__init__.py +1 -1
- experimaestro/utils/environment.py +105 -22
- experimaestro/utils/fswatcher.py +124 -0
- experimaestro/utils/jobs.py +1 -2
- experimaestro/utils/jupyter.py +1 -2
- experimaestro/utils/logging.py +72 -0
- experimaestro/version.py +2 -2
- experimaestro/webui/__init__.py +9 -0
- experimaestro/webui/app.py +117 -0
- experimaestro/{server → webui}/data/index.css +66 -11
- experimaestro/webui/data/index.css.map +1 -0
- experimaestro/{server → webui}/data/index.js +82763 -87217
- experimaestro/webui/data/index.js.map +1 -0
- experimaestro/webui/routes/__init__.py +5 -0
- experimaestro/webui/routes/auth.py +53 -0
- experimaestro/webui/routes/proxy.py +117 -0
- experimaestro/webui/server.py +200 -0
- experimaestro/webui/state_bridge.py +152 -0
- experimaestro/webui/websocket.py +413 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/METADATA +8 -9
- experimaestro-2.0.0b17.dist-info/RECORD +219 -0
- experimaestro/cli/progress.py +0 -269
- experimaestro/scheduler/state.py +0 -75
- experimaestro/scheduler/state_db.py +0 -388
- experimaestro/scheduler/state_sync.py +0 -834
- experimaestro/server/__init__.py +0 -467
- experimaestro/server/data/index.css.map +0 -1
- experimaestro/server/data/index.js.map +0 -1
- experimaestro/tests/test_cli_jobs.py +0 -615
- experimaestro/tests/test_file_progress.py +0 -425
- experimaestro/tests/test_file_progress_integration.py +0 -477
- experimaestro/tests/test_state_db.py +0 -434
- experimaestro-2.0.0b4.dist-info/RECORD +0 -181
- /experimaestro/{server → webui}/data/1815e00441357e01619e.ttf +0 -0
- /experimaestro/{server → webui}/data/2463b90d9a316e4e5294.woff2 +0 -0
- /experimaestro/{server → webui}/data/2582b0e4bcf85eceead0.ttf +0 -0
- /experimaestro/{server → webui}/data/89999bdf5d835c012025.woff2 +0 -0
- /experimaestro/{server → webui}/data/914997e1bdfc990d0897.ttf +0 -0
- /experimaestro/{server → webui}/data/c210719e60948b211a12.woff2 +0 -0
- /experimaestro/{server → webui}/data/favicon.ico +0 -0
- /experimaestro/{server → webui}/data/index.html +0 -0
- /experimaestro/{server → webui}/data/login.html +0 -0
- /experimaestro/{server → webui}/data/manifest.json +0 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/WHEEL +0 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/entry_points.txt +0 -0
- {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""Global services widget - shows all running services across experiments"""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from textual.app import ComposeResult
|
|
8
|
+
from textual.containers import Vertical
|
|
9
|
+
from textual.widgets import DataTable, Static
|
|
10
|
+
|
|
11
|
+
from experimaestro.scheduler.state_provider import StateProvider
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("xpm.tui.global_services")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class GlobalServiceSyncs(Vertical):
|
|
17
|
+
"""Widget displaying all running services across all experiments
|
|
18
|
+
|
|
19
|
+
Shows services from all experiments with their state and URL.
|
|
20
|
+
For remote monitoring, also tracks file synchronization status.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, state_provider: StateProvider) -> None:
|
|
24
|
+
super().__init__()
|
|
25
|
+
self.state_provider = state_provider
|
|
26
|
+
# service_key -> {synchronizer, ...} for remote file syncs
|
|
27
|
+
self._syncs: dict[str, dict] = {}
|
|
28
|
+
|
|
29
|
+
def compose(self) -> ComposeResult:
|
|
30
|
+
yield Static("Running Services", classes="section-title")
|
|
31
|
+
yield DataTable(id="global-services-table", cursor_type="row")
|
|
32
|
+
|
|
33
|
+
def on_mount(self) -> None:
|
|
34
|
+
"""Set up the table"""
|
|
35
|
+
table = self.query_one("#global-services-table", DataTable)
|
|
36
|
+
table.add_columns("Experiment", "Service", "State", "Sync", "URL")
|
|
37
|
+
table.cursor_type = "row"
|
|
38
|
+
self.log.info(
|
|
39
|
+
f"GlobalServiceSyncs mounted, state_provider={type(self.state_provider).__name__}"
|
|
40
|
+
)
|
|
41
|
+
# Initial refresh
|
|
42
|
+
self.refresh_services()
|
|
43
|
+
|
|
44
|
+
def refresh_services(self) -> None:
|
|
45
|
+
"""Refresh the services list from state provider"""
|
|
46
|
+
|
|
47
|
+
try:
|
|
48
|
+
table = self.query_one("#global-services-table", DataTable)
|
|
49
|
+
except Exception:
|
|
50
|
+
return
|
|
51
|
+
|
|
52
|
+
# Guard: ensure columns have been added
|
|
53
|
+
if len(table.columns) == 0:
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
table.clear()
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
# Get all services from state provider
|
|
60
|
+
all_services = self.state_provider.get_services()
|
|
61
|
+
self.log.info(
|
|
62
|
+
f"GlobalServiceSyncs.refresh_services: got {len(all_services)} services"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
for service in all_services:
|
|
66
|
+
service_id = service.id
|
|
67
|
+
state = service.state if hasattr(service, "state") else None
|
|
68
|
+
state_name = state.name if state else "UNKNOWN"
|
|
69
|
+
exp_id = getattr(service, "_experiment_id", None) or "-"
|
|
70
|
+
self.log.info(
|
|
71
|
+
f" Service: {service_id}, state={state_name}, exp={exp_id}"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# Get description
|
|
75
|
+
description = ""
|
|
76
|
+
if hasattr(service, "description"):
|
|
77
|
+
try:
|
|
78
|
+
description = service.description()
|
|
79
|
+
except Exception:
|
|
80
|
+
description = service_id
|
|
81
|
+
|
|
82
|
+
# Get URL
|
|
83
|
+
url = getattr(service, "url", None) or "-"
|
|
84
|
+
|
|
85
|
+
# Get sync status for remote monitoring
|
|
86
|
+
sync_status = "-"
|
|
87
|
+
service_key = f"{exp_id}:{service_id}"
|
|
88
|
+
if service_key in self._syncs:
|
|
89
|
+
sync_info = self._syncs[service_key]
|
|
90
|
+
synchronizer = sync_info.get("synchronizer")
|
|
91
|
+
if synchronizer:
|
|
92
|
+
if synchronizer.syncing:
|
|
93
|
+
sync_status = "⟳ Syncing"
|
|
94
|
+
else:
|
|
95
|
+
sync_status = f"✓ {synchronizer.interval:.0f}s"
|
|
96
|
+
|
|
97
|
+
# State icon
|
|
98
|
+
state_icons = {
|
|
99
|
+
"RUNNING": "▶",
|
|
100
|
+
"STOPPED": "⏹",
|
|
101
|
+
"STARTING": "⏳",
|
|
102
|
+
"STOPPING": "⏳",
|
|
103
|
+
}
|
|
104
|
+
state_icon = state_icons.get(state_name, "?")
|
|
105
|
+
|
|
106
|
+
table.add_row(
|
|
107
|
+
exp_id,
|
|
108
|
+
description or service_id,
|
|
109
|
+
f"{state_icon} {state_name}",
|
|
110
|
+
sync_status,
|
|
111
|
+
url,
|
|
112
|
+
key=service_key,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
except Exception as e:
|
|
116
|
+
logger.warning(f"Failed to refresh global services: {e}")
|
|
117
|
+
|
|
118
|
+
# Update tab title
|
|
119
|
+
self._update_tab_title()
|
|
120
|
+
|
|
121
|
+
def add_service_sync(
|
|
122
|
+
self,
|
|
123
|
+
experiment_id: str,
|
|
124
|
+
service_id: str,
|
|
125
|
+
description: str,
|
|
126
|
+
remote_path: str,
|
|
127
|
+
url: Optional[str] = None,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""Add a new service sync (called from ServicesList for remote monitoring)"""
|
|
130
|
+
from experimaestro.scheduler.remote.adaptive_sync import AdaptiveSynchronizer
|
|
131
|
+
|
|
132
|
+
service_key = f"{experiment_id}:{service_id}"
|
|
133
|
+
|
|
134
|
+
# Don't restart if already syncing
|
|
135
|
+
if service_key in self._syncs:
|
|
136
|
+
return
|
|
137
|
+
|
|
138
|
+
if not self.state_provider.is_remote:
|
|
139
|
+
return
|
|
140
|
+
|
|
141
|
+
sync_name = f"service:{description}"
|
|
142
|
+
|
|
143
|
+
synchronizer = AdaptiveSynchronizer(
|
|
144
|
+
sync_func=self.state_provider.sync_path,
|
|
145
|
+
remote_path=remote_path,
|
|
146
|
+
name=sync_name,
|
|
147
|
+
on_sync_start=lambda sk=service_key: self.app.call_from_thread(
|
|
148
|
+
self._on_sync_start, sk
|
|
149
|
+
),
|
|
150
|
+
on_sync_complete=lambda p, sk=service_key: self.app.call_from_thread(
|
|
151
|
+
self._on_sync_complete, sk, p
|
|
152
|
+
),
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
self._syncs[service_key] = {
|
|
156
|
+
"synchronizer": synchronizer,
|
|
157
|
+
"experiment_id": experiment_id,
|
|
158
|
+
"service_id": service_id,
|
|
159
|
+
"description": description,
|
|
160
|
+
"remote_path": remote_path,
|
|
161
|
+
"url": url or "-",
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
synchronizer.start()
|
|
165
|
+
logger.info(f"Started global sync for {service_key}: {remote_path}")
|
|
166
|
+
|
|
167
|
+
# Refresh to show sync status
|
|
168
|
+
self.refresh_services()
|
|
169
|
+
|
|
170
|
+
def stop_service_sync(self, experiment_id: str, service_id: str) -> None:
|
|
171
|
+
"""Stop a service sync (called when service is STOPPED)"""
|
|
172
|
+
service_key = f"{experiment_id}:{service_id}"
|
|
173
|
+
|
|
174
|
+
if service_key in self._syncs:
|
|
175
|
+
self._syncs[service_key]["synchronizer"].stop()
|
|
176
|
+
del self._syncs[service_key]
|
|
177
|
+
logger.info(f"Stopped global sync for {service_key}")
|
|
178
|
+
self.refresh_services()
|
|
179
|
+
|
|
180
|
+
def _update_tab_title(self) -> None:
|
|
181
|
+
"""Update the Services tab title with count"""
|
|
182
|
+
try:
|
|
183
|
+
self.app.update_services_tab_title()
|
|
184
|
+
except Exception:
|
|
185
|
+
pass
|
|
186
|
+
|
|
187
|
+
def has_sync(self, experiment_id: str, service_id: str) -> bool:
|
|
188
|
+
"""Check if a sync exists for this service"""
|
|
189
|
+
return f"{experiment_id}:{service_id}" in self._syncs
|
|
190
|
+
|
|
191
|
+
def get_sync_status(self, experiment_id: str, service_id: str) -> Optional[str]:
|
|
192
|
+
"""Get sync status string for display"""
|
|
193
|
+
service_key = f"{experiment_id}:{service_id}"
|
|
194
|
+
if service_key not in self._syncs:
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
sync_info = self._syncs[service_key]
|
|
198
|
+
synchronizer = sync_info["synchronizer"]
|
|
199
|
+
|
|
200
|
+
if synchronizer.syncing:
|
|
201
|
+
return "⟳"
|
|
202
|
+
else:
|
|
203
|
+
return f"✓ {synchronizer.interval:.0f}s"
|
|
204
|
+
|
|
205
|
+
def _on_sync_start(self, service_key: str) -> None:
|
|
206
|
+
"""Handle sync start"""
|
|
207
|
+
self.refresh_services()
|
|
208
|
+
|
|
209
|
+
def _on_sync_complete(self, service_key: str, local_path: Path) -> None:
|
|
210
|
+
"""Handle sync complete"""
|
|
211
|
+
self.refresh_services()
|
|
212
|
+
|
|
213
|
+
def on_unmount(self) -> None:
|
|
214
|
+
"""Stop all syncs when app closes"""
|
|
215
|
+
for service_key, info in list(self._syncs.items()):
|
|
216
|
+
if "synchronizer" in info:
|
|
217
|
+
info["synchronizer"].stop()
|
|
218
|
+
self._syncs.clear()
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def sync_count(self) -> int:
|
|
222
|
+
"""Number of active syncs (for backward compatibility)"""
|
|
223
|
+
return len(self._syncs)
|
|
224
|
+
|
|
225
|
+
@property
|
|
226
|
+
def running_service_count(self) -> int:
|
|
227
|
+
"""Number of running services"""
|
|
228
|
+
from experimaestro.scheduler.services import ServiceState
|
|
229
|
+
|
|
230
|
+
try:
|
|
231
|
+
all_services = self.state_provider.get_services()
|
|
232
|
+
return sum(
|
|
233
|
+
1
|
|
234
|
+
for s in all_services
|
|
235
|
+
if hasattr(s, "state") and s.state == ServiceState.RUNNING
|
|
236
|
+
)
|
|
237
|
+
except Exception:
|
|
238
|
+
return 0
|