experimaestro 2.0.0b8__py3-none-any.whl → 2.0.0b17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of experimaestro might be problematic. Click here for more details.
- experimaestro/__init__.py +12 -5
- experimaestro/cli/__init__.py +239 -126
- experimaestro/cli/filter.py +48 -23
- experimaestro/cli/jobs.py +253 -71
- experimaestro/cli/refactor.py +1 -2
- experimaestro/commandline.py +7 -4
- experimaestro/connectors/__init__.py +9 -1
- experimaestro/connectors/local.py +43 -3
- experimaestro/core/arguments.py +18 -18
- experimaestro/core/identifier.py +11 -11
- experimaestro/core/objects/config.py +96 -39
- experimaestro/core/objects/config_walk.py +3 -3
- experimaestro/core/{subparameters.py → partial.py} +16 -16
- experimaestro/core/partial_lock.py +394 -0
- experimaestro/core/types.py +12 -15
- experimaestro/dynamic.py +290 -0
- experimaestro/experiments/__init__.py +6 -2
- experimaestro/experiments/cli.py +217 -50
- experimaestro/experiments/configuration.py +24 -0
- experimaestro/generators.py +5 -5
- experimaestro/ipc.py +118 -1
- experimaestro/launcherfinder/__init__.py +2 -2
- experimaestro/launcherfinder/registry.py +6 -7
- experimaestro/launcherfinder/specs.py +2 -9
- experimaestro/launchers/slurm/__init__.py +2 -2
- experimaestro/launchers/slurm/base.py +62 -0
- experimaestro/locking.py +957 -1
- experimaestro/notifications.py +89 -201
- experimaestro/progress.py +63 -366
- experimaestro/rpyc.py +0 -2
- experimaestro/run.py +29 -2
- experimaestro/scheduler/__init__.py +8 -1
- experimaestro/scheduler/base.py +629 -53
- experimaestro/scheduler/dependencies.py +20 -16
- experimaestro/scheduler/experiment.py +732 -167
- experimaestro/scheduler/interfaces.py +316 -101
- experimaestro/scheduler/jobs.py +58 -20
- experimaestro/scheduler/remote/adaptive_sync.py +265 -0
- experimaestro/scheduler/remote/client.py +171 -117
- experimaestro/scheduler/remote/protocol.py +8 -193
- experimaestro/scheduler/remote/server.py +95 -71
- experimaestro/scheduler/services.py +53 -28
- experimaestro/scheduler/state_provider.py +663 -2430
- experimaestro/scheduler/state_status.py +1247 -0
- experimaestro/scheduler/transient.py +31 -0
- experimaestro/scheduler/workspace.py +1 -1
- experimaestro/scheduler/workspace_state_provider.py +1273 -0
- experimaestro/scriptbuilder.py +4 -4
- experimaestro/settings.py +36 -0
- experimaestro/tests/conftest.py +33 -5
- experimaestro/tests/connectors/bin/executable.py +1 -1
- experimaestro/tests/fixtures/pre_experiment/experiment_check_env.py +16 -0
- experimaestro/tests/fixtures/pre_experiment/experiment_check_mock.py +14 -0
- experimaestro/tests/fixtures/pre_experiment/experiment_simple.py +12 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_env.py +5 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_error.py +3 -0
- experimaestro/tests/fixtures/pre_experiment/pre_setup_mock.py +8 -0
- experimaestro/tests/launchers/bin/test.py +1 -0
- experimaestro/tests/launchers/test_slurm.py +9 -9
- experimaestro/tests/partial_reschedule.py +46 -0
- experimaestro/tests/restart.py +3 -3
- experimaestro/tests/restart_main.py +1 -0
- experimaestro/tests/scripts/notifyandwait.py +1 -0
- experimaestro/tests/task_partial.py +38 -0
- experimaestro/tests/task_tokens.py +2 -2
- experimaestro/tests/tasks/test_dynamic.py +6 -6
- experimaestro/tests/test_dependencies.py +3 -3
- experimaestro/tests/test_deprecated.py +15 -15
- experimaestro/tests/test_dynamic_locking.py +317 -0
- experimaestro/tests/test_environment.py +24 -14
- experimaestro/tests/test_experiment.py +171 -36
- experimaestro/tests/test_identifier.py +25 -25
- experimaestro/tests/test_identifier_stability.py +3 -5
- experimaestro/tests/test_multitoken.py +2 -4
- experimaestro/tests/{test_subparameters.py → test_partial.py} +25 -25
- experimaestro/tests/test_partial_paths.py +81 -138
- experimaestro/tests/test_pre_experiment.py +219 -0
- experimaestro/tests/test_progress.py +2 -8
- experimaestro/tests/test_remote_state.py +560 -99
- experimaestro/tests/test_stray_jobs.py +261 -0
- experimaestro/tests/test_tasks.py +1 -2
- experimaestro/tests/test_token_locking.py +52 -67
- experimaestro/tests/test_tokens.py +5 -6
- experimaestro/tests/test_transient.py +225 -0
- experimaestro/tests/test_workspace_state_provider.py +768 -0
- experimaestro/tests/token_reschedule.py +1 -3
- experimaestro/tests/utils.py +2 -7
- experimaestro/tokens.py +227 -372
- experimaestro/tools/diff.py +1 -0
- experimaestro/tools/documentation.py +4 -5
- experimaestro/tools/jobs.py +1 -2
- experimaestro/tui/app.py +438 -1966
- experimaestro/tui/app.tcss +162 -0
- experimaestro/tui/dialogs.py +172 -0
- experimaestro/tui/log_viewer.py +253 -3
- experimaestro/tui/messages.py +137 -0
- experimaestro/tui/utils.py +54 -0
- experimaestro/tui/widgets/__init__.py +23 -0
- experimaestro/tui/widgets/experiments.py +468 -0
- experimaestro/tui/widgets/global_services.py +238 -0
- experimaestro/tui/widgets/jobs.py +972 -0
- experimaestro/tui/widgets/log.py +156 -0
- experimaestro/tui/widgets/orphans.py +363 -0
- experimaestro/tui/widgets/runs.py +185 -0
- experimaestro/tui/widgets/services.py +314 -0
- experimaestro/tui/widgets/stray_jobs.py +528 -0
- experimaestro/utils/__init__.py +1 -1
- experimaestro/utils/environment.py +105 -22
- experimaestro/utils/fswatcher.py +124 -0
- experimaestro/utils/jobs.py +1 -2
- experimaestro/utils/jupyter.py +1 -2
- experimaestro/utils/logging.py +72 -0
- experimaestro/version.py +2 -2
- experimaestro/webui/__init__.py +9 -0
- experimaestro/webui/app.py +117 -0
- experimaestro/{server → webui}/data/index.css +66 -11
- experimaestro/webui/data/index.css.map +1 -0
- experimaestro/{server → webui}/data/index.js +82763 -87217
- experimaestro/webui/data/index.js.map +1 -0
- experimaestro/webui/routes/__init__.py +5 -0
- experimaestro/webui/routes/auth.py +53 -0
- experimaestro/webui/routes/proxy.py +117 -0
- experimaestro/webui/server.py +200 -0
- experimaestro/webui/state_bridge.py +152 -0
- experimaestro/webui/websocket.py +413 -0
- {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/METADATA +5 -6
- experimaestro-2.0.0b17.dist-info/RECORD +219 -0
- experimaestro/cli/progress.py +0 -269
- experimaestro/scheduler/state.py +0 -75
- experimaestro/scheduler/state_db.py +0 -437
- experimaestro/scheduler/state_sync.py +0 -891
- experimaestro/server/__init__.py +0 -467
- experimaestro/server/data/index.css.map +0 -1
- experimaestro/server/data/index.js.map +0 -1
- experimaestro/tests/test_cli_jobs.py +0 -615
- experimaestro/tests/test_file_progress.py +0 -425
- experimaestro/tests/test_file_progress_integration.py +0 -477
- experimaestro/tests/test_state_db.py +0 -434
- experimaestro-2.0.0b8.dist-info/RECORD +0 -187
- /experimaestro/{server → webui}/data/1815e00441357e01619e.ttf +0 -0
- /experimaestro/{server → webui}/data/2463b90d9a316e4e5294.woff2 +0 -0
- /experimaestro/{server → webui}/data/2582b0e4bcf85eceead0.ttf +0 -0
- /experimaestro/{server → webui}/data/89999bdf5d835c012025.woff2 +0 -0
- /experimaestro/{server → webui}/data/914997e1bdfc990d0897.ttf +0 -0
- /experimaestro/{server → webui}/data/c210719e60948b211a12.woff2 +0 -0
- /experimaestro/{server → webui}/data/favicon.ico +0 -0
- /experimaestro/{server → webui}/data/index.html +0 -0
- /experimaestro/{server → webui}/data/login.html +0 -0
- /experimaestro/{server → webui}/data/manifest.json +0 -0
- {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/WHEEL +0 -0
- {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/entry_points.txt +0 -0
- {experimaestro-2.0.0b8.dist-info → experimaestro-2.0.0b17.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Runs list widget for the TUI"""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Optional
|
|
5
|
+
from textual.app import ComposeResult
|
|
6
|
+
from textual.containers import Vertical
|
|
7
|
+
from textual.widgets import DataTable, Static
|
|
8
|
+
from textual.widget import Widget
|
|
9
|
+
from textual.reactive import reactive
|
|
10
|
+
from textual.binding import Binding
|
|
11
|
+
|
|
12
|
+
from experimaestro.scheduler.state_provider import StateProvider
|
|
13
|
+
from experimaestro.tui.utils import format_duration
|
|
14
|
+
from experimaestro.tui.messages import RunSelected
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class RunsList(Widget):
|
|
18
|
+
"""Widget displaying runs for selected experiment"""
|
|
19
|
+
|
|
20
|
+
BINDINGS = [
|
|
21
|
+
Binding("escape", "go_back", "Back", show=False),
|
|
22
|
+
Binding("enter", "select_run", "Select", show=False),
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
visible: reactive[bool] = reactive(False)
|
|
26
|
+
|
|
27
|
+
def __init__(self, state_provider: StateProvider) -> None:
|
|
28
|
+
super().__init__()
|
|
29
|
+
self.state_provider = state_provider
|
|
30
|
+
self.experiment_id: Optional[str] = None
|
|
31
|
+
self.current_run_id: Optional[str] = None
|
|
32
|
+
self.runs = []
|
|
33
|
+
|
|
34
|
+
def compose(self) -> ComposeResult:
|
|
35
|
+
with Vertical(id="runs-container"):
|
|
36
|
+
yield Static("", id="runs-title")
|
|
37
|
+
yield DataTable(id="runs-table", cursor_type="row")
|
|
38
|
+
|
|
39
|
+
def on_mount(self) -> None:
|
|
40
|
+
"""Initialize the runs table"""
|
|
41
|
+
table = self.query_one("#runs-table", DataTable)
|
|
42
|
+
table.add_column("Run ID", key="run_id")
|
|
43
|
+
table.add_column("Status", key="status", width=12)
|
|
44
|
+
table.add_column("Host", key="host")
|
|
45
|
+
table.add_column("Jobs", key="jobs", width=10)
|
|
46
|
+
table.add_column("Started", key="started")
|
|
47
|
+
table.add_column("Duration", key="duration", width=12)
|
|
48
|
+
|
|
49
|
+
def watch_visible(self, visible: bool) -> None:
|
|
50
|
+
"""Show/hide the runs list"""
|
|
51
|
+
if visible:
|
|
52
|
+
self.display = True
|
|
53
|
+
self.remove_class("hidden")
|
|
54
|
+
else:
|
|
55
|
+
self.display = False
|
|
56
|
+
self.add_class("hidden")
|
|
57
|
+
|
|
58
|
+
def set_experiment(self, experiment_id: str, current_run_id: Optional[str]) -> None:
|
|
59
|
+
"""Set the experiment and refresh runs"""
|
|
60
|
+
self.experiment_id = experiment_id
|
|
61
|
+
self.current_run_id = current_run_id
|
|
62
|
+
self.query_one("#runs-title", Static).update(
|
|
63
|
+
f"[bold]Runs for {experiment_id}[/bold]"
|
|
64
|
+
)
|
|
65
|
+
self.refresh_runs()
|
|
66
|
+
self.visible = True
|
|
67
|
+
# Focus the runs table
|
|
68
|
+
self.query_one("#runs-table", DataTable).focus()
|
|
69
|
+
|
|
70
|
+
def refresh_runs(self) -> None:
|
|
71
|
+
"""Refresh the runs list"""
|
|
72
|
+
table = self.query_one("#runs-table", DataTable)
|
|
73
|
+
table.clear()
|
|
74
|
+
|
|
75
|
+
if not self.experiment_id:
|
|
76
|
+
return
|
|
77
|
+
|
|
78
|
+
self.runs = self.state_provider.get_experiment_runs(self.experiment_id)
|
|
79
|
+
|
|
80
|
+
for run in self.runs:
|
|
81
|
+
# Format status with icon
|
|
82
|
+
if run.status == "active":
|
|
83
|
+
status = "▶ Active"
|
|
84
|
+
elif run.status == "completed":
|
|
85
|
+
status = "✓ Done"
|
|
86
|
+
elif run.status == "failed":
|
|
87
|
+
status = "❌ Failed"
|
|
88
|
+
else:
|
|
89
|
+
status = run.status or "-"
|
|
90
|
+
|
|
91
|
+
# Mark current run
|
|
92
|
+
run_id_display = run.run_id
|
|
93
|
+
if run.run_id == self.current_run_id:
|
|
94
|
+
run_id_display = f"★ {run.run_id}"
|
|
95
|
+
|
|
96
|
+
# Format jobs
|
|
97
|
+
jobs_text = f"{run.finished_jobs}/{run.total_jobs}"
|
|
98
|
+
if run.failed_jobs > 0:
|
|
99
|
+
jobs_text += f" ({run.failed_jobs}✗)"
|
|
100
|
+
|
|
101
|
+
# Format hostname
|
|
102
|
+
hostname = run.hostname or "-"
|
|
103
|
+
|
|
104
|
+
# Format started time (can be float timestamp or ISO string)
|
|
105
|
+
started = "-"
|
|
106
|
+
started_ts = None
|
|
107
|
+
if run.started_at:
|
|
108
|
+
if isinstance(run.started_at, str):
|
|
109
|
+
try:
|
|
110
|
+
started_dt = datetime.fromisoformat(run.started_at)
|
|
111
|
+
started = started_dt.strftime("%Y-%m-%d %H:%M")
|
|
112
|
+
started_ts = started_dt.timestamp()
|
|
113
|
+
except ValueError:
|
|
114
|
+
started = run.started_at[:16]
|
|
115
|
+
else:
|
|
116
|
+
started = datetime.fromtimestamp(run.started_at).strftime(
|
|
117
|
+
"%Y-%m-%d %H:%M"
|
|
118
|
+
)
|
|
119
|
+
started_ts = run.started_at
|
|
120
|
+
|
|
121
|
+
# Calculate duration
|
|
122
|
+
duration = "-"
|
|
123
|
+
if started_ts:
|
|
124
|
+
ended_ts = None
|
|
125
|
+
if run.ended_at:
|
|
126
|
+
if isinstance(run.ended_at, str):
|
|
127
|
+
try:
|
|
128
|
+
ended_ts = datetime.fromisoformat(run.ended_at).timestamp()
|
|
129
|
+
except ValueError:
|
|
130
|
+
pass
|
|
131
|
+
else:
|
|
132
|
+
ended_ts = run.ended_at
|
|
133
|
+
|
|
134
|
+
if ended_ts:
|
|
135
|
+
elapsed = ended_ts - started_ts
|
|
136
|
+
else:
|
|
137
|
+
import time
|
|
138
|
+
|
|
139
|
+
elapsed = time.time() - started_ts
|
|
140
|
+
duration = format_duration(elapsed)
|
|
141
|
+
|
|
142
|
+
table.add_row(
|
|
143
|
+
run_id_display,
|
|
144
|
+
status,
|
|
145
|
+
hostname,
|
|
146
|
+
jobs_text,
|
|
147
|
+
started,
|
|
148
|
+
duration,
|
|
149
|
+
key=run.run_id,
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
def _get_selected_run_id(self) -> Optional[str]:
|
|
153
|
+
"""Get the run_id from the currently selected row"""
|
|
154
|
+
table = self.query_one("#runs-table", DataTable)
|
|
155
|
+
if table.cursor_row is None:
|
|
156
|
+
return None
|
|
157
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
158
|
+
if row_key:
|
|
159
|
+
return str(row_key.value)
|
|
160
|
+
return None
|
|
161
|
+
|
|
162
|
+
def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
|
|
163
|
+
"""Handle run selection"""
|
|
164
|
+
if event.row_key and self.experiment_id:
|
|
165
|
+
run_id = str(event.row_key.value)
|
|
166
|
+
is_current = run_id == self.current_run_id
|
|
167
|
+
self.post_message(RunSelected(self.experiment_id, run_id, is_current))
|
|
168
|
+
self.visible = False
|
|
169
|
+
|
|
170
|
+
def action_select_run(self) -> None:
|
|
171
|
+
"""Select the highlighted run"""
|
|
172
|
+
run_id = self._get_selected_run_id()
|
|
173
|
+
if run_id and self.experiment_id:
|
|
174
|
+
is_current = run_id == self.current_run_id
|
|
175
|
+
self.post_message(RunSelected(self.experiment_id, run_id, is_current))
|
|
176
|
+
self.visible = False
|
|
177
|
+
|
|
178
|
+
def action_go_back(self) -> None:
|
|
179
|
+
"""Hide the runs list"""
|
|
180
|
+
self.visible = False
|
|
181
|
+
# Return focus to experiments table
|
|
182
|
+
try:
|
|
183
|
+
self.app.query_one("#experiments-table", DataTable).focus()
|
|
184
|
+
except Exception:
|
|
185
|
+
pass
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Services list widget for the TUI"""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Optional
|
|
5
|
+
from textual import work
|
|
6
|
+
from textual.app import ComposeResult
|
|
7
|
+
from textual.containers import Vertical
|
|
8
|
+
from textual.widgets import DataTable, Static
|
|
9
|
+
from textual.binding import Binding
|
|
10
|
+
|
|
11
|
+
from experimaestro.scheduler.state_provider import StateProvider
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ServicesList(Vertical):
|
|
15
|
+
"""Widget displaying services for selected experiment
|
|
16
|
+
|
|
17
|
+
Services are retrieved from StateProvider.get_services() which
|
|
18
|
+
abstracts away whether services are live (from scheduler) or recreated
|
|
19
|
+
from database state_dict. The UI treats all services uniformly.
|
|
20
|
+
|
|
21
|
+
For remote monitoring, service syncs are managed globally by GlobalServiceSyncs.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
BINDINGS = [
|
|
25
|
+
Binding("s", "start_service", "Start"),
|
|
26
|
+
Binding("x", "stop_service", "Stop"),
|
|
27
|
+
Binding("u", "copy_url", "Copy URL", show=False),
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# State icons for display
|
|
31
|
+
STATE_ICONS = {
|
|
32
|
+
"STOPPED": "⏹",
|
|
33
|
+
"STARTING": "⏳",
|
|
34
|
+
"RUNNING": "▶",
|
|
35
|
+
"STOPPING": "⏳",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
def __init__(self, state_provider: StateProvider) -> None:
|
|
39
|
+
super().__init__()
|
|
40
|
+
self.state_provider = state_provider
|
|
41
|
+
self.current_experiment: Optional[str] = None
|
|
42
|
+
self._services: dict = {} # service_id -> Service object
|
|
43
|
+
|
|
44
|
+
def compose(self) -> ComposeResult:
|
|
45
|
+
yield Static("Loading services...", id="services-loading", classes="hidden")
|
|
46
|
+
yield DataTable(id="services-table", cursor_type="row")
|
|
47
|
+
|
|
48
|
+
def on_mount(self) -> None:
|
|
49
|
+
"""Set up the services table"""
|
|
50
|
+
table = self.query_one("#services-table", DataTable)
|
|
51
|
+
table.add_columns("ID", "Description", "State", "Sync", "URL")
|
|
52
|
+
table.cursor_type = "row"
|
|
53
|
+
|
|
54
|
+
def set_experiment(self, experiment_id: Optional[str]) -> None:
|
|
55
|
+
"""Set the current experiment and refresh services"""
|
|
56
|
+
self.current_experiment = experiment_id
|
|
57
|
+
|
|
58
|
+
# Clear and show loading for remote
|
|
59
|
+
if self.state_provider.is_remote:
|
|
60
|
+
table = self.query_one("#services-table", DataTable)
|
|
61
|
+
table.clear()
|
|
62
|
+
self._services = {}
|
|
63
|
+
self.query_one("#services-loading", Static).remove_class("hidden")
|
|
64
|
+
|
|
65
|
+
# Load in background
|
|
66
|
+
self._load_services(experiment_id)
|
|
67
|
+
|
|
68
|
+
@work(thread=True, exclusive=True, group="services_load")
|
|
69
|
+
def _load_services(self, experiment_id: Optional[str]) -> None:
|
|
70
|
+
"""Load services in background thread"""
|
|
71
|
+
if not experiment_id:
|
|
72
|
+
self.app.call_from_thread(self._on_services_loaded, [])
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
services = self.state_provider.get_services(experiment_id)
|
|
76
|
+
self.app.call_from_thread(self._on_services_loaded, services)
|
|
77
|
+
|
|
78
|
+
def _on_services_loaded(self, services: list) -> None:
|
|
79
|
+
"""Handle loaded services on main thread"""
|
|
80
|
+
self.query_one("#services-loading", Static).add_class("hidden")
|
|
81
|
+
self._refresh_services_with_data(services)
|
|
82
|
+
|
|
83
|
+
def _get_global_services(self):
|
|
84
|
+
"""Get the global services sync widget"""
|
|
85
|
+
from experimaestro.tui.widgets.global_services import GlobalServiceSyncs
|
|
86
|
+
|
|
87
|
+
try:
|
|
88
|
+
return self.app.query_one(GlobalServiceSyncs)
|
|
89
|
+
except Exception:
|
|
90
|
+
return None
|
|
91
|
+
|
|
92
|
+
def _start_synchronizer_for_service(self, service) -> None:
|
|
93
|
+
"""Register a service with the global sync manager"""
|
|
94
|
+
import logging
|
|
95
|
+
|
|
96
|
+
logger = logging.getLogger("xpm.tui.services")
|
|
97
|
+
service_id = service.id
|
|
98
|
+
|
|
99
|
+
if not self.state_provider.is_remote:
|
|
100
|
+
return
|
|
101
|
+
|
|
102
|
+
if not self.current_experiment:
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
# Check if service has paths in state_dict that need syncing
|
|
106
|
+
state_dict = getattr(service, "_state_dict_data", None)
|
|
107
|
+
if state_dict is None and hasattr(service, "state_dict"):
|
|
108
|
+
try:
|
|
109
|
+
state_dict = service.state_dict()
|
|
110
|
+
except Exception:
|
|
111
|
+
logger.debug(f"Service {service_id}: state_dict() failed")
|
|
112
|
+
return
|
|
113
|
+
|
|
114
|
+
if not state_dict:
|
|
115
|
+
logger.info(f"Service {service_id}: no state_dict")
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
# Find paths in state_dict
|
|
119
|
+
paths_to_sync = self._extract_paths(state_dict)
|
|
120
|
+
if not paths_to_sync:
|
|
121
|
+
logger.info(f"Service {service_id}: no paths in state_dict: {state_dict}")
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
logger.info(f"Service {service_id}: found paths to sync: {paths_to_sync}")
|
|
125
|
+
|
|
126
|
+
# Get service description and URL
|
|
127
|
+
description = (
|
|
128
|
+
service.description() if hasattr(service, "description") else service_id
|
|
129
|
+
)
|
|
130
|
+
url = getattr(service, "url", None)
|
|
131
|
+
|
|
132
|
+
# Register with global sync manager
|
|
133
|
+
global_services = self._get_global_services()
|
|
134
|
+
if global_services:
|
|
135
|
+
global_services.add_service_sync(
|
|
136
|
+
experiment_id=self.current_experiment,
|
|
137
|
+
service_id=service_id,
|
|
138
|
+
description=description,
|
|
139
|
+
remote_path=paths_to_sync[0],
|
|
140
|
+
url=url,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
def _extract_paths(self, state_dict: dict) -> list[str]:
|
|
144
|
+
"""Extract path strings from a service state_dict"""
|
|
145
|
+
from pathlib import PosixPath, WindowsPath
|
|
146
|
+
|
|
147
|
+
paths = []
|
|
148
|
+
|
|
149
|
+
def find_paths(d):
|
|
150
|
+
if isinstance(d, (Path, PosixPath, WindowsPath)):
|
|
151
|
+
# Direct Path object
|
|
152
|
+
paths.append(str(d))
|
|
153
|
+
elif isinstance(d, dict):
|
|
154
|
+
if "__path__" in d:
|
|
155
|
+
# Serialized path format
|
|
156
|
+
paths.append(d["__path__"])
|
|
157
|
+
else:
|
|
158
|
+
for v in d.values():
|
|
159
|
+
find_paths(v)
|
|
160
|
+
elif isinstance(d, (list, tuple)):
|
|
161
|
+
for item in d:
|
|
162
|
+
find_paths(item)
|
|
163
|
+
|
|
164
|
+
find_paths(state_dict)
|
|
165
|
+
return paths
|
|
166
|
+
|
|
167
|
+
def refresh_services(self) -> None:
|
|
168
|
+
"""Refresh the services list from state provider
|
|
169
|
+
|
|
170
|
+
For remote providers, this runs in background. For local, it's synchronous.
|
|
171
|
+
"""
|
|
172
|
+
if not self.current_experiment:
|
|
173
|
+
return
|
|
174
|
+
|
|
175
|
+
if self.state_provider.is_remote:
|
|
176
|
+
self._load_services(self.current_experiment)
|
|
177
|
+
else:
|
|
178
|
+
services = self.state_provider.get_services(self.current_experiment)
|
|
179
|
+
self._refresh_services_with_data(services)
|
|
180
|
+
|
|
181
|
+
def _refresh_services_with_data(self, services: list) -> None:
|
|
182
|
+
"""Refresh the services display with provided data"""
|
|
183
|
+
import logging
|
|
184
|
+
|
|
185
|
+
logger = logging.getLogger("xpm.tui.services")
|
|
186
|
+
|
|
187
|
+
table = self.query_one("#services-table", DataTable)
|
|
188
|
+
table.clear()
|
|
189
|
+
self._services = {}
|
|
190
|
+
|
|
191
|
+
global_services = self._get_global_services()
|
|
192
|
+
|
|
193
|
+
logger.debug(
|
|
194
|
+
f"refresh_services got {len(services)} services: "
|
|
195
|
+
f"{[(s.id, getattr(s, 'url', None)) for s in services]}"
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
for service in services:
|
|
199
|
+
service_id = service.id
|
|
200
|
+
self._services[service_id] = service
|
|
201
|
+
|
|
202
|
+
state_name = service.state.name if hasattr(service, "state") else "UNKNOWN"
|
|
203
|
+
state_icon = self.STATE_ICONS.get(state_name, "?")
|
|
204
|
+
url = getattr(service, "url", None) or "-"
|
|
205
|
+
description = (
|
|
206
|
+
service.description() if hasattr(service, "description") else ""
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
# Get sync status from global services
|
|
210
|
+
sync_status = "-"
|
|
211
|
+
if global_services and self.current_experiment:
|
|
212
|
+
status = global_services.get_sync_status(
|
|
213
|
+
self.current_experiment, service_id
|
|
214
|
+
)
|
|
215
|
+
if status:
|
|
216
|
+
sync_status = status
|
|
217
|
+
|
|
218
|
+
table.add_row(
|
|
219
|
+
service_id,
|
|
220
|
+
description,
|
|
221
|
+
f"{state_icon} {state_name}",
|
|
222
|
+
sync_status,
|
|
223
|
+
url,
|
|
224
|
+
key=service_id,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
# Start synchronizer for running services with paths (remote only)
|
|
228
|
+
if state_name == "RUNNING":
|
|
229
|
+
self._start_synchronizer_for_service(service)
|
|
230
|
+
elif (
|
|
231
|
+
state_name == "STOPPED" and global_services and self.current_experiment
|
|
232
|
+
):
|
|
233
|
+
# Stop sync when service is explicitly stopped
|
|
234
|
+
global_services.stop_service_sync(self.current_experiment, service_id)
|
|
235
|
+
|
|
236
|
+
def _get_selected_service(self):
|
|
237
|
+
"""Get the currently selected Service object"""
|
|
238
|
+
table = self.query_one("#services-table", DataTable)
|
|
239
|
+
if table.cursor_row is not None and table.row_count > 0:
|
|
240
|
+
row_key = list(table.rows.keys())[table.cursor_row]
|
|
241
|
+
if row_key:
|
|
242
|
+
service_id = str(row_key.value)
|
|
243
|
+
return self._services.get(service_id)
|
|
244
|
+
return None
|
|
245
|
+
|
|
246
|
+
def action_start_service(self) -> None:
|
|
247
|
+
"""Start the selected service"""
|
|
248
|
+
import logging
|
|
249
|
+
|
|
250
|
+
logger = logging.getLogger("xpm.tui.services")
|
|
251
|
+
|
|
252
|
+
service = self._get_selected_service()
|
|
253
|
+
if not service:
|
|
254
|
+
return
|
|
255
|
+
|
|
256
|
+
logger.info(
|
|
257
|
+
f"Starting service {service.id} (type={type(service).__name__}, "
|
|
258
|
+
f"has_get_url={hasattr(service, 'get_url')}, is_live={self.state_provider.is_live})"
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
try:
|
|
262
|
+
if hasattr(service, "get_url"):
|
|
263
|
+
url = service.get_url()
|
|
264
|
+
logger.info(f"Service started, url={url}, service.url={service.url}")
|
|
265
|
+
self.notify(f"Service started: {url}", severity="information")
|
|
266
|
+
else:
|
|
267
|
+
# MockService - service state loaded from file but not the actual service
|
|
268
|
+
self.notify(
|
|
269
|
+
"Service not available (loaded from saved state)",
|
|
270
|
+
severity="warning",
|
|
271
|
+
)
|
|
272
|
+
self.refresh_services()
|
|
273
|
+
except Exception as e:
|
|
274
|
+
self.notify(f"Failed to start service: {e}", severity="error")
|
|
275
|
+
|
|
276
|
+
def action_stop_service(self) -> None:
|
|
277
|
+
"""Stop the selected service"""
|
|
278
|
+
service = self._get_selected_service()
|
|
279
|
+
if not service:
|
|
280
|
+
return
|
|
281
|
+
|
|
282
|
+
from experimaestro.scheduler.services import ServiceState
|
|
283
|
+
|
|
284
|
+
if service.state == ServiceState.STOPPED:
|
|
285
|
+
self.notify("Service is not running", severity="warning")
|
|
286
|
+
return
|
|
287
|
+
|
|
288
|
+
try:
|
|
289
|
+
if hasattr(service, "stop"):
|
|
290
|
+
service.stop()
|
|
291
|
+
self.notify(f"Service stopped: {service.id}", severity="information")
|
|
292
|
+
else:
|
|
293
|
+
self.notify("Service does not support stopping", severity="warning")
|
|
294
|
+
self.refresh_services()
|
|
295
|
+
except Exception as e:
|
|
296
|
+
self.notify(f"Failed to stop service: {e}", severity="error")
|
|
297
|
+
|
|
298
|
+
def action_copy_url(self) -> None:
|
|
299
|
+
"""Copy the service URL to clipboard"""
|
|
300
|
+
service = self._get_selected_service()
|
|
301
|
+
if not service:
|
|
302
|
+
return
|
|
303
|
+
|
|
304
|
+
url = getattr(service, "url", None)
|
|
305
|
+
if url:
|
|
306
|
+
try:
|
|
307
|
+
import pyperclip
|
|
308
|
+
|
|
309
|
+
pyperclip.copy(url)
|
|
310
|
+
self.notify(f"URL copied: {url}", severity="information")
|
|
311
|
+
except Exception as e:
|
|
312
|
+
self.notify(f"Failed to copy: {e}", severity="error")
|
|
313
|
+
else:
|
|
314
|
+
self.notify("Start the service first to get URL", severity="warning")
|