experimaestro 2.0.0b4__py3-none-any.whl → 2.0.0b17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of experimaestro might be problematic. Click here for more details.

Files changed (154) hide show
  1. experimaestro/__init__.py +12 -5
  2. experimaestro/cli/__init__.py +393 -134
  3. experimaestro/cli/filter.py +48 -23
  4. experimaestro/cli/jobs.py +253 -71
  5. experimaestro/cli/refactor.py +1 -2
  6. experimaestro/commandline.py +7 -4
  7. experimaestro/connectors/__init__.py +9 -1
  8. experimaestro/connectors/local.py +43 -3
  9. experimaestro/core/arguments.py +18 -18
  10. experimaestro/core/identifier.py +11 -11
  11. experimaestro/core/objects/config.py +96 -39
  12. experimaestro/core/objects/config_walk.py +3 -3
  13. experimaestro/core/{subparameters.py → partial.py} +16 -16
  14. experimaestro/core/partial_lock.py +394 -0
  15. experimaestro/core/types.py +12 -15
  16. experimaestro/dynamic.py +290 -0
  17. experimaestro/experiments/__init__.py +6 -2
  18. experimaestro/experiments/cli.py +223 -52
  19. experimaestro/experiments/configuration.py +24 -0
  20. experimaestro/generators.py +5 -5
  21. experimaestro/ipc.py +118 -1
  22. experimaestro/launcherfinder/__init__.py +2 -2
  23. experimaestro/launcherfinder/registry.py +6 -7
  24. experimaestro/launcherfinder/specs.py +2 -9
  25. experimaestro/launchers/slurm/__init__.py +2 -2
  26. experimaestro/launchers/slurm/base.py +62 -0
  27. experimaestro/locking.py +957 -1
  28. experimaestro/notifications.py +89 -201
  29. experimaestro/progress.py +63 -366
  30. experimaestro/rpyc.py +0 -2
  31. experimaestro/run.py +29 -2
  32. experimaestro/scheduler/__init__.py +8 -1
  33. experimaestro/scheduler/base.py +650 -53
  34. experimaestro/scheduler/dependencies.py +20 -16
  35. experimaestro/scheduler/experiment.py +764 -169
  36. experimaestro/scheduler/interfaces.py +338 -96
  37. experimaestro/scheduler/jobs.py +58 -20
  38. experimaestro/scheduler/remote/__init__.py +31 -0
  39. experimaestro/scheduler/remote/adaptive_sync.py +265 -0
  40. experimaestro/scheduler/remote/client.py +928 -0
  41. experimaestro/scheduler/remote/protocol.py +282 -0
  42. experimaestro/scheduler/remote/server.py +447 -0
  43. experimaestro/scheduler/remote/sync.py +144 -0
  44. experimaestro/scheduler/services.py +186 -35
  45. experimaestro/scheduler/state_provider.py +811 -2157
  46. experimaestro/scheduler/state_status.py +1247 -0
  47. experimaestro/scheduler/transient.py +31 -0
  48. experimaestro/scheduler/workspace.py +1 -1
  49. experimaestro/scheduler/workspace_state_provider.py +1273 -0
  50. experimaestro/scriptbuilder.py +4 -4
  51. experimaestro/settings.py +36 -0
  52. experimaestro/tests/conftest.py +33 -5
  53. experimaestro/tests/connectors/bin/executable.py +1 -1
  54. experimaestro/tests/fixtures/pre_experiment/experiment_check_env.py +16 -0
  55. experimaestro/tests/fixtures/pre_experiment/experiment_check_mock.py +14 -0
  56. experimaestro/tests/fixtures/pre_experiment/experiment_simple.py +12 -0
  57. experimaestro/tests/fixtures/pre_experiment/pre_setup_env.py +5 -0
  58. experimaestro/tests/fixtures/pre_experiment/pre_setup_error.py +3 -0
  59. experimaestro/tests/fixtures/pre_experiment/pre_setup_mock.py +8 -0
  60. experimaestro/tests/launchers/bin/test.py +1 -0
  61. experimaestro/tests/launchers/test_slurm.py +9 -9
  62. experimaestro/tests/partial_reschedule.py +46 -0
  63. experimaestro/tests/restart.py +3 -3
  64. experimaestro/tests/restart_main.py +1 -0
  65. experimaestro/tests/scripts/notifyandwait.py +1 -0
  66. experimaestro/tests/task_partial.py +38 -0
  67. experimaestro/tests/task_tokens.py +2 -2
  68. experimaestro/tests/tasks/test_dynamic.py +6 -6
  69. experimaestro/tests/test_dependencies.py +3 -3
  70. experimaestro/tests/test_deprecated.py +15 -15
  71. experimaestro/tests/test_dynamic_locking.py +317 -0
  72. experimaestro/tests/test_environment.py +24 -14
  73. experimaestro/tests/test_experiment.py +171 -36
  74. experimaestro/tests/test_identifier.py +25 -25
  75. experimaestro/tests/test_identifier_stability.py +3 -5
  76. experimaestro/tests/test_multitoken.py +2 -4
  77. experimaestro/tests/{test_subparameters.py → test_partial.py} +25 -25
  78. experimaestro/tests/test_partial_paths.py +81 -138
  79. experimaestro/tests/test_pre_experiment.py +219 -0
  80. experimaestro/tests/test_progress.py +2 -8
  81. experimaestro/tests/test_remote_state.py +1132 -0
  82. experimaestro/tests/test_stray_jobs.py +261 -0
  83. experimaestro/tests/test_tasks.py +1 -2
  84. experimaestro/tests/test_token_locking.py +52 -67
  85. experimaestro/tests/test_tokens.py +5 -6
  86. experimaestro/tests/test_transient.py +225 -0
  87. experimaestro/tests/test_workspace_state_provider.py +768 -0
  88. experimaestro/tests/token_reschedule.py +1 -3
  89. experimaestro/tests/utils.py +2 -7
  90. experimaestro/tokens.py +227 -372
  91. experimaestro/tools/diff.py +1 -0
  92. experimaestro/tools/documentation.py +4 -5
  93. experimaestro/tools/jobs.py +1 -2
  94. experimaestro/tui/app.py +459 -1895
  95. experimaestro/tui/app.tcss +162 -0
  96. experimaestro/tui/dialogs.py +172 -0
  97. experimaestro/tui/log_viewer.py +253 -3
  98. experimaestro/tui/messages.py +137 -0
  99. experimaestro/tui/utils.py +54 -0
  100. experimaestro/tui/widgets/__init__.py +23 -0
  101. experimaestro/tui/widgets/experiments.py +468 -0
  102. experimaestro/tui/widgets/global_services.py +238 -0
  103. experimaestro/tui/widgets/jobs.py +972 -0
  104. experimaestro/tui/widgets/log.py +156 -0
  105. experimaestro/tui/widgets/orphans.py +363 -0
  106. experimaestro/tui/widgets/runs.py +185 -0
  107. experimaestro/tui/widgets/services.py +314 -0
  108. experimaestro/tui/widgets/stray_jobs.py +528 -0
  109. experimaestro/utils/__init__.py +1 -1
  110. experimaestro/utils/environment.py +105 -22
  111. experimaestro/utils/fswatcher.py +124 -0
  112. experimaestro/utils/jobs.py +1 -2
  113. experimaestro/utils/jupyter.py +1 -2
  114. experimaestro/utils/logging.py +72 -0
  115. experimaestro/version.py +2 -2
  116. experimaestro/webui/__init__.py +9 -0
  117. experimaestro/webui/app.py +117 -0
  118. experimaestro/{server → webui}/data/index.css +66 -11
  119. experimaestro/webui/data/index.css.map +1 -0
  120. experimaestro/{server → webui}/data/index.js +82763 -87217
  121. experimaestro/webui/data/index.js.map +1 -0
  122. experimaestro/webui/routes/__init__.py +5 -0
  123. experimaestro/webui/routes/auth.py +53 -0
  124. experimaestro/webui/routes/proxy.py +117 -0
  125. experimaestro/webui/server.py +200 -0
  126. experimaestro/webui/state_bridge.py +152 -0
  127. experimaestro/webui/websocket.py +413 -0
  128. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/METADATA +8 -9
  129. experimaestro-2.0.0b17.dist-info/RECORD +219 -0
  130. experimaestro/cli/progress.py +0 -269
  131. experimaestro/scheduler/state.py +0 -75
  132. experimaestro/scheduler/state_db.py +0 -388
  133. experimaestro/scheduler/state_sync.py +0 -834
  134. experimaestro/server/__init__.py +0 -467
  135. experimaestro/server/data/index.css.map +0 -1
  136. experimaestro/server/data/index.js.map +0 -1
  137. experimaestro/tests/test_cli_jobs.py +0 -615
  138. experimaestro/tests/test_file_progress.py +0 -425
  139. experimaestro/tests/test_file_progress_integration.py +0 -477
  140. experimaestro/tests/test_state_db.py +0 -434
  141. experimaestro-2.0.0b4.dist-info/RECORD +0 -181
  142. /experimaestro/{server → webui}/data/1815e00441357e01619e.ttf +0 -0
  143. /experimaestro/{server → webui}/data/2463b90d9a316e4e5294.woff2 +0 -0
  144. /experimaestro/{server → webui}/data/2582b0e4bcf85eceead0.ttf +0 -0
  145. /experimaestro/{server → webui}/data/89999bdf5d835c012025.woff2 +0 -0
  146. /experimaestro/{server → webui}/data/914997e1bdfc990d0897.ttf +0 -0
  147. /experimaestro/{server → webui}/data/c210719e60948b211a12.woff2 +0 -0
  148. /experimaestro/{server → webui}/data/favicon.ico +0 -0
  149. /experimaestro/{server → webui}/data/index.html +0 -0
  150. /experimaestro/{server → webui}/data/login.html +0 -0
  151. /experimaestro/{server → webui}/data/manifest.json +0 -0
  152. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/WHEEL +0 -0
  153. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/entry_points.txt +0 -0
  154. {experimaestro-2.0.0b4.dist-info → experimaestro-2.0.0b17.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,282 @@
1
+ """JSON-RPC 2.0 protocol utilities for SSH-based remote monitoring
2
+
3
+ This module provides JSON-RPC message types, serialization utilities,
4
+ and protocol constants for communication between SSHStateProviderServer
5
+ and SSHStateProviderClient.
6
+
7
+ Message format: Newline-delimited JSON (one JSON object per line)
8
+ """
9
+
10
+ import json
11
+ import logging
12
+ from dataclasses import dataclass, field
13
+ from datetime import datetime
14
+ from enum import Enum
15
+ from typing import Any, Dict, Optional, Union
16
+
17
+ logger = logging.getLogger("xpm.remote.protocol")
18
+
19
+ # JSON-RPC 2.0 version
20
+ JSONRPC_VERSION = "2.0"
21
+
22
+ # Standard JSON-RPC error codes
23
+ PARSE_ERROR = -32700
24
+ INVALID_REQUEST = -32600
25
+ METHOD_NOT_FOUND = -32601
26
+ INVALID_PARAMS = -32602
27
+ INTERNAL_ERROR = -32603
28
+
29
+ # Custom error codes
30
+ CONNECTION_ERROR = -32001
31
+ WORKSPACE_NOT_FOUND = -32002
32
+ PERMISSION_DENIED = -32003
33
+ TIMEOUT_ERROR = -32004
34
+
35
+
36
+ class NotificationMethod(str, Enum):
37
+ """Server-to-client notification methods"""
38
+
39
+ # Generic state event notification (serialized dataclass)
40
+ STATE_EVENT = "notification.state_event"
41
+
42
+ # Control notifications
43
+ FILE_CHANGED = "notification.file_changed"
44
+ SHUTDOWN = "notification.shutdown"
45
+
46
+
47
+ class RPCMethod(str, Enum):
48
+ """Client-to-server RPC methods"""
49
+
50
+ GET_EXPERIMENTS = "get_experiments"
51
+ GET_EXPERIMENT = "get_experiment"
52
+ GET_EXPERIMENT_RUNS = "get_experiment_runs"
53
+ GET_JOBS = "get_jobs"
54
+ GET_JOB = "get_job"
55
+ GET_ALL_JOBS = "get_all_jobs"
56
+ GET_SERVICES = "get_services"
57
+ GET_TAGS_MAP = "get_tags_map"
58
+ GET_DEPENDENCIES_MAP = "get_dependencies_map"
59
+ KILL_JOB = "kill_job"
60
+ CLEAN_JOB = "clean_job"
61
+ GET_SYNC_INFO = "get_sync_info"
62
+ GET_PROCESS_INFO = "get_process_info"
63
+
64
+
65
+ @dataclass
66
+ class RPCError:
67
+ """JSON-RPC error object"""
68
+
69
+ code: int
70
+ message: str
71
+ data: Optional[Any] = None
72
+
73
+ def to_dict(self) -> Dict:
74
+ result = {"code": self.code, "message": self.message}
75
+ if self.data is not None:
76
+ result["data"] = self.data
77
+ return result
78
+
79
+ @classmethod
80
+ def from_dict(cls, d: Dict) -> "RPCError":
81
+ return cls(code=d["code"], message=d["message"], data=d.get("data"))
82
+
83
+
84
+ @dataclass
85
+ class RPCRequest:
86
+ """JSON-RPC request message"""
87
+
88
+ method: str
89
+ params: Dict = field(default_factory=dict)
90
+ id: Optional[int] = None # None for notifications
91
+
92
+ def to_dict(self) -> Dict:
93
+ result = {"jsonrpc": JSONRPC_VERSION, "method": self.method}
94
+ if self.params:
95
+ result["params"] = self.params
96
+ if self.id is not None:
97
+ result["id"] = self.id
98
+ return result
99
+
100
+ def to_json(self) -> str:
101
+ return json.dumps(self.to_dict())
102
+
103
+ @classmethod
104
+ def from_dict(cls, d: Dict) -> "RPCRequest":
105
+ return cls(method=d["method"], params=d.get("params", {}), id=d.get("id"))
106
+
107
+
108
+ @dataclass
109
+ class RPCResponse:
110
+ """JSON-RPC response message"""
111
+
112
+ id: int
113
+ result: Optional[Any] = None
114
+ error: Optional[RPCError] = None
115
+
116
+ def to_dict(self) -> Dict:
117
+ d = {"jsonrpc": JSONRPC_VERSION, "id": self.id}
118
+ if self.error is not None:
119
+ d["error"] = self.error.to_dict()
120
+ else:
121
+ d["result"] = self.result
122
+ return d
123
+
124
+ def to_json(self) -> str:
125
+ return json.dumps(self.to_dict())
126
+
127
+ @classmethod
128
+ def from_dict(cls, d: Dict) -> "RPCResponse":
129
+ error = None
130
+ if "error" in d:
131
+ error = RPCError.from_dict(d["error"])
132
+ return cls(id=d["id"], result=d.get("result"), error=error)
133
+
134
+
135
+ @dataclass
136
+ class RPCNotification:
137
+ """JSON-RPC notification (no id, no response expected)"""
138
+
139
+ method: str
140
+ params: Dict = field(default_factory=dict)
141
+
142
+ def to_dict(self) -> Dict:
143
+ result = {"jsonrpc": JSONRPC_VERSION, "method": self.method}
144
+ if self.params:
145
+ result["params"] = self.params
146
+ return result
147
+
148
+ def to_json(self) -> str:
149
+ return json.dumps(self.to_dict())
150
+
151
+ @classmethod
152
+ def from_dict(cls, d: Dict) -> "RPCNotification":
153
+ return cls(method=d["method"], params=d.get("params", {}))
154
+
155
+
156
+ def parse_message(line: str) -> Union[RPCRequest, RPCResponse, RPCNotification]:
157
+ """Parse a JSON-RPC message from a line of text
158
+
159
+ Args:
160
+ line: A single line of JSON text
161
+
162
+ Returns:
163
+ RPCRequest, RPCResponse, or RPCNotification
164
+
165
+ Raises:
166
+ ValueError: If the message is malformed
167
+ """
168
+ try:
169
+ d = json.loads(line)
170
+ except json.JSONDecodeError as e:
171
+ raise ValueError(f"Invalid JSON: {e}")
172
+
173
+ if "jsonrpc" not in d or d["jsonrpc"] != JSONRPC_VERSION:
174
+ raise ValueError("Invalid or missing jsonrpc version")
175
+
176
+ # Response: has "id" and either "result" or "error"
177
+ if "result" in d or "error" in d:
178
+ return RPCResponse.from_dict(d)
179
+
180
+ # Request or notification: has "method"
181
+ if "method" in d:
182
+ if "id" in d:
183
+ return RPCRequest.from_dict(d)
184
+ else:
185
+ return RPCNotification.from_dict(d)
186
+
187
+ raise ValueError("Cannot determine message type")
188
+
189
+
190
+ def create_error_response(id: int, code: int, message: str, data: Any = None) -> str:
191
+ """Create a JSON-RPC error response
192
+
193
+ Args:
194
+ id: Request ID
195
+ code: Error code
196
+ message: Error message
197
+ data: Optional additional data
198
+
199
+ Returns:
200
+ JSON string
201
+ """
202
+ response = RPCResponse(id=id, error=RPCError(code=code, message=message, data=data))
203
+ return response.to_json()
204
+
205
+
206
+ def create_success_response(id: int, result: Any) -> str:
207
+ """Create a JSON-RPC success response
208
+
209
+ Args:
210
+ id: Request ID
211
+ result: Result data
212
+
213
+ Returns:
214
+ JSON string
215
+ """
216
+ response = RPCResponse(id=id, result=result)
217
+ return response.to_json()
218
+
219
+
220
+ def create_notification(method: Union[str, NotificationMethod], params: Dict) -> str:
221
+ """Create a JSON-RPC notification
222
+
223
+ Args:
224
+ method: Notification method name
225
+ params: Notification parameters
226
+
227
+ Returns:
228
+ JSON string
229
+ """
230
+ if isinstance(method, NotificationMethod):
231
+ method = method.value
232
+ notification = RPCNotification(method=method, params=params)
233
+ return notification.to_json()
234
+
235
+
236
+ def create_request(method: Union[str, RPCMethod], params: Dict, id: int) -> str:
237
+ """Create a JSON-RPC request
238
+
239
+ Args:
240
+ method: Method name
241
+ params: Request parameters
242
+ id: Request ID
243
+
244
+ Returns:
245
+ JSON string
246
+ """
247
+ if isinstance(method, RPCMethod):
248
+ method = method.value
249
+ request = RPCRequest(method=method, params=params, id=id)
250
+ return request.to_json()
251
+
252
+
253
+ # Serialization helpers for data types
254
+
255
+
256
+ def serialize_datetime(dt) -> Optional[str]:
257
+ """Serialize datetime or timestamp to ISO format string
258
+
259
+ Handles:
260
+ - None: returns None
261
+ - datetime: returns ISO format string
262
+ - float/int: treats as Unix timestamp, converts to ISO format
263
+ - str: returns as-is (already serialized)
264
+ """
265
+ if dt is None:
266
+ return None
267
+ if isinstance(dt, str):
268
+ return dt # Already serialized
269
+ if isinstance(dt, (int, float)):
270
+ # Unix timestamp
271
+ return datetime.fromtimestamp(dt).isoformat()
272
+ if isinstance(dt, datetime):
273
+ return dt.isoformat()
274
+ # Try to convert to string as fallback
275
+ return str(dt)
276
+
277
+
278
+ def deserialize_datetime(s: Optional[str]) -> Optional[datetime]:
279
+ """Deserialize ISO format string to datetime"""
280
+ if s is None:
281
+ return None
282
+ return datetime.fromisoformat(s)