splime 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spl/__init__.py +14 -0
- spl/client.py +1364 -0
- spl/core/__init__.py +23 -0
- spl/core/common.py +350 -0
- spl/core/entities/__init__.py +0 -0
- spl/core/entities/adapter.py +210 -0
- spl/core/entities/artifact.py +141 -0
- spl/core/entities/control.py +45 -0
- spl/core/entities/distribution.py +65 -0
- spl/core/entities/function.py +254 -0
- spl/core/entities/local_function.py +286 -0
- spl/core/entities/misc.py +14 -0
- spl/core/entities/module.py +88 -0
- spl/core/entities/node.py +286 -0
- spl/core/entities/node_function.py +79 -0
- spl/core/entities/node_remote.py +295 -0
- spl/core/entities/pipeline.py +436 -0
- spl/core/entities/scalar.py +55 -0
- spl/core/ir/__init__.py +0 -0
- spl/core/ir/common.py +34 -0
- spl/core/ir/parse.py +79 -0
- spl/core/ir/unparse.py +29 -0
- spl/core/ir/utils.py +163 -0
- spl/daemon/__init__.py +23 -0
- spl/daemon/__main__.py +11 -0
- spl/daemon/cli.py +582 -0
- spl/daemon/client.py +43 -0
- spl/daemon/docker_environment.py +329 -0
- spl/daemon/docker_pool.py +516 -0
- spl/daemon/environment.py +228 -0
- spl/daemon/environment_base.py +479 -0
- spl/daemon/heartbeat_service.py +119 -0
- spl/daemon/metadata.py +427 -0
- spl/daemon/remote_client.py +457 -0
- spl/daemon/repositories/__init__.py +17 -0
- spl/daemon/repositories/env.py +323 -0
- spl/daemon/repositories/library.py +181 -0
- spl/daemon/repositories/object.py +997 -0
- spl/daemon/repositories/run.py +279 -0
- spl/daemon/repositories/server_connection.py +657 -0
- spl/daemon/repositories/sync_event.py +129 -0
- spl/daemon/routes/__init__.py +1 -0
- spl/daemon/routes/_helpers.py +147 -0
- spl/daemon/routes/artifacts.py +77 -0
- spl/daemon/routes/diagnostics.py +114 -0
- spl/daemon/routes/envs.py +82 -0
- spl/daemon/routes/libraries.py +129 -0
- spl/daemon/routes/objects.py +174 -0
- spl/daemon/routes/remote.py +56 -0
- spl/daemon/routes/runs.py +96 -0
- spl/daemon/routes/server_connections.py +86 -0
- spl/daemon/runtime_backend.py +368 -0
- spl/daemon/runtime_config.py +133 -0
- spl/daemon/runtime_dependencies.py +459 -0
- spl/daemon/secret_store.py +187 -0
- spl/daemon/server.py +2224 -0
- spl/daemon/server_connection.py +267 -0
- spl/daemon/services/__init__.py +1 -0
- spl/daemon/services/sync.py +76 -0
- spl/daemon/signature.py +376 -0
- spl/daemon/storage_base.py +542 -0
- spl/daemon/store.py +436 -0
- spl/daemon/worker.py +526 -0
- spl/daemon_client.py +945 -0
- spl/pipeline_widget.py +1452 -0
- spl/py.typed +0 -0
- spl/server_client.py +787 -0
- splime-0.1.2.dist-info/METADATA +189 -0
- splime-0.1.2.dist-info/RECORD +74 -0
- splime-0.1.2.dist-info/WHEEL +5 -0
- splime-0.1.2.dist-info/entry_points.txt +2 -0
- splime-0.1.2.dist-info/licenses/LICENSE +201 -0
- splime-0.1.2.dist-info/licenses/NOTICE +8 -0
- splime-0.1.2.dist-info/top_level.txt +1 -0
spl/client.py
ADDED
|
@@ -0,0 +1,1364 @@
|
|
|
1
|
+
"""User-facing client for publishing and running SPL objects on the daemon.
|
|
2
|
+
|
|
3
|
+
This module is the thin "framework side" of the daemon integration. Code that
|
|
4
|
+
already uses SPL should not need to know about HTTP endpoints, run directories,
|
|
5
|
+
or worker subprocesses. The intended workflow is:
|
|
6
|
+
|
|
7
|
+
from spl.client import SPLClient
|
|
8
|
+
|
|
9
|
+
client = SPLClient()
|
|
10
|
+
client.publish(my_function, name="sum", env="default")
|
|
11
|
+
result = client.call("sum", kwargs={"x": 1, "y": 2})
|
|
12
|
+
|
|
13
|
+
The module only imports ``spl.core`` inside export helpers. That keeps basic
|
|
14
|
+
registry operations, such as listing remote objects, usable even from a small
|
|
15
|
+
environment that has the daemon client but does not currently have all core
|
|
16
|
+
dependencies imported yet.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import sys
|
|
22
|
+
from dataclasses import dataclass, field, replace
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Literal, cast, overload
|
|
25
|
+
|
|
26
|
+
from spl.daemon_client import (
|
|
27
|
+
DEFAULT_DAEMON_HOST,
|
|
28
|
+
DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
|
|
29
|
+
DEFAULT_SERVER_URL,
|
|
30
|
+
Client,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
OfflinePolicy = Literal["queue", "wait", "fail_fast"]
|
|
35
|
+
ObjectScope = Literal["auto", "local", "server", "all"]
|
|
36
|
+
RunSource = Literal["auto", "local"]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class PublishedObject:
|
|
41
|
+
"""Metadata returned after an object is stored in the daemon registry."""
|
|
42
|
+
|
|
43
|
+
name: str
|
|
44
|
+
entrypoint: str
|
|
45
|
+
env: str
|
|
46
|
+
yaml_path: str
|
|
47
|
+
workdir: str | None = None
|
|
48
|
+
raw: dict[str, Any] = field(default_factory=dict)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class RemoteResult:
|
|
53
|
+
"""Completed run result plus downloaded artifact locations.
|
|
54
|
+
|
|
55
|
+
``payload`` is the daemon's JSON result document. It contains the actual
|
|
56
|
+
return value under ``result`` and daemon-side artifact paths under
|
|
57
|
+
``artifacts``. ``downloaded_artifacts`` is populated only when the caller
|
|
58
|
+
asks this client to download artifacts into a local directory.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
run: dict[str, Any]
|
|
62
|
+
payload: dict[str, Any]
|
|
63
|
+
mode: str = "local"
|
|
64
|
+
downloaded_artifacts: dict[str, Path] = field(default_factory=dict)
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def value(self) -> Any:
|
|
68
|
+
"""Return the user's JSON-compatible result value."""
|
|
69
|
+
|
|
70
|
+
return self.payload.get("result")
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def artifacts(self) -> dict[str, str]:
|
|
74
|
+
"""Return daemon-side artifact paths keyed by artifact name."""
|
|
75
|
+
|
|
76
|
+
return self.payload.get("artifacts", {})
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def server_side(self) -> bool:
|
|
80
|
+
"""Return whether the result came from a central-server run."""
|
|
81
|
+
|
|
82
|
+
return self.mode == "server"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class RemoteRun:
|
|
86
|
+
"""Handle for a run that was started on the daemon.
|
|
87
|
+
|
|
88
|
+
The handle is intentionally lazy. A caller can inspect state, wait for
|
|
89
|
+
completion, fetch the result, or download artifacts without remembering raw
|
|
90
|
+
endpoint names.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def __init__(
|
|
94
|
+
self,
|
|
95
|
+
client: "SPLClient",
|
|
96
|
+
state: dict[str, Any],
|
|
97
|
+
*,
|
|
98
|
+
server_side: bool = False,
|
|
99
|
+
):
|
|
100
|
+
self._client = client
|
|
101
|
+
self.state = state
|
|
102
|
+
self.server_side = server_side
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def id(self) -> str:
|
|
106
|
+
"""Return the daemon run id."""
|
|
107
|
+
|
|
108
|
+
return self.state["id"]
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def status(self) -> str:
|
|
112
|
+
"""Return the last known daemon status."""
|
|
113
|
+
|
|
114
|
+
return self.state["status"]
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def mode(self) -> str:
|
|
118
|
+
"""Return ``local`` for daemon worker runs and ``server`` for remote runs."""
|
|
119
|
+
|
|
120
|
+
return "server" if self.server_side else "local"
|
|
121
|
+
|
|
122
|
+
def refresh(self) -> dict[str, Any]:
|
|
123
|
+
"""Refresh and return the run state from the daemon."""
|
|
124
|
+
|
|
125
|
+
if self.server_side:
|
|
126
|
+
self.state = self._client._daemon.get_remote_run(self.id)
|
|
127
|
+
else:
|
|
128
|
+
self.state = self._client._daemon.get_run(self.id)
|
|
129
|
+
return self.state
|
|
130
|
+
|
|
131
|
+
def wait(
|
|
132
|
+
self,
|
|
133
|
+
*,
|
|
134
|
+
poll_interval: float = 0.25,
|
|
135
|
+
timeout_seconds: float | None = None,
|
|
136
|
+
) -> dict[str, Any]:
|
|
137
|
+
"""Wait until the run succeeds or fails, then return final state."""
|
|
138
|
+
|
|
139
|
+
if self.server_side:
|
|
140
|
+
self.state = self._client._daemon.wait_remote_run(
|
|
141
|
+
self.id,
|
|
142
|
+
poll_interval=poll_interval,
|
|
143
|
+
timeout_seconds=timeout_seconds,
|
|
144
|
+
)
|
|
145
|
+
else:
|
|
146
|
+
self.state = self._client._daemon.wait_run(
|
|
147
|
+
self.id,
|
|
148
|
+
poll_interval=poll_interval,
|
|
149
|
+
timeout_seconds=timeout_seconds,
|
|
150
|
+
)
|
|
151
|
+
return self.state
|
|
152
|
+
|
|
153
|
+
def result(self) -> dict[str, Any]:
|
|
154
|
+
"""Return the daemon result payload for this run."""
|
|
155
|
+
|
|
156
|
+
if self.server_side:
|
|
157
|
+
self.refresh()
|
|
158
|
+
return self.state.get("result") or {}
|
|
159
|
+
return self._client._daemon.result(self.id)
|
|
160
|
+
|
|
161
|
+
def artifact_names(self) -> list[str]:
|
|
162
|
+
"""Return artifact names produced by this run."""
|
|
163
|
+
|
|
164
|
+
if self.server_side:
|
|
165
|
+
return self._client._daemon.list_remote_artifacts(self.id)
|
|
166
|
+
return self._client._daemon.list_artifacts(self.id)
|
|
167
|
+
|
|
168
|
+
def download_artifacts(self, target_dir: str | Path) -> dict[str, Path]:
|
|
169
|
+
"""Download all run artifacts into ``target_dir``."""
|
|
170
|
+
|
|
171
|
+
target_path = Path(target_dir)
|
|
172
|
+
target_path.mkdir(parents=True, exist_ok=True)
|
|
173
|
+
downloaded: dict[str, Path] = {}
|
|
174
|
+
for name in self.artifact_names():
|
|
175
|
+
if self.server_side:
|
|
176
|
+
downloaded[name] = self._client._daemon.download_remote_artifact(
|
|
177
|
+
self.id,
|
|
178
|
+
name,
|
|
179
|
+
target_path,
|
|
180
|
+
)
|
|
181
|
+
else:
|
|
182
|
+
downloaded[name] = self._client._daemon.download_artifact(
|
|
183
|
+
self.id,
|
|
184
|
+
name,
|
|
185
|
+
target_path,
|
|
186
|
+
)
|
|
187
|
+
return downloaded
|
|
188
|
+
|
|
189
|
+
def collect(
|
|
190
|
+
self,
|
|
191
|
+
*,
|
|
192
|
+
artifacts_dir: str | Path | None = None,
|
|
193
|
+
poll_interval: float = 0.25,
|
|
194
|
+
timeout_seconds: float | None = None,
|
|
195
|
+
) -> RemoteResult:
|
|
196
|
+
"""Wait for completion, return result, and optionally download artifacts."""
|
|
197
|
+
|
|
198
|
+
final_state = self.wait(
|
|
199
|
+
poll_interval=poll_interval,
|
|
200
|
+
timeout_seconds=timeout_seconds,
|
|
201
|
+
)
|
|
202
|
+
if final_state["status"] != "succeeded":
|
|
203
|
+
error = final_state.get("error") or "run returned no error message"
|
|
204
|
+
raise RuntimeError(
|
|
205
|
+
f"{self.mode} run {self.id!r} ended as "
|
|
206
|
+
f"{final_state.get('status')!r}: {error}"
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
payload = self.result()
|
|
210
|
+
downloaded = (
|
|
211
|
+
self.download_artifacts(artifacts_dir)
|
|
212
|
+
if artifacts_dir is not None
|
|
213
|
+
else {}
|
|
214
|
+
)
|
|
215
|
+
return RemoteResult(
|
|
216
|
+
run=final_state,
|
|
217
|
+
payload=payload,
|
|
218
|
+
mode=self.mode,
|
|
219
|
+
downloaded_artifacts=downloaded,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class SPLClient:
|
|
224
|
+
"""High-level client used by SPL users to interact with the local daemon."""
|
|
225
|
+
|
|
226
|
+
def __init__(
|
|
227
|
+
self,
|
|
228
|
+
base_url: str | None = None,
|
|
229
|
+
*,
|
|
230
|
+
daemon_host: str = DEFAULT_DAEMON_HOST,
|
|
231
|
+
daemon_port: int | None = None,
|
|
232
|
+
daemon_home: str | Path | None = None,
|
|
233
|
+
machine_token: str | None = None,
|
|
234
|
+
user_token: str | None = None,
|
|
235
|
+
server_url: str = DEFAULT_SERVER_URL,
|
|
236
|
+
machine_id: str | None = None,
|
|
237
|
+
display_name: str | None = None,
|
|
238
|
+
capabilities: dict[str, Any] | None = None,
|
|
239
|
+
heartbeat_interval_seconds: float | None = DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
|
|
240
|
+
api_token: str | None = None,
|
|
241
|
+
):
|
|
242
|
+
self._daemon = Client(
|
|
243
|
+
base_url,
|
|
244
|
+
daemon_host=daemon_host,
|
|
245
|
+
daemon_port=daemon_port,
|
|
246
|
+
daemon_home=daemon_home,
|
|
247
|
+
api_token=api_token,
|
|
248
|
+
)
|
|
249
|
+
self.server_connection: dict[str, Any] | None = None
|
|
250
|
+
if machine_token is not None or user_token is not None:
|
|
251
|
+
if not machine_token or not user_token:
|
|
252
|
+
raise ValueError("machine_token and user_token must be provided together")
|
|
253
|
+
self.server_connection = self.connect_server(
|
|
254
|
+
machine_token=machine_token,
|
|
255
|
+
user_token=user_token,
|
|
256
|
+
server_url=server_url,
|
|
257
|
+
machine_id=machine_id,
|
|
258
|
+
display_name=display_name,
|
|
259
|
+
capabilities=capabilities,
|
|
260
|
+
heartbeat_interval_seconds=heartbeat_interval_seconds,
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
def health(self) -> dict[str, Any]:
|
|
264
|
+
"""Check that the local daemon is reachable."""
|
|
265
|
+
|
|
266
|
+
return self._daemon.health()
|
|
267
|
+
|
|
268
|
+
def connect_server(
|
|
269
|
+
self,
|
|
270
|
+
*,
|
|
271
|
+
machine_token: str,
|
|
272
|
+
user_token: str,
|
|
273
|
+
server_url: str = DEFAULT_SERVER_URL,
|
|
274
|
+
machine_id: str | None = None,
|
|
275
|
+
display_name: str | None = None,
|
|
276
|
+
capabilities: dict[str, Any] | None = None,
|
|
277
|
+
heartbeat_interval_seconds: float | None = DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
|
|
278
|
+
) -> dict[str, Any]:
|
|
279
|
+
"""Connect the local daemon to the central daemon server.
|
|
280
|
+
|
|
281
|
+
Calling this method is optional. A plain ``SPLClient()`` remains fully
|
|
282
|
+
local and never contacts the central server.
|
|
283
|
+
"""
|
|
284
|
+
|
|
285
|
+
self.server_connection = self._daemon.connect_server(
|
|
286
|
+
machine_token=machine_token,
|
|
287
|
+
user_token=user_token,
|
|
288
|
+
server_url=server_url,
|
|
289
|
+
machine_id=machine_id,
|
|
290
|
+
display_name=display_name,
|
|
291
|
+
capabilities=capabilities,
|
|
292
|
+
heartbeat_interval_seconds=heartbeat_interval_seconds,
|
|
293
|
+
)
|
|
294
|
+
return self.server_connection
|
|
295
|
+
|
|
296
|
+
def disconnect_server(self) -> dict[str, Any]:
|
|
297
|
+
"""Gracefully disconnect the local daemon from the central server."""
|
|
298
|
+
|
|
299
|
+
response = self._daemon.disconnect_server()
|
|
300
|
+
self.server_connection = None
|
|
301
|
+
return response
|
|
302
|
+
|
|
303
|
+
def current_server_connection(self) -> dict[str, Any]:
|
|
304
|
+
"""Return local daemon state for the central-server connection."""
|
|
305
|
+
|
|
306
|
+
return self._daemon.server_connection()
|
|
307
|
+
|
|
308
|
+
def machines(self) -> dict[str, Any]:
|
|
309
|
+
"""Return user's machines and mark the current local daemon machine."""
|
|
310
|
+
|
|
311
|
+
return self._daemon.server_machines()
|
|
312
|
+
|
|
313
|
+
def libraries(self, *, include_accessible: bool = True) -> list[dict[str, Any]]:
|
|
314
|
+
"""Return libraries visible to the connected central-server user."""
|
|
315
|
+
|
|
316
|
+
self._require_server_connection("listing server libraries")
|
|
317
|
+
return self._daemon.server_libraries(include_accessible=include_accessible)
|
|
318
|
+
|
|
319
|
+
def create_library(
|
|
320
|
+
self,
|
|
321
|
+
slug: str,
|
|
322
|
+
*,
|
|
323
|
+
display_name: str | None = None,
|
|
324
|
+
description: str = "",
|
|
325
|
+
visibility: str = "private",
|
|
326
|
+
default_machine: str | None = None,
|
|
327
|
+
execution: dict[str, Any] | None = None,
|
|
328
|
+
) -> dict[str, Any]:
|
|
329
|
+
"""Create a central-server library owned by the connected user."""
|
|
330
|
+
|
|
331
|
+
self._require_server_connection("creating a library")
|
|
332
|
+
payload: dict[str, Any] = {
|
|
333
|
+
"slug": slug,
|
|
334
|
+
"display_name": display_name or slug,
|
|
335
|
+
"description": description,
|
|
336
|
+
"visibility": visibility,
|
|
337
|
+
}
|
|
338
|
+
if default_machine is not None:
|
|
339
|
+
payload["default_machine_id"] = default_machine
|
|
340
|
+
if execution is not None:
|
|
341
|
+
payload["execution"] = execution
|
|
342
|
+
return self._daemon.create_server_library(payload)
|
|
343
|
+
|
|
344
|
+
def get_library(self, ref: str) -> dict[str, Any]:
|
|
345
|
+
"""Return one central-server library by slug or id."""
|
|
346
|
+
|
|
347
|
+
self._require_server_connection("reading a library")
|
|
348
|
+
return self._daemon.get_server_library(ref)
|
|
349
|
+
|
|
350
|
+
def update_library(
|
|
351
|
+
self,
|
|
352
|
+
ref: str,
|
|
353
|
+
*,
|
|
354
|
+
display_name: str | None = None,
|
|
355
|
+
description: str | None = None,
|
|
356
|
+
visibility: str | None = None,
|
|
357
|
+
default_machine: str | None = None,
|
|
358
|
+
execution: dict[str, Any] | None = None,
|
|
359
|
+
) -> dict[str, Any]:
|
|
360
|
+
"""Update mutable metadata for one central-server library."""
|
|
361
|
+
|
|
362
|
+
self._require_server_connection("updating a library")
|
|
363
|
+
payload: dict[str, Any] = {}
|
|
364
|
+
if display_name is not None:
|
|
365
|
+
payload["display_name"] = display_name
|
|
366
|
+
if description is not None:
|
|
367
|
+
payload["description"] = description
|
|
368
|
+
if visibility is not None:
|
|
369
|
+
payload["visibility"] = visibility
|
|
370
|
+
if default_machine is not None:
|
|
371
|
+
payload["default_machine_id"] = default_machine
|
|
372
|
+
if execution is not None:
|
|
373
|
+
payload["execution"] = execution
|
|
374
|
+
return self._daemon.update_server_library(ref, payload)
|
|
375
|
+
|
|
376
|
+
def delete_library(self, ref: str) -> dict[str, Any]:
|
|
377
|
+
"""Delete or archive one central-server library when supported upstream."""
|
|
378
|
+
|
|
379
|
+
self._require_server_connection("deleting a library")
|
|
380
|
+
return self._daemon.delete_server_library(ref)
|
|
381
|
+
|
|
382
|
+
def grant_library(
|
|
383
|
+
self,
|
|
384
|
+
ref: str,
|
|
385
|
+
grantee: str,
|
|
386
|
+
*,
|
|
387
|
+
grantee_type: str = "user",
|
|
388
|
+
scopes: list[str] | None = None,
|
|
389
|
+
) -> dict[str, Any]:
|
|
390
|
+
"""Grant a user or team access to one central-server library."""
|
|
391
|
+
|
|
392
|
+
self._require_server_connection("granting library access")
|
|
393
|
+
payload: dict[str, Any] = {
|
|
394
|
+
"grantee_id": grantee,
|
|
395
|
+
"grantee_type": grantee_type,
|
|
396
|
+
}
|
|
397
|
+
if scopes is not None:
|
|
398
|
+
payload["scopes"] = scopes
|
|
399
|
+
return self._daemon.grant_server_library(ref, payload)
|
|
400
|
+
|
|
401
|
+
def revoke_library_grant(self, ref: str, grantee: str) -> dict[str, Any]:
|
|
402
|
+
"""Revoke a grantee's access to one central-server library."""
|
|
403
|
+
|
|
404
|
+
self._require_server_connection("revoking library access")
|
|
405
|
+
return self._daemon.revoke_server_library_grant(ref, grantee)
|
|
406
|
+
|
|
407
|
+
def add_reference(
|
|
408
|
+
self,
|
|
409
|
+
into_library: str,
|
|
410
|
+
name: str,
|
|
411
|
+
*,
|
|
412
|
+
owner: str | None = None,
|
|
413
|
+
from_library: str = "default",
|
|
414
|
+
version: str | int | None = "latest",
|
|
415
|
+
alias: str | None = None,
|
|
416
|
+
) -> dict[str, Any]:
|
|
417
|
+
"""Add a live reference entry from another library into ``into_library``."""
|
|
418
|
+
|
|
419
|
+
self._require_server_connection("adding a library reference")
|
|
420
|
+
payload: dict[str, Any] = {
|
|
421
|
+
"name": name,
|
|
422
|
+
"from_library": from_library,
|
|
423
|
+
}
|
|
424
|
+
if owner is not None:
|
|
425
|
+
payload["from_owner"] = owner
|
|
426
|
+
if version is not None:
|
|
427
|
+
payload["version"] = version
|
|
428
|
+
if alias is not None:
|
|
429
|
+
payload["alias"] = alias
|
|
430
|
+
return self._daemon.add_server_library_reference(into_library, payload)
|
|
431
|
+
|
|
432
|
+
def copy_object(
|
|
433
|
+
self,
|
|
434
|
+
name: str,
|
|
435
|
+
*,
|
|
436
|
+
into_library: str,
|
|
437
|
+
from_owner: str | None = None,
|
|
438
|
+
from_library: str = "default",
|
|
439
|
+
version: str | int | None = "latest",
|
|
440
|
+
new_name: str | None = None,
|
|
441
|
+
) -> dict[str, Any]:
|
|
442
|
+
"""Copy an object snapshot into a library owned by the connected user."""
|
|
443
|
+
|
|
444
|
+
self._require_server_connection("copying an object into a library")
|
|
445
|
+
payload: dict[str, Any] = {
|
|
446
|
+
"name": name,
|
|
447
|
+
"from_library": from_library,
|
|
448
|
+
}
|
|
449
|
+
if from_owner is not None:
|
|
450
|
+
payload["from_owner"] = from_owner
|
|
451
|
+
if version is not None:
|
|
452
|
+
payload["version"] = version
|
|
453
|
+
if new_name is not None:
|
|
454
|
+
payload["new_name"] = new_name
|
|
455
|
+
return self._daemon.copy_server_library_object(into_library, payload)
|
|
456
|
+
|
|
457
|
+
def remove_entry(self, library: str, name: str) -> dict[str, Any]:
|
|
458
|
+
"""Remove an owned object or reference entry from a central-server library."""
|
|
459
|
+
|
|
460
|
+
self._require_server_connection("removing a library entry")
|
|
461
|
+
return self._daemon.remove_server_library_entry(library, name)
|
|
462
|
+
|
|
463
|
+
def register_env(self, name: str = "default", python: str | None = None) -> dict[str, Any]:
|
|
464
|
+
"""Register a Python executable as a daemon environment.
|
|
465
|
+
|
|
466
|
+
By default the currently running interpreter is registered. This makes
|
|
467
|
+
the simplest local workflow short:
|
|
468
|
+
|
|
469
|
+
client.register_env()
|
|
470
|
+
client.publish(my_function, env="default")
|
|
471
|
+
"""
|
|
472
|
+
|
|
473
|
+
return self._daemon.register_env(name, python or sys.executable)
|
|
474
|
+
|
|
475
|
+
def publish(
|
|
476
|
+
self,
|
|
477
|
+
obj: Any,
|
|
478
|
+
*,
|
|
479
|
+
name: str | None = None,
|
|
480
|
+
env: str = "default",
|
|
481
|
+
entrypoint: str | None = None,
|
|
482
|
+
workdir: str | None = None,
|
|
483
|
+
runtime_config: dict[str, Any] | str | Path | None = None,
|
|
484
|
+
runtime: str | None = None,
|
|
485
|
+
python: str | None = None,
|
|
486
|
+
base_image: str | None = None,
|
|
487
|
+
dependency_frame_offset: int = 0,
|
|
488
|
+
library: str | None = None,
|
|
489
|
+
create: bool = False,
|
|
490
|
+
library_display_name: str | None = None,
|
|
491
|
+
local_only: bool = False,
|
|
492
|
+
) -> PublishedObject:
|
|
493
|
+
"""Serialize a live function/pipeline and store it in the daemon.
|
|
494
|
+
|
|
495
|
+
``name`` is the daemon registry name. ``entrypoint`` is the object name
|
|
496
|
+
inside the generated SPL/YAML file. They can differ, which lets a user
|
|
497
|
+
publish the same function under several daemon aliases.
|
|
498
|
+
|
|
499
|
+
``dependency_frame_offset`` is only needed when ``publish`` itself is
|
|
500
|
+
wrapped by user helper functions. Leave it at ``0`` for direct notebook
|
|
501
|
+
use.
|
|
502
|
+
|
|
503
|
+
``library`` targets a central-server library during sync. Missing
|
|
504
|
+
non-default libraries are rejected unless ``create=True`` is passed.
|
|
505
|
+
"""
|
|
506
|
+
|
|
507
|
+
yaml_text, resolved_entrypoint = export_object_to_yaml(
|
|
508
|
+
obj,
|
|
509
|
+
entrypoint,
|
|
510
|
+
frame_offset=4 + dependency_frame_offset,
|
|
511
|
+
)
|
|
512
|
+
registry_name = name or resolved_entrypoint
|
|
513
|
+
record = self._daemon.register_object(
|
|
514
|
+
registry_name,
|
|
515
|
+
entrypoint=resolved_entrypoint,
|
|
516
|
+
env=env,
|
|
517
|
+
yaml_text=yaml_text,
|
|
518
|
+
workdir=workdir,
|
|
519
|
+
runtime_config=build_runtime_config(
|
|
520
|
+
runtime_config,
|
|
521
|
+
runtime=runtime,
|
|
522
|
+
python=python,
|
|
523
|
+
base_image=base_image,
|
|
524
|
+
),
|
|
525
|
+
library=library,
|
|
526
|
+
create_library=create,
|
|
527
|
+
library_display_name=library_display_name,
|
|
528
|
+
local_only=local_only,
|
|
529
|
+
)
|
|
530
|
+
return PublishedObject(
|
|
531
|
+
name=record["name"],
|
|
532
|
+
entrypoint=record["entrypoint"],
|
|
533
|
+
env=record["env"],
|
|
534
|
+
yaml_path=record["yaml_path"],
|
|
535
|
+
workdir=record.get("workdir"),
|
|
536
|
+
raw=record,
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
def publish_yaml(
|
|
540
|
+
self,
|
|
541
|
+
yaml: str | Path,
|
|
542
|
+
*,
|
|
543
|
+
name: str,
|
|
544
|
+
entrypoint: str,
|
|
545
|
+
env: str = "default",
|
|
546
|
+
workdir: str | None = None,
|
|
547
|
+
runtime_config: dict[str, Any] | str | Path | None = None,
|
|
548
|
+
runtime: str | None = None,
|
|
549
|
+
python: str | None = None,
|
|
550
|
+
base_image: str | None = None,
|
|
551
|
+
library: str | None = None,
|
|
552
|
+
create: bool = False,
|
|
553
|
+
library_display_name: str | None = None,
|
|
554
|
+
local_only: bool = False,
|
|
555
|
+
) -> PublishedObject:
|
|
556
|
+
"""Store an already generated SPL/YAML document in the daemon.
|
|
557
|
+
|
|
558
|
+
``yaml`` can be YAML text or a path to a YAML file. A string is treated
|
|
559
|
+
as a path when it points to an existing file; otherwise it is sent as
|
|
560
|
+
YAML text. This method covers the explicit requirement "send generated
|
|
561
|
+
YAML" and is useful when the object was exported earlier or produced by
|
|
562
|
+
another process. ``create=True`` asks the server to create the target
|
|
563
|
+
library if it does not already exist.
|
|
564
|
+
"""
|
|
565
|
+
|
|
566
|
+
yaml_text = read_yaml_input(yaml)
|
|
567
|
+
record = self._daemon.register_object(
|
|
568
|
+
name,
|
|
569
|
+
entrypoint=entrypoint,
|
|
570
|
+
env=env,
|
|
571
|
+
yaml_text=yaml_text,
|
|
572
|
+
workdir=workdir,
|
|
573
|
+
runtime_config=build_runtime_config(
|
|
574
|
+
runtime_config,
|
|
575
|
+
runtime=runtime,
|
|
576
|
+
python=python,
|
|
577
|
+
base_image=base_image,
|
|
578
|
+
),
|
|
579
|
+
library=library,
|
|
580
|
+
create_library=create,
|
|
581
|
+
library_display_name=library_display_name,
|
|
582
|
+
local_only=local_only,
|
|
583
|
+
)
|
|
584
|
+
return PublishedObject(
|
|
585
|
+
name=record["name"],
|
|
586
|
+
entrypoint=record["entrypoint"],
|
|
587
|
+
env=record["env"],
|
|
588
|
+
yaml_path=record["yaml_path"],
|
|
589
|
+
workdir=record.get("workdir"),
|
|
590
|
+
raw=record,
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
def local_objects(self, *, compact: bool = False) -> list[dict[str, Any]]:
|
|
594
|
+
"""Return local daemon objects as a stable list."""
|
|
595
|
+
|
|
596
|
+
return self._object_records(self._daemon.list_objects(compact=compact))
|
|
597
|
+
|
|
598
|
+
def server_objects(
|
|
599
|
+
self,
|
|
600
|
+
*,
|
|
601
|
+
owner: str | None = None,
|
|
602
|
+
library: str | None = None,
|
|
603
|
+
compact: bool = False,
|
|
604
|
+
) -> list[dict[str, Any]]:
|
|
605
|
+
"""Return server catalog objects as a stable list."""
|
|
606
|
+
|
|
607
|
+
return self._daemon.server_objects(
|
|
608
|
+
owner_id=owner,
|
|
609
|
+
library=library,
|
|
610
|
+
compact=compact,
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
@staticmethod
|
|
614
|
+
def _object_records(records: dict[str, Any] | list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
615
|
+
if isinstance(records, list):
|
|
616
|
+
return list(records)
|
|
617
|
+
return [
|
|
618
|
+
dict(record) if isinstance(record, dict) else {"name": name, "value": record}
|
|
619
|
+
for name, record in records.items()
|
|
620
|
+
]
|
|
621
|
+
|
|
622
|
+
@overload
|
|
623
|
+
def objects(
|
|
624
|
+
self,
|
|
625
|
+
*,
|
|
626
|
+
compact: bool = False,
|
|
627
|
+
scope: Literal["local"],
|
|
628
|
+
owner: None = None,
|
|
629
|
+
library: None = None,
|
|
630
|
+
) -> dict[str, Any]: ...
|
|
631
|
+
|
|
632
|
+
@overload
|
|
633
|
+
def objects(
|
|
634
|
+
self,
|
|
635
|
+
*,
|
|
636
|
+
compact: bool = False,
|
|
637
|
+
scope: Literal["server"],
|
|
638
|
+
owner: str | None = None,
|
|
639
|
+
library: str | None = None,
|
|
640
|
+
) -> list[dict[str, Any]]: ...
|
|
641
|
+
|
|
642
|
+
@overload
|
|
643
|
+
def objects(
|
|
644
|
+
self,
|
|
645
|
+
*,
|
|
646
|
+
compact: bool = False,
|
|
647
|
+
scope: Literal["all"],
|
|
648
|
+
owner: str | None = None,
|
|
649
|
+
library: str | None = None,
|
|
650
|
+
) -> dict[str, Any]: ...
|
|
651
|
+
|
|
652
|
+
@overload
|
|
653
|
+
def objects(
|
|
654
|
+
self,
|
|
655
|
+
*,
|
|
656
|
+
compact: bool = False,
|
|
657
|
+
scope: Literal["auto"] = "auto",
|
|
658
|
+
owner: str | None = None,
|
|
659
|
+
library: str | None = None,
|
|
660
|
+
) -> dict[str, Any] | list[dict[str, Any]]: ...
|
|
661
|
+
|
|
662
|
+
def objects(
|
|
663
|
+
self,
|
|
664
|
+
*,
|
|
665
|
+
compact: bool = False,
|
|
666
|
+
scope: ObjectScope = "auto",
|
|
667
|
+
owner: str | None = None,
|
|
668
|
+
library: str | None = None,
|
|
669
|
+
) -> dict[str, Any] | list[dict[str, Any]]:
|
|
670
|
+
"""Return objects from the local cache, server catalog, or both."""
|
|
671
|
+
|
|
672
|
+
if scope == "auto":
|
|
673
|
+
scope = (
|
|
674
|
+
"server"
|
|
675
|
+
if owner is not None or library is not None or self._has_server_connection()
|
|
676
|
+
else "local"
|
|
677
|
+
)
|
|
678
|
+
if scope == "local":
|
|
679
|
+
if owner is not None or library is not None:
|
|
680
|
+
raise ValueError("owner/library require scope='server', scope='all', or scope='auto'")
|
|
681
|
+
return self._daemon.list_objects(compact=compact)
|
|
682
|
+
if scope == "server":
|
|
683
|
+
return self._daemon.server_objects(
|
|
684
|
+
owner_id=owner,
|
|
685
|
+
library=library,
|
|
686
|
+
compact=compact,
|
|
687
|
+
)
|
|
688
|
+
if scope == "all":
|
|
689
|
+
return {
|
|
690
|
+
"local": self._daemon.list_objects(compact=compact),
|
|
691
|
+
"server": self._daemon.server_objects(
|
|
692
|
+
owner_id=owner,
|
|
693
|
+
library=library,
|
|
694
|
+
compact=compact,
|
|
695
|
+
),
|
|
696
|
+
}
|
|
697
|
+
raise ValueError("scope must be 'auto', 'local', 'server', or 'all'")
|
|
698
|
+
|
|
699
|
+
def _has_server_connection(self) -> bool:
|
|
700
|
+
if self.server_connection is not None:
|
|
701
|
+
return bool(self.server_connection.get("connected"))
|
|
702
|
+
try:
|
|
703
|
+
state = self._daemon.server_connection()
|
|
704
|
+
except Exception:
|
|
705
|
+
return False
|
|
706
|
+
if bool(state.get("connected")):
|
|
707
|
+
return True
|
|
708
|
+
connection = state.get("connection") or state.get("remote_connection") or {}
|
|
709
|
+
return connection.get("status") == "connected"
|
|
710
|
+
|
|
711
|
+
def _require_server_connection(self, operation: str) -> None:
|
|
712
|
+
try:
|
|
713
|
+
state = self._daemon.server_connection()
|
|
714
|
+
except Exception as exc:
|
|
715
|
+
raise RuntimeError(
|
|
716
|
+
f"{operation} requires a server-connected SPLClient. "
|
|
717
|
+
"Construct SPLClient(machine_token=..., user_token=...) or call "
|
|
718
|
+
"client.connect_server(...) first."
|
|
719
|
+
) from exc
|
|
720
|
+
if state.get("connected"):
|
|
721
|
+
self.server_connection = state
|
|
722
|
+
return
|
|
723
|
+
raise RuntimeError(
|
|
724
|
+
f"{operation} requires a server-connected SPLClient. "
|
|
725
|
+
"Construct SPLClient(machine_token=..., user_token=...) or call "
|
|
726
|
+
"client.connect_server(...) first."
|
|
727
|
+
)
|
|
728
|
+
|
|
729
|
+
def signature(
|
|
730
|
+
self,
|
|
731
|
+
name: str,
|
|
732
|
+
*,
|
|
733
|
+
version: int | None = None,
|
|
734
|
+
owner: str | None = None,
|
|
735
|
+
library: str | None = None,
|
|
736
|
+
function: str | None = None,
|
|
737
|
+
) -> dict[str, Any]:
|
|
738
|
+
"""Return a concise call/read signature for one daemon object."""
|
|
739
|
+
|
|
740
|
+
return self._daemon.signature(
|
|
741
|
+
name,
|
|
742
|
+
version=version,
|
|
743
|
+
owner_id=owner,
|
|
744
|
+
library=library,
|
|
745
|
+
function=function,
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
def inputs(
|
|
749
|
+
self,
|
|
750
|
+
name: str,
|
|
751
|
+
*,
|
|
752
|
+
version: int | None = None,
|
|
753
|
+
owner: str | None = None,
|
|
754
|
+
library: str | None = None,
|
|
755
|
+
function: str | None = None,
|
|
756
|
+
) -> list[dict[str, Any]]:
|
|
757
|
+
"""Return the inputs that can be passed through ``kwargs``."""
|
|
758
|
+
|
|
759
|
+
return self._daemon.inputs(
|
|
760
|
+
name,
|
|
761
|
+
version=version,
|
|
762
|
+
owner_id=owner,
|
|
763
|
+
library=library,
|
|
764
|
+
function=function,
|
|
765
|
+
)
|
|
766
|
+
|
|
767
|
+
def outputs(
|
|
768
|
+
self,
|
|
769
|
+
name: str,
|
|
770
|
+
*,
|
|
771
|
+
version: int | None = None,
|
|
772
|
+
owner: str | None = None,
|
|
773
|
+
library: str | None = None,
|
|
774
|
+
function: str | None = None,
|
|
775
|
+
) -> list[dict[str, Any]]:
|
|
776
|
+
"""Return output selectors and how to read ``result.value``."""
|
|
777
|
+
|
|
778
|
+
return self._daemon.outputs(
|
|
779
|
+
name,
|
|
780
|
+
version=version,
|
|
781
|
+
owner_id=owner,
|
|
782
|
+
library=library,
|
|
783
|
+
function=function,
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
def decomposition(
|
|
787
|
+
self,
|
|
788
|
+
name: Any,
|
|
789
|
+
*,
|
|
790
|
+
version: int | None = None,
|
|
791
|
+
owner: str | None = None,
|
|
792
|
+
library: str | None = None,
|
|
793
|
+
) -> dict[str, Any]:
|
|
794
|
+
"""Return normalized function/node/link metadata for one object."""
|
|
795
|
+
|
|
796
|
+
if self._is_node_remote(name):
|
|
797
|
+
return self._remote_decomposition_response(name, version=version)["decomposition"]
|
|
798
|
+
if owner is not None or library is not None:
|
|
799
|
+
return self._remote_decomposition_response(
|
|
800
|
+
{
|
|
801
|
+
"name": str(name),
|
|
802
|
+
"version": version,
|
|
803
|
+
"owner_id": owner,
|
|
804
|
+
"library": library,
|
|
805
|
+
}
|
|
806
|
+
)["decomposition"]
|
|
807
|
+
return self._daemon.decomposition(str(name), version=version)
|
|
808
|
+
|
|
809
|
+
def pipeline_widget(
|
|
810
|
+
self,
|
|
811
|
+
pipeline: Any,
|
|
812
|
+
*,
|
|
813
|
+
version: int | None = None,
|
|
814
|
+
title: str | None = None,
|
|
815
|
+
height: int = 560,
|
|
816
|
+
theme: str = "dark",
|
|
817
|
+
) -> Any:
|
|
818
|
+
"""Return a rich Jupyter display object for a pipeline graph.
|
|
819
|
+
|
|
820
|
+
``pipeline`` can be a registered object name, a ``PublishedObject``, or
|
|
821
|
+
a live ``spl.core.entities.pipeline.Pipeline`` instance. In notebooks,
|
|
822
|
+
use it as the last expression in a cell or call ``.display()`` on the
|
|
823
|
+
returned object.
|
|
824
|
+
"""
|
|
825
|
+
|
|
826
|
+
from spl.core.entities.node_remote import NodeRemote
|
|
827
|
+
from spl.core.entities.pipeline import Pipeline
|
|
828
|
+
from spl.pipeline_widget import PipelineGraphWidget, pipeline_to_decomposition
|
|
829
|
+
|
|
830
|
+
if isinstance(pipeline, PublishedObject):
|
|
831
|
+
pipeline = pipeline.name
|
|
832
|
+
|
|
833
|
+
if isinstance(pipeline, NodeRemote):
|
|
834
|
+
if version is not None and pipeline.version not in {"", "latest", "current", "TODO"}:
|
|
835
|
+
raise ValueError("pass the version either on NodeRemote or draw_pipeline(...), not both")
|
|
836
|
+
response = self._remote_decomposition_response(pipeline, version=version)
|
|
837
|
+
decomposition = response["decomposition"]
|
|
838
|
+
if not decomposition.get("nodes"):
|
|
839
|
+
raise ValueError(f"remote object is not a pipeline or has no nodes: {pipeline.name}")
|
|
840
|
+
record = response.get("object") or {}
|
|
841
|
+
remote = response.get("remote") or {}
|
|
842
|
+
object_name = (
|
|
843
|
+
title
|
|
844
|
+
or record.get("display_name")
|
|
845
|
+
or record.get("name")
|
|
846
|
+
or remote.get("name")
|
|
847
|
+
or pipeline.name
|
|
848
|
+
)
|
|
849
|
+
return PipelineGraphWidget(
|
|
850
|
+
decomposition,
|
|
851
|
+
{
|
|
852
|
+
**record,
|
|
853
|
+
"remote": remote,
|
|
854
|
+
"id": record.get("id") or remote.get("object_id") or pipeline.name,
|
|
855
|
+
"name": record.get("name") or remote.get("name") or pipeline.name,
|
|
856
|
+
"displayName": object_name,
|
|
857
|
+
},
|
|
858
|
+
height=height,
|
|
859
|
+
theme=theme,
|
|
860
|
+
)
|
|
861
|
+
|
|
862
|
+
if isinstance(pipeline, Pipeline):
|
|
863
|
+
if version is not None:
|
|
864
|
+
raise ValueError("version is only supported for registered objects")
|
|
865
|
+
object_name = title or pipeline.name or "Pipeline"
|
|
866
|
+
return PipelineGraphWidget(
|
|
867
|
+
pipeline_to_decomposition(pipeline),
|
|
868
|
+
{
|
|
869
|
+
"id": pipeline.name or "pipeline",
|
|
870
|
+
"name": object_name,
|
|
871
|
+
"displayName": object_name,
|
|
872
|
+
},
|
|
873
|
+
height=height,
|
|
874
|
+
theme=theme,
|
|
875
|
+
)
|
|
876
|
+
|
|
877
|
+
if isinstance(pipeline, str):
|
|
878
|
+
record = self._daemon.get_object(
|
|
879
|
+
pipeline,
|
|
880
|
+
version=version,
|
|
881
|
+
include_yaml=True,
|
|
882
|
+
)
|
|
883
|
+
decomposition = record.get("decomposition") or self.decomposition(
|
|
884
|
+
pipeline,
|
|
885
|
+
version=version,
|
|
886
|
+
)
|
|
887
|
+
if not decomposition.get("nodes"):
|
|
888
|
+
raise ValueError(f"object is not a pipeline or has no nodes: {pipeline}")
|
|
889
|
+
object_name = title or record.get("display_name") or record.get("name") or pipeline
|
|
890
|
+
return PipelineGraphWidget(
|
|
891
|
+
decomposition,
|
|
892
|
+
{
|
|
893
|
+
**record,
|
|
894
|
+
"id": record.get("id") or pipeline,
|
|
895
|
+
"name": record.get("name") or pipeline,
|
|
896
|
+
"displayName": object_name,
|
|
897
|
+
},
|
|
898
|
+
height=height,
|
|
899
|
+
theme=theme,
|
|
900
|
+
)
|
|
901
|
+
|
|
902
|
+
raise TypeError(
|
|
903
|
+
"pipeline_widget expects an object name, PublishedObject, "
|
|
904
|
+
"spl.core Pipeline, or NodeRemote"
|
|
905
|
+
)
|
|
906
|
+
|
|
907
|
+
def draw_pipeline(
|
|
908
|
+
self,
|
|
909
|
+
pipeline: Any,
|
|
910
|
+
*,
|
|
911
|
+
version: int | None = None,
|
|
912
|
+
title: str | None = None,
|
|
913
|
+
height: int = 560,
|
|
914
|
+
theme: str = "dark",
|
|
915
|
+
) -> Any:
|
|
916
|
+
"""Alias for ``pipeline_widget`` with a notebook-oriented name."""
|
|
917
|
+
|
|
918
|
+
return self.pipeline_widget(
|
|
919
|
+
pipeline,
|
|
920
|
+
version=version,
|
|
921
|
+
title=title,
|
|
922
|
+
height=height,
|
|
923
|
+
theme=theme,
|
|
924
|
+
)
|
|
925
|
+
|
|
926
|
+
def describe(
|
|
927
|
+
self,
|
|
928
|
+
name: str,
|
|
929
|
+
*,
|
|
930
|
+
version: int | None = None,
|
|
931
|
+
owner: str | None = None,
|
|
932
|
+
library: str | None = None,
|
|
933
|
+
function: str | None = None,
|
|
934
|
+
) -> str:
|
|
935
|
+
"""Return a readable object description for notebooks and logs."""
|
|
936
|
+
|
|
937
|
+
signature = self.signature(
|
|
938
|
+
name,
|
|
939
|
+
version=version,
|
|
940
|
+
owner=owner,
|
|
941
|
+
library=library,
|
|
942
|
+
function=function,
|
|
943
|
+
)
|
|
944
|
+
display_name = signature.get("display_name") or signature["name"]
|
|
945
|
+
lines = [
|
|
946
|
+
(
|
|
947
|
+
f"{display_name} "
|
|
948
|
+
f"v{signature['version']} ({signature['kind']})"
|
|
949
|
+
)
|
|
950
|
+
]
|
|
951
|
+
if signature.get("description"):
|
|
952
|
+
lines.append(signature["description"])
|
|
953
|
+
|
|
954
|
+
if (
|
|
955
|
+
function is None
|
|
956
|
+
and signature.get("kind") == "pipeline"
|
|
957
|
+
and signature.get("internal_functions")
|
|
958
|
+
):
|
|
959
|
+
lines.append("Functions:")
|
|
960
|
+
for item in signature["internal_functions"]:
|
|
961
|
+
lines.append(f" - {item['name']}")
|
|
962
|
+
|
|
963
|
+
lines.append("Inputs:")
|
|
964
|
+
if signature["inputs"]:
|
|
965
|
+
for item in signature["inputs"]:
|
|
966
|
+
required = "required" if item["required"] else "optional"
|
|
967
|
+
default = (
|
|
968
|
+
""
|
|
969
|
+
if item["default"] is None
|
|
970
|
+
else f", default={item['default']}"
|
|
971
|
+
)
|
|
972
|
+
lines.append(
|
|
973
|
+
f" - {item['name']}: {item['type'] or 'Any'} "
|
|
974
|
+
f"({required}{default})"
|
|
975
|
+
)
|
|
976
|
+
else:
|
|
977
|
+
lines.append(" - none")
|
|
978
|
+
|
|
979
|
+
lines.append("Outputs:")
|
|
980
|
+
if signature["outputs"]:
|
|
981
|
+
for item in signature["outputs"]:
|
|
982
|
+
selector = (
|
|
983
|
+
f'output="{item["selector"]}"'
|
|
984
|
+
if item["selector"] is not None
|
|
985
|
+
else "no output selector"
|
|
986
|
+
)
|
|
987
|
+
lines.append(
|
|
988
|
+
f" - {item['name']}: {selector}; read {item['read']}"
|
|
989
|
+
)
|
|
990
|
+
else:
|
|
991
|
+
lines.append(" - none")
|
|
992
|
+
|
|
993
|
+
lines.append(f"Example: {signature['call']['example']}")
|
|
994
|
+
lines.append(f"Read: {signature['call']['read']}")
|
|
995
|
+
return "\n".join(lines)
|
|
996
|
+
|
|
997
|
+
def envs(self) -> dict[str, Any]:
|
|
998
|
+
"""Return registered daemon environments."""
|
|
999
|
+
|
|
1000
|
+
return self._daemon.list_envs()
|
|
1001
|
+
|
|
1002
|
+
def environment_builds(self) -> list[dict[str, Any]]:
|
|
1003
|
+
"""Return cached daemon venv builds."""
|
|
1004
|
+
|
|
1005
|
+
return self._daemon.list_environment_builds()
|
|
1006
|
+
|
|
1007
|
+
def rebuild_environment(
|
|
1008
|
+
self,
|
|
1009
|
+
spec_hash: str,
|
|
1010
|
+
*,
|
|
1011
|
+
wait: bool = False,
|
|
1012
|
+
) -> dict[str, Any]:
|
|
1013
|
+
"""Force a cached daemon venv build to be recreated."""
|
|
1014
|
+
|
|
1015
|
+
return self._daemon.rebuild_environment_build(spec_hash, wait=wait)
|
|
1016
|
+
|
|
1017
|
+
def runs(self) -> list[dict[str, Any]]:
|
|
1018
|
+
"""Return known daemon runs, newest first."""
|
|
1019
|
+
|
|
1020
|
+
return self._daemon.list_runs()
|
|
1021
|
+
|
|
1022
|
+
def start(
|
|
1023
|
+
self,
|
|
1024
|
+
name: str,
|
|
1025
|
+
*,
|
|
1026
|
+
args: list[Any] | None = None,
|
|
1027
|
+
kwargs: dict[str, Any] | None = None,
|
|
1028
|
+
output: str | None = None,
|
|
1029
|
+
timeout_seconds: float | None = None,
|
|
1030
|
+
target_machine: str | None = None,
|
|
1031
|
+
owner: str | None = None,
|
|
1032
|
+
library: str | None = None,
|
|
1033
|
+
offline_policy: OfflinePolicy | None = None,
|
|
1034
|
+
function: str | None = None,
|
|
1035
|
+
source: RunSource = "auto",
|
|
1036
|
+
) -> RemoteRun:
|
|
1037
|
+
"""Start a run and return a handle immediately.
|
|
1038
|
+
|
|
1039
|
+
The default path is local daemon execution. Passing ``target_machine``,
|
|
1040
|
+
``owner``, or ``library`` intentionally selects central-server remote
|
|
1041
|
+
execution through the connected daemon.
|
|
1042
|
+
"""
|
|
1043
|
+
|
|
1044
|
+
remote = target_machine is not None or owner is not None or library is not None
|
|
1045
|
+
state = self._daemon.run(
|
|
1046
|
+
name,
|
|
1047
|
+
args=args,
|
|
1048
|
+
kwargs=kwargs,
|
|
1049
|
+
output=output,
|
|
1050
|
+
timeout_seconds=timeout_seconds,
|
|
1051
|
+
target_machine=target_machine,
|
|
1052
|
+
object_owner_id=owner,
|
|
1053
|
+
library=library,
|
|
1054
|
+
offline_policy=offline_policy,
|
|
1055
|
+
function=function,
|
|
1056
|
+
source=source,
|
|
1057
|
+
remote=remote or None,
|
|
1058
|
+
)
|
|
1059
|
+
return RemoteRun(self, state, server_side=remote)
|
|
1060
|
+
|
|
1061
|
+
def queue(
|
|
1062
|
+
self,
|
|
1063
|
+
name: str,
|
|
1064
|
+
*,
|
|
1065
|
+
args: list[Any] | None = None,
|
|
1066
|
+
kwargs: dict[str, Any] | None = None,
|
|
1067
|
+
output: str | None = None,
|
|
1068
|
+
timeout_seconds: float | None = None,
|
|
1069
|
+
target_machine: str,
|
|
1070
|
+
owner: str | None = None,
|
|
1071
|
+
library: str | None = None,
|
|
1072
|
+
function: str | None = None,
|
|
1073
|
+
source: RunSource = "auto",
|
|
1074
|
+
) -> RemoteRun:
|
|
1075
|
+
"""Queue a server-side run and return its task handle without waiting."""
|
|
1076
|
+
|
|
1077
|
+
return self.start(
|
|
1078
|
+
name,
|
|
1079
|
+
args=args,
|
|
1080
|
+
kwargs=kwargs,
|
|
1081
|
+
output=output,
|
|
1082
|
+
timeout_seconds=timeout_seconds,
|
|
1083
|
+
target_machine=target_machine,
|
|
1084
|
+
owner=owner,
|
|
1085
|
+
library=library,
|
|
1086
|
+
function=function,
|
|
1087
|
+
offline_policy="queue",
|
|
1088
|
+
source=source,
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
def call(
|
|
1092
|
+
self,
|
|
1093
|
+
name: str,
|
|
1094
|
+
*,
|
|
1095
|
+
args: list[Any] | None = None,
|
|
1096
|
+
kwargs: dict[str, Any] | None = None,
|
|
1097
|
+
output: str | None = None,
|
|
1098
|
+
timeout_seconds: float | None = None,
|
|
1099
|
+
artifacts_dir: str | Path | None = None,
|
|
1100
|
+
target_machine: str | None = None,
|
|
1101
|
+
owner: str | None = None,
|
|
1102
|
+
library: str | None = None,
|
|
1103
|
+
offline_policy: OfflinePolicy | None = None,
|
|
1104
|
+
function: str | None = None,
|
|
1105
|
+
source: RunSource = "auto",
|
|
1106
|
+
) -> RemoteResult:
|
|
1107
|
+
"""Run an object, wait for completion, and return result/artifacts.
|
|
1108
|
+
|
|
1109
|
+
With only ``name``/``args``/``kwargs`` this is a local daemon worker
|
|
1110
|
+
call. Passing ``target_machine``, ``owner``, or ``library`` makes it a
|
|
1111
|
+
server-side remote run through the daemon. The returned
|
|
1112
|
+
``RemoteResult.mode`` is therefore either ``"local"`` or ``"server"``.
|
|
1113
|
+
"""
|
|
1114
|
+
|
|
1115
|
+
run = self.start(
|
|
1116
|
+
name,
|
|
1117
|
+
args=args,
|
|
1118
|
+
kwargs=kwargs,
|
|
1119
|
+
output=output,
|
|
1120
|
+
timeout_seconds=timeout_seconds,
|
|
1121
|
+
target_machine=target_machine,
|
|
1122
|
+
owner=owner,
|
|
1123
|
+
library=library,
|
|
1124
|
+
offline_policy=offline_policy,
|
|
1125
|
+
function=function,
|
|
1126
|
+
source=source,
|
|
1127
|
+
)
|
|
1128
|
+
return run.collect(
|
|
1129
|
+
artifacts_dir=artifacts_dir,
|
|
1130
|
+
timeout_seconds=timeout_seconds,
|
|
1131
|
+
)
|
|
1132
|
+
|
|
1133
|
+
def run_node(
|
|
1134
|
+
self,
|
|
1135
|
+
node: Any,
|
|
1136
|
+
kwargs: dict[str, Any],
|
|
1137
|
+
*,
|
|
1138
|
+
timeout_seconds: float | None = None,
|
|
1139
|
+
) -> Any:
|
|
1140
|
+
"""Run a ``NodeRemote`` through the local daemon and central server."""
|
|
1141
|
+
|
|
1142
|
+
payload = self._remote_node_payload(node)
|
|
1143
|
+
response = self._daemon.run_remote_node(
|
|
1144
|
+
payload,
|
|
1145
|
+
kwargs=kwargs,
|
|
1146
|
+
timeout_seconds=timeout_seconds,
|
|
1147
|
+
)
|
|
1148
|
+
return response.get("value")
|
|
1149
|
+
|
|
1150
|
+
def run_node_result(
|
|
1151
|
+
self,
|
|
1152
|
+
node: Any,
|
|
1153
|
+
*,
|
|
1154
|
+
kwargs: dict[str, Any] | None = None,
|
|
1155
|
+
timeout_seconds: float | None = None,
|
|
1156
|
+
) -> RemoteResult:
|
|
1157
|
+
"""Run a ``NodeRemote`` and return run metadata plus the selected value."""
|
|
1158
|
+
|
|
1159
|
+
payload = self._remote_node_payload(node)
|
|
1160
|
+
response = self._daemon.run_remote_node(
|
|
1161
|
+
payload,
|
|
1162
|
+
kwargs=kwargs or {},
|
|
1163
|
+
timeout_seconds=timeout_seconds,
|
|
1164
|
+
)
|
|
1165
|
+
value = response.get("value")
|
|
1166
|
+
raw_payload = response.get("payload")
|
|
1167
|
+
result_payload = dict(raw_payload) if isinstance(raw_payload, dict) else {}
|
|
1168
|
+
result_payload["result"] = value
|
|
1169
|
+
result_payload.setdefault("artifacts", response.get("artifacts") or {})
|
|
1170
|
+
|
|
1171
|
+
run = response.get("run")
|
|
1172
|
+
if not isinstance(run, dict):
|
|
1173
|
+
run = {
|
|
1174
|
+
"id": response.get("run_id"),
|
|
1175
|
+
"status": response.get("status") or "succeeded",
|
|
1176
|
+
}
|
|
1177
|
+
return RemoteResult(
|
|
1178
|
+
run=run,
|
|
1179
|
+
payload=result_payload,
|
|
1180
|
+
mode="server",
|
|
1181
|
+
downloaded_artifacts={},
|
|
1182
|
+
)
|
|
1183
|
+
|
|
1184
|
+
def _is_node_remote(self, value: Any) -> bool:
|
|
1185
|
+
try:
|
|
1186
|
+
from spl.core.entities.node_remote import NodeRemote
|
|
1187
|
+
except Exception:
|
|
1188
|
+
return False
|
|
1189
|
+
return isinstance(value, NodeRemote)
|
|
1190
|
+
|
|
1191
|
+
def _remote_node_payload(
|
|
1192
|
+
self,
|
|
1193
|
+
node: Any,
|
|
1194
|
+
*,
|
|
1195
|
+
version: int | str | None = None,
|
|
1196
|
+
) -> dict[str, Any]:
|
|
1197
|
+
payload = {
|
|
1198
|
+
"uuid": str(node.uuid),
|
|
1199
|
+
"url": getattr(node, "url", ""),
|
|
1200
|
+
"name": node.name,
|
|
1201
|
+
"version": node.version if version is None else version,
|
|
1202
|
+
}
|
|
1203
|
+
for attr in ("target_machine", "owner_id", "library"):
|
|
1204
|
+
value = getattr(node, attr, None)
|
|
1205
|
+
if value is not None:
|
|
1206
|
+
payload[attr] = value
|
|
1207
|
+
return payload
|
|
1208
|
+
|
|
1209
|
+
def _remote_decomposition_response(
|
|
1210
|
+
self,
|
|
1211
|
+
remote: Any,
|
|
1212
|
+
*,
|
|
1213
|
+
version: int | None = None,
|
|
1214
|
+
) -> dict[str, Any]:
|
|
1215
|
+
ref = (
|
|
1216
|
+
self._remote_node_payload(remote, version=version)
|
|
1217
|
+
if self._is_node_remote(remote)
|
|
1218
|
+
else dict(remote)
|
|
1219
|
+
)
|
|
1220
|
+
if version is not None:
|
|
1221
|
+
ref["version"] = version
|
|
1222
|
+
return self._daemon.resolve_remote_decomposition(ref)
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
def export_object_to_yaml(
|
|
1226
|
+
obj: Any,
|
|
1227
|
+
entrypoint: str | None = None,
|
|
1228
|
+
*,
|
|
1229
|
+
frame_offset: int = 3,
|
|
1230
|
+
) -> tuple[str, str]:
|
|
1231
|
+
"""Serialize a live SPL object to YAML text.
|
|
1232
|
+
|
|
1233
|
+
The existing core exporter writes to a file and assumes it was called
|
|
1234
|
+
directly from the user's module/notebook. This helper uses the same core IR
|
|
1235
|
+
utilities with an explicit frame offset. That keeps notebook-defined
|
|
1236
|
+
functions working without changing ``spl.core``.
|
|
1237
|
+
"""
|
|
1238
|
+
|
|
1239
|
+
export_obj, resolved_entrypoint = prepare_export_object(obj, entrypoint)
|
|
1240
|
+
return export_objects_to_yaml([export_obj], frame_offset=frame_offset), resolved_entrypoint
|
|
1241
|
+
|
|
1242
|
+
|
|
1243
|
+
def read_yaml_input(yaml: str | Path) -> str:
|
|
1244
|
+
"""Read YAML text from a path-like value or return raw YAML text.
|
|
1245
|
+
|
|
1246
|
+
Notebook examples often use ``Path('spl/demo/_bundle.yaml')``. Shell-style
|
|
1247
|
+
snippets often use the same path as a string. Supporting both keeps the
|
|
1248
|
+
user API small without adding a separate ``publish_yaml_file`` method.
|
|
1249
|
+
"""
|
|
1250
|
+
|
|
1251
|
+
if isinstance(yaml, Path):
|
|
1252
|
+
return yaml.read_text(encoding="utf-8")
|
|
1253
|
+
|
|
1254
|
+
possible_path = Path(yaml)
|
|
1255
|
+
if "\n" not in yaml and possible_path.exists():
|
|
1256
|
+
return possible_path.read_text(encoding="utf-8")
|
|
1257
|
+
|
|
1258
|
+
return yaml
|
|
1259
|
+
|
|
1260
|
+
|
|
1261
|
+
def build_runtime_config(
|
|
1262
|
+
runtime_config: dict[str, Any] | str | Path | None = None,
|
|
1263
|
+
*,
|
|
1264
|
+
runtime: str | None = None,
|
|
1265
|
+
python: str | None = None,
|
|
1266
|
+
base_image: str | None = None,
|
|
1267
|
+
) -> dict[str, Any] | None:
|
|
1268
|
+
"""Build a daemon runtime config from explicit options or a sidecar file."""
|
|
1269
|
+
|
|
1270
|
+
config: dict[str, Any]
|
|
1271
|
+
if runtime_config is None:
|
|
1272
|
+
config = {}
|
|
1273
|
+
elif isinstance(runtime_config, dict):
|
|
1274
|
+
config = dict(runtime_config)
|
|
1275
|
+
else:
|
|
1276
|
+
import yaml
|
|
1277
|
+
|
|
1278
|
+
loaded = yaml.safe_load(Path(runtime_config).read_text(encoding="utf-8"))
|
|
1279
|
+
if loaded is None:
|
|
1280
|
+
config = {}
|
|
1281
|
+
elif isinstance(loaded, dict):
|
|
1282
|
+
config = loaded
|
|
1283
|
+
else:
|
|
1284
|
+
raise ValueError("runtime_config file must contain a YAML mapping")
|
|
1285
|
+
|
|
1286
|
+
if "runtime" in config and isinstance(config["runtime"], dict):
|
|
1287
|
+
target = dict(config["runtime"])
|
|
1288
|
+
config = {"runtime": target}
|
|
1289
|
+
else:
|
|
1290
|
+
target = config
|
|
1291
|
+
|
|
1292
|
+
if runtime is not None:
|
|
1293
|
+
target["mode"] = runtime
|
|
1294
|
+
if python is not None:
|
|
1295
|
+
target["python"] = python
|
|
1296
|
+
if base_image is not None:
|
|
1297
|
+
target["base_image"] = base_image
|
|
1298
|
+
|
|
1299
|
+
if not config and runtime is None and python is None and base_image is None:
|
|
1300
|
+
return None
|
|
1301
|
+
return config
|
|
1302
|
+
|
|
1303
|
+
|
|
1304
|
+
def export_objects_to_yaml(xs: list[Any], *, frame_offset: int = 2) -> str:
|
|
1305
|
+
"""Serialize SPL objects to one YAML bundle.
|
|
1306
|
+
|
|
1307
|
+
``frame_offset`` is passed to the existing dependency scanner. Use ``2``
|
|
1308
|
+
when this helper is called directly by user code, and ``3`` when it is
|
|
1309
|
+
called through ``SPLClient.publish``. This mirrors the hard-coded offset in
|
|
1310
|
+
``spl.core.ir.utils.spl_export_to_file`` while allowing this client wrapper
|
|
1311
|
+
to stay compatible with notebook globals such as ``np``, ``sympy`` and
|
|
1312
|
+
``XGBRegressor``.
|
|
1313
|
+
"""
|
|
1314
|
+
|
|
1315
|
+
import yaml
|
|
1316
|
+
|
|
1317
|
+
from spl.core.entities.control import DSPLSelfImport
|
|
1318
|
+
from spl.core.ir.parse import get_top_level_deps
|
|
1319
|
+
|
|
1320
|
+
top_level_deps = get_top_level_deps(frame_offset, xs)
|
|
1321
|
+
|
|
1322
|
+
mapping = {
|
|
1323
|
+
root: DSPLSelfImport(name=cast(Any, root).name)
|
|
1324
|
+
for (root, _) in top_level_deps
|
|
1325
|
+
if hasattr(root, "name")
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
normalized_deps = {
|
|
1329
|
+
root: [mapping.get(dependency, dependency) for dependency in dependencies]
|
|
1330
|
+
for root, dependencies in top_level_deps
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
return yaml.dump_all(
|
|
1334
|
+
[[root, *dependencies] for root, dependencies in normalized_deps.items()],
|
|
1335
|
+
sort_keys=False,
|
|
1336
|
+
allow_unicode=True,
|
|
1337
|
+
)
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
def prepare_export_object(obj: Any, entrypoint: str | None) -> tuple[Any, str]:
|
|
1341
|
+
"""Return an object ready for core export and the exported entrypoint name."""
|
|
1342
|
+
|
|
1343
|
+
from spl.core.entities.pipeline import Pipeline
|
|
1344
|
+
|
|
1345
|
+
if isinstance(obj, Pipeline):
|
|
1346
|
+
if entrypoint is None:
|
|
1347
|
+
if obj.name is None:
|
|
1348
|
+
raise ValueError(
|
|
1349
|
+
"unnamed pipeline requires entrypoint; "
|
|
1350
|
+
"use pipeline.render(name) or publish(..., entrypoint='name')"
|
|
1351
|
+
)
|
|
1352
|
+
return obj, obj.name
|
|
1353
|
+
return replace(obj, name=entrypoint), entrypoint
|
|
1354
|
+
|
|
1355
|
+
if callable(obj) and hasattr(obj, "__name__"):
|
|
1356
|
+
function_name = obj.__name__
|
|
1357
|
+
if entrypoint is not None and entrypoint != function_name:
|
|
1358
|
+
raise ValueError(
|
|
1359
|
+
"function entrypoint must match function.__name__; "
|
|
1360
|
+
"use publish(..., name='daemon_alias') for daemon aliases"
|
|
1361
|
+
)
|
|
1362
|
+
return obj, function_name
|
|
1363
|
+
|
|
1364
|
+
raise TypeError("SPL client can publish a Python function or spl.core Pipeline")
|