dlt-runtime 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dlt_runtime/__init__.py +1 -0
- dlt_runtime/__plugin__.py +21 -0
- dlt_runtime/_runtime_command.py +1561 -0
- dlt_runtime/commands.py +516 -0
- dlt_runtime/exceptions.py +23 -0
- dlt_runtime/py.typed +0 -0
- dlt_runtime/runtime.py +206 -0
- dlt_runtime/runtime_clients/api/__init__.py +8 -0
- dlt_runtime/runtime_clients/api/api/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/configurations/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/configurations/create_configuration.py +274 -0
- dlt_runtime/runtime_clients/api/api/configurations/get_configuration.py +260 -0
- dlt_runtime/runtime_clients/api/api/configurations/get_latest_configuration.py +246 -0
- dlt_runtime/runtime_clients/api/api/configurations/list_configurations.py +286 -0
- dlt_runtime/runtime_clients/api/api/default/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/default/ping.py +147 -0
- dlt_runtime/runtime_clients/api/api/deployments/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/deployments/create_deployment.py +270 -0
- dlt_runtime/runtime_clients/api/api/deployments/get_deployment.py +260 -0
- dlt_runtime/runtime_clients/api/api/deployments/get_latest_deployment.py +246 -0
- dlt_runtime/runtime_clients/api/api/deployments/list_deployments.py +286 -0
- dlt_runtime/runtime_clients/api/api/me/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/me/me.py +169 -0
- dlt_runtime/runtime_clients/api/api/organizations/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/organizations/get_organization.py +246 -0
- dlt_runtime/runtime_clients/api/api/runs/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/runs/cancel_run.py +260 -0
- dlt_runtime/runtime_clients/api/api/runs/create_run.py +316 -0
- dlt_runtime/runtime_clients/api/api/runs/get_latest_run.py +274 -0
- dlt_runtime/runtime_clients/api/api/runs/get_run.py +288 -0
- dlt_runtime/runtime_clients/api/api/runs/get_run_logs.py +288 -0
- dlt_runtime/runtime_clients/api/api/runs/list_runs.py +331 -0
- dlt_runtime/runtime_clients/api/api/scripts/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/scripts/create_or_update_script.py +292 -0
- dlt_runtime/runtime_clients/api/api/scripts/disable_public_url.py +264 -0
- dlt_runtime/runtime_clients/api/api/scripts/enable_public_url.py +268 -0
- dlt_runtime/runtime_clients/api/api/scripts/get_latest_script_version.py +260 -0
- dlt_runtime/runtime_clients/api/api/scripts/get_script.py +260 -0
- dlt_runtime/runtime_clients/api/api/scripts/get_script_version.py +274 -0
- dlt_runtime/runtime_clients/api/api/scripts/list_script_versions.py +300 -0
- dlt_runtime/runtime_clients/api/api/scripts/list_scripts.py +282 -0
- dlt_runtime/runtime_clients/api/api/scripts/update_script.py +294 -0
- dlt_runtime/runtime_clients/api/api/workspaces/__init__.py +1 -0
- dlt_runtime/runtime_clients/api/api/workspaces/get_workspace.py +246 -0
- dlt_runtime/runtime_clients/api/client.py +286 -0
- dlt_runtime/runtime_clients/api/errors.py +16 -0
- dlt_runtime/runtime_clients/api/models/__init__.py +73 -0
- dlt_runtime/runtime_clients/api/models/configuration_response.py +145 -0
- dlt_runtime/runtime_clients/api/models/create_configuration_body.py +83 -0
- dlt_runtime/runtime_clients/api/models/create_deployment_body.py +83 -0
- dlt_runtime/runtime_clients/api/models/create_run_request.py +106 -0
- dlt_runtime/runtime_clients/api/models/create_script_request.py +142 -0
- dlt_runtime/runtime_clients/api/models/deployment_response.py +137 -0
- dlt_runtime/runtime_clients/api/models/detailed_run_response.py +308 -0
- dlt_runtime/runtime_clients/api/models/detailed_script_response.py +325 -0
- dlt_runtime/runtime_clients/api/models/error_response_400.py +97 -0
- dlt_runtime/runtime_clients/api/models/error_response_400_extra.py +44 -0
- dlt_runtime/runtime_clients/api/models/error_response_401.py +97 -0
- dlt_runtime/runtime_clients/api/models/error_response_401_extra.py +44 -0
- dlt_runtime/runtime_clients/api/models/error_response_403.py +97 -0
- dlt_runtime/runtime_clients/api/models/error_response_403_extra.py +44 -0
- dlt_runtime/runtime_clients/api/models/error_response_404.py +97 -0
- dlt_runtime/runtime_clients/api/models/error_response_404_extra.py +44 -0
- dlt_runtime/runtime_clients/api/models/list_configurations_response_200.py +107 -0
- dlt_runtime/runtime_clients/api/models/list_deployments_response_200.py +107 -0
- dlt_runtime/runtime_clients/api/models/list_runs_response_200.py +107 -0
- dlt_runtime/runtime_clients/api/models/list_script_versions_response_200.py +107 -0
- dlt_runtime/runtime_clients/api/models/list_scripts_response_200.py +107 -0
- dlt_runtime/runtime_clients/api/models/logs_response.py +93 -0
- dlt_runtime/runtime_clients/api/models/me_response.py +106 -0
- dlt_runtime/runtime_clients/api/models/organization_response.py +114 -0
- dlt_runtime/runtime_clients/api/models/ping_response.py +67 -0
- dlt_runtime/runtime_clients/api/models/run_mode.py +9 -0
- dlt_runtime/runtime_clients/api/models/run_response.py +293 -0
- dlt_runtime/runtime_clients/api/models/run_status.py +16 -0
- dlt_runtime/runtime_clients/api/models/run_trigger_type.py +9 -0
- dlt_runtime/runtime_clients/api/models/script_response.py +286 -0
- dlt_runtime/runtime_clients/api/models/script_type.py +9 -0
- dlt_runtime/runtime_clients/api/models/script_version_response.py +192 -0
- dlt_runtime/runtime_clients/api/models/update_script_request.py +178 -0
- dlt_runtime/runtime_clients/api/models/workspace_response.py +114 -0
- dlt_runtime/runtime_clients/api/types.py +54 -0
- dlt_runtime/runtime_clients/auth/__init__.py +8 -0
- dlt_runtime/runtime_clients/auth/api/__init__.py +1 -0
- dlt_runtime/runtime_clients/auth/api/default/__init__.py +1 -0
- dlt_runtime/runtime_clients/auth/api/default/ping.py +127 -0
- dlt_runtime/runtime_clients/auth/api/github/__init__.py +1 -0
- dlt_runtime/runtime_clients/auth/api/github/github_oauth_complete.py +166 -0
- dlt_runtime/runtime_clients/auth/api/github/github_oauth_exchange.py +166 -0
- dlt_runtime/runtime_clients/auth/api/github/github_oauth_start.py +170 -0
- dlt_runtime/runtime_clients/auth/client.py +286 -0
- dlt_runtime/runtime_clients/auth/errors.py +16 -0
- dlt_runtime/runtime_clients/auth/models/__init__.py +19 -0
- dlt_runtime/runtime_clients/auth/models/error_response_400.py +97 -0
- dlt_runtime/runtime_clients/auth/models/error_response_400_extra.py +44 -0
- dlt_runtime/runtime_clients/auth/models/github_device_flow_login_request.py +59 -0
- dlt_runtime/runtime_clients/auth/models/github_device_flow_start_response.py +83 -0
- dlt_runtime/runtime_clients/auth/models/github_oauth_exchange_request.py +59 -0
- dlt_runtime/runtime_clients/auth/models/login_response.py +76 -0
- dlt_runtime/runtime_clients/auth/models/ping_response.py +67 -0
- dlt_runtime/runtime_clients/auth/types.py +54 -0
- dlt_runtime/version.py +6 -0
- dlt_runtime-0.20.0.dist-info/METADATA +66 -0
- dlt_runtime-0.20.0.dist-info/RECORD +107 -0
- dlt_runtime-0.20.0.dist-info/WHEEL +4 -0
- dlt_runtime-0.20.0.dist-info/entry_points.txt +2 -0
- dlt_runtime-0.20.0.dist-info/licenses/LICENSE.txt +203 -0
|
@@ -0,0 +1,1561 @@
|
|
|
1
|
+
# Python internals
|
|
2
|
+
import argparse
|
|
3
|
+
import time
|
|
4
|
+
import webbrowser
|
|
5
|
+
from functools import partial
|
|
6
|
+
from io import BytesIO
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Optional, Set, Union
|
|
9
|
+
from uuid import UUID
|
|
10
|
+
|
|
11
|
+
# Other libraries
|
|
12
|
+
from cron_descriptor import FormatException, get_description
|
|
13
|
+
from dlt._workspace._workspace_context import active
|
|
14
|
+
from dlt._workspace.cli import echo as fmt
|
|
15
|
+
from dlt._workspace.cli.exceptions import CliCommandInnerException
|
|
16
|
+
from dlt._workspace.cli.utils import track_command as dlt_track_command
|
|
17
|
+
from dlt._workspace.deployment.file_selector import (
|
|
18
|
+
ConfigurationFileSelector,
|
|
19
|
+
WorkspaceFileSelector,
|
|
20
|
+
)
|
|
21
|
+
from dlt._workspace.deployment.package_builder import PackageBuilder
|
|
22
|
+
from dlt.common.json import json
|
|
23
|
+
from tabulate import tabulate
|
|
24
|
+
|
|
25
|
+
from dlt_runtime.exceptions import (
|
|
26
|
+
LocalWorkspaceIdNotSet,
|
|
27
|
+
RuntimeNotAuthenticated,
|
|
28
|
+
WorkspaceIdMismatch,
|
|
29
|
+
)
|
|
30
|
+
from dlt_runtime.runtime import RuntimeAuthService, get_auth_client
|
|
31
|
+
from dlt_runtime.runtime_clients.api.api.configurations import (
|
|
32
|
+
create_configuration,
|
|
33
|
+
get_configuration,
|
|
34
|
+
get_latest_configuration,
|
|
35
|
+
list_configurations,
|
|
36
|
+
)
|
|
37
|
+
from dlt_runtime.runtime_clients.api.api.deployments import (
|
|
38
|
+
create_deployment,
|
|
39
|
+
get_deployment,
|
|
40
|
+
get_latest_deployment,
|
|
41
|
+
list_deployments,
|
|
42
|
+
)
|
|
43
|
+
from dlt_runtime.runtime_clients.api.api.runs import (
|
|
44
|
+
cancel_run,
|
|
45
|
+
create_run,
|
|
46
|
+
get_run,
|
|
47
|
+
get_run_logs,
|
|
48
|
+
list_runs,
|
|
49
|
+
)
|
|
50
|
+
from dlt_runtime.runtime_clients.api.api.scripts import (
|
|
51
|
+
create_or_update_script,
|
|
52
|
+
disable_public_url,
|
|
53
|
+
enable_public_url,
|
|
54
|
+
get_script,
|
|
55
|
+
list_scripts,
|
|
56
|
+
)
|
|
57
|
+
from dlt_runtime.runtime_clients.api.client import Client as ApiClient
|
|
58
|
+
from dlt_runtime.runtime_clients.api.models.create_deployment_body import (
|
|
59
|
+
CreateDeploymentBody,
|
|
60
|
+
)
|
|
61
|
+
from dlt_runtime.runtime_clients.api.models.detailed_run_response import (
|
|
62
|
+
DetailedRunResponse,
|
|
63
|
+
)
|
|
64
|
+
from dlt_runtime.runtime_clients.api.models.run_status import RunStatus
|
|
65
|
+
from dlt_runtime.runtime_clients.api.models.script_type import ScriptType
|
|
66
|
+
from dlt_runtime.runtime_clients.api.types import UNSET, File, Response
|
|
67
|
+
from dlt_runtime.runtime_clients.auth.api.github import (
|
|
68
|
+
github_oauth_complete,
|
|
69
|
+
github_oauth_start,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
DEPLOYMENT_HEADERS = CONFIGURATION_HEADERS = {
|
|
73
|
+
"version": fmt.bold("Version #"),
|
|
74
|
+
"date_added": fmt.bold("Created at"),
|
|
75
|
+
"file_count": fmt.bold("File count"),
|
|
76
|
+
"content_hash": fmt.bold("Content hash"),
|
|
77
|
+
}
|
|
78
|
+
JOB_HEADERS = {
|
|
79
|
+
"name": fmt.bold("Job name"),
|
|
80
|
+
"version": fmt.bold("Version #"),
|
|
81
|
+
"entry_point": fmt.bold("Script path"),
|
|
82
|
+
"date_added": fmt.bold("Created at"),
|
|
83
|
+
"schedule": fmt.bold("Schedule"),
|
|
84
|
+
"script_url": fmt.bold("Script URL"),
|
|
85
|
+
}
|
|
86
|
+
JOB_RUN_HEADERS = {
|
|
87
|
+
"job_name": fmt.bold("Job name"),
|
|
88
|
+
"number": fmt.bold("Run #"),
|
|
89
|
+
"status": fmt.bold("Status"),
|
|
90
|
+
"profile": fmt.bold("Profile"),
|
|
91
|
+
"time_started": fmt.bold("Started at"),
|
|
92
|
+
"time_ended": fmt.bold("Ended at"),
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
track_command = partial(dlt_track_command, "runtime", track_before=False)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _to_uuid(value: Union[str, UUID]) -> UUID:
|
|
100
|
+
if isinstance(value, UUID):
|
|
101
|
+
return value
|
|
102
|
+
try:
|
|
103
|
+
return UUID(value)
|
|
104
|
+
except ValueError:
|
|
105
|
+
raise RuntimeError(f"Invalid UUID: {value}")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _exception_from_response(message: str, response: Response[Any]) -> BaseException:
|
|
109
|
+
status = response.status_code
|
|
110
|
+
try:
|
|
111
|
+
details = json.loads(response.content.decode("utf-8"))["detail"]
|
|
112
|
+
except Exception:
|
|
113
|
+
details = response.content.decode("utf-8")
|
|
114
|
+
|
|
115
|
+
if status < 500:
|
|
116
|
+
message += f". {details.capitalize()} (HTTP {status})"
|
|
117
|
+
return CliCommandInnerException(cmd="runtime", msg=message)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _check_cron_expression(cron_expression: Optional[str]) -> None:
|
|
121
|
+
if cron_expression:
|
|
122
|
+
try:
|
|
123
|
+
get_description(cron_expression)
|
|
124
|
+
except FormatException as exc:
|
|
125
|
+
raise CliCommandInnerException(
|
|
126
|
+
cmd="runtime",
|
|
127
|
+
msg=f"Invalid cron expression: {cron_expression} ({exc})",
|
|
128
|
+
inner_exc=exc,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _extract_keys(data: dict[str, Any], keys_dict: dict[str, str]) -> dict[str, Any]:
|
|
133
|
+
return {key: data[key] for key in keys_dict.keys() if key in data}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@track_command(operation="login")
|
|
137
|
+
def login(minimal_logging: bool = True) -> RuntimeAuthService:
|
|
138
|
+
auth_service = RuntimeAuthService(run_context=active())
|
|
139
|
+
try:
|
|
140
|
+
auth_info = auth_service.authenticate()
|
|
141
|
+
if not minimal_logging:
|
|
142
|
+
fmt.echo("Already logged in as %s" % fmt.bold(auth_info.email))
|
|
143
|
+
_connect(auth_service=auth_service, minimal_logging=minimal_logging)
|
|
144
|
+
return auth_service
|
|
145
|
+
except RuntimeNotAuthenticated:
|
|
146
|
+
client = get_auth_client()
|
|
147
|
+
start_kwargs = {}
|
|
148
|
+
if auth_service.workspace_run_context.runtime_config.invite_code:
|
|
149
|
+
start_kwargs["invite_code"] = (
|
|
150
|
+
auth_service.workspace_run_context.runtime_config.invite_code
|
|
151
|
+
)
|
|
152
|
+
# start device flow
|
|
153
|
+
login_request = github_oauth_start.sync(client=client, **start_kwargs)
|
|
154
|
+
if not isinstance(
|
|
155
|
+
login_request, github_oauth_start.GithubDeviceFlowStartResponse
|
|
156
|
+
):
|
|
157
|
+
raise RuntimeError("Failed to log in with Github OAuth")
|
|
158
|
+
fmt.echo(
|
|
159
|
+
"Logging in with Github OAuth. Please go to %s and enter the code %s"
|
|
160
|
+
% (
|
|
161
|
+
fmt.bold(login_request.verification_uri),
|
|
162
|
+
fmt.bold(login_request.user_code),
|
|
163
|
+
)
|
|
164
|
+
)
|
|
165
|
+
fmt.echo("Waiting for response from Github...")
|
|
166
|
+
|
|
167
|
+
while True:
|
|
168
|
+
time.sleep(login_request.interval)
|
|
169
|
+
token_response = github_oauth_complete.sync(
|
|
170
|
+
client=client,
|
|
171
|
+
body=github_oauth_complete.GithubDeviceFlowLoginRequest(
|
|
172
|
+
device_code=login_request.device_code
|
|
173
|
+
),
|
|
174
|
+
)
|
|
175
|
+
if isinstance(token_response, github_oauth_complete.LoginResponse):
|
|
176
|
+
auth_info = auth_service.login(token_response.jwt)
|
|
177
|
+
fmt.echo("Logged in as %s" % fmt.bold(auth_info.email))
|
|
178
|
+
_connect(auth_service=auth_service)
|
|
179
|
+
return auth_service
|
|
180
|
+
elif isinstance(token_response, github_oauth_complete.ErrorResponse400):
|
|
181
|
+
raise RuntimeError("Failed to complete authentication with Github")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@track_command(operation="logout")
|
|
185
|
+
def logout() -> None:
|
|
186
|
+
auth_service = RuntimeAuthService(run_context=active())
|
|
187
|
+
auth_service.logout()
|
|
188
|
+
fmt.echo("Logged out")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _connect(
|
|
192
|
+
auth_service: Optional[RuntimeAuthService] = None, minimal_logging: bool = False
|
|
193
|
+
) -> None:
|
|
194
|
+
if auth_service is None:
|
|
195
|
+
auth_service = RuntimeAuthService(run_context=active())
|
|
196
|
+
auth_service.authenticate()
|
|
197
|
+
|
|
198
|
+
try:
|
|
199
|
+
auth_service.connect()
|
|
200
|
+
except LocalWorkspaceIdNotSet:
|
|
201
|
+
should_overwrite = fmt.confirm(
|
|
202
|
+
"No workspace id found in local config. Do you want to connect local workspace to the"
|
|
203
|
+
" remote one?",
|
|
204
|
+
default=True,
|
|
205
|
+
)
|
|
206
|
+
if should_overwrite:
|
|
207
|
+
auth_service.overwrite_local_workspace_id()
|
|
208
|
+
fmt.echo("Using remote workspace id")
|
|
209
|
+
else:
|
|
210
|
+
raise RuntimeError("Local workspace is not connected to the remote one")
|
|
211
|
+
except WorkspaceIdMismatch as e:
|
|
212
|
+
fmt.warning(
|
|
213
|
+
"Workspace id in local config (%s) is not the same as remote workspace id (%s)"
|
|
214
|
+
% (e.local_workspace_id, e.remote_workspace_id)
|
|
215
|
+
)
|
|
216
|
+
should_overwrite = fmt.confirm(
|
|
217
|
+
"Do you want to overwrite the local workspace id with the remote one?",
|
|
218
|
+
default=True,
|
|
219
|
+
)
|
|
220
|
+
if should_overwrite:
|
|
221
|
+
auth_service.overwrite_local_workspace_id()
|
|
222
|
+
fmt.echo("Local workspace id overwritten with remote workspace id")
|
|
223
|
+
else:
|
|
224
|
+
raise RuntimeError("Unable to synchronise remote and local workspaces")
|
|
225
|
+
if not minimal_logging:
|
|
226
|
+
fmt.echo("Authorized to workspace %s" % fmt.bold(auth_service.workspace_id))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@track_command(operation="deploy")
|
|
230
|
+
def deploy(*, auth_service: RuntimeAuthService, api_client: ApiClient) -> None:
|
|
231
|
+
_sync_deployment(auth_service=auth_service, api_client=api_client)
|
|
232
|
+
_sync_configuration(auth_service=auth_service, api_client=api_client)
|
|
233
|
+
fmt.echo("Deployment and configuration synchronized successfully")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@track_command(operation="deployment", suboperation="sync")
|
|
237
|
+
def sync_deployment(
|
|
238
|
+
minimal_logging: bool = True,
|
|
239
|
+
*,
|
|
240
|
+
auth_service: RuntimeAuthService,
|
|
241
|
+
api_client: ApiClient,
|
|
242
|
+
) -> None:
|
|
243
|
+
_sync_deployment(
|
|
244
|
+
minimal_logging=minimal_logging,
|
|
245
|
+
auth_service=auth_service,
|
|
246
|
+
api_client=api_client,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _sync_deployment(
|
|
251
|
+
minimal_logging: bool = True,
|
|
252
|
+
*,
|
|
253
|
+
auth_service: RuntimeAuthService,
|
|
254
|
+
api_client: ApiClient,
|
|
255
|
+
) -> None:
|
|
256
|
+
content_stream = BytesIO()
|
|
257
|
+
package_builder = PackageBuilder(context=active())
|
|
258
|
+
package_hash = package_builder.write_package_to_stream(
|
|
259
|
+
file_selector=WorkspaceFileSelector(active()), output_stream=content_stream
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
latest_deployment = get_latest_deployment.sync_detailed(
|
|
263
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
264
|
+
client=api_client,
|
|
265
|
+
)
|
|
266
|
+
if isinstance(latest_deployment.parsed, get_latest_deployment.DeploymentResponse):
|
|
267
|
+
if latest_deployment.parsed.content_hash == package_hash:
|
|
268
|
+
if not minimal_logging:
|
|
269
|
+
fmt.echo("No changes detected in the deployment, skipping file upload")
|
|
270
|
+
content_stream.close()
|
|
271
|
+
return
|
|
272
|
+
elif isinstance(latest_deployment.parsed, get_latest_deployment.ErrorResponse404):
|
|
273
|
+
if not minimal_logging:
|
|
274
|
+
fmt.echo("No deployment found in this workspace, creating new deployment")
|
|
275
|
+
else:
|
|
276
|
+
content_stream.close()
|
|
277
|
+
raise _exception_from_response(
|
|
278
|
+
"Failed to get latest deployment", latest_deployment
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
create_deployment_result = create_deployment.sync_detailed(
|
|
282
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
283
|
+
client=api_client,
|
|
284
|
+
body=CreateDeploymentBody(
|
|
285
|
+
file=File(
|
|
286
|
+
payload=content_stream,
|
|
287
|
+
file_name="workspace.tar.gz",
|
|
288
|
+
mime_type="application/x-tar",
|
|
289
|
+
)
|
|
290
|
+
),
|
|
291
|
+
)
|
|
292
|
+
if isinstance(
|
|
293
|
+
create_deployment_result.parsed, create_deployment.DeploymentResponse
|
|
294
|
+
):
|
|
295
|
+
if not minimal_logging:
|
|
296
|
+
fmt.echo(
|
|
297
|
+
tabulate(
|
|
298
|
+
[
|
|
299
|
+
_extract_keys(
|
|
300
|
+
create_deployment_result.parsed.to_dict(),
|
|
301
|
+
DEPLOYMENT_HEADERS,
|
|
302
|
+
)
|
|
303
|
+
],
|
|
304
|
+
headers=DEPLOYMENT_HEADERS,
|
|
305
|
+
)
|
|
306
|
+
)
|
|
307
|
+
else:
|
|
308
|
+
raise _exception_from_response(
|
|
309
|
+
"Failed to create deployment", create_deployment_result
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
@track_command(operation="configuration", suboperation="sync")
|
|
314
|
+
def sync_configuration(
|
|
315
|
+
minimal_logging: bool = True,
|
|
316
|
+
*,
|
|
317
|
+
auth_service: RuntimeAuthService,
|
|
318
|
+
api_client: ApiClient,
|
|
319
|
+
) -> None:
|
|
320
|
+
_sync_configuration(
|
|
321
|
+
minimal_logging=minimal_logging,
|
|
322
|
+
auth_service=auth_service,
|
|
323
|
+
api_client=api_client,
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _sync_configuration(
|
|
328
|
+
minimal_logging: bool = True,
|
|
329
|
+
*,
|
|
330
|
+
auth_service: RuntimeAuthService,
|
|
331
|
+
api_client: ApiClient,
|
|
332
|
+
) -> None:
|
|
333
|
+
content_stream = BytesIO()
|
|
334
|
+
package_builder = PackageBuilder(context=active())
|
|
335
|
+
package_hash = package_builder.write_package_to_stream(
|
|
336
|
+
file_selector=ConfigurationFileSelector(active()), output_stream=content_stream
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
latest_configuration = get_latest_configuration.sync_detailed(
|
|
340
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
341
|
+
client=api_client,
|
|
342
|
+
)
|
|
343
|
+
if isinstance(
|
|
344
|
+
latest_configuration.parsed, get_latest_configuration.ConfigurationResponse
|
|
345
|
+
):
|
|
346
|
+
if latest_configuration.parsed.content_hash == package_hash:
|
|
347
|
+
if not minimal_logging:
|
|
348
|
+
fmt.echo(
|
|
349
|
+
"No changes detected in the configuration, skipping file upload"
|
|
350
|
+
)
|
|
351
|
+
content_stream.close()
|
|
352
|
+
return
|
|
353
|
+
elif isinstance(
|
|
354
|
+
latest_configuration.parsed, get_latest_configuration.ErrorResponse404
|
|
355
|
+
):
|
|
356
|
+
if not minimal_logging:
|
|
357
|
+
fmt.echo(
|
|
358
|
+
"No configuration found in this workspace, creating new configuration"
|
|
359
|
+
)
|
|
360
|
+
else:
|
|
361
|
+
content_stream.close()
|
|
362
|
+
raise _exception_from_response(
|
|
363
|
+
"Failed to get latest configuration", latest_configuration
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
create_configuration_result = create_configuration.sync_detailed(
|
|
367
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
368
|
+
client=api_client,
|
|
369
|
+
body=create_configuration.CreateConfigurationBody(
|
|
370
|
+
file=File(
|
|
371
|
+
payload=content_stream,
|
|
372
|
+
file_name="configurations.tar.gz",
|
|
373
|
+
mime_type="application/x-tar",
|
|
374
|
+
)
|
|
375
|
+
),
|
|
376
|
+
)
|
|
377
|
+
if isinstance(
|
|
378
|
+
create_configuration_result.parsed, create_configuration.ConfigurationResponse
|
|
379
|
+
):
|
|
380
|
+
if not minimal_logging:
|
|
381
|
+
fmt.echo(
|
|
382
|
+
tabulate(
|
|
383
|
+
[
|
|
384
|
+
_extract_keys(
|
|
385
|
+
create_configuration_result.parsed.to_dict(),
|
|
386
|
+
CONFIGURATION_HEADERS,
|
|
387
|
+
)
|
|
388
|
+
],
|
|
389
|
+
headers=CONFIGURATION_HEADERS,
|
|
390
|
+
)
|
|
391
|
+
)
|
|
392
|
+
else:
|
|
393
|
+
raise _exception_from_response(
|
|
394
|
+
"Failed to create configuration", create_configuration_result
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _preprocess_run_outut(
|
|
399
|
+
run: dict[str, Any], headers: dict[str, str]
|
|
400
|
+
) -> dict[str, Any]:
|
|
401
|
+
result = _extract_keys(run, headers)
|
|
402
|
+
result["job_name"] = run["script"]["name"]
|
|
403
|
+
return {key: result[key] for key in headers.keys() if key in result}
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
@track_command(operation="job-runs", suboperation="info")
|
|
407
|
+
def get_job_run_info(
|
|
408
|
+
script_path_or_job_name: Optional[str] = None,
|
|
409
|
+
run_number: Optional[int] = None,
|
|
410
|
+
*,
|
|
411
|
+
auth_service: RuntimeAuthService,
|
|
412
|
+
api_client: ApiClient,
|
|
413
|
+
) -> None:
|
|
414
|
+
if script_path_or_job_name is None:
|
|
415
|
+
raise CliCommandInnerException(
|
|
416
|
+
cmd="runtime",
|
|
417
|
+
msg="Script path or job name is required",
|
|
418
|
+
)
|
|
419
|
+
if run_number is None:
|
|
420
|
+
run = _get_latest_run(api_client, auth_service, script_path_or_job_name)
|
|
421
|
+
run_id = run.id
|
|
422
|
+
else:
|
|
423
|
+
run_id = _resolve_run_id_by_number(
|
|
424
|
+
api_client=api_client,
|
|
425
|
+
auth_service=auth_service,
|
|
426
|
+
script_path_or_job_name=script_path_or_job_name,
|
|
427
|
+
run_number=run_number,
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
get_run_result = get_run.sync_detailed(
|
|
431
|
+
client=api_client,
|
|
432
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
433
|
+
run_id=_to_uuid(run_id),
|
|
434
|
+
)
|
|
435
|
+
if isinstance(get_run_result.parsed, get_run.DetailedRunResponse):
|
|
436
|
+
fmt.echo(
|
|
437
|
+
tabulate(
|
|
438
|
+
[_extract_keys(get_run_result.parsed.to_dict(), JOB_RUN_HEADERS)],
|
|
439
|
+
headers=JOB_RUN_HEADERS,
|
|
440
|
+
)
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
else:
|
|
444
|
+
raise _exception_from_response("Failed to get run status", get_run_result)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
@track_command(operation="logs")
|
|
448
|
+
def logs(
|
|
449
|
+
script_path_or_job_name: Optional[str] = None,
|
|
450
|
+
run_number: Optional[int] = None,
|
|
451
|
+
follow: bool = False,
|
|
452
|
+
*,
|
|
453
|
+
auth_service: RuntimeAuthService,
|
|
454
|
+
api_client: ApiClient,
|
|
455
|
+
) -> None:
|
|
456
|
+
_fetch_run_logs(
|
|
457
|
+
script_path_or_job_name,
|
|
458
|
+
run_number,
|
|
459
|
+
follow,
|
|
460
|
+
auth_service=auth_service,
|
|
461
|
+
api_client=api_client,
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
@track_command(operation="job-runs", suboperation="logs")
|
|
466
|
+
def job_run_logs(
|
|
467
|
+
script_path_or_job_name: Optional[str] = None,
|
|
468
|
+
run_number: Optional[int] = None,
|
|
469
|
+
follow: bool = False,
|
|
470
|
+
*,
|
|
471
|
+
auth_service: RuntimeAuthService,
|
|
472
|
+
api_client: ApiClient,
|
|
473
|
+
) -> None:
|
|
474
|
+
_fetch_run_logs(
|
|
475
|
+
script_path_or_job_name,
|
|
476
|
+
run_number,
|
|
477
|
+
follow,
|
|
478
|
+
auth_service=auth_service,
|
|
479
|
+
api_client=api_client,
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _fetch_run_logs(
|
|
484
|
+
script_path_or_job_name: Optional[str] = None,
|
|
485
|
+
run_number: Optional[int] = None,
|
|
486
|
+
follow: bool = False,
|
|
487
|
+
*,
|
|
488
|
+
auth_service: RuntimeAuthService,
|
|
489
|
+
api_client: ApiClient,
|
|
490
|
+
) -> None:
|
|
491
|
+
"""Get logs for a run of job (latest if run number not provided)."""
|
|
492
|
+
if script_path_or_job_name is None:
|
|
493
|
+
raise CliCommandInnerException(
|
|
494
|
+
cmd="runtime",
|
|
495
|
+
msg="Script path or job name is required",
|
|
496
|
+
)
|
|
497
|
+
if run_number is None:
|
|
498
|
+
run = _get_latest_run(api_client, auth_service, script_path_or_job_name)
|
|
499
|
+
run_id = run.id
|
|
500
|
+
else:
|
|
501
|
+
run_id = _resolve_run_id_by_number(
|
|
502
|
+
api_client=api_client,
|
|
503
|
+
auth_service=auth_service,
|
|
504
|
+
script_path_or_job_name=script_path_or_job_name,
|
|
505
|
+
run_number=run_number,
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
if follow:
|
|
509
|
+
_follow_job_run(
|
|
510
|
+
run_id,
|
|
511
|
+
{RunStatus.FAILED, RunStatus.CANCELLED, RunStatus.COMPLETED},
|
|
512
|
+
None,
|
|
513
|
+
True,
|
|
514
|
+
auth_service=auth_service,
|
|
515
|
+
api_client=api_client,
|
|
516
|
+
)
|
|
517
|
+
else:
|
|
518
|
+
get_run_logs_result = get_run_logs.sync_detailed(
|
|
519
|
+
client=api_client,
|
|
520
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
521
|
+
run_id=run_id,
|
|
522
|
+
)
|
|
523
|
+
if isinstance(get_run_logs_result.parsed, get_run_logs.LogsResponse):
|
|
524
|
+
run = get_run_logs_result.parsed.run
|
|
525
|
+
run_info = f"Run # {run.number} of job {run.script.name}"
|
|
526
|
+
fmt.echo(f"========== Run logs for {run_info} ==========")
|
|
527
|
+
fmt.echo(get_run_logs_result.parsed.logs)
|
|
528
|
+
fmt.echo(f"========== End of run logs for {run_info} ==========")
|
|
529
|
+
else:
|
|
530
|
+
raise _exception_from_response(
|
|
531
|
+
"Failed to get run logs.", get_run_logs_result
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
@track_command(operation="job-runs", suboperation="list")
|
|
536
|
+
def get_runs(
|
|
537
|
+
script_path_or_job_name: Optional[str] = None,
|
|
538
|
+
*,
|
|
539
|
+
auth_service: RuntimeAuthService,
|
|
540
|
+
api_client: ApiClient,
|
|
541
|
+
) -> None:
|
|
542
|
+
script_id: Optional[UUID] = None
|
|
543
|
+
if script_path_or_job_name:
|
|
544
|
+
script = get_script.sync_detailed(
|
|
545
|
+
client=api_client,
|
|
546
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
547
|
+
script_id_or_name=script_path_or_job_name,
|
|
548
|
+
)
|
|
549
|
+
if isinstance(script.parsed, get_script.DetailedScriptResponse):
|
|
550
|
+
script_id = script.parsed.id
|
|
551
|
+
else:
|
|
552
|
+
raise _exception_from_response(
|
|
553
|
+
f"Failed to get script with name {script_path_or_job_name} from runtime. Did you"
|
|
554
|
+
" create one?",
|
|
555
|
+
script,
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
list_runs_result = list_runs.sync_detailed(
|
|
559
|
+
client=api_client,
|
|
560
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
561
|
+
script_id=script_id,
|
|
562
|
+
)
|
|
563
|
+
if (
|
|
564
|
+
isinstance(list_runs_result.parsed, list_runs.ListRunsResponse200)
|
|
565
|
+
and list_runs_result.parsed.items
|
|
566
|
+
):
|
|
567
|
+
fmt.echo(
|
|
568
|
+
tabulate(
|
|
569
|
+
[
|
|
570
|
+
_preprocess_run_outut(run.to_dict(), JOB_RUN_HEADERS)
|
|
571
|
+
for run in reversed(list_runs_result.parsed.items)
|
|
572
|
+
],
|
|
573
|
+
headers=JOB_RUN_HEADERS,
|
|
574
|
+
)
|
|
575
|
+
)
|
|
576
|
+
else:
|
|
577
|
+
raise _exception_from_response(
|
|
578
|
+
"Failed to list workspace runs", list_runs_result
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
@track_command(operation="deployment", suboperation="list")
|
|
583
|
+
def get_deployments(*, auth_service: RuntimeAuthService, api_client: ApiClient) -> None:
|
|
584
|
+
list_deployments_result = list_deployments.sync_detailed(
|
|
585
|
+
client=api_client,
|
|
586
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
587
|
+
)
|
|
588
|
+
if isinstance(
|
|
589
|
+
list_deployments_result.parsed, list_deployments.ListDeploymentsResponse200
|
|
590
|
+
):
|
|
591
|
+
if not list_deployments_result.parsed.items:
|
|
592
|
+
fmt.echo("No deployments found in this workspace")
|
|
593
|
+
return
|
|
594
|
+
fmt.echo(
|
|
595
|
+
tabulate(
|
|
596
|
+
[
|
|
597
|
+
_extract_keys(deployment.to_dict(), DEPLOYMENT_HEADERS)
|
|
598
|
+
for deployment in reversed(list_deployments_result.parsed.items)
|
|
599
|
+
],
|
|
600
|
+
headers=DEPLOYMENT_HEADERS,
|
|
601
|
+
)
|
|
602
|
+
)
|
|
603
|
+
else:
|
|
604
|
+
raise _exception_from_response(
|
|
605
|
+
"Failed to list deployments", list_deployments_result
|
|
606
|
+
)
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
@track_command(operation="deployment", suboperation="info")
|
|
610
|
+
def get_deployment_info(
|
|
611
|
+
deployment_version_no: Optional[int] = None,
|
|
612
|
+
*,
|
|
613
|
+
auth_service: RuntimeAuthService,
|
|
614
|
+
api_client: ApiClient,
|
|
615
|
+
) -> None:
|
|
616
|
+
if deployment_version_no is None:
|
|
617
|
+
get_deployment_result = get_latest_deployment.sync_detailed(
|
|
618
|
+
client=api_client,
|
|
619
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
620
|
+
)
|
|
621
|
+
else:
|
|
622
|
+
get_deployment_result = get_deployment.sync_detailed(
|
|
623
|
+
client=api_client,
|
|
624
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
625
|
+
deployment_id_or_version=deployment_version_no,
|
|
626
|
+
)
|
|
627
|
+
if isinstance(get_deployment_result.parsed, get_deployment.DeploymentResponse):
|
|
628
|
+
fmt.echo(
|
|
629
|
+
tabulate(
|
|
630
|
+
[
|
|
631
|
+
_extract_keys(
|
|
632
|
+
get_deployment_result.parsed.to_dict(), DEPLOYMENT_HEADERS
|
|
633
|
+
)
|
|
634
|
+
],
|
|
635
|
+
headers=DEPLOYMENT_HEADERS,
|
|
636
|
+
)
|
|
637
|
+
)
|
|
638
|
+
else:
|
|
639
|
+
raise _exception_from_response(
|
|
640
|
+
"Failed to get deployment info", get_deployment_result
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
@track_command(operation="cancel")
|
|
645
|
+
def cancel(
|
|
646
|
+
script_path_or_job_name: Optional[str] = None,
|
|
647
|
+
run_number: Optional[int] = None,
|
|
648
|
+
*,
|
|
649
|
+
auth_service: RuntimeAuthService,
|
|
650
|
+
api_client: ApiClient,
|
|
651
|
+
) -> None:
|
|
652
|
+
_request_run_cancel(
|
|
653
|
+
script_path_or_job_name,
|
|
654
|
+
run_number,
|
|
655
|
+
auth_service=auth_service,
|
|
656
|
+
api_client=api_client,
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
@track_command(operation="job-runs", suboperation="cancel")
|
|
661
|
+
def cancel_job_run(
|
|
662
|
+
script_path_or_job_name: Optional[str] = None,
|
|
663
|
+
run_number: Optional[int] = None,
|
|
664
|
+
*,
|
|
665
|
+
auth_service: RuntimeAuthService,
|
|
666
|
+
api_client: ApiClient,
|
|
667
|
+
) -> None:
|
|
668
|
+
_request_run_cancel(
|
|
669
|
+
script_path_or_job_name,
|
|
670
|
+
run_number,
|
|
671
|
+
auth_service=auth_service,
|
|
672
|
+
api_client=api_client,
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def _request_run_cancel(
|
|
677
|
+
script_path_or_job_name: Optional[str] = None,
|
|
678
|
+
run_number: Optional[int] = None,
|
|
679
|
+
*,
|
|
680
|
+
auth_service: RuntimeAuthService,
|
|
681
|
+
api_client: ApiClient,
|
|
682
|
+
) -> None:
|
|
683
|
+
"""Request the cancellation of a run, for a script or workspace if script is not provided"""
|
|
684
|
+
if script_path_or_job_name is None:
|
|
685
|
+
raise CliCommandInnerException(
|
|
686
|
+
cmd="runtime",
|
|
687
|
+
msg="Script path or job name is required",
|
|
688
|
+
)
|
|
689
|
+
if run_number is None:
|
|
690
|
+
run = _get_latest_run(api_client, auth_service, script_path_or_job_name)
|
|
691
|
+
run_id = run.id
|
|
692
|
+
run_no = run.number
|
|
693
|
+
else:
|
|
694
|
+
run_id = _resolve_run_id_by_number(
|
|
695
|
+
api_client=api_client,
|
|
696
|
+
auth_service=auth_service,
|
|
697
|
+
script_path_or_job_name=script_path_or_job_name,
|
|
698
|
+
run_number=run_number,
|
|
699
|
+
)
|
|
700
|
+
run_no = run_number
|
|
701
|
+
|
|
702
|
+
cancel_run_result = cancel_run.sync_detailed(
|
|
703
|
+
client=api_client,
|
|
704
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
705
|
+
run_id=_to_uuid(run_id),
|
|
706
|
+
)
|
|
707
|
+
if isinstance(cancel_run_result.parsed, cancel_run.DetailedRunResponse):
|
|
708
|
+
fmt.echo(f"Successfully requested cancellation of run # {run_no}")
|
|
709
|
+
else:
|
|
710
|
+
raise _exception_from_response(
|
|
711
|
+
"Failed to request cancellation of run", cancel_run_result
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def _get_latest_run(
|
|
716
|
+
api_client: ApiClient,
|
|
717
|
+
auth_service: RuntimeAuthService,
|
|
718
|
+
script_id_or_name: Optional[str] = None,
|
|
719
|
+
) -> DetailedRunResponse:
|
|
720
|
+
"""Get the latest run for a script or workspace if script is not provided"""
|
|
721
|
+
if script_id_or_name:
|
|
722
|
+
script = get_script.sync_detailed(
|
|
723
|
+
client=api_client,
|
|
724
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
725
|
+
script_id_or_name=script_id_or_name,
|
|
726
|
+
)
|
|
727
|
+
if isinstance(script.parsed, get_script.DetailedScriptResponse):
|
|
728
|
+
fmt.echo(f"Job {script.parsed.name} found on runtime.")
|
|
729
|
+
runs = list_runs.sync_detailed(
|
|
730
|
+
client=api_client,
|
|
731
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
732
|
+
script_id=script.parsed.id,
|
|
733
|
+
limit=1,
|
|
734
|
+
)
|
|
735
|
+
if isinstance(runs.parsed, list_runs.ListRunsResponse200):
|
|
736
|
+
if not runs.parsed.items:
|
|
737
|
+
raise _exception_from_response(
|
|
738
|
+
"No runs executed in for this job", runs
|
|
739
|
+
)
|
|
740
|
+
else:
|
|
741
|
+
return runs.parsed.items[0]
|
|
742
|
+
raise _exception_from_response(
|
|
743
|
+
f"Failed to get runs for script with name or id {script_id_or_name}",
|
|
744
|
+
runs,
|
|
745
|
+
)
|
|
746
|
+
else:
|
|
747
|
+
raise _exception_from_response(
|
|
748
|
+
f"Failed to get script with name or id {script_id_or_name}", script
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
else:
|
|
752
|
+
runs = list_runs.sync_detailed(
|
|
753
|
+
client=api_client,
|
|
754
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
755
|
+
limit=1,
|
|
756
|
+
)
|
|
757
|
+
if isinstance(runs.parsed, list_runs.ListRunsResponse200):
|
|
758
|
+
if not runs.parsed.items:
|
|
759
|
+
raise _exception_from_response(
|
|
760
|
+
"No runs executed in this workspace", runs
|
|
761
|
+
)
|
|
762
|
+
else:
|
|
763
|
+
return runs.parsed.items[0]
|
|
764
|
+
raise _exception_from_response("Failed to get runs for workspace", runs)
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
@track_command(operation="configuration", suboperation="list")
|
|
768
|
+
def get_configurations(
|
|
769
|
+
*, auth_service: RuntimeAuthService, api_client: ApiClient
|
|
770
|
+
) -> None:
|
|
771
|
+
list_configurations_result = list_configurations.sync_detailed(
|
|
772
|
+
client=api_client,
|
|
773
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
774
|
+
)
|
|
775
|
+
if isinstance(
|
|
776
|
+
list_configurations_result.parsed,
|
|
777
|
+
list_configurations.ListConfigurationsResponse200,
|
|
778
|
+
) and isinstance(list_configurations_result.parsed.items, list):
|
|
779
|
+
fmt.echo(
|
|
780
|
+
tabulate(
|
|
781
|
+
[
|
|
782
|
+
_extract_keys(configuration.to_dict(), CONFIGURATION_HEADERS)
|
|
783
|
+
for configuration in reversed(
|
|
784
|
+
list_configurations_result.parsed.items
|
|
785
|
+
)
|
|
786
|
+
],
|
|
787
|
+
headers=CONFIGURATION_HEADERS,
|
|
788
|
+
)
|
|
789
|
+
)
|
|
790
|
+
else:
|
|
791
|
+
raise _exception_from_response(
|
|
792
|
+
"Failed to list configurations", list_configurations_result
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
@track_command(operation="configuration", suboperation="info")
|
|
797
|
+
def get_configuration_info(
|
|
798
|
+
configuration_version_no: Optional[int] = None,
|
|
799
|
+
*,
|
|
800
|
+
auth_service: RuntimeAuthService,
|
|
801
|
+
api_client: ApiClient,
|
|
802
|
+
) -> None:
|
|
803
|
+
if configuration_version_no is None:
|
|
804
|
+
get_configuration_result = get_latest_configuration.sync_detailed(
|
|
805
|
+
client=api_client,
|
|
806
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
807
|
+
)
|
|
808
|
+
else:
|
|
809
|
+
get_configuration_result = get_configuration.sync_detailed(
|
|
810
|
+
client=api_client,
|
|
811
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
812
|
+
configuration_id_or_version=configuration_version_no,
|
|
813
|
+
)
|
|
814
|
+
if isinstance(
|
|
815
|
+
get_configuration_result.parsed, get_configuration.ConfigurationResponse
|
|
816
|
+
):
|
|
817
|
+
fmt.echo(
|
|
818
|
+
tabulate(
|
|
819
|
+
[
|
|
820
|
+
_extract_keys(
|
|
821
|
+
get_configuration_result.parsed.to_dict(), CONFIGURATION_HEADERS
|
|
822
|
+
)
|
|
823
|
+
],
|
|
824
|
+
headers=CONFIGURATION_HEADERS,
|
|
825
|
+
)
|
|
826
|
+
)
|
|
827
|
+
else:
|
|
828
|
+
raise _exception_from_response(
|
|
829
|
+
"Failed to get configuration info", get_configuration_result
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _ensure_profile_warning(required_profile: str) -> bool:
|
|
834
|
+
"""Warn if recommended profile is not set up."""
|
|
835
|
+
try:
|
|
836
|
+
ctx = active()
|
|
837
|
+
available = set(ctx.available_profiles())
|
|
838
|
+
if required_profile not in available:
|
|
839
|
+
if required_profile == "access":
|
|
840
|
+
fmt.warning(
|
|
841
|
+
"No 'access' profile detected. Only default config/secrets will be used. "
|
|
842
|
+
"Dashboard/notebook sharing may be limited."
|
|
843
|
+
)
|
|
844
|
+
elif required_profile == "prod":
|
|
845
|
+
fmt.warning(
|
|
846
|
+
"No 'prod' profile detected. Only default config/secrets will be used."
|
|
847
|
+
)
|
|
848
|
+
return False
|
|
849
|
+
return True
|
|
850
|
+
except Exception:
|
|
851
|
+
# Fallback silent; lack of profiles is non-fatal
|
|
852
|
+
return False
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def _resolve_run_id_by_number(
|
|
856
|
+
*,
|
|
857
|
+
api_client: ApiClient,
|
|
858
|
+
auth_service: RuntimeAuthService,
|
|
859
|
+
script_path_or_job_name: str,
|
|
860
|
+
run_number: int,
|
|
861
|
+
) -> UUID:
|
|
862
|
+
script = get_script.sync_detailed(
|
|
863
|
+
client=api_client,
|
|
864
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
865
|
+
script_id_or_name=script_path_or_job_name,
|
|
866
|
+
)
|
|
867
|
+
if not isinstance(script.parsed, get_script.DetailedScriptResponse):
|
|
868
|
+
raise _exception_from_response(
|
|
869
|
+
f"Failed to get script with name or id {script_path_or_job_name}", script
|
|
870
|
+
)
|
|
871
|
+
runs = list_runs.sync_detailed(
|
|
872
|
+
client=api_client,
|
|
873
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
874
|
+
script_id=script.parsed.id,
|
|
875
|
+
)
|
|
876
|
+
if (
|
|
877
|
+
not isinstance(runs.parsed, list_runs.ListRunsResponse200)
|
|
878
|
+
or not runs.parsed.items
|
|
879
|
+
):
|
|
880
|
+
raise _exception_from_response("Failed to get runs for script", runs)
|
|
881
|
+
for r in runs.parsed.items:
|
|
882
|
+
if r.number == run_number:
|
|
883
|
+
return r.id
|
|
884
|
+
raise CliCommandInnerException(
|
|
885
|
+
cmd="runtime",
|
|
886
|
+
msg=f"Run number {run_number} not found for script/job {script_path_or_job_name}",
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
# Convenience commands
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
@track_command(operation="launch")
|
|
894
|
+
def launch(
|
|
895
|
+
script_path: str,
|
|
896
|
+
detach: bool = False,
|
|
897
|
+
*,
|
|
898
|
+
auth_service: RuntimeAuthService,
|
|
899
|
+
api_client: ApiClient,
|
|
900
|
+
) -> None:
|
|
901
|
+
_ensure_profile_warning("prod")
|
|
902
|
+
# Sync and run
|
|
903
|
+
_sync_deployment(auth_service=auth_service, api_client=api_client)
|
|
904
|
+
_sync_configuration(auth_service=auth_service, api_client=api_client)
|
|
905
|
+
run_id = _run_script(
|
|
906
|
+
script_path,
|
|
907
|
+
is_interactive=False,
|
|
908
|
+
auth_service=auth_service,
|
|
909
|
+
api_client=api_client,
|
|
910
|
+
)
|
|
911
|
+
if not detach:
|
|
912
|
+
# Show status and then logs for latest run
|
|
913
|
+
_follow_run_status(
|
|
914
|
+
run_id, True, auth_service=auth_service, api_client=api_client
|
|
915
|
+
)
|
|
916
|
+
_follow_run_logs(run_id, auth_service=auth_service, api_client=api_client)
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
@track_command(operation="serve")
|
|
920
|
+
def serve(
|
|
921
|
+
script_path: str, *, auth_service: RuntimeAuthService, api_client: ApiClient
|
|
922
|
+
) -> None:
|
|
923
|
+
_ensure_profile_warning("access")
|
|
924
|
+
# Try to detect marimo notebook
|
|
925
|
+
try:
|
|
926
|
+
script_path_obj = Path(active().run_dir) / script_path
|
|
927
|
+
if script_path_obj.exists() and script_path_obj.is_file():
|
|
928
|
+
content = script_path_obj.read_text(encoding="utf-8", errors="ignore")
|
|
929
|
+
if "import marimo" not in content and "from marimo" not in content:
|
|
930
|
+
fmt.warning(
|
|
931
|
+
"Could not detect a marimo notebook in the provided script. "
|
|
932
|
+
"Proceeding to serve as an interactive app."
|
|
933
|
+
)
|
|
934
|
+
else:
|
|
935
|
+
raise CliCommandInnerException(
|
|
936
|
+
cmd="runtime",
|
|
937
|
+
msg="Provided script path does not exist locally",
|
|
938
|
+
)
|
|
939
|
+
except Exception as e:
|
|
940
|
+
raise CliCommandInnerException(
|
|
941
|
+
cmd="runtime",
|
|
942
|
+
msg="Failed to read script file",
|
|
943
|
+
inner_exc=e,
|
|
944
|
+
)
|
|
945
|
+
|
|
946
|
+
# Sync and run interactive
|
|
947
|
+
_sync_deployment(auth_service=auth_service, api_client=api_client)
|
|
948
|
+
_sync_configuration(auth_service=auth_service, api_client=api_client)
|
|
949
|
+
run_id = _run_script(
|
|
950
|
+
script_path,
|
|
951
|
+
is_interactive=True,
|
|
952
|
+
auth_service=auth_service,
|
|
953
|
+
api_client=api_client,
|
|
954
|
+
)
|
|
955
|
+
# Follow until ready: show status
|
|
956
|
+
_follow_run_status(run_id, False, auth_service=auth_service, api_client=api_client)
|
|
957
|
+
|
|
958
|
+
# Open the application URL
|
|
959
|
+
try:
|
|
960
|
+
res = get_script.sync_detailed(
|
|
961
|
+
client=api_client,
|
|
962
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
963
|
+
script_id_or_name=script_path,
|
|
964
|
+
)
|
|
965
|
+
if isinstance(res.parsed, get_script.DetailedScriptResponse):
|
|
966
|
+
url = res.parsed.script_url
|
|
967
|
+
fmt.echo(f"Opening {url}")
|
|
968
|
+
# Python internals
|
|
969
|
+
import webbrowser
|
|
970
|
+
|
|
971
|
+
webbrowser.open(url, new=2, autoraise=True)
|
|
972
|
+
except Exception:
|
|
973
|
+
# Non-fatal if we cannot resolve or open URL
|
|
974
|
+
fmt.warning(f"Failed to open application URL for script {script_path}")
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
@track_command(operation="publish")
|
|
978
|
+
def publish(
|
|
979
|
+
script_path: str,
|
|
980
|
+
cancel: bool = False,
|
|
981
|
+
*,
|
|
982
|
+
auth_service: RuntimeAuthService,
|
|
983
|
+
api_client: ApiClient,
|
|
984
|
+
) -> None:
|
|
985
|
+
"""Enable or disable a public link for an interactive script."""
|
|
986
|
+
_ensure_profile_warning("access")
|
|
987
|
+
|
|
988
|
+
script = get_script.sync_detailed(
|
|
989
|
+
client=api_client,
|
|
990
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
991
|
+
script_id_or_name=script_path,
|
|
992
|
+
)
|
|
993
|
+
if not isinstance(script.parsed, get_script.DetailedScriptResponse):
|
|
994
|
+
raise _exception_from_response(
|
|
995
|
+
f"Failed to get script with name or id {script_path}", script
|
|
996
|
+
)
|
|
997
|
+
|
|
998
|
+
if cancel:
|
|
999
|
+
# disabling public link
|
|
1000
|
+
if not script.parsed.public_url:
|
|
1001
|
+
fmt.echo(f"Public link for script {script_path} already disabled")
|
|
1002
|
+
return
|
|
1003
|
+
disable_public_url_result = disable_public_url.sync_detailed(
|
|
1004
|
+
client=api_client,
|
|
1005
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1006
|
+
script_id_or_name=script_path,
|
|
1007
|
+
)
|
|
1008
|
+
if isinstance(
|
|
1009
|
+
disable_public_url_result.parsed, disable_public_url.ScriptResponse
|
|
1010
|
+
):
|
|
1011
|
+
fmt.echo(f"Public link for script {script_path} disabled successfully")
|
|
1012
|
+
else:
|
|
1013
|
+
raise _exception_from_response(
|
|
1014
|
+
"Failed to disable public link", disable_public_url_result
|
|
1015
|
+
)
|
|
1016
|
+
return
|
|
1017
|
+
|
|
1018
|
+
# enabling public link
|
|
1019
|
+
if script.parsed.public_url:
|
|
1020
|
+
fmt.echo(
|
|
1021
|
+
f"Public link for script {script_path} already enabled: {script.parsed.public_url}"
|
|
1022
|
+
)
|
|
1023
|
+
return
|
|
1024
|
+
enable_public_url_result = enable_public_url.sync_detailed(
|
|
1025
|
+
client=api_client,
|
|
1026
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1027
|
+
script_id_or_name=script_path,
|
|
1028
|
+
)
|
|
1029
|
+
if isinstance(enable_public_url_result.parsed, enable_public_url.ScriptResponse):
|
|
1030
|
+
fmt.echo(
|
|
1031
|
+
f"Public link for script {script_path} enabled successfully: {enable_public_url_result.parsed.public_url}"
|
|
1032
|
+
)
|
|
1033
|
+
else:
|
|
1034
|
+
raise _exception_from_response(
|
|
1035
|
+
"Failed to enable public link", enable_public_url_result
|
|
1036
|
+
)
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
@track_command(operation="serve", suboperation="publish")
|
|
1040
|
+
def enable_public_link(
|
|
1041
|
+
script_path: str, *, auth_service: RuntimeAuthService, api_client: ApiClient
|
|
1042
|
+
) -> None:
|
|
1043
|
+
_ensure_profile_warning("access")
|
|
1044
|
+
|
|
1045
|
+
script = get_script.sync_detailed(
|
|
1046
|
+
client=api_client,
|
|
1047
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1048
|
+
script_id_or_name=script_path,
|
|
1049
|
+
)
|
|
1050
|
+
if not isinstance(script.parsed, get_script.DetailedScriptResponse):
|
|
1051
|
+
raise _exception_from_response(
|
|
1052
|
+
f"Failed to get script with name or id {script_path}", script
|
|
1053
|
+
)
|
|
1054
|
+
if script.parsed.public_url:
|
|
1055
|
+
fmt.echo(
|
|
1056
|
+
f"Public link for script {script_path} already enabled: {script.parsed.public_url}"
|
|
1057
|
+
)
|
|
1058
|
+
return
|
|
1059
|
+
|
|
1060
|
+
enable_public_url_result = enable_public_url.sync_detailed(
|
|
1061
|
+
client=api_client,
|
|
1062
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1063
|
+
script_id_or_name=script_path,
|
|
1064
|
+
)
|
|
1065
|
+
if isinstance(enable_public_url_result.parsed, enable_public_url.ScriptResponse):
|
|
1066
|
+
fmt.echo(
|
|
1067
|
+
f"Public link for script {script_path} enabled successfully: {enable_public_url_result.parsed.public_url}"
|
|
1068
|
+
)
|
|
1069
|
+
else:
|
|
1070
|
+
raise _exception_from_response(
|
|
1071
|
+
"Failed to enable public link", enable_public_url_result
|
|
1072
|
+
)
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
@track_command(operation="serve", suboperation="unpublish")
|
|
1076
|
+
def disable_public_link(
|
|
1077
|
+
script_path: str, *, auth_service: RuntimeAuthService, api_client: ApiClient
|
|
1078
|
+
) -> None:
|
|
1079
|
+
_ensure_profile_warning("access")
|
|
1080
|
+
|
|
1081
|
+
script = get_script.sync_detailed(
|
|
1082
|
+
client=api_client,
|
|
1083
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1084
|
+
script_id_or_name=script_path,
|
|
1085
|
+
)
|
|
1086
|
+
if not isinstance(script.parsed, get_script.DetailedScriptResponse):
|
|
1087
|
+
raise _exception_from_response(
|
|
1088
|
+
f"Failed to get script with name or id {script_path}", script
|
|
1089
|
+
)
|
|
1090
|
+
if not script.parsed.public_url:
|
|
1091
|
+
fmt.echo(f"Public link for script {script_path} already disabled")
|
|
1092
|
+
return
|
|
1093
|
+
|
|
1094
|
+
disable_public_url_result = disable_public_url.sync_detailed(
|
|
1095
|
+
client=api_client,
|
|
1096
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1097
|
+
script_id_or_name=script_path,
|
|
1098
|
+
)
|
|
1099
|
+
if isinstance(disable_public_url_result.parsed, disable_public_url.ScriptResponse):
|
|
1100
|
+
fmt.echo(f"Public link for script {script_path} disabled successfully")
|
|
1101
|
+
else:
|
|
1102
|
+
raise _exception_from_response(
|
|
1103
|
+
"Failed to disable public link", disable_public_url_result
|
|
1104
|
+
)
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def _run_script(
|
|
1108
|
+
script_file_name: str,
|
|
1109
|
+
is_interactive: bool = False,
|
|
1110
|
+
profile: Optional[str] = None,
|
|
1111
|
+
*,
|
|
1112
|
+
auth_service: RuntimeAuthService,
|
|
1113
|
+
api_client: ApiClient,
|
|
1114
|
+
) -> UUID:
|
|
1115
|
+
script_path = Path(active().run_dir) / script_file_name
|
|
1116
|
+
if not script_path.exists():
|
|
1117
|
+
raise RuntimeError(f"Script file {script_file_name} not found")
|
|
1118
|
+
|
|
1119
|
+
create_script_result = create_or_update_script.sync_detailed(
|
|
1120
|
+
client=api_client,
|
|
1121
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1122
|
+
body=create_or_update_script.CreateScriptRequest(
|
|
1123
|
+
name=script_file_name,
|
|
1124
|
+
description=f"The {script_file_name} script",
|
|
1125
|
+
entry_point=script_file_name,
|
|
1126
|
+
script_type=ScriptType.INTERACTIVE if is_interactive else ScriptType.BATCH,
|
|
1127
|
+
profile=profile,
|
|
1128
|
+
schedule=None,
|
|
1129
|
+
),
|
|
1130
|
+
)
|
|
1131
|
+
if not isinstance(
|
|
1132
|
+
create_script_result.parsed, create_or_update_script.ScriptResponse
|
|
1133
|
+
):
|
|
1134
|
+
raise _exception_from_response("Failed to create script", create_script_result)
|
|
1135
|
+
else:
|
|
1136
|
+
fmt.echo(
|
|
1137
|
+
f"Job {script_file_name} created or updated successfully, version #:"
|
|
1138
|
+
f" {create_script_result.parsed.version}"
|
|
1139
|
+
)
|
|
1140
|
+
|
|
1141
|
+
create_run_result = create_run.sync_detailed(
|
|
1142
|
+
client=api_client,
|
|
1143
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1144
|
+
body=create_run.CreateRunRequest(
|
|
1145
|
+
script_id_or_name_or_secret=script_file_name,
|
|
1146
|
+
profile=None,
|
|
1147
|
+
),
|
|
1148
|
+
)
|
|
1149
|
+
if isinstance(create_run_result.parsed, create_run.DetailedRunResponse):
|
|
1150
|
+
fmt.echo("Job %s run successfully" % (fmt.bold(str(script_file_name))))
|
|
1151
|
+
if is_interactive:
|
|
1152
|
+
url = create_script_result.parsed.script_url
|
|
1153
|
+
fmt.echo(f"Job is accessible on {url}")
|
|
1154
|
+
return create_run_result.parsed.id
|
|
1155
|
+
else:
|
|
1156
|
+
raise _exception_from_response("Failed to run script", create_run_result)
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
def _follow_run_status(
|
|
1160
|
+
run_id: UUID,
|
|
1161
|
+
is_batch: bool,
|
|
1162
|
+
*,
|
|
1163
|
+
auth_service: RuntimeAuthService,
|
|
1164
|
+
api_client: ApiClient,
|
|
1165
|
+
) -> None:
|
|
1166
|
+
final_states = {RunStatus.FAILED, RunStatus.CANCELLED}
|
|
1167
|
+
if is_batch:
|
|
1168
|
+
final_states.add(RunStatus.STARTING)
|
|
1169
|
+
else:
|
|
1170
|
+
final_states.add(RunStatus.RUNNING)
|
|
1171
|
+
return _follow_job_run(
|
|
1172
|
+
run_id, final_states, auth_service=auth_service, api_client=api_client
|
|
1173
|
+
)
|
|
1174
|
+
|
|
1175
|
+
|
|
1176
|
+
def _follow_run_logs(
|
|
1177
|
+
run_id: UUID, *, auth_service: RuntimeAuthService, api_client: ApiClient
|
|
1178
|
+
) -> None:
|
|
1179
|
+
final_states = {RunStatus.FAILED, RunStatus.CANCELLED, RunStatus.COMPLETED}
|
|
1180
|
+
return _follow_job_run(
|
|
1181
|
+
run_id,
|
|
1182
|
+
final_states,
|
|
1183
|
+
RunStatus.STARTING,
|
|
1184
|
+
True,
|
|
1185
|
+
auth_service=auth_service,
|
|
1186
|
+
api_client=api_client,
|
|
1187
|
+
)
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def _follow_job_run(
|
|
1191
|
+
run_id: UUID,
|
|
1192
|
+
final_states: Set[RunStatus],
|
|
1193
|
+
start_status: Optional[RunStatus] = None,
|
|
1194
|
+
follow_logs: bool = False,
|
|
1195
|
+
*,
|
|
1196
|
+
auth_service: RuntimeAuthService,
|
|
1197
|
+
api_client: ApiClient,
|
|
1198
|
+
) -> None:
|
|
1199
|
+
status = start_status
|
|
1200
|
+
print_from_line_idx = 0
|
|
1201
|
+
|
|
1202
|
+
if follow_logs:
|
|
1203
|
+
fmt.echo("========== Run logs ==========")
|
|
1204
|
+
while True:
|
|
1205
|
+
get_run_result = get_run.sync_detailed(
|
|
1206
|
+
client=api_client,
|
|
1207
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1208
|
+
run_id=run_id,
|
|
1209
|
+
)
|
|
1210
|
+
if not isinstance(get_run_result.parsed, get_run.DetailedRunResponse):
|
|
1211
|
+
raise _exception_from_response("Failed to get run info", get_run_result)
|
|
1212
|
+
new_status = get_run_result.parsed.status
|
|
1213
|
+
if new_status != status:
|
|
1214
|
+
if not follow_logs:
|
|
1215
|
+
fmt.echo(f"Run status: {new_status}")
|
|
1216
|
+
status = new_status
|
|
1217
|
+
|
|
1218
|
+
if follow_logs:
|
|
1219
|
+
get_run_logs_result = get_run_logs.sync_detailed(
|
|
1220
|
+
client=api_client,
|
|
1221
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1222
|
+
run_id=run_id,
|
|
1223
|
+
)
|
|
1224
|
+
if not isinstance(get_run_logs_result.parsed, get_run_logs.LogsResponse):
|
|
1225
|
+
raise _exception_from_response(
|
|
1226
|
+
"Failed to get run logs", get_run_logs_result
|
|
1227
|
+
)
|
|
1228
|
+
if isinstance(get_run_logs_result.parsed.logs, str):
|
|
1229
|
+
log_lines = get_run_logs_result.parsed.logs.split("\n")
|
|
1230
|
+
for line in log_lines[print_from_line_idx:]:
|
|
1231
|
+
fmt.echo(line)
|
|
1232
|
+
print_from_line_idx = len(log_lines)
|
|
1233
|
+
|
|
1234
|
+
if status in final_states:
|
|
1235
|
+
if follow_logs:
|
|
1236
|
+
fmt.echo("========== End of run logs ==========")
|
|
1237
|
+
fmt.echo(f"Run status: {new_status}")
|
|
1238
|
+
break
|
|
1239
|
+
time.sleep(2)
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
@track_command(operation="schedule")
|
|
1243
|
+
def schedule(
|
|
1244
|
+
script_path: str,
|
|
1245
|
+
cron: Optional[str],
|
|
1246
|
+
*,
|
|
1247
|
+
auth_service: RuntimeAuthService,
|
|
1248
|
+
api_client: ApiClient,
|
|
1249
|
+
) -> None:
|
|
1250
|
+
if not cron:
|
|
1251
|
+
raise CliCommandInnerException(
|
|
1252
|
+
cmd="runtime",
|
|
1253
|
+
msg=(
|
|
1254
|
+
"Cron schedule must be provided: dlt runtime schedule <SCRIPT_PATH> <SCHEDULE_CRON>"
|
|
1255
|
+
),
|
|
1256
|
+
)
|
|
1257
|
+
_check_cron_expression(cron)
|
|
1258
|
+
_ensure_profile_warning("prod")
|
|
1259
|
+
|
|
1260
|
+
# Ensure deployment/configuration in place
|
|
1261
|
+
_sync_deployment(auth_service=auth_service, api_client=api_client)
|
|
1262
|
+
_sync_configuration(auth_service=auth_service, api_client=api_client)
|
|
1263
|
+
|
|
1264
|
+
# Upsert script with schedule
|
|
1265
|
+
upsert = create_or_update_script.sync_detailed(
|
|
1266
|
+
client=api_client,
|
|
1267
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1268
|
+
body=create_or_update_script.CreateScriptRequest(
|
|
1269
|
+
name=script_path,
|
|
1270
|
+
description=f"The {script_path} scheduled job",
|
|
1271
|
+
entry_point=script_path,
|
|
1272
|
+
script_type=ScriptType.BATCH,
|
|
1273
|
+
schedule=cron,
|
|
1274
|
+
),
|
|
1275
|
+
)
|
|
1276
|
+
if isinstance(upsert.parsed, create_or_update_script.ScriptResponse):
|
|
1277
|
+
fmt.echo(
|
|
1278
|
+
f"Scheduled {fmt.bold(script_path)} with cron {fmt.bold(cron)}. Job version #:"
|
|
1279
|
+
f" {upsert.parsed.version}"
|
|
1280
|
+
)
|
|
1281
|
+
else:
|
|
1282
|
+
raise _exception_from_response("Failed to schedule script", upsert)
|
|
1283
|
+
|
|
1284
|
+
|
|
1285
|
+
@track_command(operation="schedule", suboperation="cancel")
|
|
1286
|
+
def schedule_cancel(
|
|
1287
|
+
script_path: str,
|
|
1288
|
+
cancel_current: bool = False,
|
|
1289
|
+
*,
|
|
1290
|
+
auth_service: RuntimeAuthService,
|
|
1291
|
+
api_client: ApiClient,
|
|
1292
|
+
) -> None:
|
|
1293
|
+
existing_script = get_script.sync_detailed(
|
|
1294
|
+
client=api_client,
|
|
1295
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1296
|
+
script_id_or_name=script_path,
|
|
1297
|
+
)
|
|
1298
|
+
if isinstance(existing_script.parsed, get_script.DetailedScriptResponse):
|
|
1299
|
+
if not isinstance(existing_script.parsed.schedule, str):
|
|
1300
|
+
fmt.error(f"{script_path} is not a scheduled job")
|
|
1301
|
+
return
|
|
1302
|
+
else:
|
|
1303
|
+
raise _exception_from_response("Failed to get job", existing_script)
|
|
1304
|
+
|
|
1305
|
+
# Unset schedule
|
|
1306
|
+
upsert = create_or_update_script.sync_detailed(
|
|
1307
|
+
client=api_client,
|
|
1308
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1309
|
+
body=create_or_update_script.CreateScriptRequest(
|
|
1310
|
+
name=script_path,
|
|
1311
|
+
description=f"The {script_path} job",
|
|
1312
|
+
entry_point=script_path,
|
|
1313
|
+
script_type=ScriptType.BATCH,
|
|
1314
|
+
schedule=None,
|
|
1315
|
+
),
|
|
1316
|
+
)
|
|
1317
|
+
if isinstance(upsert.parsed, create_or_update_script.ScriptResponse):
|
|
1318
|
+
fmt.echo(f"Cancelled schedule for {fmt.bold(script_path)}")
|
|
1319
|
+
else:
|
|
1320
|
+
raise _exception_from_response("Failed to cancel schedule", upsert)
|
|
1321
|
+
if cancel_current:
|
|
1322
|
+
try:
|
|
1323
|
+
_request_run_cancel(
|
|
1324
|
+
script_path, auth_service=auth_service, api_client=api_client
|
|
1325
|
+
)
|
|
1326
|
+
except CliCommandInnerException as e:
|
|
1327
|
+
if "terminal state" not in e.args[0]:
|
|
1328
|
+
raise e
|
|
1329
|
+
|
|
1330
|
+
|
|
1331
|
+
@track_command(operation="dashboard")
|
|
1332
|
+
def open_dashboard(*, auth_service: RuntimeAuthService, api_client: ApiClient) -> None:
|
|
1333
|
+
job = get_script.sync_detailed(
|
|
1334
|
+
client=api_client,
|
|
1335
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1336
|
+
script_id_or_name="dashboard",
|
|
1337
|
+
)
|
|
1338
|
+
if isinstance(job.parsed, get_script.DetailedScriptResponse):
|
|
1339
|
+
if not job.parsed.script_url:
|
|
1340
|
+
fmt.error("Failed to get the URL for the dashboard")
|
|
1341
|
+
return
|
|
1342
|
+
fmt.echo(f"Dashboard is available at {job.parsed.script_url}")
|
|
1343
|
+
webbrowser.open(job.parsed.script_url)
|
|
1344
|
+
else:
|
|
1345
|
+
raise _exception_from_response("Failed to get dashboard job", job)
|
|
1346
|
+
|
|
1347
|
+
|
|
1348
|
+
@track_command(operation="info")
|
|
1349
|
+
def runtime_info(*, auth_service: RuntimeAuthService, api_client: ApiClient) -> None:
|
|
1350
|
+
# jobs
|
|
1351
|
+
scr = list_scripts.sync_detailed(
|
|
1352
|
+
client=api_client,
|
|
1353
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1354
|
+
)
|
|
1355
|
+
job_count = (
|
|
1356
|
+
len(scr.parsed.items)
|
|
1357
|
+
if isinstance(scr.parsed, list_scripts.ListScriptsResponse200)
|
|
1358
|
+
and scr.parsed.items
|
|
1359
|
+
else 0
|
|
1360
|
+
)
|
|
1361
|
+
fmt.echo(f"# registered jobs: {job_count}. Run `dlt runtime job list` to see all")
|
|
1362
|
+
|
|
1363
|
+
# last job run
|
|
1364
|
+
|
|
1365
|
+
latest_run = _get_latest_run(api_client, auth_service)
|
|
1366
|
+
if isinstance(latest_run, DetailedRunResponse):
|
|
1367
|
+
fmt.echo(
|
|
1368
|
+
f"Latest job run: {latest_run.script.name} ({latest_run.status}), started at"
|
|
1369
|
+
f" {latest_run.time_started}, ended at {latest_run.time_ended}"
|
|
1370
|
+
)
|
|
1371
|
+
else:
|
|
1372
|
+
raise _exception_from_response("Failed to get latest run", latest_run)
|
|
1373
|
+
|
|
1374
|
+
# deployments
|
|
1375
|
+
latest_deployment = get_latest_deployment.sync_detailed(
|
|
1376
|
+
client=api_client,
|
|
1377
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1378
|
+
)
|
|
1379
|
+
if isinstance(latest_deployment.parsed, get_latest_deployment.DeploymentResponse):
|
|
1380
|
+
fmt.echo(
|
|
1381
|
+
f"Current deployment version: {latest_deployment.parsed.version}, last updated at"
|
|
1382
|
+
f" {latest_deployment.parsed.date_added}. Run `dlt runtime deployment info` to see"
|
|
1383
|
+
" detailed deployment information"
|
|
1384
|
+
)
|
|
1385
|
+
else:
|
|
1386
|
+
raise _exception_from_response(
|
|
1387
|
+
"Failed to get latest deployment", latest_deployment
|
|
1388
|
+
)
|
|
1389
|
+
|
|
1390
|
+
# configurations
|
|
1391
|
+
latest_configuration = get_latest_configuration.sync_detailed(
|
|
1392
|
+
client=api_client,
|
|
1393
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1394
|
+
)
|
|
1395
|
+
if isinstance(
|
|
1396
|
+
latest_configuration.parsed, get_latest_configuration.ConfigurationResponse
|
|
1397
|
+
):
|
|
1398
|
+
fmt.echo(
|
|
1399
|
+
f"Current configuration version: {latest_configuration.parsed.version}, last updated at"
|
|
1400
|
+
f" {latest_configuration.parsed.date_added}. Run `dlt runtime configuration info` to"
|
|
1401
|
+
" see detailed configuration information"
|
|
1402
|
+
)
|
|
1403
|
+
else:
|
|
1404
|
+
raise _exception_from_response(
|
|
1405
|
+
"Failed to get latest configuration", latest_configuration
|
|
1406
|
+
)
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
# Power user: jobs and job-runs
|
|
1410
|
+
|
|
1411
|
+
|
|
1412
|
+
@track_command(operation="jobs", suboperation="list")
|
|
1413
|
+
def jobs_list(*, auth_service: RuntimeAuthService, api_client: ApiClient) -> None:
|
|
1414
|
+
res = list_scripts.sync_detailed(
|
|
1415
|
+
client=api_client,
|
|
1416
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1417
|
+
)
|
|
1418
|
+
if isinstance(res.parsed, list_scripts.ListScriptsResponse200) and isinstance(
|
|
1419
|
+
res.parsed.items, list
|
|
1420
|
+
):
|
|
1421
|
+
fmt.echo(
|
|
1422
|
+
tabulate(
|
|
1423
|
+
[
|
|
1424
|
+
_extract_keys(script.to_dict(), JOB_HEADERS)
|
|
1425
|
+
for script in reversed(res.parsed.items)
|
|
1426
|
+
],
|
|
1427
|
+
headers=JOB_HEADERS,
|
|
1428
|
+
)
|
|
1429
|
+
)
|
|
1430
|
+
else:
|
|
1431
|
+
raise _exception_from_response("Failed to list jobs", res)
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
@track_command(operation="jobs", suboperation="info")
|
|
1435
|
+
def job_info(
|
|
1436
|
+
script_path_or_job_name: Optional[str] = None,
|
|
1437
|
+
*,
|
|
1438
|
+
auth_service: RuntimeAuthService,
|
|
1439
|
+
api_client: ApiClient,
|
|
1440
|
+
) -> None:
|
|
1441
|
+
if not script_path_or_job_name:
|
|
1442
|
+
raise CliCommandInnerException(
|
|
1443
|
+
cmd="runtime",
|
|
1444
|
+
msg="Script path or job name is required",
|
|
1445
|
+
)
|
|
1446
|
+
res = get_script.sync_detailed(
|
|
1447
|
+
client=api_client,
|
|
1448
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1449
|
+
script_id_or_name=script_path_or_job_name,
|
|
1450
|
+
)
|
|
1451
|
+
if isinstance(res.parsed, get_script.DetailedScriptResponse):
|
|
1452
|
+
print(res.parsed.to_dict())
|
|
1453
|
+
fmt.echo(
|
|
1454
|
+
tabulate(
|
|
1455
|
+
[_extract_keys(res.parsed.to_dict(), JOB_HEADERS)], headers=JOB_HEADERS
|
|
1456
|
+
)
|
|
1457
|
+
)
|
|
1458
|
+
else:
|
|
1459
|
+
raise _exception_from_response("Failed to get job info", res)
|
|
1460
|
+
|
|
1461
|
+
|
|
1462
|
+
@track_command(operation="jobs", suboperation="create")
|
|
1463
|
+
def job_create(
|
|
1464
|
+
script_path: Optional[str],
|
|
1465
|
+
args: argparse.Namespace,
|
|
1466
|
+
*,
|
|
1467
|
+
auth_service: RuntimeAuthService,
|
|
1468
|
+
api_client: ApiClient,
|
|
1469
|
+
) -> None:
|
|
1470
|
+
if not script_path:
|
|
1471
|
+
raise CliCommandInnerException(
|
|
1472
|
+
cmd="runtime",
|
|
1473
|
+
msg="Script path is required to be a first argument",
|
|
1474
|
+
)
|
|
1475
|
+
script_path_obj = Path(active().run_dir) / script_path
|
|
1476
|
+
if not script_path_obj.exists():
|
|
1477
|
+
raise CliCommandInnerException(
|
|
1478
|
+
cmd="runtime",
|
|
1479
|
+
msg="Script path is required to be a first argument. Provided path does not exist",
|
|
1480
|
+
)
|
|
1481
|
+
if args.schedule:
|
|
1482
|
+
_check_cron_expression(args.schedule)
|
|
1483
|
+
job_name = args.name or script_path
|
|
1484
|
+
job_type = ScriptType.INTERACTIVE if args.interactive else ScriptType.BATCH
|
|
1485
|
+
job_description = args.description or f"The {job_name} job"
|
|
1486
|
+
|
|
1487
|
+
# warn if the job exists already with different parameters
|
|
1488
|
+
res = get_script.sync_detailed(
|
|
1489
|
+
client=api_client,
|
|
1490
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1491
|
+
script_id_or_name=job_name,
|
|
1492
|
+
)
|
|
1493
|
+
if isinstance(res.parsed, get_script.DetailedScriptResponse):
|
|
1494
|
+
if args.name and res.parsed.entry_point != script_path:
|
|
1495
|
+
fmt.warning(
|
|
1496
|
+
f"Warning: Job {job_name} already exists for different script path"
|
|
1497
|
+
f" ({res.parsed.entry_point} -> {script_path}). Overwriting..."
|
|
1498
|
+
)
|
|
1499
|
+
elif res.parsed.schedule != args.schedule:
|
|
1500
|
+
fmt.warning(
|
|
1501
|
+
f"Warning: Job {job_name} already exists with different schedule"
|
|
1502
|
+
f" ({res.parsed.schedule} -> {args.schedule}). Overwriting..."
|
|
1503
|
+
)
|
|
1504
|
+
elif res.parsed.script_type != job_type:
|
|
1505
|
+
fmt.warning(
|
|
1506
|
+
f"Warning: Job {job_name} already exists with different interactive mode"
|
|
1507
|
+
f" ({res.parsed.script_type} -> {job_type}). Overwriting..."
|
|
1508
|
+
)
|
|
1509
|
+
|
|
1510
|
+
upsert = create_or_update_script.sync_detailed(
|
|
1511
|
+
client=api_client,
|
|
1512
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1513
|
+
body=create_or_update_script.CreateScriptRequest(
|
|
1514
|
+
name=job_name,
|
|
1515
|
+
description=job_description,
|
|
1516
|
+
entry_point=script_path,
|
|
1517
|
+
script_type=job_type,
|
|
1518
|
+
profile=None,
|
|
1519
|
+
schedule=args.schedule,
|
|
1520
|
+
),
|
|
1521
|
+
)
|
|
1522
|
+
if isinstance(upsert.parsed, create_or_update_script.ScriptResponse):
|
|
1523
|
+
fmt.echo(
|
|
1524
|
+
tabulate(
|
|
1525
|
+
[_extract_keys(upsert.parsed.to_dict(), JOB_HEADERS)],
|
|
1526
|
+
headers=JOB_HEADERS,
|
|
1527
|
+
)
|
|
1528
|
+
)
|
|
1529
|
+
else:
|
|
1530
|
+
raise _exception_from_response("Failed to create job", upsert)
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
@track_command(operation="job-runs", suboperation="create")
|
|
1534
|
+
def create_job_run(
|
|
1535
|
+
script_path_or_job_name: Optional[str] = None,
|
|
1536
|
+
*,
|
|
1537
|
+
auth_service: RuntimeAuthService,
|
|
1538
|
+
api_client: ApiClient,
|
|
1539
|
+
) -> None:
|
|
1540
|
+
if script_path_or_job_name is None:
|
|
1541
|
+
raise CliCommandInnerException(
|
|
1542
|
+
cmd="runtime",
|
|
1543
|
+
msg="Script path or job name is required",
|
|
1544
|
+
)
|
|
1545
|
+
res = create_run.sync_detailed(
|
|
1546
|
+
client=api_client,
|
|
1547
|
+
workspace_id=_to_uuid(auth_service.workspace_id),
|
|
1548
|
+
body=create_run.CreateRunRequest(
|
|
1549
|
+
script_id_or_name_or_secret=script_path_or_job_name,
|
|
1550
|
+
profile=None,
|
|
1551
|
+
),
|
|
1552
|
+
)
|
|
1553
|
+
if isinstance(res.parsed, create_run.DetailedRunResponse):
|
|
1554
|
+
fmt.echo(
|
|
1555
|
+
tabulate(
|
|
1556
|
+
[_extract_keys(res.parsed.to_dict(), JOB_RUN_HEADERS)],
|
|
1557
|
+
headers=JOB_RUN_HEADERS,
|
|
1558
|
+
)
|
|
1559
|
+
)
|
|
1560
|
+
else:
|
|
1561
|
+
raise _exception_from_response("Failed to start run", res)
|