outerbounds 0.3.91__py3-none-any.whl → 0.3.93__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- outerbounds/command_groups/apps_cli.py +535 -0
- outerbounds/command_groups/cli.py +8 -4
- outerbounds/command_groups/local_setup_cli.py +74 -3
- outerbounds/command_groups/tutorials_cli.py +111 -0
- outerbounds/utils/schema.py +2 -2
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/METADATA +4 -4
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/RECORD +9 -7
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/WHEEL +0 -0
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,535 @@
|
|
1
|
+
import json
|
2
|
+
import os
|
3
|
+
from os import path
|
4
|
+
from outerbounds._vendor import click
|
5
|
+
import requests
|
6
|
+
|
7
|
+
from ..utils import metaflowconfig
|
8
|
+
from ..utils.schema import (
|
9
|
+
CommandStatus,
|
10
|
+
OuterboundsCommandResponse,
|
11
|
+
OuterboundsCommandStatus,
|
12
|
+
)
|
13
|
+
|
14
|
+
|
15
|
+
@click.group()
|
16
|
+
def cli(**kwargs):
|
17
|
+
pass
|
18
|
+
|
19
|
+
|
20
|
+
@click.group(help="Manage apps")
|
21
|
+
def app(**kwargs):
|
22
|
+
pass
|
23
|
+
|
24
|
+
|
25
|
+
@app.command(help="Start an app using a port and a name")
|
26
|
+
@click.option(
|
27
|
+
"-d",
|
28
|
+
"--config-dir",
|
29
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
30
|
+
help="Path to Metaflow configuration directory",
|
31
|
+
show_default=True,
|
32
|
+
)
|
33
|
+
@click.option(
|
34
|
+
"-p",
|
35
|
+
"--profile",
|
36
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
37
|
+
help="The named metaflow profile in which your workstation exists",
|
38
|
+
)
|
39
|
+
@click.option(
|
40
|
+
"--port",
|
41
|
+
required=True,
|
42
|
+
help="Port number where you want to start your app",
|
43
|
+
type=int,
|
44
|
+
)
|
45
|
+
@click.option(
|
46
|
+
"--name",
|
47
|
+
required=True,
|
48
|
+
help="Name of your app",
|
49
|
+
type=str,
|
50
|
+
)
|
51
|
+
@click.option(
|
52
|
+
"-o",
|
53
|
+
"--output",
|
54
|
+
default="",
|
55
|
+
help="Show output in the specified format.",
|
56
|
+
type=click.Choice(["json", ""]),
|
57
|
+
)
|
58
|
+
def start(config_dir=None, profile=None, port=-1, name="", output=""):
|
59
|
+
start_app_response = OuterboundsCommandResponse()
|
60
|
+
|
61
|
+
validate_workstation_step = CommandStatus(
|
62
|
+
"ValidateRunningOnWorkstation",
|
63
|
+
OuterboundsCommandStatus.OK,
|
64
|
+
"Command is being run on a workstation.",
|
65
|
+
)
|
66
|
+
|
67
|
+
list_workstations_step = CommandStatus(
|
68
|
+
"ListWorkstations",
|
69
|
+
OuterboundsCommandStatus.OK,
|
70
|
+
"List of workstations fetched.",
|
71
|
+
)
|
72
|
+
|
73
|
+
validate_port_exists = CommandStatus(
|
74
|
+
"ValidatePortExists",
|
75
|
+
OuterboundsCommandStatus.OK,
|
76
|
+
"Port exists on workstation",
|
77
|
+
)
|
78
|
+
|
79
|
+
start_app_step = CommandStatus(
|
80
|
+
"StartApp",
|
81
|
+
OuterboundsCommandStatus.OK,
|
82
|
+
f"App {name} started on port {port}!",
|
83
|
+
)
|
84
|
+
|
85
|
+
if "WORKSTATION_ID" not in os.environ:
|
86
|
+
validate_workstation_step.update(
|
87
|
+
OuterboundsCommandStatus.FAIL,
|
88
|
+
"All outerbounds app commands can only be run from a workstation.",
|
89
|
+
"",
|
90
|
+
)
|
91
|
+
start_app_response.add_step(validate_workstation_step)
|
92
|
+
click.secho(
|
93
|
+
"All outerbounds app commands can only be run from a workstation.",
|
94
|
+
fg="red",
|
95
|
+
err=True,
|
96
|
+
)
|
97
|
+
|
98
|
+
if output == "json":
|
99
|
+
click.echo(json.dumps(start_app_response.as_dict(), indent=4))
|
100
|
+
return
|
101
|
+
|
102
|
+
try:
|
103
|
+
try:
|
104
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
105
|
+
config_dir, profile
|
106
|
+
)
|
107
|
+
api_url = metaflowconfig.get_sanitized_url_from_config(
|
108
|
+
config_dir, profile, "OBP_API_SERVER"
|
109
|
+
)
|
110
|
+
|
111
|
+
workstations_response = requests.get(
|
112
|
+
f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
|
113
|
+
)
|
114
|
+
workstations_response.raise_for_status()
|
115
|
+
start_app_response.add_step(list_workstations_step)
|
116
|
+
except:
|
117
|
+
click.secho("Failed to list workstations!", fg="red", err=True)
|
118
|
+
list_workstations_step.update(
|
119
|
+
OuterboundsCommandStatus.FAIL, "Failed to list workstations!", ""
|
120
|
+
)
|
121
|
+
start_app_response.add_step(list_workstations_step)
|
122
|
+
if output == "json":
|
123
|
+
click.echo(json.dumps(start_app_response.as_dict(), indent=4))
|
124
|
+
return
|
125
|
+
|
126
|
+
workstations_json = workstations_response.json()["workstations"]
|
127
|
+
for workstation in workstations_json:
|
128
|
+
if workstation["instance_id"] == os.environ["WORKSTATION_ID"]:
|
129
|
+
if "named_ports" in workstation["spec"]:
|
130
|
+
for named_port in workstation["spec"]["named_ports"]:
|
131
|
+
if int(named_port["port"]) == port:
|
132
|
+
start_app_response.add_step(validate_port_exists)
|
133
|
+
if named_port["enabled"] and named_port["name"] == name:
|
134
|
+
click.secho(
|
135
|
+
f"App {name} started on port {port}!",
|
136
|
+
fg="green",
|
137
|
+
err=True,
|
138
|
+
)
|
139
|
+
click.secho(
|
140
|
+
f"App URL: {api_url.replace('api', 'ui')}/apps/{os.environ['WORKSTATION_ID']}/{name}/",
|
141
|
+
fg="green",
|
142
|
+
err=True,
|
143
|
+
)
|
144
|
+
start_app_response.add_step(start_app_step)
|
145
|
+
if output == "json":
|
146
|
+
click.echo(
|
147
|
+
json.dumps(
|
148
|
+
start_app_response.as_dict(), indent=4
|
149
|
+
)
|
150
|
+
)
|
151
|
+
return
|
152
|
+
else:
|
153
|
+
try:
|
154
|
+
response = requests.put(
|
155
|
+
f"{api_url}/v1/workstations/update/{os.environ['WORKSTATION_ID']}/namedports",
|
156
|
+
headers={"x-api-key": metaflow_token},
|
157
|
+
json={
|
158
|
+
"port": port,
|
159
|
+
"name": name,
|
160
|
+
"enabled": True,
|
161
|
+
},
|
162
|
+
)
|
163
|
+
|
164
|
+
response.raise_for_status()
|
165
|
+
click.secho(
|
166
|
+
f"App {name} started on port {port}!",
|
167
|
+
fg="green",
|
168
|
+
err=True,
|
169
|
+
)
|
170
|
+
except:
|
171
|
+
click.secho(
|
172
|
+
f"Failed to start app {name} on port {port}!",
|
173
|
+
fg="red",
|
174
|
+
err=True,
|
175
|
+
)
|
176
|
+
start_app_step.update(
|
177
|
+
OuterboundsCommandStatus.FAIL,
|
178
|
+
f"Failed to start app {name} on port {port}!",
|
179
|
+
"",
|
180
|
+
)
|
181
|
+
|
182
|
+
start_app_response.add_step(start_app_step)
|
183
|
+
if output == "json":
|
184
|
+
click.echo(
|
185
|
+
json.dumps(
|
186
|
+
start_app_response.as_dict(), indent=4
|
187
|
+
)
|
188
|
+
)
|
189
|
+
return
|
190
|
+
|
191
|
+
click.secho(
|
192
|
+
f"Port {port} not found on workstation {os.environ['WORKSTATION_ID']}",
|
193
|
+
fg="red",
|
194
|
+
err=True,
|
195
|
+
)
|
196
|
+
validate_port_exists.update(
|
197
|
+
OuterboundsCommandStatus.FAIL,
|
198
|
+
f"Port {port} not found on workstation {os.environ['WORKSTATION_ID']}",
|
199
|
+
"",
|
200
|
+
)
|
201
|
+
start_app_response.add_step(validate_port_exists)
|
202
|
+
if output == "json":
|
203
|
+
click.echo(json.dumps(start_app_response.as_dict(), indent=4))
|
204
|
+
except Exception as e:
|
205
|
+
click.secho(f"Failed to start app {name} on port {port}!", fg="red", err=True)
|
206
|
+
start_app_step.update(
|
207
|
+
OuterboundsCommandStatus.FAIL,
|
208
|
+
f"Failed to start app {name} on port {port}!",
|
209
|
+
"",
|
210
|
+
)
|
211
|
+
start_app_response.add_step(start_app_step)
|
212
|
+
if output == "json":
|
213
|
+
click.secho(json.dumps(start_app_response.as_dict(), indent=4))
|
214
|
+
|
215
|
+
|
216
|
+
@app.command(help="Stop an app using its port number")
|
217
|
+
@click.option(
|
218
|
+
"-d",
|
219
|
+
"--config-dir",
|
220
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
221
|
+
help="Path to Metaflow configuration directory",
|
222
|
+
show_default=True,
|
223
|
+
)
|
224
|
+
@click.option(
|
225
|
+
"-p",
|
226
|
+
"--profile",
|
227
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
228
|
+
help="The named metaflow profile in which your workstation exists",
|
229
|
+
)
|
230
|
+
@click.option(
|
231
|
+
"--port",
|
232
|
+
required=False,
|
233
|
+
default=-1,
|
234
|
+
help="Port number where you want to start your app.",
|
235
|
+
type=int,
|
236
|
+
)
|
237
|
+
@click.option(
|
238
|
+
"--name",
|
239
|
+
required=False,
|
240
|
+
help="Name of your app",
|
241
|
+
default="",
|
242
|
+
type=str,
|
243
|
+
)
|
244
|
+
@click.option(
|
245
|
+
"-o",
|
246
|
+
"--output",
|
247
|
+
default="",
|
248
|
+
help="Show output in the specified format.",
|
249
|
+
type=click.Choice(["json", ""]),
|
250
|
+
)
|
251
|
+
def stop(config_dir=None, profile=None, port=-1, name="", output=""):
|
252
|
+
if port == -1 and not name:
|
253
|
+
click.secho(
|
254
|
+
"Please provide either a port number or a name to stop the app.",
|
255
|
+
fg="red",
|
256
|
+
err=True,
|
257
|
+
)
|
258
|
+
return
|
259
|
+
|
260
|
+
stop_app_response = OuterboundsCommandResponse()
|
261
|
+
|
262
|
+
validate_workstation_step = CommandStatus(
|
263
|
+
"ValidateRunningOnWorkstation",
|
264
|
+
OuterboundsCommandStatus.OK,
|
265
|
+
"Command is being run on a workstation.",
|
266
|
+
)
|
267
|
+
|
268
|
+
list_workstations_step = CommandStatus(
|
269
|
+
"ListWorkstations",
|
270
|
+
OuterboundsCommandStatus.OK,
|
271
|
+
"List of workstations fetched.",
|
272
|
+
)
|
273
|
+
|
274
|
+
validate_port_exists = CommandStatus(
|
275
|
+
"ValidatePortExists",
|
276
|
+
OuterboundsCommandStatus.OK,
|
277
|
+
"Port exists on workstation",
|
278
|
+
)
|
279
|
+
|
280
|
+
stop_app_step = CommandStatus(
|
281
|
+
"StopApp",
|
282
|
+
OuterboundsCommandStatus.OK,
|
283
|
+
f"App stopped on port {port}!",
|
284
|
+
)
|
285
|
+
|
286
|
+
if "WORKSTATION_ID" not in os.environ:
|
287
|
+
validate_workstation_step.update(
|
288
|
+
OuterboundsCommandStatus.FAIL,
|
289
|
+
"All outerbounds app commands can only be run from a workstation.",
|
290
|
+
"",
|
291
|
+
)
|
292
|
+
stop_app_response.add_step(validate_workstation_step)
|
293
|
+
click.secho(
|
294
|
+
"All outerbounds app commands can only be run from a workstation.",
|
295
|
+
fg="red",
|
296
|
+
err=True,
|
297
|
+
)
|
298
|
+
|
299
|
+
if output == "json":
|
300
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
301
|
+
return
|
302
|
+
|
303
|
+
try:
|
304
|
+
try:
|
305
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
306
|
+
config_dir, profile
|
307
|
+
)
|
308
|
+
api_url = metaflowconfig.get_sanitized_url_from_config(
|
309
|
+
config_dir, profile, "OBP_API_SERVER"
|
310
|
+
)
|
311
|
+
|
312
|
+
workstations_response = requests.get(
|
313
|
+
f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
|
314
|
+
)
|
315
|
+
workstations_response.raise_for_status()
|
316
|
+
stop_app_response.add_step(list_workstations_step)
|
317
|
+
except:
|
318
|
+
click.secho("Failed to list workstations!", fg="red", err=True)
|
319
|
+
list_workstations_step.update(
|
320
|
+
OuterboundsCommandStatus.FAIL, "Failed to list workstations!", ""
|
321
|
+
)
|
322
|
+
stop_app_response.add_step(list_workstations_step)
|
323
|
+
if output == "json":
|
324
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
325
|
+
return
|
326
|
+
|
327
|
+
workstations_json = workstations_response.json()["workstations"]
|
328
|
+
for workstation in workstations_json:
|
329
|
+
if workstation["instance_id"] == os.environ["WORKSTATION_ID"]:
|
330
|
+
if "named_ports" in workstation["spec"]:
|
331
|
+
for named_port in workstation["spec"]["named_ports"]:
|
332
|
+
if (
|
333
|
+
int(named_port["port"]) == port
|
334
|
+
or named_port["name"] == name
|
335
|
+
):
|
336
|
+
stop_app_response.add_step(validate_port_exists)
|
337
|
+
if not named_port["enabled"]:
|
338
|
+
click.secho(
|
339
|
+
f"App {named_port['name']} stopped on port {port}!",
|
340
|
+
fg="green",
|
341
|
+
err=True,
|
342
|
+
)
|
343
|
+
stop_app_response.add_step(stop_app_step)
|
344
|
+
if output == "json":
|
345
|
+
click.echo(
|
346
|
+
json.dumps(
|
347
|
+
stop_app_response.as_dict(), indent=4
|
348
|
+
)
|
349
|
+
)
|
350
|
+
return
|
351
|
+
else:
|
352
|
+
try:
|
353
|
+
response = requests.put(
|
354
|
+
f"{api_url}/v1/workstations/update/{os.environ['WORKSTATION_ID']}/namedports",
|
355
|
+
headers={"x-api-key": metaflow_token},
|
356
|
+
json={
|
357
|
+
"port": port,
|
358
|
+
"name": named_port["name"],
|
359
|
+
"enabled": False,
|
360
|
+
},
|
361
|
+
)
|
362
|
+
response.raise_for_status()
|
363
|
+
click.secho(
|
364
|
+
f"App stopped on port {port}!",
|
365
|
+
fg="green",
|
366
|
+
err=True,
|
367
|
+
)
|
368
|
+
except:
|
369
|
+
click.secho(
|
370
|
+
f"Failed to stop app on port {port}!",
|
371
|
+
fg="red",
|
372
|
+
err=True,
|
373
|
+
)
|
374
|
+
stop_app_step.update(
|
375
|
+
OuterboundsCommandStatus.FAIL,
|
376
|
+
f"Failed to stop app on port {port}!",
|
377
|
+
"",
|
378
|
+
)
|
379
|
+
|
380
|
+
stop_app_response.add_step(stop_app_step)
|
381
|
+
if output == "json":
|
382
|
+
click.echo(
|
383
|
+
json.dumps(
|
384
|
+
stop_app_response.as_dict(), indent=4
|
385
|
+
)
|
386
|
+
)
|
387
|
+
return
|
388
|
+
|
389
|
+
err_message = (
|
390
|
+
f"Port {port} not found on workstation {os.environ['WORKSTATION_ID']}"
|
391
|
+
)
|
392
|
+
if port == -1:
|
393
|
+
err_message = (
|
394
|
+
f"App {name} not found on workstation {os.environ['WORKSTATION_ID']}"
|
395
|
+
)
|
396
|
+
|
397
|
+
click.secho(
|
398
|
+
err_message,
|
399
|
+
fg="red",
|
400
|
+
err=True,
|
401
|
+
)
|
402
|
+
|
403
|
+
validate_port_exists.update(
|
404
|
+
OuterboundsCommandStatus.FAIL,
|
405
|
+
err_message,
|
406
|
+
"",
|
407
|
+
)
|
408
|
+
stop_app_response.add_step(validate_port_exists)
|
409
|
+
if output == "json":
|
410
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
411
|
+
except Exception as e:
|
412
|
+
click.secho(f"Failed to stop app on port {port}!", fg="red", err=True)
|
413
|
+
stop_app_step.update(
|
414
|
+
OuterboundsCommandStatus.FAIL, f"Failed to stop on port {port}!", ""
|
415
|
+
)
|
416
|
+
stop_app_response.add_step(stop_app_step)
|
417
|
+
if output == "json":
|
418
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
419
|
+
|
420
|
+
|
421
|
+
@app.command(help="Stop an app using its port number")
|
422
|
+
@click.option(
|
423
|
+
"-d",
|
424
|
+
"--config-dir",
|
425
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
426
|
+
help="Path to Metaflow configuration directory",
|
427
|
+
show_default=True,
|
428
|
+
)
|
429
|
+
@click.option(
|
430
|
+
"-p",
|
431
|
+
"--profile",
|
432
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
433
|
+
help="The named metaflow profile in which your workstation exists",
|
434
|
+
)
|
435
|
+
@click.option(
|
436
|
+
"-o",
|
437
|
+
"--output",
|
438
|
+
default="",
|
439
|
+
help="Show output in the specified format.",
|
440
|
+
type=click.Choice(["json", ""]),
|
441
|
+
)
|
442
|
+
def list(config_dir=None, profile=None, output=""):
|
443
|
+
list_app_response = OuterboundsCommandResponse()
|
444
|
+
|
445
|
+
validate_workstation_step = CommandStatus(
|
446
|
+
"ValidateRunningOnWorkstation",
|
447
|
+
OuterboundsCommandStatus.OK,
|
448
|
+
"Command is being run on a workstation.",
|
449
|
+
)
|
450
|
+
|
451
|
+
list_workstations_step = CommandStatus(
|
452
|
+
"ListWorkstations",
|
453
|
+
OuterboundsCommandStatus.OK,
|
454
|
+
"List of workstations fetched.",
|
455
|
+
)
|
456
|
+
|
457
|
+
if "WORKSTATION_ID" not in os.environ:
|
458
|
+
validate_workstation_step.update(
|
459
|
+
OuterboundsCommandStatus.FAIL,
|
460
|
+
"All outerbounds app commands can only be run from a workstation.",
|
461
|
+
"",
|
462
|
+
)
|
463
|
+
list_app_response.add_step(validate_workstation_step)
|
464
|
+
click.secho(
|
465
|
+
"All outerbounds app commands can only be run from a workstation.",
|
466
|
+
fg="red",
|
467
|
+
err=True,
|
468
|
+
)
|
469
|
+
|
470
|
+
if output == "json":
|
471
|
+
click.echo(json.dumps(list_app_response.as_dict(), indent=4))
|
472
|
+
return
|
473
|
+
|
474
|
+
try:
|
475
|
+
try:
|
476
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
477
|
+
config_dir, profile
|
478
|
+
)
|
479
|
+
api_url = metaflowconfig.get_sanitized_url_from_config(
|
480
|
+
config_dir, profile, "OBP_API_SERVER"
|
481
|
+
)
|
482
|
+
|
483
|
+
workstations_response = requests.get(
|
484
|
+
f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
|
485
|
+
)
|
486
|
+
workstations_response.raise_for_status()
|
487
|
+
list_app_response.add_step(list_workstations_step)
|
488
|
+
except:
|
489
|
+
click.secho("Failed to list workstations!", fg="red", err=True)
|
490
|
+
list_workstations_step.update(
|
491
|
+
OuterboundsCommandStatus.FAIL, "Failed to list workstations!", ""
|
492
|
+
)
|
493
|
+
list_app_response.add_step(list_workstations_step)
|
494
|
+
if output == "json":
|
495
|
+
click.echo(json.dumps(list_app_response.as_dict(), indent=4))
|
496
|
+
return
|
497
|
+
|
498
|
+
workstations_json = workstations_response.json()["workstations"]
|
499
|
+
for workstation in workstations_json:
|
500
|
+
if workstation["instance_id"] == os.environ["WORKSTATION_ID"]:
|
501
|
+
if "named_ports" in workstation["spec"]:
|
502
|
+
for named_port in workstation["spec"]["named_ports"]:
|
503
|
+
if named_port["enabled"]:
|
504
|
+
click.secho(
|
505
|
+
f"App Name: {named_port['name']}", fg="green", err=True
|
506
|
+
)
|
507
|
+
click.secho(
|
508
|
+
f"App Port on Workstation: {named_port['port']}",
|
509
|
+
fg="green",
|
510
|
+
err=True,
|
511
|
+
)
|
512
|
+
click.secho(f"App Status: Deployed", fg="green", err=True)
|
513
|
+
click.secho(
|
514
|
+
f"App URL: {api_url.replace('api', 'ui')}/apps/{os.environ['WORKSTATION_ID']}/{named_port['name']}/",
|
515
|
+
fg="green",
|
516
|
+
err=True,
|
517
|
+
)
|
518
|
+
else:
|
519
|
+
click.secho(
|
520
|
+
f"App Port on Workstation: {named_port['port']}",
|
521
|
+
fg="yellow",
|
522
|
+
err=True,
|
523
|
+
)
|
524
|
+
click.secho(
|
525
|
+
f"App Status: Not Deployed", fg="yellow", err=True
|
526
|
+
)
|
527
|
+
|
528
|
+
click.echo("\n", err=True)
|
529
|
+
except Exception as e:
|
530
|
+
click.secho(f"Failed to list apps!", fg="red", err=True)
|
531
|
+
if output == "json":
|
532
|
+
click.echo(json.dumps(list_app_response.as_dict(), indent=4))
|
533
|
+
|
534
|
+
|
535
|
+
cli.add_command(app, name="app")
|
@@ -1,12 +1,16 @@
|
|
1
1
|
from outerbounds._vendor import click
|
2
|
-
from . import local_setup_cli
|
3
|
-
from . import workstations_cli
|
4
|
-
from . import perimeters_cli
|
2
|
+
from . import local_setup_cli, workstations_cli, perimeters_cli, apps_cli, tutorials_cli
|
5
3
|
|
6
4
|
|
7
5
|
@click.command(
|
8
6
|
cls=click.CommandCollection,
|
9
|
-
sources=[
|
7
|
+
sources=[
|
8
|
+
local_setup_cli.cli,
|
9
|
+
workstations_cli.cli,
|
10
|
+
perimeters_cli.cli,
|
11
|
+
apps_cli.cli,
|
12
|
+
tutorials_cli.cli,
|
13
|
+
],
|
10
14
|
)
|
11
15
|
def cli(**kwargs):
|
12
16
|
pass
|
@@ -5,6 +5,7 @@ import os
|
|
5
5
|
import re
|
6
6
|
import subprocess
|
7
7
|
import sys
|
8
|
+
import time
|
8
9
|
import zlib
|
9
10
|
from base64 import b64decode, b64encode
|
10
11
|
from importlib.machinery import PathFinder
|
@@ -15,6 +16,10 @@ from outerbounds._vendor import click
|
|
15
16
|
import requests
|
16
17
|
from requests.exceptions import HTTPError
|
17
18
|
|
19
|
+
from google.oauth2 import service_account
|
20
|
+
import google.auth
|
21
|
+
import google.auth.jwt
|
22
|
+
|
18
23
|
from ..utils import kubeconfig, metaflowconfig
|
19
24
|
from ..utils.schema import (
|
20
25
|
CommandStatus,
|
@@ -762,6 +767,48 @@ def get_gha_jwt(audience: str):
|
|
762
767
|
sys.exit(1)
|
763
768
|
|
764
769
|
|
770
|
+
def get_gcp_jwt(audience: str, service_account_token_file_path: str = ""):
|
771
|
+
try:
|
772
|
+
if service_account_token_file_path != "":
|
773
|
+
credentials = service_account.Credentials.from_service_account_file(
|
774
|
+
service_account_token_file_path
|
775
|
+
)
|
776
|
+
else:
|
777
|
+
credentials, project = google.auth.default()
|
778
|
+
except Exception as e:
|
779
|
+
click.secho(
|
780
|
+
f"Failed to get Google Cloud service account credentials. Error: {str(e)}",
|
781
|
+
fg="red",
|
782
|
+
)
|
783
|
+
sys.exit(1)
|
784
|
+
|
785
|
+
# Ensure the credentials are service account credentials to sign the JWT
|
786
|
+
if isinstance(credentials, service_account.Credentials):
|
787
|
+
try:
|
788
|
+
payload = {
|
789
|
+
"iat": int(time.time()), # Issued at time
|
790
|
+
"exp": int(time.time()) + 172800, # 48 hours expiration time
|
791
|
+
"sub": credentials.service_account_email,
|
792
|
+
"aud": audience,
|
793
|
+
"sa_email": credentials.service_account_email,
|
794
|
+
}
|
795
|
+
|
796
|
+
signed_jwt = google.auth.jwt.encode(credentials.signer, payload)
|
797
|
+
return signed_jwt.decode("utf-8")
|
798
|
+
except Exception as e:
|
799
|
+
click.secho(
|
800
|
+
f"Failed to sign JWT token using Google Cloud service account credentials. Error: {str(e)}",
|
801
|
+
fg="red",
|
802
|
+
)
|
803
|
+
sys.exit(1)
|
804
|
+
|
805
|
+
click.secho(
|
806
|
+
"The provided credentials are not service account credentials. Please provide a valid service account credentials file or set valid service account credentials in the environment using GOOGLE_APPLICATION_CREDENTIALS.",
|
807
|
+
fg="red",
|
808
|
+
)
|
809
|
+
sys.exit(1)
|
810
|
+
|
811
|
+
|
765
812
|
def get_origin_token(
|
766
813
|
service_principal_name: str,
|
767
814
|
deployment: str,
|
@@ -932,6 +979,17 @@ def configure(
|
|
932
979
|
is_flag=True,
|
933
980
|
help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
|
934
981
|
)
|
982
|
+
@click.option(
|
983
|
+
"--gcp",
|
984
|
+
is_flag=True,
|
985
|
+
help="Set if the command is being run which a Google Cloud service account credential. If both --jwt-token and --gcp are specified the -gcp flag will be ignored.",
|
986
|
+
)
|
987
|
+
@click.option(
|
988
|
+
"--gcp-service-account-toke-file-path",
|
989
|
+
default="",
|
990
|
+
help="The full path to the Google Cloud service account token file. If --gcp is set and this value is not specified, the default GOOGLE_APPLICATION_CREDENTIALS environment variable will be used.",
|
991
|
+
required=True,
|
992
|
+
)
|
935
993
|
@click.option(
|
936
994
|
"-d",
|
937
995
|
"--config-dir",
|
@@ -963,18 +1021,31 @@ def service_principal_configure(
|
|
963
1021
|
perimeter: str,
|
964
1022
|
jwt_token="",
|
965
1023
|
github_actions=False,
|
1024
|
+
gcp=False,
|
1025
|
+
gcp_service_account_toke_file_path="",
|
966
1026
|
config_dir=None,
|
967
1027
|
profile=None,
|
968
1028
|
echo=None,
|
969
1029
|
force=False,
|
970
1030
|
):
|
971
1031
|
audience = f"https://{deployment_domain}"
|
972
|
-
|
973
|
-
|
1032
|
+
|
1033
|
+
# ensure only one of github_actions or gcp is set
|
1034
|
+
if github_actions and gcp:
|
1035
|
+
click.secho(
|
1036
|
+
"Both --github-actions and --gcp flags cannot be set at the same time.",
|
1037
|
+
fg="red",
|
1038
|
+
)
|
1039
|
+
sys.exit(1)
|
974
1040
|
|
975
1041
|
if jwt_token == "":
|
1042
|
+
if github_actions:
|
1043
|
+
jwt_token = get_gha_jwt(audience)
|
1044
|
+
elif gcp:
|
1045
|
+
jwt_token = get_gcp_jwt(audience, gcp_service_account_toke_file_path)
|
1046
|
+
else:
|
976
1047
|
click.secho(
|
977
|
-
"No JWT token provided. Please provider either a valid jwt token or set --github-actions",
|
1048
|
+
"No JWT token provided. Please provider either a valid jwt token or set --github-actions or --gcp flag.",
|
978
1049
|
fg="red",
|
979
1050
|
)
|
980
1051
|
sys.exit(1)
|
@@ -0,0 +1,111 @@
|
|
1
|
+
import os
|
2
|
+
from outerbounds._vendor import click
|
3
|
+
import requests
|
4
|
+
|
5
|
+
import tarfile
|
6
|
+
import hashlib
|
7
|
+
import tempfile
|
8
|
+
|
9
|
+
|
10
|
+
@click.group()
|
11
|
+
def cli(**kwargs):
|
12
|
+
pass
|
13
|
+
|
14
|
+
|
15
|
+
@click.group(help="Manage tutorials curated by Outerbounds.", hidden=True)
|
16
|
+
def tutorials(**kwargs):
|
17
|
+
pass
|
18
|
+
|
19
|
+
|
20
|
+
@tutorials.command(help="Pull Outerbounds tutorials.")
|
21
|
+
@click.option(
|
22
|
+
"--url",
|
23
|
+
required=True,
|
24
|
+
help="URL to pull the tutorials from.",
|
25
|
+
type=str,
|
26
|
+
)
|
27
|
+
@click.option(
|
28
|
+
"--destination-dir",
|
29
|
+
help="Show output in the specified format.",
|
30
|
+
type=str,
|
31
|
+
required=True,
|
32
|
+
)
|
33
|
+
@click.option(
|
34
|
+
"--force-overwrite",
|
35
|
+
is_flag=True,
|
36
|
+
help="Overwrite all existing files across all tutorials.",
|
37
|
+
type=bool,
|
38
|
+
required=False,
|
39
|
+
default=False,
|
40
|
+
)
|
41
|
+
def pull(url="", destination_dir="", force_overwrite=False):
|
42
|
+
try:
|
43
|
+
secure_download_and_extract(
|
44
|
+
url, destination_dir, force_overwrite=force_overwrite
|
45
|
+
)
|
46
|
+
click.secho("Tutorials pulled successfully.", fg="green", err=True)
|
47
|
+
except Exception as e:
|
48
|
+
print(e)
|
49
|
+
click.secho(f"Failed to pull tutorials: {e}", fg="red", err=True)
|
50
|
+
|
51
|
+
|
52
|
+
def secure_download_and_extract(
|
53
|
+
url, dest_dir, expected_hash=None, force_overwrite=False
|
54
|
+
):
|
55
|
+
"""
|
56
|
+
Download a tar.gz file from a URL, verify its integrity, and extract its contents.
|
57
|
+
|
58
|
+
:param url: URL of the tar.gz file to download
|
59
|
+
:param dest_dir: Destination directory to extract the contents
|
60
|
+
:param expected_hash: Expected SHA256 hash of the file (optional)
|
61
|
+
"""
|
62
|
+
|
63
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
64
|
+
temp_file = os.path.join(
|
65
|
+
temp_dir, hashlib.md5(url.encode()).hexdigest() + ".tar.gz"
|
66
|
+
)
|
67
|
+
|
68
|
+
# Download the file
|
69
|
+
try:
|
70
|
+
response = requests.get(url, stream=True, verify=True)
|
71
|
+
response.raise_for_status()
|
72
|
+
|
73
|
+
with open(temp_file, "wb") as f:
|
74
|
+
for chunk in response.iter_content(chunk_size=8192):
|
75
|
+
f.write(chunk)
|
76
|
+
except requests.exceptions.RequestException as e:
|
77
|
+
raise Exception(f"Failed to download file: {e}")
|
78
|
+
|
79
|
+
if expected_hash:
|
80
|
+
with open(temp_file, "rb") as f:
|
81
|
+
file_hash = hashlib.sha256(f.read()).hexdigest()
|
82
|
+
if file_hash != expected_hash:
|
83
|
+
raise Exception("File integrity check failed")
|
84
|
+
|
85
|
+
os.makedirs(dest_dir, exist_ok=True)
|
86
|
+
|
87
|
+
try:
|
88
|
+
with tarfile.open(temp_file, "r:gz") as tar:
|
89
|
+
# Keep track of new journeys to extract.
|
90
|
+
to_extract = []
|
91
|
+
members = tar.getmembers()
|
92
|
+
for member in members:
|
93
|
+
member_path = os.path.join(dest_dir, member.name)
|
94
|
+
# Check for any files trying to write outside the destination
|
95
|
+
if not os.path.abspath(member_path).startswith(
|
96
|
+
os.path.abspath(dest_dir)
|
97
|
+
):
|
98
|
+
raise Exception("Attempted path traversal in tar file")
|
99
|
+
if not os.path.exists(member_path):
|
100
|
+
# The user might have modified the existing files, leave them untouched.
|
101
|
+
to_extract.append(member)
|
102
|
+
|
103
|
+
if force_overwrite:
|
104
|
+
tar.extractall(path=dest_dir)
|
105
|
+
else:
|
106
|
+
tar.extractall(path=dest_dir, members=to_extract)
|
107
|
+
except tarfile.TarError as e:
|
108
|
+
raise Exception(f"Failed to extract tar file: {e}")
|
109
|
+
|
110
|
+
|
111
|
+
cli.add_command(tutorials, name="tutorials")
|
outerbounds/utils/schema.py
CHANGED
@@ -59,14 +59,14 @@ class OuterboundsCommandResponse:
|
|
59
59
|
if step.status == OuterboundsCommandStatus.FAIL:
|
60
60
|
self.status = OuterboundsCommandStatus.FAIL
|
61
61
|
self._code = 500
|
62
|
-
self._message = "
|
62
|
+
self._message = "Encountered an error when trying to run command."
|
63
63
|
elif (
|
64
64
|
step.status == OuterboundsCommandStatus.WARN
|
65
65
|
and self.status != OuterboundsCommandStatus.FAIL
|
66
66
|
):
|
67
67
|
self.status = OuterboundsCommandStatus.WARN
|
68
68
|
self._code = 200
|
69
|
-
self._message = "
|
69
|
+
self._message = "Encountered one or more warnings when running the command."
|
70
70
|
|
71
71
|
def as_dict(self):
|
72
72
|
self._data["steps"] = [step.as_dict() for step in self._steps]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: outerbounds
|
3
|
-
Version: 0.3.
|
3
|
+
Version: 0.3.93
|
4
4
|
Summary: More Data Science, Less Administration
|
5
5
|
License: Proprietary
|
6
6
|
Keywords: data science,machine learning,MLOps
|
@@ -24,9 +24,9 @@ Requires-Dist: google-api-core (>=2.16.1,<3.0.0) ; extra == "gcp"
|
|
24
24
|
Requires-Dist: google-auth (>=2.27.0,<3.0.0) ; extra == "gcp"
|
25
25
|
Requires-Dist: google-cloud-secret-manager (>=2.20.0,<3.0.0) ; extra == "gcp"
|
26
26
|
Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
|
27
|
-
Requires-Dist: ob-metaflow (==2.12.
|
28
|
-
Requires-Dist: ob-metaflow-extensions (==1.1.
|
29
|
-
Requires-Dist: ob-metaflow-stubs (==5.
|
27
|
+
Requires-Dist: ob-metaflow (==2.12.17.1)
|
28
|
+
Requires-Dist: ob-metaflow-extensions (==1.1.82)
|
29
|
+
Requires-Dist: ob-metaflow-stubs (==5.5)
|
30
30
|
Requires-Dist: opentelemetry-distro (==0.41b0)
|
31
31
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http (==1.20.0)
|
32
32
|
Requires-Dist: opentelemetry-instrumentation-requests (==0.41b0)
|
@@ -41,17 +41,19 @@ outerbounds/_vendor/yaml/serializer.py,sha256=8wFZRy9SsQSktF_f9OOroroqsh4qVUe53r
|
|
41
41
|
outerbounds/_vendor/yaml/tokens.py,sha256=JBSu38wihGr4l73JwbfMA7Ks1-X84g8-NskTz7KwPmA,2578
|
42
42
|
outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
|
43
43
|
outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
|
44
|
-
outerbounds/command_groups/
|
45
|
-
outerbounds/command_groups/
|
44
|
+
outerbounds/command_groups/apps_cli.py,sha256=k3zsBh7GupBqM5sbEU8OHSC1TBVdTBJybvPX4DX5Ack,20052
|
45
|
+
outerbounds/command_groups/cli.py,sha256=q0hdJO4biD3iEOdyJcxnRkeleA8AKAhx842kQ49I6kk,365
|
46
|
+
outerbounds/command_groups/local_setup_cli.py,sha256=AgK4fe-q1uRm20OsMGFGDezheGOPMBoESC98JT9HzqQ,39046
|
46
47
|
outerbounds/command_groups/perimeters_cli.py,sha256=mrJfFIRYFOjuiz-9h4OKg2JT8Utmbs72z6wvPzDss3s,18685
|
48
|
+
outerbounds/command_groups/tutorials_cli.py,sha256=UInFyiMqtscHFfi8YQwiY_6Sdw9quJOtRu5OukEBccw,3522
|
47
49
|
outerbounds/command_groups/workstations_cli.py,sha256=V5Jbj1cVb4IRllI7fOgNgL6OekRpuFDv6CEhDb4xC6w,22016
|
48
50
|
outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
49
51
|
outerbounds/utils/kubeconfig.py,sha256=yvcyRXGR4AhQuqUDqmbGxEOHw5ixMFV0AZIDg1LI_Qo,7981
|
50
52
|
outerbounds/utils/metaflowconfig.py,sha256=l2vJbgPkLISU-XPGZFaC8ZKmYFyJemlD6bwB-EKUsAw,5770
|
51
|
-
outerbounds/utils/schema.py,sha256=
|
53
|
+
outerbounds/utils/schema.py,sha256=lMUr9kNgn9wy-sO_t_Tlxmbt63yLeN4b0xQXbDUDj4A,2331
|
52
54
|
outerbounds/utils/utils.py,sha256=4Z8cszNob_8kDYCLNTrP-wWads_S_MdL3Uj3ju4mEsk,501
|
53
55
|
outerbounds/vendor.py,sha256=gRLRJNXtZBeUpPEog0LOeIsl6GosaFFbCxUvR4bW6IQ,5093
|
54
|
-
outerbounds-0.3.
|
55
|
-
outerbounds-0.3.
|
56
|
-
outerbounds-0.3.
|
57
|
-
outerbounds-0.3.
|
56
|
+
outerbounds-0.3.93.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
|
57
|
+
outerbounds-0.3.93.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
|
58
|
+
outerbounds-0.3.93.dist-info/METADATA,sha256=DbAq4c18up77upq4UVgOtzl1AL_cjHkdUC1SBHnx0os,1632
|
59
|
+
outerbounds-0.3.93.dist-info/RECORD,,
|
File without changes
|
File without changes
|