outerbounds 0.3.91__py3-none-any.whl → 0.3.93__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerbounds/command_groups/apps_cli.py +535 -0
- outerbounds/command_groups/cli.py +8 -4
- outerbounds/command_groups/local_setup_cli.py +74 -3
- outerbounds/command_groups/tutorials_cli.py +111 -0
- outerbounds/utils/schema.py +2 -2
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/METADATA +4 -4
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/RECORD +9 -7
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/WHEEL +0 -0
- {outerbounds-0.3.91.dist-info → outerbounds-0.3.93.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,535 @@
|
|
1
|
+
import json
|
2
|
+
import os
|
3
|
+
from os import path
|
4
|
+
from outerbounds._vendor import click
|
5
|
+
import requests
|
6
|
+
|
7
|
+
from ..utils import metaflowconfig
|
8
|
+
from ..utils.schema import (
|
9
|
+
CommandStatus,
|
10
|
+
OuterboundsCommandResponse,
|
11
|
+
OuterboundsCommandStatus,
|
12
|
+
)
|
13
|
+
|
14
|
+
|
15
|
+
@click.group()
|
16
|
+
def cli(**kwargs):
|
17
|
+
pass
|
18
|
+
|
19
|
+
|
20
|
+
@click.group(help="Manage apps")
|
21
|
+
def app(**kwargs):
|
22
|
+
pass
|
23
|
+
|
24
|
+
|
25
|
+
@app.command(help="Start an app using a port and a name")
|
26
|
+
@click.option(
|
27
|
+
"-d",
|
28
|
+
"--config-dir",
|
29
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
30
|
+
help="Path to Metaflow configuration directory",
|
31
|
+
show_default=True,
|
32
|
+
)
|
33
|
+
@click.option(
|
34
|
+
"-p",
|
35
|
+
"--profile",
|
36
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
37
|
+
help="The named metaflow profile in which your workstation exists",
|
38
|
+
)
|
39
|
+
@click.option(
|
40
|
+
"--port",
|
41
|
+
required=True,
|
42
|
+
help="Port number where you want to start your app",
|
43
|
+
type=int,
|
44
|
+
)
|
45
|
+
@click.option(
|
46
|
+
"--name",
|
47
|
+
required=True,
|
48
|
+
help="Name of your app",
|
49
|
+
type=str,
|
50
|
+
)
|
51
|
+
@click.option(
|
52
|
+
"-o",
|
53
|
+
"--output",
|
54
|
+
default="",
|
55
|
+
help="Show output in the specified format.",
|
56
|
+
type=click.Choice(["json", ""]),
|
57
|
+
)
|
58
|
+
def start(config_dir=None, profile=None, port=-1, name="", output=""):
|
59
|
+
start_app_response = OuterboundsCommandResponse()
|
60
|
+
|
61
|
+
validate_workstation_step = CommandStatus(
|
62
|
+
"ValidateRunningOnWorkstation",
|
63
|
+
OuterboundsCommandStatus.OK,
|
64
|
+
"Command is being run on a workstation.",
|
65
|
+
)
|
66
|
+
|
67
|
+
list_workstations_step = CommandStatus(
|
68
|
+
"ListWorkstations",
|
69
|
+
OuterboundsCommandStatus.OK,
|
70
|
+
"List of workstations fetched.",
|
71
|
+
)
|
72
|
+
|
73
|
+
validate_port_exists = CommandStatus(
|
74
|
+
"ValidatePortExists",
|
75
|
+
OuterboundsCommandStatus.OK,
|
76
|
+
"Port exists on workstation",
|
77
|
+
)
|
78
|
+
|
79
|
+
start_app_step = CommandStatus(
|
80
|
+
"StartApp",
|
81
|
+
OuterboundsCommandStatus.OK,
|
82
|
+
f"App {name} started on port {port}!",
|
83
|
+
)
|
84
|
+
|
85
|
+
if "WORKSTATION_ID" not in os.environ:
|
86
|
+
validate_workstation_step.update(
|
87
|
+
OuterboundsCommandStatus.FAIL,
|
88
|
+
"All outerbounds app commands can only be run from a workstation.",
|
89
|
+
"",
|
90
|
+
)
|
91
|
+
start_app_response.add_step(validate_workstation_step)
|
92
|
+
click.secho(
|
93
|
+
"All outerbounds app commands can only be run from a workstation.",
|
94
|
+
fg="red",
|
95
|
+
err=True,
|
96
|
+
)
|
97
|
+
|
98
|
+
if output == "json":
|
99
|
+
click.echo(json.dumps(start_app_response.as_dict(), indent=4))
|
100
|
+
return
|
101
|
+
|
102
|
+
try:
|
103
|
+
try:
|
104
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
105
|
+
config_dir, profile
|
106
|
+
)
|
107
|
+
api_url = metaflowconfig.get_sanitized_url_from_config(
|
108
|
+
config_dir, profile, "OBP_API_SERVER"
|
109
|
+
)
|
110
|
+
|
111
|
+
workstations_response = requests.get(
|
112
|
+
f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
|
113
|
+
)
|
114
|
+
workstations_response.raise_for_status()
|
115
|
+
start_app_response.add_step(list_workstations_step)
|
116
|
+
except:
|
117
|
+
click.secho("Failed to list workstations!", fg="red", err=True)
|
118
|
+
list_workstations_step.update(
|
119
|
+
OuterboundsCommandStatus.FAIL, "Failed to list workstations!", ""
|
120
|
+
)
|
121
|
+
start_app_response.add_step(list_workstations_step)
|
122
|
+
if output == "json":
|
123
|
+
click.echo(json.dumps(start_app_response.as_dict(), indent=4))
|
124
|
+
return
|
125
|
+
|
126
|
+
workstations_json = workstations_response.json()["workstations"]
|
127
|
+
for workstation in workstations_json:
|
128
|
+
if workstation["instance_id"] == os.environ["WORKSTATION_ID"]:
|
129
|
+
if "named_ports" in workstation["spec"]:
|
130
|
+
for named_port in workstation["spec"]["named_ports"]:
|
131
|
+
if int(named_port["port"]) == port:
|
132
|
+
start_app_response.add_step(validate_port_exists)
|
133
|
+
if named_port["enabled"] and named_port["name"] == name:
|
134
|
+
click.secho(
|
135
|
+
f"App {name} started on port {port}!",
|
136
|
+
fg="green",
|
137
|
+
err=True,
|
138
|
+
)
|
139
|
+
click.secho(
|
140
|
+
f"App URL: {api_url.replace('api', 'ui')}/apps/{os.environ['WORKSTATION_ID']}/{name}/",
|
141
|
+
fg="green",
|
142
|
+
err=True,
|
143
|
+
)
|
144
|
+
start_app_response.add_step(start_app_step)
|
145
|
+
if output == "json":
|
146
|
+
click.echo(
|
147
|
+
json.dumps(
|
148
|
+
start_app_response.as_dict(), indent=4
|
149
|
+
)
|
150
|
+
)
|
151
|
+
return
|
152
|
+
else:
|
153
|
+
try:
|
154
|
+
response = requests.put(
|
155
|
+
f"{api_url}/v1/workstations/update/{os.environ['WORKSTATION_ID']}/namedports",
|
156
|
+
headers={"x-api-key": metaflow_token},
|
157
|
+
json={
|
158
|
+
"port": port,
|
159
|
+
"name": name,
|
160
|
+
"enabled": True,
|
161
|
+
},
|
162
|
+
)
|
163
|
+
|
164
|
+
response.raise_for_status()
|
165
|
+
click.secho(
|
166
|
+
f"App {name} started on port {port}!",
|
167
|
+
fg="green",
|
168
|
+
err=True,
|
169
|
+
)
|
170
|
+
except:
|
171
|
+
click.secho(
|
172
|
+
f"Failed to start app {name} on port {port}!",
|
173
|
+
fg="red",
|
174
|
+
err=True,
|
175
|
+
)
|
176
|
+
start_app_step.update(
|
177
|
+
OuterboundsCommandStatus.FAIL,
|
178
|
+
f"Failed to start app {name} on port {port}!",
|
179
|
+
"",
|
180
|
+
)
|
181
|
+
|
182
|
+
start_app_response.add_step(start_app_step)
|
183
|
+
if output == "json":
|
184
|
+
click.echo(
|
185
|
+
json.dumps(
|
186
|
+
start_app_response.as_dict(), indent=4
|
187
|
+
)
|
188
|
+
)
|
189
|
+
return
|
190
|
+
|
191
|
+
click.secho(
|
192
|
+
f"Port {port} not found on workstation {os.environ['WORKSTATION_ID']}",
|
193
|
+
fg="red",
|
194
|
+
err=True,
|
195
|
+
)
|
196
|
+
validate_port_exists.update(
|
197
|
+
OuterboundsCommandStatus.FAIL,
|
198
|
+
f"Port {port} not found on workstation {os.environ['WORKSTATION_ID']}",
|
199
|
+
"",
|
200
|
+
)
|
201
|
+
start_app_response.add_step(validate_port_exists)
|
202
|
+
if output == "json":
|
203
|
+
click.echo(json.dumps(start_app_response.as_dict(), indent=4))
|
204
|
+
except Exception as e:
|
205
|
+
click.secho(f"Failed to start app {name} on port {port}!", fg="red", err=True)
|
206
|
+
start_app_step.update(
|
207
|
+
OuterboundsCommandStatus.FAIL,
|
208
|
+
f"Failed to start app {name} on port {port}!",
|
209
|
+
"",
|
210
|
+
)
|
211
|
+
start_app_response.add_step(start_app_step)
|
212
|
+
if output == "json":
|
213
|
+
click.secho(json.dumps(start_app_response.as_dict(), indent=4))
|
214
|
+
|
215
|
+
|
216
|
+
@app.command(help="Stop an app using its port number")
|
217
|
+
@click.option(
|
218
|
+
"-d",
|
219
|
+
"--config-dir",
|
220
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
221
|
+
help="Path to Metaflow configuration directory",
|
222
|
+
show_default=True,
|
223
|
+
)
|
224
|
+
@click.option(
|
225
|
+
"-p",
|
226
|
+
"--profile",
|
227
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
228
|
+
help="The named metaflow profile in which your workstation exists",
|
229
|
+
)
|
230
|
+
@click.option(
|
231
|
+
"--port",
|
232
|
+
required=False,
|
233
|
+
default=-1,
|
234
|
+
help="Port number where you want to start your app.",
|
235
|
+
type=int,
|
236
|
+
)
|
237
|
+
@click.option(
|
238
|
+
"--name",
|
239
|
+
required=False,
|
240
|
+
help="Name of your app",
|
241
|
+
default="",
|
242
|
+
type=str,
|
243
|
+
)
|
244
|
+
@click.option(
|
245
|
+
"-o",
|
246
|
+
"--output",
|
247
|
+
default="",
|
248
|
+
help="Show output in the specified format.",
|
249
|
+
type=click.Choice(["json", ""]),
|
250
|
+
)
|
251
|
+
def stop(config_dir=None, profile=None, port=-1, name="", output=""):
|
252
|
+
if port == -1 and not name:
|
253
|
+
click.secho(
|
254
|
+
"Please provide either a port number or a name to stop the app.",
|
255
|
+
fg="red",
|
256
|
+
err=True,
|
257
|
+
)
|
258
|
+
return
|
259
|
+
|
260
|
+
stop_app_response = OuterboundsCommandResponse()
|
261
|
+
|
262
|
+
validate_workstation_step = CommandStatus(
|
263
|
+
"ValidateRunningOnWorkstation",
|
264
|
+
OuterboundsCommandStatus.OK,
|
265
|
+
"Command is being run on a workstation.",
|
266
|
+
)
|
267
|
+
|
268
|
+
list_workstations_step = CommandStatus(
|
269
|
+
"ListWorkstations",
|
270
|
+
OuterboundsCommandStatus.OK,
|
271
|
+
"List of workstations fetched.",
|
272
|
+
)
|
273
|
+
|
274
|
+
validate_port_exists = CommandStatus(
|
275
|
+
"ValidatePortExists",
|
276
|
+
OuterboundsCommandStatus.OK,
|
277
|
+
"Port exists on workstation",
|
278
|
+
)
|
279
|
+
|
280
|
+
stop_app_step = CommandStatus(
|
281
|
+
"StopApp",
|
282
|
+
OuterboundsCommandStatus.OK,
|
283
|
+
f"App stopped on port {port}!",
|
284
|
+
)
|
285
|
+
|
286
|
+
if "WORKSTATION_ID" not in os.environ:
|
287
|
+
validate_workstation_step.update(
|
288
|
+
OuterboundsCommandStatus.FAIL,
|
289
|
+
"All outerbounds app commands can only be run from a workstation.",
|
290
|
+
"",
|
291
|
+
)
|
292
|
+
stop_app_response.add_step(validate_workstation_step)
|
293
|
+
click.secho(
|
294
|
+
"All outerbounds app commands can only be run from a workstation.",
|
295
|
+
fg="red",
|
296
|
+
err=True,
|
297
|
+
)
|
298
|
+
|
299
|
+
if output == "json":
|
300
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
301
|
+
return
|
302
|
+
|
303
|
+
try:
|
304
|
+
try:
|
305
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
306
|
+
config_dir, profile
|
307
|
+
)
|
308
|
+
api_url = metaflowconfig.get_sanitized_url_from_config(
|
309
|
+
config_dir, profile, "OBP_API_SERVER"
|
310
|
+
)
|
311
|
+
|
312
|
+
workstations_response = requests.get(
|
313
|
+
f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
|
314
|
+
)
|
315
|
+
workstations_response.raise_for_status()
|
316
|
+
stop_app_response.add_step(list_workstations_step)
|
317
|
+
except:
|
318
|
+
click.secho("Failed to list workstations!", fg="red", err=True)
|
319
|
+
list_workstations_step.update(
|
320
|
+
OuterboundsCommandStatus.FAIL, "Failed to list workstations!", ""
|
321
|
+
)
|
322
|
+
stop_app_response.add_step(list_workstations_step)
|
323
|
+
if output == "json":
|
324
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
325
|
+
return
|
326
|
+
|
327
|
+
workstations_json = workstations_response.json()["workstations"]
|
328
|
+
for workstation in workstations_json:
|
329
|
+
if workstation["instance_id"] == os.environ["WORKSTATION_ID"]:
|
330
|
+
if "named_ports" in workstation["spec"]:
|
331
|
+
for named_port in workstation["spec"]["named_ports"]:
|
332
|
+
if (
|
333
|
+
int(named_port["port"]) == port
|
334
|
+
or named_port["name"] == name
|
335
|
+
):
|
336
|
+
stop_app_response.add_step(validate_port_exists)
|
337
|
+
if not named_port["enabled"]:
|
338
|
+
click.secho(
|
339
|
+
f"App {named_port['name']} stopped on port {port}!",
|
340
|
+
fg="green",
|
341
|
+
err=True,
|
342
|
+
)
|
343
|
+
stop_app_response.add_step(stop_app_step)
|
344
|
+
if output == "json":
|
345
|
+
click.echo(
|
346
|
+
json.dumps(
|
347
|
+
stop_app_response.as_dict(), indent=4
|
348
|
+
)
|
349
|
+
)
|
350
|
+
return
|
351
|
+
else:
|
352
|
+
try:
|
353
|
+
response = requests.put(
|
354
|
+
f"{api_url}/v1/workstations/update/{os.environ['WORKSTATION_ID']}/namedports",
|
355
|
+
headers={"x-api-key": metaflow_token},
|
356
|
+
json={
|
357
|
+
"port": port,
|
358
|
+
"name": named_port["name"],
|
359
|
+
"enabled": False,
|
360
|
+
},
|
361
|
+
)
|
362
|
+
response.raise_for_status()
|
363
|
+
click.secho(
|
364
|
+
f"App stopped on port {port}!",
|
365
|
+
fg="green",
|
366
|
+
err=True,
|
367
|
+
)
|
368
|
+
except:
|
369
|
+
click.secho(
|
370
|
+
f"Failed to stop app on port {port}!",
|
371
|
+
fg="red",
|
372
|
+
err=True,
|
373
|
+
)
|
374
|
+
stop_app_step.update(
|
375
|
+
OuterboundsCommandStatus.FAIL,
|
376
|
+
f"Failed to stop app on port {port}!",
|
377
|
+
"",
|
378
|
+
)
|
379
|
+
|
380
|
+
stop_app_response.add_step(stop_app_step)
|
381
|
+
if output == "json":
|
382
|
+
click.echo(
|
383
|
+
json.dumps(
|
384
|
+
stop_app_response.as_dict(), indent=4
|
385
|
+
)
|
386
|
+
)
|
387
|
+
return
|
388
|
+
|
389
|
+
err_message = (
|
390
|
+
f"Port {port} not found on workstation {os.environ['WORKSTATION_ID']}"
|
391
|
+
)
|
392
|
+
if port == -1:
|
393
|
+
err_message = (
|
394
|
+
f"App {name} not found on workstation {os.environ['WORKSTATION_ID']}"
|
395
|
+
)
|
396
|
+
|
397
|
+
click.secho(
|
398
|
+
err_message,
|
399
|
+
fg="red",
|
400
|
+
err=True,
|
401
|
+
)
|
402
|
+
|
403
|
+
validate_port_exists.update(
|
404
|
+
OuterboundsCommandStatus.FAIL,
|
405
|
+
err_message,
|
406
|
+
"",
|
407
|
+
)
|
408
|
+
stop_app_response.add_step(validate_port_exists)
|
409
|
+
if output == "json":
|
410
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
411
|
+
except Exception as e:
|
412
|
+
click.secho(f"Failed to stop app on port {port}!", fg="red", err=True)
|
413
|
+
stop_app_step.update(
|
414
|
+
OuterboundsCommandStatus.FAIL, f"Failed to stop on port {port}!", ""
|
415
|
+
)
|
416
|
+
stop_app_response.add_step(stop_app_step)
|
417
|
+
if output == "json":
|
418
|
+
click.echo(json.dumps(stop_app_response.as_dict(), indent=4))
|
419
|
+
|
420
|
+
|
421
|
+
@app.command(help="Stop an app using its port number")
|
422
|
+
@click.option(
|
423
|
+
"-d",
|
424
|
+
"--config-dir",
|
425
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
426
|
+
help="Path to Metaflow configuration directory",
|
427
|
+
show_default=True,
|
428
|
+
)
|
429
|
+
@click.option(
|
430
|
+
"-p",
|
431
|
+
"--profile",
|
432
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
433
|
+
help="The named metaflow profile in which your workstation exists",
|
434
|
+
)
|
435
|
+
@click.option(
|
436
|
+
"-o",
|
437
|
+
"--output",
|
438
|
+
default="",
|
439
|
+
help="Show output in the specified format.",
|
440
|
+
type=click.Choice(["json", ""]),
|
441
|
+
)
|
442
|
+
def list(config_dir=None, profile=None, output=""):
|
443
|
+
list_app_response = OuterboundsCommandResponse()
|
444
|
+
|
445
|
+
validate_workstation_step = CommandStatus(
|
446
|
+
"ValidateRunningOnWorkstation",
|
447
|
+
OuterboundsCommandStatus.OK,
|
448
|
+
"Command is being run on a workstation.",
|
449
|
+
)
|
450
|
+
|
451
|
+
list_workstations_step = CommandStatus(
|
452
|
+
"ListWorkstations",
|
453
|
+
OuterboundsCommandStatus.OK,
|
454
|
+
"List of workstations fetched.",
|
455
|
+
)
|
456
|
+
|
457
|
+
if "WORKSTATION_ID" not in os.environ:
|
458
|
+
validate_workstation_step.update(
|
459
|
+
OuterboundsCommandStatus.FAIL,
|
460
|
+
"All outerbounds app commands can only be run from a workstation.",
|
461
|
+
"",
|
462
|
+
)
|
463
|
+
list_app_response.add_step(validate_workstation_step)
|
464
|
+
click.secho(
|
465
|
+
"All outerbounds app commands can only be run from a workstation.",
|
466
|
+
fg="red",
|
467
|
+
err=True,
|
468
|
+
)
|
469
|
+
|
470
|
+
if output == "json":
|
471
|
+
click.echo(json.dumps(list_app_response.as_dict(), indent=4))
|
472
|
+
return
|
473
|
+
|
474
|
+
try:
|
475
|
+
try:
|
476
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
477
|
+
config_dir, profile
|
478
|
+
)
|
479
|
+
api_url = metaflowconfig.get_sanitized_url_from_config(
|
480
|
+
config_dir, profile, "OBP_API_SERVER"
|
481
|
+
)
|
482
|
+
|
483
|
+
workstations_response = requests.get(
|
484
|
+
f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
|
485
|
+
)
|
486
|
+
workstations_response.raise_for_status()
|
487
|
+
list_app_response.add_step(list_workstations_step)
|
488
|
+
except:
|
489
|
+
click.secho("Failed to list workstations!", fg="red", err=True)
|
490
|
+
list_workstations_step.update(
|
491
|
+
OuterboundsCommandStatus.FAIL, "Failed to list workstations!", ""
|
492
|
+
)
|
493
|
+
list_app_response.add_step(list_workstations_step)
|
494
|
+
if output == "json":
|
495
|
+
click.echo(json.dumps(list_app_response.as_dict(), indent=4))
|
496
|
+
return
|
497
|
+
|
498
|
+
workstations_json = workstations_response.json()["workstations"]
|
499
|
+
for workstation in workstations_json:
|
500
|
+
if workstation["instance_id"] == os.environ["WORKSTATION_ID"]:
|
501
|
+
if "named_ports" in workstation["spec"]:
|
502
|
+
for named_port in workstation["spec"]["named_ports"]:
|
503
|
+
if named_port["enabled"]:
|
504
|
+
click.secho(
|
505
|
+
f"App Name: {named_port['name']}", fg="green", err=True
|
506
|
+
)
|
507
|
+
click.secho(
|
508
|
+
f"App Port on Workstation: {named_port['port']}",
|
509
|
+
fg="green",
|
510
|
+
err=True,
|
511
|
+
)
|
512
|
+
click.secho(f"App Status: Deployed", fg="green", err=True)
|
513
|
+
click.secho(
|
514
|
+
f"App URL: {api_url.replace('api', 'ui')}/apps/{os.environ['WORKSTATION_ID']}/{named_port['name']}/",
|
515
|
+
fg="green",
|
516
|
+
err=True,
|
517
|
+
)
|
518
|
+
else:
|
519
|
+
click.secho(
|
520
|
+
f"App Port on Workstation: {named_port['port']}",
|
521
|
+
fg="yellow",
|
522
|
+
err=True,
|
523
|
+
)
|
524
|
+
click.secho(
|
525
|
+
f"App Status: Not Deployed", fg="yellow", err=True
|
526
|
+
)
|
527
|
+
|
528
|
+
click.echo("\n", err=True)
|
529
|
+
except Exception as e:
|
530
|
+
click.secho(f"Failed to list apps!", fg="red", err=True)
|
531
|
+
if output == "json":
|
532
|
+
click.echo(json.dumps(list_app_response.as_dict(), indent=4))
|
533
|
+
|
534
|
+
|
535
|
+
cli.add_command(app, name="app")
|
@@ -1,12 +1,16 @@
|
|
1
1
|
from outerbounds._vendor import click
|
2
|
-
from . import local_setup_cli
|
3
|
-
from . import workstations_cli
|
4
|
-
from . import perimeters_cli
|
2
|
+
from . import local_setup_cli, workstations_cli, perimeters_cli, apps_cli, tutorials_cli
|
5
3
|
|
6
4
|
|
7
5
|
@click.command(
|
8
6
|
cls=click.CommandCollection,
|
9
|
-
sources=[
|
7
|
+
sources=[
|
8
|
+
local_setup_cli.cli,
|
9
|
+
workstations_cli.cli,
|
10
|
+
perimeters_cli.cli,
|
11
|
+
apps_cli.cli,
|
12
|
+
tutorials_cli.cli,
|
13
|
+
],
|
10
14
|
)
|
11
15
|
def cli(**kwargs):
|
12
16
|
pass
|
@@ -5,6 +5,7 @@ import os
|
|
5
5
|
import re
|
6
6
|
import subprocess
|
7
7
|
import sys
|
8
|
+
import time
|
8
9
|
import zlib
|
9
10
|
from base64 import b64decode, b64encode
|
10
11
|
from importlib.machinery import PathFinder
|
@@ -15,6 +16,10 @@ from outerbounds._vendor import click
|
|
15
16
|
import requests
|
16
17
|
from requests.exceptions import HTTPError
|
17
18
|
|
19
|
+
from google.oauth2 import service_account
|
20
|
+
import google.auth
|
21
|
+
import google.auth.jwt
|
22
|
+
|
18
23
|
from ..utils import kubeconfig, metaflowconfig
|
19
24
|
from ..utils.schema import (
|
20
25
|
CommandStatus,
|
@@ -762,6 +767,48 @@ def get_gha_jwt(audience: str):
|
|
762
767
|
sys.exit(1)
|
763
768
|
|
764
769
|
|
770
|
+
def get_gcp_jwt(audience: str, service_account_token_file_path: str = ""):
|
771
|
+
try:
|
772
|
+
if service_account_token_file_path != "":
|
773
|
+
credentials = service_account.Credentials.from_service_account_file(
|
774
|
+
service_account_token_file_path
|
775
|
+
)
|
776
|
+
else:
|
777
|
+
credentials, project = google.auth.default()
|
778
|
+
except Exception as e:
|
779
|
+
click.secho(
|
780
|
+
f"Failed to get Google Cloud service account credentials. Error: {str(e)}",
|
781
|
+
fg="red",
|
782
|
+
)
|
783
|
+
sys.exit(1)
|
784
|
+
|
785
|
+
# Ensure the credentials are service account credentials to sign the JWT
|
786
|
+
if isinstance(credentials, service_account.Credentials):
|
787
|
+
try:
|
788
|
+
payload = {
|
789
|
+
"iat": int(time.time()), # Issued at time
|
790
|
+
"exp": int(time.time()) + 172800, # 48 hours expiration time
|
791
|
+
"sub": credentials.service_account_email,
|
792
|
+
"aud": audience,
|
793
|
+
"sa_email": credentials.service_account_email,
|
794
|
+
}
|
795
|
+
|
796
|
+
signed_jwt = google.auth.jwt.encode(credentials.signer, payload)
|
797
|
+
return signed_jwt.decode("utf-8")
|
798
|
+
except Exception as e:
|
799
|
+
click.secho(
|
800
|
+
f"Failed to sign JWT token using Google Cloud service account credentials. Error: {str(e)}",
|
801
|
+
fg="red",
|
802
|
+
)
|
803
|
+
sys.exit(1)
|
804
|
+
|
805
|
+
click.secho(
|
806
|
+
"The provided credentials are not service account credentials. Please provide a valid service account credentials file or set valid service account credentials in the environment using GOOGLE_APPLICATION_CREDENTIALS.",
|
807
|
+
fg="red",
|
808
|
+
)
|
809
|
+
sys.exit(1)
|
810
|
+
|
811
|
+
|
765
812
|
def get_origin_token(
|
766
813
|
service_principal_name: str,
|
767
814
|
deployment: str,
|
@@ -932,6 +979,17 @@ def configure(
|
|
932
979
|
is_flag=True,
|
933
980
|
help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
|
934
981
|
)
|
982
|
+
@click.option(
|
983
|
+
"--gcp",
|
984
|
+
is_flag=True,
|
985
|
+
help="Set if the command is being run which a Google Cloud service account credential. If both --jwt-token and --gcp are specified the -gcp flag will be ignored.",
|
986
|
+
)
|
987
|
+
@click.option(
|
988
|
+
"--gcp-service-account-toke-file-path",
|
989
|
+
default="",
|
990
|
+
help="The full path to the Google Cloud service account token file. If --gcp is set and this value is not specified, the default GOOGLE_APPLICATION_CREDENTIALS environment variable will be used.",
|
991
|
+
required=True,
|
992
|
+
)
|
935
993
|
@click.option(
|
936
994
|
"-d",
|
937
995
|
"--config-dir",
|
@@ -963,18 +1021,31 @@ def service_principal_configure(
|
|
963
1021
|
perimeter: str,
|
964
1022
|
jwt_token="",
|
965
1023
|
github_actions=False,
|
1024
|
+
gcp=False,
|
1025
|
+
gcp_service_account_toke_file_path="",
|
966
1026
|
config_dir=None,
|
967
1027
|
profile=None,
|
968
1028
|
echo=None,
|
969
1029
|
force=False,
|
970
1030
|
):
|
971
1031
|
audience = f"https://{deployment_domain}"
|
972
|
-
|
973
|
-
|
1032
|
+
|
1033
|
+
# ensure only one of github_actions or gcp is set
|
1034
|
+
if github_actions and gcp:
|
1035
|
+
click.secho(
|
1036
|
+
"Both --github-actions and --gcp flags cannot be set at the same time.",
|
1037
|
+
fg="red",
|
1038
|
+
)
|
1039
|
+
sys.exit(1)
|
974
1040
|
|
975
1041
|
if jwt_token == "":
|
1042
|
+
if github_actions:
|
1043
|
+
jwt_token = get_gha_jwt(audience)
|
1044
|
+
elif gcp:
|
1045
|
+
jwt_token = get_gcp_jwt(audience, gcp_service_account_toke_file_path)
|
1046
|
+
else:
|
976
1047
|
click.secho(
|
977
|
-
"No JWT token provided. Please provider either a valid jwt token or set --github-actions",
|
1048
|
+
"No JWT token provided. Please provider either a valid jwt token or set --github-actions or --gcp flag.",
|
978
1049
|
fg="red",
|
979
1050
|
)
|
980
1051
|
sys.exit(1)
|
@@ -0,0 +1,111 @@
|
|
1
|
+
import os
|
2
|
+
from outerbounds._vendor import click
|
3
|
+
import requests
|
4
|
+
|
5
|
+
import tarfile
|
6
|
+
import hashlib
|
7
|
+
import tempfile
|
8
|
+
|
9
|
+
|
10
|
+
@click.group()
|
11
|
+
def cli(**kwargs):
|
12
|
+
pass
|
13
|
+
|
14
|
+
|
15
|
+
@click.group(help="Manage tutorials curated by Outerbounds.", hidden=True)
|
16
|
+
def tutorials(**kwargs):
|
17
|
+
pass
|
18
|
+
|
19
|
+
|
20
|
+
@tutorials.command(help="Pull Outerbounds tutorials.")
|
21
|
+
@click.option(
|
22
|
+
"--url",
|
23
|
+
required=True,
|
24
|
+
help="URL to pull the tutorials from.",
|
25
|
+
type=str,
|
26
|
+
)
|
27
|
+
@click.option(
|
28
|
+
"--destination-dir",
|
29
|
+
help="Show output in the specified format.",
|
30
|
+
type=str,
|
31
|
+
required=True,
|
32
|
+
)
|
33
|
+
@click.option(
|
34
|
+
"--force-overwrite",
|
35
|
+
is_flag=True,
|
36
|
+
help="Overwrite all existing files across all tutorials.",
|
37
|
+
type=bool,
|
38
|
+
required=False,
|
39
|
+
default=False,
|
40
|
+
)
|
41
|
+
def pull(url="", destination_dir="", force_overwrite=False):
|
42
|
+
try:
|
43
|
+
secure_download_and_extract(
|
44
|
+
url, destination_dir, force_overwrite=force_overwrite
|
45
|
+
)
|
46
|
+
click.secho("Tutorials pulled successfully.", fg="green", err=True)
|
47
|
+
except Exception as e:
|
48
|
+
print(e)
|
49
|
+
click.secho(f"Failed to pull tutorials: {e}", fg="red", err=True)
|
50
|
+
|
51
|
+
|
52
|
+
def secure_download_and_extract(
|
53
|
+
url, dest_dir, expected_hash=None, force_overwrite=False
|
54
|
+
):
|
55
|
+
"""
|
56
|
+
Download a tar.gz file from a URL, verify its integrity, and extract its contents.
|
57
|
+
|
58
|
+
:param url: URL of the tar.gz file to download
|
59
|
+
:param dest_dir: Destination directory to extract the contents
|
60
|
+
:param expected_hash: Expected SHA256 hash of the file (optional)
|
61
|
+
"""
|
62
|
+
|
63
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
64
|
+
temp_file = os.path.join(
|
65
|
+
temp_dir, hashlib.md5(url.encode()).hexdigest() + ".tar.gz"
|
66
|
+
)
|
67
|
+
|
68
|
+
# Download the file
|
69
|
+
try:
|
70
|
+
response = requests.get(url, stream=True, verify=True)
|
71
|
+
response.raise_for_status()
|
72
|
+
|
73
|
+
with open(temp_file, "wb") as f:
|
74
|
+
for chunk in response.iter_content(chunk_size=8192):
|
75
|
+
f.write(chunk)
|
76
|
+
except requests.exceptions.RequestException as e:
|
77
|
+
raise Exception(f"Failed to download file: {e}")
|
78
|
+
|
79
|
+
if expected_hash:
|
80
|
+
with open(temp_file, "rb") as f:
|
81
|
+
file_hash = hashlib.sha256(f.read()).hexdigest()
|
82
|
+
if file_hash != expected_hash:
|
83
|
+
raise Exception("File integrity check failed")
|
84
|
+
|
85
|
+
os.makedirs(dest_dir, exist_ok=True)
|
86
|
+
|
87
|
+
try:
|
88
|
+
with tarfile.open(temp_file, "r:gz") as tar:
|
89
|
+
# Keep track of new journeys to extract.
|
90
|
+
to_extract = []
|
91
|
+
members = tar.getmembers()
|
92
|
+
for member in members:
|
93
|
+
member_path = os.path.join(dest_dir, member.name)
|
94
|
+
# Check for any files trying to write outside the destination
|
95
|
+
if not os.path.abspath(member_path).startswith(
|
96
|
+
os.path.abspath(dest_dir)
|
97
|
+
):
|
98
|
+
raise Exception("Attempted path traversal in tar file")
|
99
|
+
if not os.path.exists(member_path):
|
100
|
+
# The user might have modified the existing files, leave them untouched.
|
101
|
+
to_extract.append(member)
|
102
|
+
|
103
|
+
if force_overwrite:
|
104
|
+
tar.extractall(path=dest_dir)
|
105
|
+
else:
|
106
|
+
tar.extractall(path=dest_dir, members=to_extract)
|
107
|
+
except tarfile.TarError as e:
|
108
|
+
raise Exception(f"Failed to extract tar file: {e}")
|
109
|
+
|
110
|
+
|
111
|
+
cli.add_command(tutorials, name="tutorials")
|
outerbounds/utils/schema.py
CHANGED
@@ -59,14 +59,14 @@ class OuterboundsCommandResponse:
|
|
59
59
|
if step.status == OuterboundsCommandStatus.FAIL:
|
60
60
|
self.status = OuterboundsCommandStatus.FAIL
|
61
61
|
self._code = 500
|
62
|
-
self._message = "
|
62
|
+
self._message = "Encountered an error when trying to run command."
|
63
63
|
elif (
|
64
64
|
step.status == OuterboundsCommandStatus.WARN
|
65
65
|
and self.status != OuterboundsCommandStatus.FAIL
|
66
66
|
):
|
67
67
|
self.status = OuterboundsCommandStatus.WARN
|
68
68
|
self._code = 200
|
69
|
-
self._message = "
|
69
|
+
self._message = "Encountered one or more warnings when running the command."
|
70
70
|
|
71
71
|
def as_dict(self):
|
72
72
|
self._data["steps"] = [step.as_dict() for step in self._steps]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: outerbounds
|
3
|
-
Version: 0.3.
|
3
|
+
Version: 0.3.93
|
4
4
|
Summary: More Data Science, Less Administration
|
5
5
|
License: Proprietary
|
6
6
|
Keywords: data science,machine learning,MLOps
|
@@ -24,9 +24,9 @@ Requires-Dist: google-api-core (>=2.16.1,<3.0.0) ; extra == "gcp"
|
|
24
24
|
Requires-Dist: google-auth (>=2.27.0,<3.0.0) ; extra == "gcp"
|
25
25
|
Requires-Dist: google-cloud-secret-manager (>=2.20.0,<3.0.0) ; extra == "gcp"
|
26
26
|
Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
|
27
|
-
Requires-Dist: ob-metaflow (==2.12.
|
28
|
-
Requires-Dist: ob-metaflow-extensions (==1.1.
|
29
|
-
Requires-Dist: ob-metaflow-stubs (==5.
|
27
|
+
Requires-Dist: ob-metaflow (==2.12.17.1)
|
28
|
+
Requires-Dist: ob-metaflow-extensions (==1.1.82)
|
29
|
+
Requires-Dist: ob-metaflow-stubs (==5.5)
|
30
30
|
Requires-Dist: opentelemetry-distro (==0.41b0)
|
31
31
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http (==1.20.0)
|
32
32
|
Requires-Dist: opentelemetry-instrumentation-requests (==0.41b0)
|
@@ -41,17 +41,19 @@ outerbounds/_vendor/yaml/serializer.py,sha256=8wFZRy9SsQSktF_f9OOroroqsh4qVUe53r
|
|
41
41
|
outerbounds/_vendor/yaml/tokens.py,sha256=JBSu38wihGr4l73JwbfMA7Ks1-X84g8-NskTz7KwPmA,2578
|
42
42
|
outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
|
43
43
|
outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
|
44
|
-
outerbounds/command_groups/
|
45
|
-
outerbounds/command_groups/
|
44
|
+
outerbounds/command_groups/apps_cli.py,sha256=k3zsBh7GupBqM5sbEU8OHSC1TBVdTBJybvPX4DX5Ack,20052
|
45
|
+
outerbounds/command_groups/cli.py,sha256=q0hdJO4biD3iEOdyJcxnRkeleA8AKAhx842kQ49I6kk,365
|
46
|
+
outerbounds/command_groups/local_setup_cli.py,sha256=AgK4fe-q1uRm20OsMGFGDezheGOPMBoESC98JT9HzqQ,39046
|
46
47
|
outerbounds/command_groups/perimeters_cli.py,sha256=mrJfFIRYFOjuiz-9h4OKg2JT8Utmbs72z6wvPzDss3s,18685
|
48
|
+
outerbounds/command_groups/tutorials_cli.py,sha256=UInFyiMqtscHFfi8YQwiY_6Sdw9quJOtRu5OukEBccw,3522
|
47
49
|
outerbounds/command_groups/workstations_cli.py,sha256=V5Jbj1cVb4IRllI7fOgNgL6OekRpuFDv6CEhDb4xC6w,22016
|
48
50
|
outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
49
51
|
outerbounds/utils/kubeconfig.py,sha256=yvcyRXGR4AhQuqUDqmbGxEOHw5ixMFV0AZIDg1LI_Qo,7981
|
50
52
|
outerbounds/utils/metaflowconfig.py,sha256=l2vJbgPkLISU-XPGZFaC8ZKmYFyJemlD6bwB-EKUsAw,5770
|
51
|
-
outerbounds/utils/schema.py,sha256=
|
53
|
+
outerbounds/utils/schema.py,sha256=lMUr9kNgn9wy-sO_t_Tlxmbt63yLeN4b0xQXbDUDj4A,2331
|
52
54
|
outerbounds/utils/utils.py,sha256=4Z8cszNob_8kDYCLNTrP-wWads_S_MdL3Uj3ju4mEsk,501
|
53
55
|
outerbounds/vendor.py,sha256=gRLRJNXtZBeUpPEog0LOeIsl6GosaFFbCxUvR4bW6IQ,5093
|
54
|
-
outerbounds-0.3.
|
55
|
-
outerbounds-0.3.
|
56
|
-
outerbounds-0.3.
|
57
|
-
outerbounds-0.3.
|
56
|
+
outerbounds-0.3.93.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
|
57
|
+
outerbounds-0.3.93.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
|
58
|
+
outerbounds-0.3.93.dist-info/METADATA,sha256=DbAq4c18up77upq4UVgOtzl1AL_cjHkdUC1SBHnx0os,1632
|
59
|
+
outerbounds-0.3.93.dist-info/RECORD,,
|
File without changes
|
File without changes
|