outerbounds 0.3.55rc8__py3-none-any.whl → 0.3.133__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerbounds/_vendor/PyYAML.LICENSE +20 -0
- outerbounds/_vendor/__init__.py +0 -0
- outerbounds/_vendor/_yaml/__init__.py +34 -0
- outerbounds/_vendor/click/__init__.py +73 -0
- outerbounds/_vendor/click/_compat.py +626 -0
- outerbounds/_vendor/click/_termui_impl.py +717 -0
- outerbounds/_vendor/click/_textwrap.py +49 -0
- outerbounds/_vendor/click/_winconsole.py +279 -0
- outerbounds/_vendor/click/core.py +2998 -0
- outerbounds/_vendor/click/decorators.py +497 -0
- outerbounds/_vendor/click/exceptions.py +287 -0
- outerbounds/_vendor/click/formatting.py +301 -0
- outerbounds/_vendor/click/globals.py +68 -0
- outerbounds/_vendor/click/parser.py +529 -0
- outerbounds/_vendor/click/py.typed +0 -0
- outerbounds/_vendor/click/shell_completion.py +580 -0
- outerbounds/_vendor/click/termui.py +787 -0
- outerbounds/_vendor/click/testing.py +479 -0
- outerbounds/_vendor/click/types.py +1073 -0
- outerbounds/_vendor/click/utils.py +580 -0
- outerbounds/_vendor/click.LICENSE +28 -0
- outerbounds/_vendor/vendor_any.txt +2 -0
- outerbounds/_vendor/yaml/__init__.py +471 -0
- outerbounds/_vendor/yaml/_yaml.cpython-311-darwin.so +0 -0
- outerbounds/_vendor/yaml/composer.py +146 -0
- outerbounds/_vendor/yaml/constructor.py +862 -0
- outerbounds/_vendor/yaml/cyaml.py +177 -0
- outerbounds/_vendor/yaml/dumper.py +138 -0
- outerbounds/_vendor/yaml/emitter.py +1239 -0
- outerbounds/_vendor/yaml/error.py +94 -0
- outerbounds/_vendor/yaml/events.py +104 -0
- outerbounds/_vendor/yaml/loader.py +62 -0
- outerbounds/_vendor/yaml/nodes.py +51 -0
- outerbounds/_vendor/yaml/parser.py +629 -0
- outerbounds/_vendor/yaml/reader.py +208 -0
- outerbounds/_vendor/yaml/representer.py +378 -0
- outerbounds/_vendor/yaml/resolver.py +245 -0
- outerbounds/_vendor/yaml/scanner.py +1555 -0
- outerbounds/_vendor/yaml/serializer.py +127 -0
- outerbounds/_vendor/yaml/tokens.py +129 -0
- outerbounds/command_groups/apps_cli.py +450 -0
- outerbounds/command_groups/cli.py +9 -5
- outerbounds/command_groups/local_setup_cli.py +247 -36
- outerbounds/command_groups/perimeters_cli.py +212 -32
- outerbounds/command_groups/tutorials_cli.py +111 -0
- outerbounds/command_groups/workstations_cli.py +2 -2
- outerbounds/utils/kubeconfig.py +2 -2
- outerbounds/utils/metaflowconfig.py +93 -16
- outerbounds/utils/schema.py +2 -2
- outerbounds/utils/utils.py +19 -0
- outerbounds/vendor.py +159 -0
- {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/METADATA +17 -6
- outerbounds-0.3.133.dist-info/RECORD +59 -0
- {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/WHEEL +1 -1
- outerbounds-0.3.55rc8.dist-info/RECORD +0 -15
- {outerbounds-0.3.55rc8.dist-info → outerbounds-0.3.133.dist-info}/entry_points.txt +0 -0
@@ -1,23 +1,15 @@
|
|
1
|
-
import base64
|
2
|
-
import hashlib
|
3
1
|
import json
|
4
2
|
import os
|
5
|
-
import re
|
6
|
-
import subprocess
|
7
3
|
import sys
|
8
|
-
import
|
9
|
-
from base64 import b64decode, b64encode
|
10
|
-
from importlib.machinery import PathFinder
|
4
|
+
from io import StringIO
|
11
5
|
from os import path
|
12
|
-
from
|
13
|
-
from
|
14
|
-
|
15
|
-
import boto3
|
16
|
-
import click
|
6
|
+
from typing import Any, Dict
|
7
|
+
from outerbounds._vendor import click
|
17
8
|
import requests
|
18
|
-
|
9
|
+
import configparser
|
19
10
|
|
20
|
-
from ..utils import
|
11
|
+
from ..utils import metaflowconfig
|
12
|
+
from ..utils.utils import safe_write_to_disk
|
21
13
|
from ..utils.schema import (
|
22
14
|
CommandStatus,
|
23
15
|
OuterboundsCommandResponse,
|
@@ -32,7 +24,12 @@ def cli(**kwargs):
|
|
32
24
|
pass
|
33
25
|
|
34
26
|
|
35
|
-
@
|
27
|
+
@click.group(help="Manage perimeters")
|
28
|
+
def perimeter(**kwargs):
|
29
|
+
pass
|
30
|
+
|
31
|
+
|
32
|
+
@perimeter.command(help="Switch current perimeter")
|
36
33
|
@click.option(
|
37
34
|
"-d",
|
38
35
|
"--config-dir",
|
@@ -61,7 +58,7 @@ def cli(**kwargs):
|
|
61
58
|
help="Force change the existing perimeter",
|
62
59
|
default=False,
|
63
60
|
)
|
64
|
-
def
|
61
|
+
def switch(config_dir=None, profile=None, output="", id=None, force=False):
|
65
62
|
switch_perimeter_response = OuterboundsCommandResponse()
|
66
63
|
|
67
64
|
switch_perimeter_step = CommandStatus(
|
@@ -77,7 +74,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
|
|
77
74
|
id, perimeters, output, switch_perimeter_response, switch_perimeter_step
|
78
75
|
)
|
79
76
|
|
80
|
-
path_to_config = get_ob_config_file_path(config_dir, profile)
|
77
|
+
path_to_config = metaflowconfig.get_ob_config_file_path(config_dir, profile)
|
81
78
|
|
82
79
|
import fcntl
|
83
80
|
|
@@ -121,12 +118,34 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
|
|
121
118
|
)
|
122
119
|
|
123
120
|
switch_perimeter_response.add_step(switch_perimeter_step)
|
121
|
+
|
122
|
+
ensure_cloud_creds_step = CommandStatus(
|
123
|
+
"EnsureCloudCredentials",
|
124
|
+
OuterboundsCommandStatus.OK,
|
125
|
+
"Cloud credentials were successfully updated.",
|
126
|
+
)
|
127
|
+
|
128
|
+
try:
|
129
|
+
ensure_cloud_credentials_for_shell(config_dir, profile)
|
130
|
+
except:
|
131
|
+
click.secho(
|
132
|
+
"Failed to update cloud credentials.",
|
133
|
+
fg="red",
|
134
|
+
err=True,
|
135
|
+
)
|
136
|
+
ensure_cloud_creds_step.update(
|
137
|
+
status=OuterboundsCommandStatus.FAIL,
|
138
|
+
reason="Failed to update cloud credentials.",
|
139
|
+
mitigation="",
|
140
|
+
)
|
141
|
+
|
142
|
+
switch_perimeter_response.add_step(ensure_cloud_creds_step)
|
143
|
+
|
124
144
|
if output == "json":
|
125
145
|
click.echo(json.dumps(switch_perimeter_response.as_dict(), indent=4))
|
126
|
-
return
|
127
146
|
|
128
147
|
|
129
|
-
@
|
148
|
+
@perimeter.command(help="Show current perimeter")
|
130
149
|
@click.option(
|
131
150
|
"-d",
|
132
151
|
"--config-dir",
|
@@ -148,7 +167,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
|
|
148
167
|
help="Show output in the specified format.",
|
149
168
|
type=click.Choice(["json", ""]),
|
150
169
|
)
|
151
|
-
def
|
170
|
+
def show_current(config_dir=None, profile=None, output=""):
|
152
171
|
show_current_perimeter_response = OuterboundsCommandResponse()
|
153
172
|
|
154
173
|
show_current_perimeter_step = CommandStatus(
|
@@ -194,7 +213,7 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
|
|
194
213
|
click.echo(json.dumps(show_current_perimeter_response.as_dict(), indent=4))
|
195
214
|
|
196
215
|
|
197
|
-
@
|
216
|
+
@perimeter.command(help="List all available perimeters")
|
198
217
|
@click.option(
|
199
218
|
"-d",
|
200
219
|
"--config-dir",
|
@@ -215,7 +234,7 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
|
|
215
234
|
help="Show output in the specified format.",
|
216
235
|
type=click.Choice(["json", ""]),
|
217
236
|
)
|
218
|
-
def
|
237
|
+
def list(config_dir=None, profile=None, output=""):
|
219
238
|
list_perimeters_response = OuterboundsCommandResponse()
|
220
239
|
|
221
240
|
list_perimeters_step = CommandStatus(
|
@@ -272,6 +291,58 @@ def list_perimeters(config_dir=None, profile=None, output=""):
|
|
272
291
|
click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
|
273
292
|
|
274
293
|
|
294
|
+
@perimeter.command(
|
295
|
+
help="Ensure credentials for cloud are synced with perimeter", hidden=True
|
296
|
+
)
|
297
|
+
@click.option(
|
298
|
+
"-d",
|
299
|
+
"--config-dir",
|
300
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
301
|
+
help="Path to Metaflow configuration directory",
|
302
|
+
show_default=True,
|
303
|
+
)
|
304
|
+
@click.option(
|
305
|
+
"-p",
|
306
|
+
"--profile",
|
307
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
308
|
+
help="The named metaflow profile in which your workstation exists",
|
309
|
+
)
|
310
|
+
@click.option(
|
311
|
+
"-o",
|
312
|
+
"--output",
|
313
|
+
default="",
|
314
|
+
help="Show output in the specified format.",
|
315
|
+
type=click.Choice(["json", ""]),
|
316
|
+
)
|
317
|
+
def ensure_cloud_creds(config_dir=None, profile=None, output=""):
|
318
|
+
ensure_cloud_creds_step = CommandStatus(
|
319
|
+
"EnsureCloudCredentials",
|
320
|
+
OuterboundsCommandStatus.OK,
|
321
|
+
"Cloud credentials were successfully updated.",
|
322
|
+
)
|
323
|
+
|
324
|
+
ensure_cloud_creds_response = OuterboundsCommandResponse()
|
325
|
+
|
326
|
+
try:
|
327
|
+
ensure_cloud_credentials_for_shell(config_dir, profile)
|
328
|
+
click.secho("Cloud credentials updated successfully.", fg="green", err=True)
|
329
|
+
except:
|
330
|
+
click.secho(
|
331
|
+
"Failed to update cloud credentials.",
|
332
|
+
fg="red",
|
333
|
+
err=True,
|
334
|
+
)
|
335
|
+
ensure_cloud_creds_step.update(
|
336
|
+
status=OuterboundsCommandStatus.FAIL,
|
337
|
+
reason="Failed to update cloud credentials.",
|
338
|
+
mitigation="",
|
339
|
+
)
|
340
|
+
|
341
|
+
ensure_cloud_creds_response.add_step(ensure_cloud_creds_step)
|
342
|
+
if output == "json":
|
343
|
+
click.echo(json.dumps(ensure_cloud_creds_response.as_dict(), indent=4))
|
344
|
+
|
345
|
+
|
275
346
|
def get_list_perimeters_api_response(config_dir, profile):
|
276
347
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
277
348
|
api_url = metaflowconfig.get_sanitized_url_from_config(
|
@@ -285,15 +356,6 @@ def get_list_perimeters_api_response(config_dir, profile):
|
|
285
356
|
return perimeters_response.json()["perimeters"]
|
286
357
|
|
287
358
|
|
288
|
-
def get_ob_config_file_path(config_dir: str, profile: str) -> str:
|
289
|
-
# If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
|
290
|
-
# If neither are set, use ~/.metaflowconfig
|
291
|
-
obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
|
292
|
-
|
293
|
-
ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
|
294
|
-
return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
|
295
|
-
|
296
|
-
|
297
359
|
def get_perimeters_from_api_or_fail_command(
|
298
360
|
config_dir: str,
|
299
361
|
profile: str,
|
@@ -328,7 +390,7 @@ def get_ob_config_or_fail_command(
|
|
328
390
|
command_response: OuterboundsCommandResponse,
|
329
391
|
command_step: CommandStatus,
|
330
392
|
) -> Dict[str, str]:
|
331
|
-
path_to_config = get_ob_config_file_path(config_dir, profile)
|
393
|
+
path_to_config = metaflowconfig.get_ob_config_file_path(config_dir, profile)
|
332
394
|
|
333
395
|
if not os.path.exists(path_to_config):
|
334
396
|
click.secho(
|
@@ -368,6 +430,23 @@ def get_ob_config_or_fail_command(
|
|
368
430
|
return ob_config_dict
|
369
431
|
|
370
432
|
|
433
|
+
def ensure_cloud_credentials_for_shell(config_dir, profile):
|
434
|
+
if "WORKSTATION_ID" not in os.environ:
|
435
|
+
# Naive check to see if we're running in workstation. No need to ensure anything
|
436
|
+
# if this is not a workstation.
|
437
|
+
return
|
438
|
+
|
439
|
+
mf_config = metaflowconfig.init_config(config_dir, profile)
|
440
|
+
|
441
|
+
# Currently we only support GCP. TODO: utkarsh to add support for AWS and Azure
|
442
|
+
if "METAFLOW_DEFAULT_GCP_CLIENT_PROVIDER" in mf_config:
|
443
|
+
# This is a GCP deployment.
|
444
|
+
ensure_gcp_cloud_creds(config_dir, profile)
|
445
|
+
elif "METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER" in mf_config:
|
446
|
+
# This is an AWS deployment.
|
447
|
+
ensure_aws_cloud_creds(config_dir, profile)
|
448
|
+
|
449
|
+
|
371
450
|
def confirm_user_has_access_to_perimeter_or_fail(
|
372
451
|
perimeter_id: str,
|
373
452
|
perimeters: Dict[str, Any],
|
@@ -390,3 +469,104 @@ def confirm_user_has_access_to_perimeter_or_fail(
|
|
390
469
|
if output == "json":
|
391
470
|
click.echo(json.dumps(command_response.as_dict(), indent=4))
|
392
471
|
sys.exit(1)
|
472
|
+
|
473
|
+
|
474
|
+
def ensure_gcp_cloud_creds(config_dir, profile):
|
475
|
+
token_info = get_gcp_auth_credentials(config_dir, profile)
|
476
|
+
auth_url = metaflowconfig.get_sanitized_url_from_config(
|
477
|
+
config_dir, profile, "OBP_AUTH_SERVER"
|
478
|
+
)
|
479
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
480
|
+
|
481
|
+
try:
|
482
|
+
# GOOGLE_APPLICATION_CREDENTIALS is a well known gcloud environment variable
|
483
|
+
credentials_file_loc = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
|
484
|
+
except KeyError:
|
485
|
+
# This is most likely an old workstation when these params weren't set. Do nothing.
|
486
|
+
# Alternatively, user might have deliberately unset it to use their own auth.
|
487
|
+
return
|
488
|
+
|
489
|
+
credentials_json = {
|
490
|
+
"type": "external_account",
|
491
|
+
"audience": f"//iam.googleapis.com/projects/{token_info['gcpProjectNumber']}/locations/global/workloadIdentityPools/{token_info['gcpWorkloadIdentityPool']}/providers/{token_info['gcpWorkloadIdentityPoolProvider']}",
|
492
|
+
"subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
|
493
|
+
"token_url": "https://sts.googleapis.com/v1/token",
|
494
|
+
"service_account_impersonation_url": f"https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{token_info['gcpServiceAccountEmail']}:generateAccessToken",
|
495
|
+
"credential_source": {
|
496
|
+
"url": f"{auth_url}/generate/gcp",
|
497
|
+
"headers": {"x-api-key": metaflow_token},
|
498
|
+
"format": {"type": "json", "subject_token_field_name": "token"},
|
499
|
+
},
|
500
|
+
}
|
501
|
+
|
502
|
+
safe_write_to_disk(credentials_file_loc, json.dumps(credentials_json))
|
503
|
+
|
504
|
+
|
505
|
+
def ensure_aws_cloud_creds(config_dir, profile):
|
506
|
+
token_info = get_aws_auth_credentials(config_dir, profile)
|
507
|
+
|
508
|
+
try:
|
509
|
+
token_file_loc = os.environ["OBP_AWS_WEB_IDENTITY_TOKEN_FILE"]
|
510
|
+
|
511
|
+
# AWS_CONFIG_FILE is a well known aws cli environment variable
|
512
|
+
config_file_loc = os.environ["AWS_CONFIG_FILE"]
|
513
|
+
except KeyError:
|
514
|
+
# This is most likely an old workstation when these params weren't set. Do nothing.
|
515
|
+
# Alternatively, user might have deliberately unset it to use their own auth.
|
516
|
+
return
|
517
|
+
|
518
|
+
aws_config = configparser.ConfigParser()
|
519
|
+
aws_config.read(config_file_loc)
|
520
|
+
|
521
|
+
aws_config["profile task"] = {
|
522
|
+
"role_arn": token_info["role_arn"],
|
523
|
+
"web_identity_token_file": token_file_loc,
|
524
|
+
}
|
525
|
+
|
526
|
+
if token_info.get("cspr_role_arn"):
|
527
|
+
# If CSPR role is present, then we need to use the task role (in the task profile)
|
528
|
+
# to assume the CSPR role.
|
529
|
+
aws_config["profile outerbounds"] = {
|
530
|
+
"role_arn": token_info["cspr_role_arn"],
|
531
|
+
"source_profile": "task",
|
532
|
+
}
|
533
|
+
else:
|
534
|
+
# If no CSPR role is present, just use the task profile as the outerbounds profile.
|
535
|
+
aws_config["profile outerbounds"] = aws_config["profile task"]
|
536
|
+
|
537
|
+
aws_config_string = StringIO()
|
538
|
+
aws_config.write(aws_config_string)
|
539
|
+
|
540
|
+
safe_write_to_disk(token_file_loc, token_info["token"])
|
541
|
+
safe_write_to_disk(config_file_loc, aws_config_string.getvalue())
|
542
|
+
|
543
|
+
|
544
|
+
def get_aws_auth_credentials(config_dir, profile): # pragma: no cover
|
545
|
+
token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
546
|
+
auth_server_url = metaflowconfig.get_sanitized_url_from_config(
|
547
|
+
config_dir, profile, "OBP_AUTH_SERVER"
|
548
|
+
)
|
549
|
+
|
550
|
+
response = requests.get(
|
551
|
+
"{}/generate/aws".format(auth_server_url), headers={"x-api-key": token}
|
552
|
+
)
|
553
|
+
response.raise_for_status()
|
554
|
+
|
555
|
+
return response.json()
|
556
|
+
|
557
|
+
|
558
|
+
def get_gcp_auth_credentials(config_dir, profile):
|
559
|
+
token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
560
|
+
auth_server_url = metaflowconfig.get_sanitized_url_from_config(
|
561
|
+
config_dir, profile, "OBP_AUTH_SERVER"
|
562
|
+
)
|
563
|
+
|
564
|
+
response = requests.get(
|
565
|
+
"{}/generate/gcp".format(auth_server_url), headers={"x-api-key": token}
|
566
|
+
)
|
567
|
+
response.raise_for_status()
|
568
|
+
|
569
|
+
return response.json()
|
570
|
+
|
571
|
+
|
572
|
+
cli.add_command(perimeter, name="perimeter")
|
@@ -0,0 +1,111 @@
|
|
1
|
+
import os
|
2
|
+
from outerbounds._vendor import click
|
3
|
+
import requests
|
4
|
+
|
5
|
+
import tarfile
|
6
|
+
import hashlib
|
7
|
+
import tempfile
|
8
|
+
|
9
|
+
|
10
|
+
@click.group()
|
11
|
+
def cli(**kwargs):
|
12
|
+
pass
|
13
|
+
|
14
|
+
|
15
|
+
@click.group(help="Manage tutorials curated by Outerbounds.", hidden=True)
|
16
|
+
def tutorials(**kwargs):
|
17
|
+
pass
|
18
|
+
|
19
|
+
|
20
|
+
@tutorials.command(help="Pull Outerbounds tutorials.")
|
21
|
+
@click.option(
|
22
|
+
"--url",
|
23
|
+
required=True,
|
24
|
+
help="URL to pull the tutorials from.",
|
25
|
+
type=str,
|
26
|
+
)
|
27
|
+
@click.option(
|
28
|
+
"--destination-dir",
|
29
|
+
help="Show output in the specified format.",
|
30
|
+
type=str,
|
31
|
+
required=True,
|
32
|
+
)
|
33
|
+
@click.option(
|
34
|
+
"--force-overwrite",
|
35
|
+
is_flag=True,
|
36
|
+
help="Overwrite all existing files across all tutorials.",
|
37
|
+
type=bool,
|
38
|
+
required=False,
|
39
|
+
default=False,
|
40
|
+
)
|
41
|
+
def pull(url="", destination_dir="", force_overwrite=False):
|
42
|
+
try:
|
43
|
+
secure_download_and_extract(
|
44
|
+
url, destination_dir, force_overwrite=force_overwrite
|
45
|
+
)
|
46
|
+
click.secho("Tutorials pulled successfully.", fg="green", err=True)
|
47
|
+
except Exception as e:
|
48
|
+
print(e)
|
49
|
+
click.secho(f"Failed to pull tutorials: {e}", fg="red", err=True)
|
50
|
+
|
51
|
+
|
52
|
+
def secure_download_and_extract(
|
53
|
+
url, dest_dir, expected_hash=None, force_overwrite=False
|
54
|
+
):
|
55
|
+
"""
|
56
|
+
Download a tar.gz file from a URL, verify its integrity, and extract its contents.
|
57
|
+
|
58
|
+
:param url: URL of the tar.gz file to download
|
59
|
+
:param dest_dir: Destination directory to extract the contents
|
60
|
+
:param expected_hash: Expected SHA256 hash of the file (optional)
|
61
|
+
"""
|
62
|
+
|
63
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
64
|
+
temp_file = os.path.join(
|
65
|
+
temp_dir, hashlib.md5(url.encode()).hexdigest() + ".tar.gz"
|
66
|
+
)
|
67
|
+
|
68
|
+
# Download the file
|
69
|
+
try:
|
70
|
+
response = requests.get(url, stream=True, verify=True)
|
71
|
+
response.raise_for_status()
|
72
|
+
|
73
|
+
with open(temp_file, "wb") as f:
|
74
|
+
for chunk in response.iter_content(chunk_size=8192):
|
75
|
+
f.write(chunk)
|
76
|
+
except requests.exceptions.RequestException as e:
|
77
|
+
raise Exception(f"Failed to download file: {e}")
|
78
|
+
|
79
|
+
if expected_hash:
|
80
|
+
with open(temp_file, "rb") as f:
|
81
|
+
file_hash = hashlib.sha256(f.read()).hexdigest()
|
82
|
+
if file_hash != expected_hash:
|
83
|
+
raise Exception("File integrity check failed")
|
84
|
+
|
85
|
+
os.makedirs(dest_dir, exist_ok=True)
|
86
|
+
|
87
|
+
try:
|
88
|
+
with tarfile.open(temp_file, "r:gz") as tar:
|
89
|
+
# Keep track of new journeys to extract.
|
90
|
+
to_extract = []
|
91
|
+
members = tar.getmembers()
|
92
|
+
for member in members:
|
93
|
+
member_path = os.path.join(dest_dir, member.name)
|
94
|
+
# Check for any files trying to write outside the destination
|
95
|
+
if not os.path.abspath(member_path).startswith(
|
96
|
+
os.path.abspath(dest_dir)
|
97
|
+
):
|
98
|
+
raise Exception("Attempted path traversal in tar file")
|
99
|
+
if not os.path.exists(member_path):
|
100
|
+
# The user might have modified the existing files, leave them untouched.
|
101
|
+
to_extract.append(member)
|
102
|
+
|
103
|
+
if force_overwrite:
|
104
|
+
tar.extractall(path=dest_dir)
|
105
|
+
else:
|
106
|
+
tar.extractall(path=dest_dir, members=to_extract)
|
107
|
+
except tarfile.TarError as e:
|
108
|
+
raise Exception(f"Failed to extract tar file: {e}")
|
109
|
+
|
110
|
+
|
111
|
+
cli.add_command(tutorials, name="tutorials")
|
outerbounds/utils/kubeconfig.py
CHANGED
@@ -1,25 +1,49 @@
|
|
1
|
+
from outerbounds._vendor import click
|
1
2
|
import json
|
2
3
|
import os
|
3
4
|
import requests
|
4
5
|
from os import path
|
5
|
-
import
|
6
|
+
from typing import Dict, Union
|
7
|
+
import sys
|
8
|
+
|
9
|
+
"""
|
10
|
+
key: perimeter specific URL to fetch the remote metaflow config from
|
11
|
+
value: the remote metaflow config
|
12
|
+
"""
|
13
|
+
CACHED_REMOTE_METAFLOW_CONFIG: Dict[str, Dict[str, str]] = {}
|
14
|
+
|
6
15
|
|
16
|
+
CURRENT_PERIMETER_KEY = "OB_CURRENT_PERIMETER"
|
17
|
+
CURRENT_PERIMETER_URL = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
|
18
|
+
CURRENT_PERIMETER_URL_LEGACY_KEY = (
|
19
|
+
"OB_CURRENT_PERIMETER_URL" # For backwards compatibility with workstations.
|
20
|
+
)
|
7
21
|
|
8
|
-
|
22
|
+
|
23
|
+
def init_config(config_dir, profile) -> Dict[str, str]:
|
24
|
+
global CACHED_REMOTE_METAFLOW_CONFIG
|
9
25
|
config = read_metaflow_config_from_filesystem(config_dir, profile)
|
10
26
|
|
11
|
-
#
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
27
|
+
# Either user has an ob_config.json file with the perimeter URL
|
28
|
+
# or the default config on the filesystem has the config URL in it.
|
29
|
+
perimeter_specifc_url = get_perimeter_config_url_if_set_in_ob_config(
|
30
|
+
config_dir, profile
|
31
|
+
) or config.get("OBP_METAFLOW_CONFIG_URL", "")
|
32
|
+
|
33
|
+
if perimeter_specifc_url != "":
|
34
|
+
if perimeter_specifc_url in CACHED_REMOTE_METAFLOW_CONFIG:
|
35
|
+
return CACHED_REMOTE_METAFLOW_CONFIG[perimeter_specifc_url]
|
36
|
+
|
37
|
+
remote_config = init_config_from_url(config_dir, profile, perimeter_specifc_url)
|
38
|
+
remote_config["OBP_METAFLOW_CONFIG_URL"] = perimeter_specifc_url
|
39
|
+
|
40
|
+
CACHED_REMOTE_METAFLOW_CONFIG[perimeter_specifc_url] = remote_config
|
17
41
|
return remote_config
|
18
|
-
|
42
|
+
|
19
43
|
return config
|
20
44
|
|
21
45
|
|
22
|
-
def init_config_from_url(config_dir, profile, url) ->
|
46
|
+
def init_config_from_url(config_dir, profile, url) -> Dict[str, str]:
|
23
47
|
config = read_metaflow_config_from_filesystem(config_dir, profile)
|
24
48
|
|
25
49
|
if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
@@ -34,13 +58,9 @@ def init_config_from_url(config_dir, profile, url) -> dict:
|
|
34
58
|
return remote_config
|
35
59
|
|
36
60
|
|
37
|
-
def read_metaflow_config_from_filesystem(config_dir
|
38
|
-
profile = profile or os.environ.get("METAFLOW_PROFILE")
|
39
|
-
config_dir = config_dir or os.path.expanduser(
|
40
|
-
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
41
|
-
)
|
42
|
-
|
61
|
+
def read_metaflow_config_from_filesystem(config_dir, profile) -> Dict[str, str]:
|
43
62
|
config_filename = f"config_{profile}.json" if profile else "config.json"
|
63
|
+
|
44
64
|
path_to_config = os.path.join(config_dir, config_filename)
|
45
65
|
|
46
66
|
if os.path.exists(path_to_config):
|
@@ -83,3 +103,60 @@ def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> st
|
|
83
103
|
|
84
104
|
url_in_config = url_in_config.rstrip("/")
|
85
105
|
return url_in_config
|
106
|
+
|
107
|
+
|
108
|
+
def get_remote_metaflow_config_for_perimeter(
|
109
|
+
origin_token: str, perimeter: str, api_server: str
|
110
|
+
):
|
111
|
+
try:
|
112
|
+
response = requests.get(
|
113
|
+
f"{api_server}/v1/perimeters/{perimeter}/metaflowconfigs/default",
|
114
|
+
headers={"x-api-key": origin_token},
|
115
|
+
)
|
116
|
+
response.raise_for_status()
|
117
|
+
config = response.json()["config"]
|
118
|
+
config["METAFLOW_SERVICE_AUTH_KEY"] = origin_token
|
119
|
+
return config
|
120
|
+
except Exception as e:
|
121
|
+
click.secho(
|
122
|
+
f"Failed to get metaflow config from {api_server}. Error: {str(e)}",
|
123
|
+
fg="red",
|
124
|
+
)
|
125
|
+
sys.exit(1)
|
126
|
+
|
127
|
+
|
128
|
+
def get_ob_config_file_path(config_dir: str, profile: str) -> str:
|
129
|
+
# If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
|
130
|
+
# If neither are set, use ~/.metaflowconfig
|
131
|
+
obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
|
132
|
+
|
133
|
+
ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
|
134
|
+
return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
|
135
|
+
|
136
|
+
|
137
|
+
def get_perimeter_config_url_if_set_in_ob_config(
|
138
|
+
config_dir: str, profile: str
|
139
|
+
) -> Union[str, None]:
|
140
|
+
file_path = get_ob_config_file_path(config_dir, profile)
|
141
|
+
|
142
|
+
if os.path.exists(file_path):
|
143
|
+
with open(file_path, "r") as f:
|
144
|
+
ob_config = json.loads(f.read())
|
145
|
+
|
146
|
+
if CURRENT_PERIMETER_URL in ob_config:
|
147
|
+
return ob_config[CURRENT_PERIMETER_URL]
|
148
|
+
elif CURRENT_PERIMETER_URL_LEGACY_KEY in ob_config:
|
149
|
+
return ob_config[CURRENT_PERIMETER_URL_LEGACY_KEY]
|
150
|
+
else:
|
151
|
+
raise ValueError(
|
152
|
+
"{} does not contain the key {}".format(
|
153
|
+
file_path, CURRENT_PERIMETER_KEY
|
154
|
+
)
|
155
|
+
)
|
156
|
+
elif "OBP_CONFIG_DIR" in os.environ:
|
157
|
+
raise FileNotFoundError(
|
158
|
+
"Environment variable OBP_CONFIG_DIR is set to {} but this directory does not contain an ob_config.json file.".format(
|
159
|
+
os.environ["OBP_CONFIG_DIR"]
|
160
|
+
)
|
161
|
+
)
|
162
|
+
return None
|
outerbounds/utils/schema.py
CHANGED
@@ -59,14 +59,14 @@ class OuterboundsCommandResponse:
|
|
59
59
|
if step.status == OuterboundsCommandStatus.FAIL:
|
60
60
|
self.status = OuterboundsCommandStatus.FAIL
|
61
61
|
self._code = 500
|
62
|
-
self._message = "
|
62
|
+
self._message = "Encountered an error when trying to run command."
|
63
63
|
elif (
|
64
64
|
step.status == OuterboundsCommandStatus.WARN
|
65
65
|
and self.status != OuterboundsCommandStatus.FAIL
|
66
66
|
):
|
67
67
|
self.status = OuterboundsCommandStatus.WARN
|
68
68
|
self._code = 200
|
69
|
-
self._message = "
|
69
|
+
self._message = "Encountered one or more warnings when running the command."
|
70
70
|
|
71
71
|
def as_dict(self):
|
72
72
|
self._data["steps"] = [step.as_dict() for step in self._steps]
|
@@ -0,0 +1,19 @@
|
|
1
|
+
import tempfile
|
2
|
+
import os
|
3
|
+
|
4
|
+
"""
|
5
|
+
Writes the given data to file_loc. Ensures that the directory exists
|
6
|
+
and uses a temporary file to ensure that the file is written atomically.
|
7
|
+
"""
|
8
|
+
|
9
|
+
|
10
|
+
def safe_write_to_disk(file_loc, data):
|
11
|
+
# Ensure the directory exists
|
12
|
+
os.makedirs(os.path.dirname(file_loc), exist_ok=True)
|
13
|
+
|
14
|
+
with tempfile.NamedTemporaryFile(
|
15
|
+
"w", dir=os.path.dirname(file_loc), delete=False
|
16
|
+
) as f:
|
17
|
+
f.write(data)
|
18
|
+
tmp_file = f.name
|
19
|
+
os.rename(tmp_file, file_loc)
|