outerbounds 0.3.68__py3-none-any.whl → 0.3.104__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerbounds/_vendor/PyYAML.LICENSE +20 -0
- outerbounds/_vendor/__init__.py +0 -0
- outerbounds/_vendor/_yaml/__init__.py +34 -0
- outerbounds/_vendor/click/__init__.py +73 -0
- outerbounds/_vendor/click/_compat.py +626 -0
- outerbounds/_vendor/click/_termui_impl.py +717 -0
- outerbounds/_vendor/click/_textwrap.py +49 -0
- outerbounds/_vendor/click/_winconsole.py +279 -0
- outerbounds/_vendor/click/core.py +2998 -0
- outerbounds/_vendor/click/decorators.py +497 -0
- outerbounds/_vendor/click/exceptions.py +287 -0
- outerbounds/_vendor/click/formatting.py +301 -0
- outerbounds/_vendor/click/globals.py +68 -0
- outerbounds/_vendor/click/parser.py +529 -0
- outerbounds/_vendor/click/py.typed +0 -0
- outerbounds/_vendor/click/shell_completion.py +580 -0
- outerbounds/_vendor/click/termui.py +787 -0
- outerbounds/_vendor/click/testing.py +479 -0
- outerbounds/_vendor/click/types.py +1073 -0
- outerbounds/_vendor/click/utils.py +580 -0
- outerbounds/_vendor/click.LICENSE +28 -0
- outerbounds/_vendor/vendor_any.txt +2 -0
- outerbounds/_vendor/yaml/__init__.py +471 -0
- outerbounds/_vendor/yaml/_yaml.cpython-311-darwin.so +0 -0
- outerbounds/_vendor/yaml/composer.py +146 -0
- outerbounds/_vendor/yaml/constructor.py +862 -0
- outerbounds/_vendor/yaml/cyaml.py +177 -0
- outerbounds/_vendor/yaml/dumper.py +138 -0
- outerbounds/_vendor/yaml/emitter.py +1239 -0
- outerbounds/_vendor/yaml/error.py +94 -0
- outerbounds/_vendor/yaml/events.py +104 -0
- outerbounds/_vendor/yaml/loader.py +62 -0
- outerbounds/_vendor/yaml/nodes.py +51 -0
- outerbounds/_vendor/yaml/parser.py +629 -0
- outerbounds/_vendor/yaml/reader.py +208 -0
- outerbounds/_vendor/yaml/representer.py +378 -0
- outerbounds/_vendor/yaml/resolver.py +245 -0
- outerbounds/_vendor/yaml/scanner.py +1555 -0
- outerbounds/_vendor/yaml/serializer.py +127 -0
- outerbounds/_vendor/yaml/tokens.py +129 -0
- outerbounds/command_groups/apps_cli.py +586 -0
- outerbounds/command_groups/cli.py +9 -5
- outerbounds/command_groups/local_setup_cli.py +1 -5
- outerbounds/command_groups/perimeters_cli.py +198 -25
- outerbounds/command_groups/tutorials_cli.py +111 -0
- outerbounds/command_groups/workstations_cli.py +2 -2
- outerbounds/utils/kubeconfig.py +2 -2
- outerbounds/utils/metaflowconfig.py +68 -9
- outerbounds/utils/schema.py +2 -2
- outerbounds/utils/utils.py +19 -0
- outerbounds/vendor.py +159 -0
- {outerbounds-0.3.68.dist-info → outerbounds-0.3.104.dist-info}/METADATA +14 -7
- outerbounds-0.3.104.dist-info/RECORD +59 -0
- {outerbounds-0.3.68.dist-info → outerbounds-0.3.104.dist-info}/WHEEL +1 -1
- outerbounds-0.3.68.dist-info/RECORD +0 -15
- {outerbounds-0.3.68.dist-info → outerbounds-0.3.104.dist-info}/entry_points.txt +0 -0
@@ -1,22 +1,15 @@
|
|
1
|
-
import base64
|
2
|
-
import hashlib
|
3
1
|
import json
|
4
2
|
import os
|
5
|
-
import re
|
6
|
-
import subprocess
|
7
3
|
import sys
|
8
|
-
import
|
9
|
-
from base64 import b64decode, b64encode
|
10
|
-
from importlib.machinery import PathFinder
|
4
|
+
from io import StringIO
|
11
5
|
from os import path
|
12
|
-
from
|
13
|
-
from
|
14
|
-
|
15
|
-
import click
|
6
|
+
from typing import Any, Dict
|
7
|
+
from outerbounds._vendor import click
|
16
8
|
import requests
|
17
|
-
|
9
|
+
import configparser
|
18
10
|
|
19
|
-
from ..utils import
|
11
|
+
from ..utils import metaflowconfig
|
12
|
+
from ..utils.utils import safe_write_to_disk
|
20
13
|
from ..utils.schema import (
|
21
14
|
CommandStatus,
|
22
15
|
OuterboundsCommandResponse,
|
@@ -81,7 +74,7 @@ def switch(config_dir=None, profile=None, output="", id=None, force=False):
|
|
81
74
|
id, perimeters, output, switch_perimeter_response, switch_perimeter_step
|
82
75
|
)
|
83
76
|
|
84
|
-
path_to_config = get_ob_config_file_path(config_dir, profile)
|
77
|
+
path_to_config = metaflowconfig.get_ob_config_file_path(config_dir, profile)
|
85
78
|
|
86
79
|
import fcntl
|
87
80
|
|
@@ -125,9 +118,31 @@ def switch(config_dir=None, profile=None, output="", id=None, force=False):
|
|
125
118
|
)
|
126
119
|
|
127
120
|
switch_perimeter_response.add_step(switch_perimeter_step)
|
121
|
+
|
122
|
+
ensure_cloud_creds_step = CommandStatus(
|
123
|
+
"EnsureCloudCredentials",
|
124
|
+
OuterboundsCommandStatus.OK,
|
125
|
+
"Cloud credentials were successfully updated.",
|
126
|
+
)
|
127
|
+
|
128
|
+
try:
|
129
|
+
ensure_cloud_credentials_for_shell(config_dir, profile)
|
130
|
+
except:
|
131
|
+
click.secho(
|
132
|
+
"Failed to update cloud credentials.",
|
133
|
+
fg="red",
|
134
|
+
err=True,
|
135
|
+
)
|
136
|
+
ensure_cloud_creds_step.update(
|
137
|
+
status=OuterboundsCommandStatus.FAIL,
|
138
|
+
reason="Failed to update cloud credentials.",
|
139
|
+
mitigation="",
|
140
|
+
)
|
141
|
+
|
142
|
+
switch_perimeter_response.add_step(ensure_cloud_creds_step)
|
143
|
+
|
128
144
|
if output == "json":
|
129
145
|
click.echo(json.dumps(switch_perimeter_response.as_dict(), indent=4))
|
130
|
-
return
|
131
146
|
|
132
147
|
|
133
148
|
@perimeter.command(help="Show current perimeter")
|
@@ -276,6 +291,58 @@ def list(config_dir=None, profile=None, output=""):
|
|
276
291
|
click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
|
277
292
|
|
278
293
|
|
294
|
+
@perimeter.command(
|
295
|
+
help="Ensure credentials for cloud are synced with perimeter", hidden=True
|
296
|
+
)
|
297
|
+
@click.option(
|
298
|
+
"-d",
|
299
|
+
"--config-dir",
|
300
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
301
|
+
help="Path to Metaflow configuration directory",
|
302
|
+
show_default=True,
|
303
|
+
)
|
304
|
+
@click.option(
|
305
|
+
"-p",
|
306
|
+
"--profile",
|
307
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
308
|
+
help="The named metaflow profile in which your workstation exists",
|
309
|
+
)
|
310
|
+
@click.option(
|
311
|
+
"-o",
|
312
|
+
"--output",
|
313
|
+
default="",
|
314
|
+
help="Show output in the specified format.",
|
315
|
+
type=click.Choice(["json", ""]),
|
316
|
+
)
|
317
|
+
def ensure_cloud_creds(config_dir=None, profile=None, output=""):
|
318
|
+
ensure_cloud_creds_step = CommandStatus(
|
319
|
+
"EnsureCloudCredentials",
|
320
|
+
OuterboundsCommandStatus.OK,
|
321
|
+
"Cloud credentials were successfully updated.",
|
322
|
+
)
|
323
|
+
|
324
|
+
ensure_cloud_creds_response = OuterboundsCommandResponse()
|
325
|
+
|
326
|
+
try:
|
327
|
+
ensure_cloud_credentials_for_shell(config_dir, profile)
|
328
|
+
click.secho("Cloud credentials updated successfully.", fg="green", err=True)
|
329
|
+
except:
|
330
|
+
click.secho(
|
331
|
+
"Failed to update cloud credentials.",
|
332
|
+
fg="red",
|
333
|
+
err=True,
|
334
|
+
)
|
335
|
+
ensure_cloud_creds_step.update(
|
336
|
+
status=OuterboundsCommandStatus.FAIL,
|
337
|
+
reason="Failed to update cloud credentials.",
|
338
|
+
mitigation="",
|
339
|
+
)
|
340
|
+
|
341
|
+
ensure_cloud_creds_response.add_step(ensure_cloud_creds_step)
|
342
|
+
if output == "json":
|
343
|
+
click.echo(json.dumps(ensure_cloud_creds_response.as_dict(), indent=4))
|
344
|
+
|
345
|
+
|
279
346
|
def get_list_perimeters_api_response(config_dir, profile):
|
280
347
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
281
348
|
api_url = metaflowconfig.get_sanitized_url_from_config(
|
@@ -289,15 +356,6 @@ def get_list_perimeters_api_response(config_dir, profile):
|
|
289
356
|
return perimeters_response.json()["perimeters"]
|
290
357
|
|
291
358
|
|
292
|
-
def get_ob_config_file_path(config_dir: str, profile: str) -> str:
|
293
|
-
# If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
|
294
|
-
# If neither are set, use ~/.metaflowconfig
|
295
|
-
obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
|
296
|
-
|
297
|
-
ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
|
298
|
-
return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
|
299
|
-
|
300
|
-
|
301
359
|
def get_perimeters_from_api_or_fail_command(
|
302
360
|
config_dir: str,
|
303
361
|
profile: str,
|
@@ -332,7 +390,7 @@ def get_ob_config_or_fail_command(
|
|
332
390
|
command_response: OuterboundsCommandResponse,
|
333
391
|
command_step: CommandStatus,
|
334
392
|
) -> Dict[str, str]:
|
335
|
-
path_to_config = get_ob_config_file_path(config_dir, profile)
|
393
|
+
path_to_config = metaflowconfig.get_ob_config_file_path(config_dir, profile)
|
336
394
|
|
337
395
|
if not os.path.exists(path_to_config):
|
338
396
|
click.secho(
|
@@ -372,6 +430,23 @@ def get_ob_config_or_fail_command(
|
|
372
430
|
return ob_config_dict
|
373
431
|
|
374
432
|
|
433
|
+
def ensure_cloud_credentials_for_shell(config_dir, profile):
|
434
|
+
if "WORKSTATION_ID" not in os.environ:
|
435
|
+
# Naive check to see if we're running in workstation. No need to ensure anything
|
436
|
+
# if this is not a workstation.
|
437
|
+
return
|
438
|
+
|
439
|
+
mf_config = metaflowconfig.init_config(config_dir, profile)
|
440
|
+
|
441
|
+
# Currently we only support GCP. TODO: utkarsh to add support for AWS and Azure
|
442
|
+
if "METAFLOW_DEFAULT_GCP_CLIENT_PROVIDER" in mf_config:
|
443
|
+
# This is a GCP deployment.
|
444
|
+
ensure_gcp_cloud_creds(config_dir, profile)
|
445
|
+
elif "METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER" in mf_config:
|
446
|
+
# This is an AWS deployment.
|
447
|
+
ensure_aws_cloud_creds(config_dir, profile)
|
448
|
+
|
449
|
+
|
375
450
|
def confirm_user_has_access_to_perimeter_or_fail(
|
376
451
|
perimeter_id: str,
|
377
452
|
perimeters: Dict[str, Any],
|
@@ -396,4 +471,102 @@ def confirm_user_has_access_to_perimeter_or_fail(
|
|
396
471
|
sys.exit(1)
|
397
472
|
|
398
473
|
|
474
|
+
def ensure_gcp_cloud_creds(config_dir, profile):
|
475
|
+
token_info = get_gcp_auth_credentials(config_dir, profile)
|
476
|
+
auth_url = metaflowconfig.get_sanitized_url_from_config(
|
477
|
+
config_dir, profile, "OBP_AUTH_SERVER"
|
478
|
+
)
|
479
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
480
|
+
|
481
|
+
try:
|
482
|
+
# GOOGLE_APPLICATION_CREDENTIALS is a well known gcloud environment variable
|
483
|
+
credentials_file_loc = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
|
484
|
+
except KeyError:
|
485
|
+
# This is most likely an old workstation when these params weren't set. Do nothing.
|
486
|
+
# Alternatively, user might have deliberately unset it to use their own auth.
|
487
|
+
return
|
488
|
+
|
489
|
+
credentials_json = {
|
490
|
+
"type": "external_account",
|
491
|
+
"audience": f"//iam.googleapis.com/projects/{token_info['gcpProjectNumber']}/locations/global/workloadIdentityPools/{token_info['gcpWorkloadIdentityPool']}/providers/{token_info['gcpWorkloadIdentityPoolProvider']}",
|
492
|
+
"subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
|
493
|
+
"token_url": "https://sts.googleapis.com/v1/token",
|
494
|
+
"service_account_impersonation_url": f"https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{token_info['gcpServiceAccountEmail']}:generateAccessToken",
|
495
|
+
"credential_source": {
|
496
|
+
"url": f"{auth_url}/generate/gcp",
|
497
|
+
"headers": {"x-api-key": metaflow_token},
|
498
|
+
"format": {"type": "json", "subject_token_field_name": "token"},
|
499
|
+
},
|
500
|
+
}
|
501
|
+
|
502
|
+
safe_write_to_disk(credentials_file_loc, json.dumps(credentials_json))
|
503
|
+
|
504
|
+
|
505
|
+
def ensure_aws_cloud_creds(config_dir, profile):
|
506
|
+
token_info = get_aws_auth_credentials(config_dir, profile)
|
507
|
+
|
508
|
+
try:
|
509
|
+
token_file_loc = os.environ["OBP_AWS_WEB_IDENTITY_TOKEN_FILE"]
|
510
|
+
|
511
|
+
# AWS_CONFIG_FILE is a well known aws cli environment variable
|
512
|
+
config_file_loc = os.environ["AWS_CONFIG_FILE"]
|
513
|
+
except KeyError:
|
514
|
+
# This is most likely an old workstation when these params weren't set. Do nothing.
|
515
|
+
# Alternatively, user might have deliberately unset it to use their own auth.
|
516
|
+
return
|
517
|
+
|
518
|
+
aws_config = configparser.ConfigParser()
|
519
|
+
aws_config.read(config_file_loc)
|
520
|
+
|
521
|
+
aws_config["profile task"] = {
|
522
|
+
"role_arn": token_info["role_arn"],
|
523
|
+
"web_identity_token_file": token_file_loc,
|
524
|
+
}
|
525
|
+
|
526
|
+
if token_info.get("cspr_role_arn"):
|
527
|
+
# If CSPR role is present, then we need to use the task role (in the task profile)
|
528
|
+
# to assume the CSPR role.
|
529
|
+
aws_config["profile outerbounds"] = {
|
530
|
+
"role_arn": token_info["cspr_role_arn"],
|
531
|
+
"source_profile": "task",
|
532
|
+
}
|
533
|
+
else:
|
534
|
+
# If no CSPR role is present, just use the task profile as the outerbounds profile.
|
535
|
+
aws_config["profile outerbounds"] = aws_config["profile task"]
|
536
|
+
|
537
|
+
aws_config_string = StringIO()
|
538
|
+
aws_config.write(aws_config_string)
|
539
|
+
|
540
|
+
safe_write_to_disk(token_file_loc, token_info["token"])
|
541
|
+
safe_write_to_disk(config_file_loc, aws_config_string.getvalue())
|
542
|
+
|
543
|
+
|
544
|
+
def get_aws_auth_credentials(config_dir, profile): # pragma: no cover
|
545
|
+
token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
546
|
+
auth_server_url = metaflowconfig.get_sanitized_url_from_config(
|
547
|
+
config_dir, profile, "OBP_AUTH_SERVER"
|
548
|
+
)
|
549
|
+
|
550
|
+
response = requests.get(
|
551
|
+
"{}/generate/aws".format(auth_server_url), headers={"x-api-key": token}
|
552
|
+
)
|
553
|
+
response.raise_for_status()
|
554
|
+
|
555
|
+
return response.json()
|
556
|
+
|
557
|
+
|
558
|
+
def get_gcp_auth_credentials(config_dir, profile):
|
559
|
+
token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
560
|
+
auth_server_url = metaflowconfig.get_sanitized_url_from_config(
|
561
|
+
config_dir, profile, "OBP_AUTH_SERVER"
|
562
|
+
)
|
563
|
+
|
564
|
+
response = requests.get(
|
565
|
+
"{}/generate/gcp".format(auth_server_url), headers={"x-api-key": token}
|
566
|
+
)
|
567
|
+
response.raise_for_status()
|
568
|
+
|
569
|
+
return response.json()
|
570
|
+
|
571
|
+
|
399
572
|
cli.add_command(perimeter, name="perimeter")
|
@@ -0,0 +1,111 @@
|
|
1
|
+
import os
|
2
|
+
from outerbounds._vendor import click
|
3
|
+
import requests
|
4
|
+
|
5
|
+
import tarfile
|
6
|
+
import hashlib
|
7
|
+
import tempfile
|
8
|
+
|
9
|
+
|
10
|
+
@click.group()
|
11
|
+
def cli(**kwargs):
|
12
|
+
pass
|
13
|
+
|
14
|
+
|
15
|
+
@click.group(help="Manage tutorials curated by Outerbounds.", hidden=True)
|
16
|
+
def tutorials(**kwargs):
|
17
|
+
pass
|
18
|
+
|
19
|
+
|
20
|
+
@tutorials.command(help="Pull Outerbounds tutorials.")
|
21
|
+
@click.option(
|
22
|
+
"--url",
|
23
|
+
required=True,
|
24
|
+
help="URL to pull the tutorials from.",
|
25
|
+
type=str,
|
26
|
+
)
|
27
|
+
@click.option(
|
28
|
+
"--destination-dir",
|
29
|
+
help="Show output in the specified format.",
|
30
|
+
type=str,
|
31
|
+
required=True,
|
32
|
+
)
|
33
|
+
@click.option(
|
34
|
+
"--force-overwrite",
|
35
|
+
is_flag=True,
|
36
|
+
help="Overwrite all existing files across all tutorials.",
|
37
|
+
type=bool,
|
38
|
+
required=False,
|
39
|
+
default=False,
|
40
|
+
)
|
41
|
+
def pull(url="", destination_dir="", force_overwrite=False):
|
42
|
+
try:
|
43
|
+
secure_download_and_extract(
|
44
|
+
url, destination_dir, force_overwrite=force_overwrite
|
45
|
+
)
|
46
|
+
click.secho("Tutorials pulled successfully.", fg="green", err=True)
|
47
|
+
except Exception as e:
|
48
|
+
print(e)
|
49
|
+
click.secho(f"Failed to pull tutorials: {e}", fg="red", err=True)
|
50
|
+
|
51
|
+
|
52
|
+
def secure_download_and_extract(
|
53
|
+
url, dest_dir, expected_hash=None, force_overwrite=False
|
54
|
+
):
|
55
|
+
"""
|
56
|
+
Download a tar.gz file from a URL, verify its integrity, and extract its contents.
|
57
|
+
|
58
|
+
:param url: URL of the tar.gz file to download
|
59
|
+
:param dest_dir: Destination directory to extract the contents
|
60
|
+
:param expected_hash: Expected SHA256 hash of the file (optional)
|
61
|
+
"""
|
62
|
+
|
63
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
64
|
+
temp_file = os.path.join(
|
65
|
+
temp_dir, hashlib.md5(url.encode()).hexdigest() + ".tar.gz"
|
66
|
+
)
|
67
|
+
|
68
|
+
# Download the file
|
69
|
+
try:
|
70
|
+
response = requests.get(url, stream=True, verify=True)
|
71
|
+
response.raise_for_status()
|
72
|
+
|
73
|
+
with open(temp_file, "wb") as f:
|
74
|
+
for chunk in response.iter_content(chunk_size=8192):
|
75
|
+
f.write(chunk)
|
76
|
+
except requests.exceptions.RequestException as e:
|
77
|
+
raise Exception(f"Failed to download file: {e}")
|
78
|
+
|
79
|
+
if expected_hash:
|
80
|
+
with open(temp_file, "rb") as f:
|
81
|
+
file_hash = hashlib.sha256(f.read()).hexdigest()
|
82
|
+
if file_hash != expected_hash:
|
83
|
+
raise Exception("File integrity check failed")
|
84
|
+
|
85
|
+
os.makedirs(dest_dir, exist_ok=True)
|
86
|
+
|
87
|
+
try:
|
88
|
+
with tarfile.open(temp_file, "r:gz") as tar:
|
89
|
+
# Keep track of new journeys to extract.
|
90
|
+
to_extract = []
|
91
|
+
members = tar.getmembers()
|
92
|
+
for member in members:
|
93
|
+
member_path = os.path.join(dest_dir, member.name)
|
94
|
+
# Check for any files trying to write outside the destination
|
95
|
+
if not os.path.abspath(member_path).startswith(
|
96
|
+
os.path.abspath(dest_dir)
|
97
|
+
):
|
98
|
+
raise Exception("Attempted path traversal in tar file")
|
99
|
+
if not os.path.exists(member_path):
|
100
|
+
# The user might have modified the existing files, leave them untouched.
|
101
|
+
to_extract.append(member)
|
102
|
+
|
103
|
+
if force_overwrite:
|
104
|
+
tar.extractall(path=dest_dir)
|
105
|
+
else:
|
106
|
+
tar.extractall(path=dest_dir, members=to_extract)
|
107
|
+
except tarfile.TarError as e:
|
108
|
+
raise Exception(f"Failed to extract tar file: {e}")
|
109
|
+
|
110
|
+
|
111
|
+
cli.add_command(tutorials, name="tutorials")
|
outerbounds/utils/kubeconfig.py
CHANGED
@@ -1,23 +1,45 @@
|
|
1
|
-
import click
|
1
|
+
from outerbounds._vendor import click
|
2
2
|
import json
|
3
3
|
import os
|
4
4
|
import requests
|
5
5
|
from os import path
|
6
|
-
from typing import Dict
|
6
|
+
from typing import Dict, Union
|
7
7
|
import sys
|
8
8
|
|
9
|
+
"""
|
10
|
+
key: perimeter specific URL to fetch the remote metaflow config from
|
11
|
+
value: the remote metaflow config
|
12
|
+
"""
|
13
|
+
CACHED_REMOTE_METAFLOW_CONFIG: Dict[str, Dict[str, str]] = {}
|
14
|
+
|
15
|
+
|
16
|
+
CURRENT_PERIMETER_KEY = "OB_CURRENT_PERIMETER"
|
17
|
+
CURRENT_PERIMETER_URL = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
|
18
|
+
CURRENT_PERIMETER_URL_LEGACY_KEY = (
|
19
|
+
"OB_CURRENT_PERIMETER_URL" # For backwards compatibility with workstations.
|
20
|
+
)
|
21
|
+
|
9
22
|
|
10
23
|
def init_config(config_dir, profile) -> Dict[str, str]:
|
24
|
+
global CACHED_REMOTE_METAFLOW_CONFIG
|
11
25
|
config = read_metaflow_config_from_filesystem(config_dir, profile)
|
12
26
|
|
13
|
-
#
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
27
|
+
# Either user has an ob_config.json file with the perimeter URL
|
28
|
+
# or the default config on the filesystem has the config URL in it.
|
29
|
+
perimeter_specifc_url = get_perimeter_config_url_if_set_in_ob_config(
|
30
|
+
config_dir, profile
|
31
|
+
) or config.get("OBP_METAFLOW_CONFIG_URL", "")
|
32
|
+
|
33
|
+
if perimeter_specifc_url != "":
|
34
|
+
if perimeter_specifc_url in CACHED_REMOTE_METAFLOW_CONFIG:
|
35
|
+
return CACHED_REMOTE_METAFLOW_CONFIG[perimeter_specifc_url]
|
36
|
+
|
37
|
+
remote_config = init_config_from_url(config_dir, profile, perimeter_specifc_url)
|
38
|
+
remote_config["OBP_METAFLOW_CONFIG_URL"] = perimeter_specifc_url
|
39
|
+
|
40
|
+
CACHED_REMOTE_METAFLOW_CONFIG[perimeter_specifc_url] = remote_config
|
19
41
|
return remote_config
|
20
|
-
|
42
|
+
|
21
43
|
return config
|
22
44
|
|
23
45
|
|
@@ -101,3 +123,40 @@ def get_remote_metaflow_config_for_perimeter(
|
|
101
123
|
fg="red",
|
102
124
|
)
|
103
125
|
sys.exit(1)
|
126
|
+
|
127
|
+
|
128
|
+
def get_ob_config_file_path(config_dir: str, profile: str) -> str:
|
129
|
+
# If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
|
130
|
+
# If neither are set, use ~/.metaflowconfig
|
131
|
+
obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
|
132
|
+
|
133
|
+
ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
|
134
|
+
return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
|
135
|
+
|
136
|
+
|
137
|
+
def get_perimeter_config_url_if_set_in_ob_config(
|
138
|
+
config_dir: str, profile: str
|
139
|
+
) -> Union[str, None]:
|
140
|
+
file_path = get_ob_config_file_path(config_dir, profile)
|
141
|
+
|
142
|
+
if os.path.exists(file_path):
|
143
|
+
with open(file_path, "r") as f:
|
144
|
+
ob_config = json.loads(f.read())
|
145
|
+
|
146
|
+
if CURRENT_PERIMETER_URL in ob_config:
|
147
|
+
return ob_config[CURRENT_PERIMETER_URL]
|
148
|
+
elif CURRENT_PERIMETER_URL_LEGACY_KEY in ob_config:
|
149
|
+
return ob_config[CURRENT_PERIMETER_URL_LEGACY_KEY]
|
150
|
+
else:
|
151
|
+
raise ValueError(
|
152
|
+
"{} does not contain the key {}".format(
|
153
|
+
file_path, CURRENT_PERIMETER_KEY
|
154
|
+
)
|
155
|
+
)
|
156
|
+
elif "OBP_CONFIG_DIR" in os.environ:
|
157
|
+
raise FileNotFoundError(
|
158
|
+
"Environment variable OBP_CONFIG_DIR is set to {} but this directory does not contain an ob_config.json file.".format(
|
159
|
+
os.environ["OBP_CONFIG_DIR"]
|
160
|
+
)
|
161
|
+
)
|
162
|
+
return None
|
outerbounds/utils/schema.py
CHANGED
@@ -59,14 +59,14 @@ class OuterboundsCommandResponse:
|
|
59
59
|
if step.status == OuterboundsCommandStatus.FAIL:
|
60
60
|
self.status = OuterboundsCommandStatus.FAIL
|
61
61
|
self._code = 500
|
62
|
-
self._message = "
|
62
|
+
self._message = "Encountered an error when trying to run command."
|
63
63
|
elif (
|
64
64
|
step.status == OuterboundsCommandStatus.WARN
|
65
65
|
and self.status != OuterboundsCommandStatus.FAIL
|
66
66
|
):
|
67
67
|
self.status = OuterboundsCommandStatus.WARN
|
68
68
|
self._code = 200
|
69
|
-
self._message = "
|
69
|
+
self._message = "Encountered one or more warnings when running the command."
|
70
70
|
|
71
71
|
def as_dict(self):
|
72
72
|
self._data["steps"] = [step.as_dict() for step in self._steps]
|
@@ -0,0 +1,19 @@
|
|
1
|
+
import tempfile
|
2
|
+
import os
|
3
|
+
|
4
|
+
"""
|
5
|
+
Writes the given data to file_loc. Ensures that the directory exists
|
6
|
+
and uses a temporary file to ensure that the file is written atomically.
|
7
|
+
"""
|
8
|
+
|
9
|
+
|
10
|
+
def safe_write_to_disk(file_loc, data):
|
11
|
+
# Ensure the directory exists
|
12
|
+
os.makedirs(os.path.dirname(file_loc), exist_ok=True)
|
13
|
+
|
14
|
+
with tempfile.NamedTemporaryFile(
|
15
|
+
"w", dir=os.path.dirname(file_loc), delete=False
|
16
|
+
) as f:
|
17
|
+
f.write(data)
|
18
|
+
tmp_file = f.name
|
19
|
+
os.rename(tmp_file, file_loc)
|