outerbounds 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerbounds/command_groups/cli.py +3 -1
- outerbounds/command_groups/local_setup_cli.py +232 -14
- outerbounds/command_groups/perimeters_cli.py +400 -0
- outerbounds/command_groups/workstations_cli.py +122 -19
- outerbounds/utils/metaflowconfig.py +67 -40
- outerbounds/utils/schema.py +12 -1
- {outerbounds-0.3.58.dist-info → outerbounds-0.3.60.dist-info}/METADATA +4 -4
- outerbounds-0.3.60.dist-info/RECORD +15 -0
- outerbounds-0.3.58.dist-info/RECORD +0 -14
- {outerbounds-0.3.58.dist-info → outerbounds-0.3.60.dist-info}/WHEEL +0 -0
- {outerbounds-0.3.58.dist-info → outerbounds-0.3.60.dist-info}/entry_points.txt +0 -0
@@ -1,10 +1,12 @@
|
|
1
1
|
import click
|
2
2
|
from . import local_setup_cli
|
3
3
|
from . import workstations_cli
|
4
|
+
from . import perimeters_cli
|
4
5
|
|
5
6
|
|
6
7
|
@click.command(
|
7
|
-
cls=click.CommandCollection,
|
8
|
+
cls=click.CommandCollection,
|
9
|
+
sources=[local_setup_cli.cli, workstations_cli.cli, perimeters_cli.cli],
|
8
10
|
)
|
9
11
|
def cli(**kwargs):
|
10
12
|
pass
|
@@ -43,6 +43,8 @@ BAD_EXTENSION_MESSAGE = (
|
|
43
43
|
"Mis-installation of the Outerbounds Platform extension package has been detected."
|
44
44
|
)
|
45
45
|
|
46
|
+
PERIMETER_CONFIG_URL_KEY = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
|
47
|
+
|
46
48
|
|
47
49
|
class Narrator:
|
48
50
|
def __init__(self, verbose):
|
@@ -235,20 +237,21 @@ class ConfigEntrySpec:
|
|
235
237
|
def get_config_specs():
|
236
238
|
return [
|
237
239
|
ConfigEntrySpec(
|
238
|
-
"METAFLOW_DATASTORE_SYSROOT_S3",
|
240
|
+
"METAFLOW_DATASTORE_SYSROOT_S3",
|
241
|
+
r"s3://[a-z0-9\-]+/metaflow(-[a-z0-9\-]+)?[/]?",
|
242
|
+
),
|
243
|
+
ConfigEntrySpec(
|
244
|
+
"METAFLOW_DATATOOLS_S3ROOT", r"s3://[a-z0-9\-]+/data(-[a-z0-9\-]+)?[/]?"
|
239
245
|
),
|
240
|
-
ConfigEntrySpec("METAFLOW_DATATOOLS_S3ROOT", "s3://[a-z0-9\-]+/data[/]?"),
|
241
246
|
ConfigEntrySpec("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "obp", expected="obp"),
|
242
247
|
ConfigEntrySpec("METAFLOW_DEFAULT_DATASTORE", "s3", expected="s3"),
|
243
248
|
ConfigEntrySpec("METAFLOW_DEFAULT_METADATA", "service", expected="service"),
|
244
|
-
ConfigEntrySpec(
|
245
|
-
|
246
|
-
),
|
247
|
-
ConfigEntrySpec("
|
248
|
-
ConfigEntrySpec("
|
249
|
-
ConfigEntrySpec("
|
250
|
-
ConfigEntrySpec("METAFLOW_UI_URL", "https://ui\..*"),
|
251
|
-
ConfigEntrySpec("OBP_AUTH_SERVER", "auth\..*"),
|
249
|
+
ConfigEntrySpec("METAFLOW_KUBERNETES_NAMESPACE", r"jobs-.*"),
|
250
|
+
ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", r"eval \$\(.*"),
|
251
|
+
ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", r"[a-zA-Z0-9!_\-\.]+"),
|
252
|
+
ConfigEntrySpec("METAFLOW_SERVICE_URL", r"https://metadata\..*"),
|
253
|
+
ConfigEntrySpec("METAFLOW_UI_URL", r"https://ui\..*"),
|
254
|
+
ConfigEntrySpec("OBP_AUTH_SERVER", r"auth\..*"),
|
252
255
|
]
|
253
256
|
|
254
257
|
|
@@ -261,7 +264,12 @@ def check_metaflow_config(narrator: Narrator) -> CommandStatus:
|
|
261
264
|
mitigation="",
|
262
265
|
)
|
263
266
|
|
264
|
-
|
267
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
268
|
+
config_dir = os.path.expanduser(
|
269
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
270
|
+
)
|
271
|
+
|
272
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
265
273
|
for spec in get_config_specs():
|
266
274
|
narrator.announce_check("config entry " + spec.name)
|
267
275
|
if spec.name not in config:
|
@@ -304,7 +312,12 @@ def check_metaflow_token(narrator: Narrator) -> CommandStatus:
|
|
304
312
|
mitigation="",
|
305
313
|
)
|
306
314
|
|
307
|
-
|
315
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
316
|
+
config_dir = os.path.expanduser(
|
317
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
318
|
+
)
|
319
|
+
|
320
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
308
321
|
try:
|
309
322
|
if "OBP_AUTH_SERVER" in config:
|
310
323
|
k8s_response = requests.get(
|
@@ -363,7 +376,13 @@ def check_workstation_api_accessible(narrator: Narrator) -> CommandStatus:
|
|
363
376
|
)
|
364
377
|
|
365
378
|
try:
|
366
|
-
|
379
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
380
|
+
config_dir = os.path.expanduser(
|
381
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
382
|
+
)
|
383
|
+
|
384
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
385
|
+
|
367
386
|
missing_keys = []
|
368
387
|
if "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
369
388
|
missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
|
@@ -422,7 +441,13 @@ def check_kubeconfig_valid_for_workstations(narrator: Narrator) -> CommandStatus
|
|
422
441
|
)
|
423
442
|
|
424
443
|
try:
|
425
|
-
|
444
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
445
|
+
config_dir = os.path.expanduser(
|
446
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
447
|
+
)
|
448
|
+
|
449
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
450
|
+
|
426
451
|
missing_keys = []
|
427
452
|
if "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
428
453
|
missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
|
@@ -585,6 +610,13 @@ class ConfigurationWriter:
|
|
585
610
|
self.decoded_config = None
|
586
611
|
self.out_dir = out_dir
|
587
612
|
self.profile = profile
|
613
|
+
self.selected_perimeter = None
|
614
|
+
|
615
|
+
ob_config_dir = path.expanduser(os.getenv("OBP_CONFIG_DIR", out_dir))
|
616
|
+
self.ob_config_path = path.join(
|
617
|
+
ob_config_dir,
|
618
|
+
"ob_config_{}.json".format(profile) if profile else "ob_config.json",
|
619
|
+
)
|
588
620
|
|
589
621
|
def decode(self):
|
590
622
|
self.decoded_config = deserialize(self.encoded_config)
|
@@ -592,6 +624,9 @@ class ConfigurationWriter:
|
|
592
624
|
def process_decoded_config(self):
|
593
625
|
config_type = self.decoded_config.get("OB_CONFIG_TYPE", "inline")
|
594
626
|
if config_type == "inline":
|
627
|
+
if "OBP_PERIMETER" in self.decoded_config:
|
628
|
+
self.selected_perimeter = self.decoded_config["OBP_PERIMETER"]
|
629
|
+
|
595
630
|
if "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
|
596
631
|
self.decoded_config = {
|
597
632
|
"OBP_METAFLOW_CONFIG_URL": self.decoded_config[
|
@@ -648,6 +683,18 @@ class ConfigurationWriter:
|
|
648
683
|
with open(config_path, "w") as fd:
|
649
684
|
json.dump(self.existing, fd, indent=4)
|
650
685
|
|
686
|
+
# Every time a config is initialized, we should also reset the corresponding ob_config[_profile].json
|
687
|
+
remote_config = metaflowconfig.init_config(self.out_dir, self.profile)
|
688
|
+
if self.selected_perimeter and "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
|
689
|
+
with open(self.ob_config_path, "w") as fd:
|
690
|
+
ob_config_dict = {
|
691
|
+
"OB_CURRENT_PERIMETER": self.selected_perimeter,
|
692
|
+
PERIMETER_CONFIG_URL_KEY: self.decoded_config[
|
693
|
+
"OBP_METAFLOW_CONFIG_URL"
|
694
|
+
],
|
695
|
+
}
|
696
|
+
json.dump(ob_config_dict, fd, indent=4)
|
697
|
+
|
651
698
|
def confirm_overwrite_config(self, config_path):
|
652
699
|
if os.path.exists(config_path):
|
653
700
|
if not click.confirm(
|
@@ -670,6 +717,64 @@ class ConfigurationWriter:
|
|
670
717
|
return True
|
671
718
|
|
672
719
|
|
720
|
+
def get_gha_jwt(audience: str):
|
721
|
+
# These are specific environment variables that are set by GitHub Actions.
|
722
|
+
if (
|
723
|
+
"ACTIONS_ID_TOKEN_REQUEST_TOKEN" in os.environ
|
724
|
+
and "ACTIONS_ID_TOKEN_REQUEST_URL" in os.environ
|
725
|
+
):
|
726
|
+
try:
|
727
|
+
response = requests.get(
|
728
|
+
url=os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"],
|
729
|
+
headers={
|
730
|
+
"Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"
|
731
|
+
},
|
732
|
+
params={"audience": audience},
|
733
|
+
)
|
734
|
+
response.raise_for_status()
|
735
|
+
return response.json()["value"]
|
736
|
+
except Exception as e:
|
737
|
+
click.secho(
|
738
|
+
"Failed to fetch JWT token from GitHub Actions. Please make sure you are permission 'id-token: write' is set on the GHA jobs level.",
|
739
|
+
fg="red",
|
740
|
+
)
|
741
|
+
sys.exit(1)
|
742
|
+
|
743
|
+
click.secho(
|
744
|
+
"The --github-actions flag was set, but we didn't not find '$ACTIONS_ID_TOKEN_REQUEST_TOKEN' and '$ACTIONS_ID_TOKEN_REQUEST_URL' environment variables. Please make sure you are running this command in a GitHub Actions environment and with correct permissions as per https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers",
|
745
|
+
fg="red",
|
746
|
+
)
|
747
|
+
sys.exit(1)
|
748
|
+
|
749
|
+
|
750
|
+
def get_origin_token(
|
751
|
+
service_principal_name: str,
|
752
|
+
deployment: str,
|
753
|
+
perimeter: str,
|
754
|
+
token: str,
|
755
|
+
auth_server: str,
|
756
|
+
):
|
757
|
+
try:
|
758
|
+
response = requests.get(
|
759
|
+
f"{auth_server}/generate/service-principal",
|
760
|
+
headers={"x-api-key": token},
|
761
|
+
data=json.dumps(
|
762
|
+
{
|
763
|
+
"servicePrincipalName": service_principal_name,
|
764
|
+
"deploymentName": deployment,
|
765
|
+
"perimeter": perimeter,
|
766
|
+
}
|
767
|
+
),
|
768
|
+
)
|
769
|
+
response.raise_for_status()
|
770
|
+
return response.json()["token"]
|
771
|
+
except Exception as e:
|
772
|
+
click.secho(
|
773
|
+
f"Failed to get origin token from {auth_server}. Error: {str(e)}", fg="red"
|
774
|
+
)
|
775
|
+
sys.exit(1)
|
776
|
+
|
777
|
+
|
673
778
|
@click.group(help="The Outerbounds Platform CLI", no_args_is_help=True)
|
674
779
|
def cli(**kwargs):
|
675
780
|
pass
|
@@ -778,3 +883,116 @@ def configure(
|
|
778
883
|
except Exception as e:
|
779
884
|
click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
|
780
885
|
click.secho("Error: {}".format(str(e)))
|
886
|
+
|
887
|
+
|
888
|
+
@cli.command(
|
889
|
+
help="Authenticate service principals using JWT minted by their IDPs and configure Metaflow"
|
890
|
+
)
|
891
|
+
@click.option(
|
892
|
+
"-n",
|
893
|
+
"--name",
|
894
|
+
default="",
|
895
|
+
help="The name of service principals to authenticate",
|
896
|
+
required=True,
|
897
|
+
)
|
898
|
+
@click.option(
|
899
|
+
"--deployment-domain",
|
900
|
+
default="",
|
901
|
+
help="The full domain of the target Outerbounds Platform deployment (eg. 'foo.obp.outerbounds.com')",
|
902
|
+
required=True,
|
903
|
+
)
|
904
|
+
@click.option(
|
905
|
+
"-p",
|
906
|
+
"--perimeter",
|
907
|
+
default="default",
|
908
|
+
help="The name of the perimeter to authenticate the service principal in",
|
909
|
+
)
|
910
|
+
@click.option(
|
911
|
+
"-t",
|
912
|
+
"--jwt-token",
|
913
|
+
default="",
|
914
|
+
help="The JWT token that will be used to authenticate against the OBP Auth Server.",
|
915
|
+
)
|
916
|
+
@click.option(
|
917
|
+
"--github-actions",
|
918
|
+
is_flag=True,
|
919
|
+
help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
|
920
|
+
)
|
921
|
+
@click.option(
|
922
|
+
"-d",
|
923
|
+
"--config-dir",
|
924
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
925
|
+
help="Path to Metaflow configuration directory",
|
926
|
+
show_default=True,
|
927
|
+
)
|
928
|
+
@click.option(
|
929
|
+
"--profile",
|
930
|
+
default="",
|
931
|
+
help="Configure a named profile. Activate the profile by setting "
|
932
|
+
"`METAFLOW_PROFILE` environment variable.",
|
933
|
+
)
|
934
|
+
@click.option(
|
935
|
+
"-e",
|
936
|
+
"--echo",
|
937
|
+
is_flag=True,
|
938
|
+
help="Print decoded configuration to stdout",
|
939
|
+
)
|
940
|
+
@click.option(
|
941
|
+
"-f",
|
942
|
+
"--force",
|
943
|
+
is_flag=True,
|
944
|
+
help="Force overwrite of existing configuration",
|
945
|
+
)
|
946
|
+
def service_principal_configure(
|
947
|
+
name: str,
|
948
|
+
deployment_domain: str,
|
949
|
+
perimeter: str,
|
950
|
+
jwt_token="",
|
951
|
+
github_actions=False,
|
952
|
+
config_dir=None,
|
953
|
+
profile=None,
|
954
|
+
echo=None,
|
955
|
+
force=False,
|
956
|
+
):
|
957
|
+
audience = f"https://{deployment_domain}"
|
958
|
+
if jwt_token == "" and github_actions:
|
959
|
+
jwt_token = get_gha_jwt(audience)
|
960
|
+
|
961
|
+
if jwt_token == "":
|
962
|
+
click.secho(
|
963
|
+
"No JWT token provided. Please provider either a valid jwt token or set --github-actions",
|
964
|
+
fg="red",
|
965
|
+
)
|
966
|
+
sys.exit(1)
|
967
|
+
|
968
|
+
auth_server = f"https://auth.{deployment_domain}"
|
969
|
+
deployment_name = deployment_domain.split(".")[0]
|
970
|
+
origin_token = get_origin_token(
|
971
|
+
name, deployment_name, perimeter, jwt_token, auth_server
|
972
|
+
)
|
973
|
+
|
974
|
+
api_server = f"https://api.{deployment_domain}"
|
975
|
+
metaflow_config = metaflowconfig.get_remote_metaflow_config_for_perimeter(
|
976
|
+
origin_token, perimeter, api_server
|
977
|
+
)
|
978
|
+
|
979
|
+
writer = ConfigurationWriter(serialize(metaflow_config), config_dir, profile)
|
980
|
+
try:
|
981
|
+
writer.decode()
|
982
|
+
except:
|
983
|
+
click.secho("Decoding the configuration text failed.", fg="red")
|
984
|
+
sys.exit(1)
|
985
|
+
try:
|
986
|
+
writer.process_decoded_config()
|
987
|
+
except DecodedConfigProcessingError as e:
|
988
|
+
click.secho("Resolving the configuration remotely failed.", fg="red")
|
989
|
+
click.secho(str(e), fg="magenta")
|
990
|
+
sys.exit(1)
|
991
|
+
try:
|
992
|
+
if echo == True:
|
993
|
+
writer.display()
|
994
|
+
if force or writer.confirm_overwrite():
|
995
|
+
writer.write_config()
|
996
|
+
except Exception as e:
|
997
|
+
click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
|
998
|
+
click.secho("Error: {}".format(str(e)))
|
@@ -0,0 +1,400 @@
|
|
1
|
+
import base64
|
2
|
+
import hashlib
|
3
|
+
import json
|
4
|
+
import os
|
5
|
+
import re
|
6
|
+
import subprocess
|
7
|
+
import sys
|
8
|
+
import zlib
|
9
|
+
from base64 import b64decode, b64encode
|
10
|
+
from importlib.machinery import PathFinder
|
11
|
+
from os import path
|
12
|
+
from pathlib import Path
|
13
|
+
from typing import Any, Callable, Dict, List
|
14
|
+
|
15
|
+
import boto3
|
16
|
+
import click
|
17
|
+
import requests
|
18
|
+
from requests.exceptions import HTTPError
|
19
|
+
|
20
|
+
from ..utils import kubeconfig, metaflowconfig
|
21
|
+
from ..utils.schema import (
|
22
|
+
CommandStatus,
|
23
|
+
OuterboundsCommandResponse,
|
24
|
+
OuterboundsCommandStatus,
|
25
|
+
)
|
26
|
+
|
27
|
+
from .local_setup_cli import PERIMETER_CONFIG_URL_KEY
|
28
|
+
|
29
|
+
|
30
|
+
@click.group()
|
31
|
+
def cli(**kwargs):
|
32
|
+
pass
|
33
|
+
|
34
|
+
|
35
|
+
@click.group(help="Manage perimeters")
|
36
|
+
def perimeter(**kwargs):
|
37
|
+
pass
|
38
|
+
|
39
|
+
|
40
|
+
@perimeter.command(help="Switch current perimeter")
|
41
|
+
@click.option(
|
42
|
+
"-d",
|
43
|
+
"--config-dir",
|
44
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
45
|
+
help="Path to Metaflow configuration directory",
|
46
|
+
show_default=True,
|
47
|
+
)
|
48
|
+
@click.option(
|
49
|
+
"-p",
|
50
|
+
"--profile",
|
51
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
52
|
+
help="The named metaflow profile in which your workstation exists",
|
53
|
+
)
|
54
|
+
@click.option(
|
55
|
+
"-o",
|
56
|
+
"--output",
|
57
|
+
default="",
|
58
|
+
help="Show output in the specified format.",
|
59
|
+
type=click.Choice(["json", ""]),
|
60
|
+
)
|
61
|
+
@click.option("--id", default="", type=str, help="Perimeter name to switch to")
|
62
|
+
@click.option(
|
63
|
+
"-f",
|
64
|
+
"--force",
|
65
|
+
is_flag=True,
|
66
|
+
help="Force change the existing perimeter",
|
67
|
+
default=False,
|
68
|
+
)
|
69
|
+
def switch(config_dir=None, profile=None, output="", id=None, force=False):
|
70
|
+
switch_perimeter_response = OuterboundsCommandResponse()
|
71
|
+
|
72
|
+
switch_perimeter_step = CommandStatus(
|
73
|
+
"SwitchPerimeter",
|
74
|
+
OuterboundsCommandStatus.OK,
|
75
|
+
"Perimeter was successfully switched!",
|
76
|
+
)
|
77
|
+
|
78
|
+
perimeters = get_perimeters_from_api_or_fail_command(
|
79
|
+
config_dir, profile, output, switch_perimeter_response, switch_perimeter_step
|
80
|
+
)
|
81
|
+
confirm_user_has_access_to_perimeter_or_fail(
|
82
|
+
id, perimeters, output, switch_perimeter_response, switch_perimeter_step
|
83
|
+
)
|
84
|
+
|
85
|
+
path_to_config = get_ob_config_file_path(config_dir, profile)
|
86
|
+
|
87
|
+
import fcntl
|
88
|
+
|
89
|
+
try:
|
90
|
+
if os.path.exists(path_to_config):
|
91
|
+
if not force:
|
92
|
+
fd = os.open(path_to_config, os.O_WRONLY)
|
93
|
+
# Try to acquire an exclusive lock
|
94
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
95
|
+
else:
|
96
|
+
click.secho(
|
97
|
+
"Force flag is set. Perimeter will be switched, but can have unintended consequences on other running processes.",
|
98
|
+
fg="yellow",
|
99
|
+
err=True,
|
100
|
+
)
|
101
|
+
|
102
|
+
ob_config_dict = {
|
103
|
+
"OB_CURRENT_PERIMETER": str(id),
|
104
|
+
PERIMETER_CONFIG_URL_KEY: perimeters[id]["remote_config_url"],
|
105
|
+
}
|
106
|
+
|
107
|
+
# Now that we have the lock, we can safely write to the file
|
108
|
+
with open(path_to_config, "w") as file:
|
109
|
+
json.dump(ob_config_dict, file, indent=4)
|
110
|
+
|
111
|
+
click.secho("Perimeter switched to {}".format(id), fg="green", err=True)
|
112
|
+
except BlockingIOError:
|
113
|
+
# This exception is raised if the file is already locked (non-blocking mode)
|
114
|
+
# Note that its the metaflow package (the extension actually) that acquires a shared read lock
|
115
|
+
# on the file whenever a process imports metaflow.
|
116
|
+
# In the future we might want to get smarter about it and show which process is holding the lock.
|
117
|
+
click.secho(
|
118
|
+
"Can't switch perimeter while Metaflow is in use. Please make sure there are no running python processes or notebooks using metaflow.",
|
119
|
+
fg="red",
|
120
|
+
err=True,
|
121
|
+
)
|
122
|
+
switch_perimeter_step.update(
|
123
|
+
status=OuterboundsCommandStatus.FAIL,
|
124
|
+
reason="Can't switch perimeter while Metaflow is in use.",
|
125
|
+
mitigation="Please make sure there are no running python processes or notebooks using metaflow.",
|
126
|
+
)
|
127
|
+
|
128
|
+
switch_perimeter_response.add_step(switch_perimeter_step)
|
129
|
+
if output == "json":
|
130
|
+
click.echo(json.dumps(switch_perimeter_response.as_dict(), indent=4))
|
131
|
+
return
|
132
|
+
|
133
|
+
|
134
|
+
@perimeter.command(help="Show current perimeter")
|
135
|
+
@click.option(
|
136
|
+
"-d",
|
137
|
+
"--config-dir",
|
138
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
139
|
+
help="Path to Metaflow configuration directory",
|
140
|
+
show_default=True,
|
141
|
+
)
|
142
|
+
@click.option(
|
143
|
+
"-p",
|
144
|
+
"--profile",
|
145
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
146
|
+
help="Configure a named profile. Activate the profile by setting "
|
147
|
+
"`METAFLOW_PROFILE` environment variable.",
|
148
|
+
)
|
149
|
+
@click.option(
|
150
|
+
"-o",
|
151
|
+
"--output",
|
152
|
+
default="",
|
153
|
+
help="Show output in the specified format.",
|
154
|
+
type=click.Choice(["json", ""]),
|
155
|
+
)
|
156
|
+
def show_current(config_dir=None, profile=None, output=""):
|
157
|
+
show_current_perimeter_response = OuterboundsCommandResponse()
|
158
|
+
|
159
|
+
show_current_perimeter_step = CommandStatus(
|
160
|
+
"ShowCurrentPerimeter",
|
161
|
+
OuterboundsCommandStatus.OK,
|
162
|
+
"Current Perimeter Fetch Successful.",
|
163
|
+
)
|
164
|
+
|
165
|
+
ob_config_dict = get_ob_config_or_fail_command(
|
166
|
+
config_dir,
|
167
|
+
profile,
|
168
|
+
output,
|
169
|
+
show_current_perimeter_response,
|
170
|
+
show_current_perimeter_step,
|
171
|
+
)
|
172
|
+
|
173
|
+
perimeters = get_perimeters_from_api_or_fail_command(
|
174
|
+
config_dir,
|
175
|
+
profile,
|
176
|
+
output,
|
177
|
+
show_current_perimeter_response,
|
178
|
+
show_current_perimeter_step,
|
179
|
+
)
|
180
|
+
confirm_user_has_access_to_perimeter_or_fail(
|
181
|
+
ob_config_dict["OB_CURRENT_PERIMETER"],
|
182
|
+
perimeters,
|
183
|
+
output,
|
184
|
+
show_current_perimeter_response,
|
185
|
+
show_current_perimeter_step,
|
186
|
+
)
|
187
|
+
|
188
|
+
click.secho(
|
189
|
+
"Current Perimeter: {}".format(ob_config_dict["OB_CURRENT_PERIMETER"]),
|
190
|
+
fg="green",
|
191
|
+
err=True,
|
192
|
+
)
|
193
|
+
|
194
|
+
show_current_perimeter_response.add_or_update_data(
|
195
|
+
"current_perimeter", ob_config_dict["OB_CURRENT_PERIMETER"]
|
196
|
+
)
|
197
|
+
|
198
|
+
if output == "json":
|
199
|
+
click.echo(json.dumps(show_current_perimeter_response.as_dict(), indent=4))
|
200
|
+
|
201
|
+
|
202
|
+
@perimeter.command(help="List all available perimeters")
|
203
|
+
@click.option(
|
204
|
+
"-d",
|
205
|
+
"--config-dir",
|
206
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
207
|
+
help="Path to Metaflow configuration directory",
|
208
|
+
show_default=True,
|
209
|
+
)
|
210
|
+
@click.option(
|
211
|
+
"-p",
|
212
|
+
"--profile",
|
213
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
214
|
+
help="The named metaflow profile in which your workstation exists",
|
215
|
+
)
|
216
|
+
@click.option(
|
217
|
+
"-o",
|
218
|
+
"--output",
|
219
|
+
default="",
|
220
|
+
help="Show output in the specified format.",
|
221
|
+
type=click.Choice(["json", ""]),
|
222
|
+
)
|
223
|
+
def list(config_dir=None, profile=None, output=""):
|
224
|
+
list_perimeters_response = OuterboundsCommandResponse()
|
225
|
+
|
226
|
+
list_perimeters_step = CommandStatus(
|
227
|
+
"ListPerimeters", OuterboundsCommandStatus.OK, "Perimeter Fetch Successful."
|
228
|
+
)
|
229
|
+
|
230
|
+
if "WORKSTATION_ID" in os.environ and (
|
231
|
+
"OBP_DEFAULT_PERIMETER" not in os.environ
|
232
|
+
or "OBP_DEFAULT_PERIMETER_URL" not in os.environ
|
233
|
+
):
|
234
|
+
list_perimeters_response.update(
|
235
|
+
OuterboundsCommandStatus.NOT_SUPPORTED,
|
236
|
+
500,
|
237
|
+
"Perimeters are not supported on old workstations.",
|
238
|
+
)
|
239
|
+
click.secho(
|
240
|
+
"Perimeters are not supported on old workstations.", err=True, fg="red"
|
241
|
+
)
|
242
|
+
if output == "json":
|
243
|
+
click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
|
244
|
+
return
|
245
|
+
|
246
|
+
ob_config_dict = get_ob_config_or_fail_command(
|
247
|
+
config_dir, profile, output, list_perimeters_response, list_perimeters_step
|
248
|
+
)
|
249
|
+
active_perimeter = ob_config_dict["OB_CURRENT_PERIMETER"]
|
250
|
+
|
251
|
+
perimeters = get_perimeters_from_api_or_fail_command(
|
252
|
+
config_dir, profile, output, list_perimeters_response, list_perimeters_step
|
253
|
+
)
|
254
|
+
|
255
|
+
perimeter_list = []
|
256
|
+
for perimeter in perimeters.values():
|
257
|
+
status = "OK"
|
258
|
+
perimeter_list.append(
|
259
|
+
{
|
260
|
+
"id": perimeter["perimeter"],
|
261
|
+
"active": perimeter["perimeter"] == active_perimeter,
|
262
|
+
"status": status,
|
263
|
+
}
|
264
|
+
)
|
265
|
+
if perimeter["perimeter"] != active_perimeter:
|
266
|
+
click.secho("Perimeter: {}".format(perimeter["perimeter"]), err=True)
|
267
|
+
else:
|
268
|
+
click.secho(
|
269
|
+
"Perimeter: {} (active)".format(perimeter["perimeter"]),
|
270
|
+
fg="green",
|
271
|
+
err=True,
|
272
|
+
)
|
273
|
+
|
274
|
+
list_perimeters_response.add_or_update_data("perimeters", perimeter_list)
|
275
|
+
|
276
|
+
if output == "json":
|
277
|
+
click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
|
278
|
+
|
279
|
+
|
280
|
+
def get_list_perimeters_api_response(config_dir, profile):
|
281
|
+
metaflow_token = metaflowconfig.get_metaflow_token_from_config(config_dir, profile)
|
282
|
+
api_url = metaflowconfig.get_sanitized_url_from_config(
|
283
|
+
config_dir, profile, "OBP_API_SERVER"
|
284
|
+
)
|
285
|
+
perimeters_response = requests.get(
|
286
|
+
f"{api_url}/v1/me/perimeters?privilege=Execute",
|
287
|
+
headers={"x-api-key": metaflow_token},
|
288
|
+
)
|
289
|
+
perimeters_response.raise_for_status()
|
290
|
+
return perimeters_response.json()["perimeters"]
|
291
|
+
|
292
|
+
|
293
|
+
def get_ob_config_file_path(config_dir: str, profile: str) -> str:
|
294
|
+
# If OBP_CONFIG_DIR is set, use that, otherwise use METAFLOW_HOME
|
295
|
+
# If neither are set, use ~/.metaflowconfig
|
296
|
+
obp_config_dir = path.expanduser(os.environ.get("OBP_CONFIG_DIR", config_dir))
|
297
|
+
|
298
|
+
ob_config_filename = f"ob_config_{profile}.json" if profile else "ob_config.json"
|
299
|
+
return os.path.expanduser(os.path.join(obp_config_dir, ob_config_filename))
|
300
|
+
|
301
|
+
|
302
|
+
def get_perimeters_from_api_or_fail_command(
|
303
|
+
config_dir: str,
|
304
|
+
profile: str,
|
305
|
+
output: str,
|
306
|
+
command_response: OuterboundsCommandResponse,
|
307
|
+
command_step: CommandStatus,
|
308
|
+
) -> Dict[str, Dict[str, str]]:
|
309
|
+
try:
|
310
|
+
perimeters = get_list_perimeters_api_response(config_dir, profile)
|
311
|
+
except:
|
312
|
+
click.secho(
|
313
|
+
"Failed to fetch perimeters from API.",
|
314
|
+
fg="red",
|
315
|
+
err=True,
|
316
|
+
)
|
317
|
+
command_step.update(
|
318
|
+
status=OuterboundsCommandStatus.FAIL,
|
319
|
+
reason="Failed to fetch perimeters from API",
|
320
|
+
mitigation="",
|
321
|
+
)
|
322
|
+
command_response.add_step(command_step)
|
323
|
+
if output == "json":
|
324
|
+
click.echo(json.dumps(command_response.as_dict(), indent=4))
|
325
|
+
sys.exit(1)
|
326
|
+
return {p["perimeter"]: p for p in perimeters}
|
327
|
+
|
328
|
+
|
329
|
+
def get_ob_config_or_fail_command(
|
330
|
+
config_dir: str,
|
331
|
+
profile: str,
|
332
|
+
output: str,
|
333
|
+
command_response: OuterboundsCommandResponse,
|
334
|
+
command_step: CommandStatus,
|
335
|
+
) -> Dict[str, str]:
|
336
|
+
path_to_config = get_ob_config_file_path(config_dir, profile)
|
337
|
+
|
338
|
+
if not os.path.exists(path_to_config):
|
339
|
+
click.secho(
|
340
|
+
"Config file not found at {}".format(path_to_config), fg="red", err=True
|
341
|
+
)
|
342
|
+
command_step.update(
|
343
|
+
status=OuterboundsCommandStatus.FAIL,
|
344
|
+
reason="Config file not found",
|
345
|
+
mitigation="Please make sure the config file exists at {}".format(
|
346
|
+
path_to_config
|
347
|
+
),
|
348
|
+
)
|
349
|
+
command_response.add_step(command_step)
|
350
|
+
if output == "json":
|
351
|
+
click.echo(json.dumps(command_response.as_dict(), indent=4))
|
352
|
+
sys.exit(1)
|
353
|
+
|
354
|
+
with open(path_to_config, "r") as file:
|
355
|
+
ob_config_dict = json.load(file)
|
356
|
+
|
357
|
+
if "OB_CURRENT_PERIMETER" not in ob_config_dict:
|
358
|
+
click.secho(
|
359
|
+
"OB_CURRENT_PERIMETER not found in Config file: {}".format(path_to_config),
|
360
|
+
fg="red",
|
361
|
+
err=True,
|
362
|
+
)
|
363
|
+
command_step.update(
|
364
|
+
status=OuterboundsCommandStatus.FAIL,
|
365
|
+
reason="OB_CURRENT_PERIMETER not found in Config file: {}",
|
366
|
+
mitigation="",
|
367
|
+
)
|
368
|
+
command_response.add_step(command_step)
|
369
|
+
if output == "json":
|
370
|
+
click.echo(json.dumps(command_response.as_dict(), indent=4))
|
371
|
+
sys.exit(1)
|
372
|
+
|
373
|
+
return ob_config_dict
|
374
|
+
|
375
|
+
|
376
|
+
def confirm_user_has_access_to_perimeter_or_fail(
|
377
|
+
perimeter_id: str,
|
378
|
+
perimeters: Dict[str, Any],
|
379
|
+
output: str,
|
380
|
+
command_response: OuterboundsCommandResponse,
|
381
|
+
command_step: CommandStatus,
|
382
|
+
):
|
383
|
+
if perimeter_id not in perimeters:
|
384
|
+
click.secho(
|
385
|
+
f"You do not have access to perimeter {perimeter_id} or it does not exist.",
|
386
|
+
fg="red",
|
387
|
+
err=True,
|
388
|
+
)
|
389
|
+
command_step.update(
|
390
|
+
status=OuterboundsCommandStatus.FAIL,
|
391
|
+
reason=f"You do not have access to perimeter {perimeter_id} or it does not exist.",
|
392
|
+
mitigation="",
|
393
|
+
)
|
394
|
+
command_response.add_step(command_step)
|
395
|
+
if output == "json":
|
396
|
+
click.echo(json.dumps(command_response.as_dict(), indent=4))
|
397
|
+
sys.exit(1)
|
398
|
+
|
399
|
+
|
400
|
+
cli.add_command(perimeter, name="perimeter")
|
@@ -19,6 +19,10 @@ from ..utils.schema import (
|
|
19
19
|
OuterboundsCommandStatus,
|
20
20
|
)
|
21
21
|
from tempfile import NamedTemporaryFile
|
22
|
+
from .perimeters_cli import (
|
23
|
+
get_perimeters_from_api_or_fail_command,
|
24
|
+
confirm_user_has_access_to_perimeter_or_fail,
|
25
|
+
)
|
22
26
|
|
23
27
|
KUBECTL_INSTALL_MITIGATION = "Please install kubectl manually from https://kubernetes.io/docs/tasks/tools/#kubectl"
|
24
28
|
|
@@ -89,7 +93,7 @@ def generate_workstation_token(config_dir=None, profile=None):
|
|
89
93
|
@click.option(
|
90
94
|
"-p",
|
91
95
|
"--profile",
|
92
|
-
default="",
|
96
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
93
97
|
help="The named metaflow profile in which your workstation exists",
|
94
98
|
)
|
95
99
|
@click.option(
|
@@ -110,7 +114,6 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
|
|
110
114
|
kubeconfig_configure_step = CommandStatus(
|
111
115
|
"ConfigureKubeConfig", OuterboundsCommandStatus.OK, "Kubeconfig is configured"
|
112
116
|
)
|
113
|
-
|
114
117
|
try:
|
115
118
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
116
119
|
config_dir, profile
|
@@ -191,10 +194,25 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
|
|
191
194
|
@click.option(
|
192
195
|
"-p",
|
193
196
|
"--profile",
|
194
|
-
default="",
|
197
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
195
198
|
help="The named metaflow profile in which your workstation exists",
|
196
199
|
)
|
197
|
-
|
200
|
+
@click.option(
|
201
|
+
"-o",
|
202
|
+
"--output",
|
203
|
+
default="json",
|
204
|
+
help="Show output in the specified format.",
|
205
|
+
type=click.Choice(["json"]),
|
206
|
+
)
|
207
|
+
def list_workstations(config_dir=None, profile=None, output="json"):
|
208
|
+
list_response = OuterboundsCommandResponse()
|
209
|
+
list_step = CommandStatus(
|
210
|
+
"listWorkstations",
|
211
|
+
OuterboundsCommandStatus.OK,
|
212
|
+
"Workstation list successfully fetched!",
|
213
|
+
)
|
214
|
+
list_response.add_or_update_data("workstations", [])
|
215
|
+
|
198
216
|
try:
|
199
217
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
200
218
|
config_dir, profile
|
@@ -205,17 +223,23 @@ def list_workstations(config_dir=None, profile=None):
|
|
205
223
|
workstations_response = requests.get(
|
206
224
|
f"{api_url}/v1/workstations", headers={"x-api-key": metaflow_token}
|
207
225
|
)
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
click.
|
214
|
-
"Error: {}".format(json.dumps(workstations_response.json(), indent=4))
|
215
|
-
)
|
226
|
+
workstations_response.raise_for_status()
|
227
|
+
list_response.add_or_update_data(
|
228
|
+
"workstations", workstations_response.json()["workstations"]
|
229
|
+
)
|
230
|
+
if output == "json":
|
231
|
+
click.echo(json.dumps(list_response.as_dict(), indent=4))
|
216
232
|
except Exception as e:
|
217
|
-
|
218
|
-
|
233
|
+
list_step.update(
|
234
|
+
OuterboundsCommandStatus.FAIL, "Failed to list workstations", ""
|
235
|
+
)
|
236
|
+
list_response.add_step(list_step)
|
237
|
+
if output == "json":
|
238
|
+
list_response.add_or_update_data("error", str(e))
|
239
|
+
click.echo(json.dumps(list_response.as_dict(), indent=4))
|
240
|
+
else:
|
241
|
+
click.secho("Failed to list workstations", fg="red", err=True)
|
242
|
+
click.secho("Error: {}".format(str(e)), fg="red", err=True)
|
219
243
|
|
220
244
|
|
221
245
|
@cli.command(help="Hibernate workstation", hidden=True)
|
@@ -235,7 +259,7 @@ def list_workstations(config_dir=None, profile=None):
|
|
235
259
|
@click.option(
|
236
260
|
"-w",
|
237
261
|
"--workstation",
|
238
|
-
default="",
|
262
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
239
263
|
help="The ID of the workstation to hibernate",
|
240
264
|
)
|
241
265
|
def hibernate_workstation(config_dir=None, profile=None, workstation=None):
|
@@ -243,6 +267,8 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
|
|
243
267
|
click.secho("Please specify a workstation ID", fg="red")
|
244
268
|
return
|
245
269
|
try:
|
270
|
+
if not profile:
|
271
|
+
profile = metaflowconfig.get_metaflow_profile()
|
246
272
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
247
273
|
config_dir, profile
|
248
274
|
)
|
@@ -267,7 +293,7 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
|
|
267
293
|
)
|
268
294
|
except Exception as e:
|
269
295
|
click.secho("Failed to hibernate workstation", fg="red")
|
270
|
-
click.secho("Error: {}".format(str(e)))
|
296
|
+
click.secho("Error: {}".format(str(e)), fg="red")
|
271
297
|
|
272
298
|
|
273
299
|
@cli.command(help="Restart workstation to the int", hidden=True)
|
@@ -281,7 +307,7 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
|
|
281
307
|
@click.option(
|
282
308
|
"-p",
|
283
309
|
"--profile",
|
284
|
-
default="",
|
310
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
285
311
|
help="The named metaflow profile in which your workstation exists",
|
286
312
|
)
|
287
313
|
@click.option(
|
@@ -319,7 +345,7 @@ def restart_workstation(config_dir=None, profile=None, workstation=None):
|
|
319
345
|
)
|
320
346
|
except Exception as e:
|
321
347
|
click.secho("Failed to restart workstation", fg="red")
|
322
|
-
click.secho("Error: {}".format(str(e)))
|
348
|
+
click.secho("Error: {}".format(str(e)), fg="red")
|
323
349
|
|
324
350
|
|
325
351
|
@cli.command(help="Install dependencies needed by workstations", hidden=True)
|
@@ -486,8 +512,85 @@ def add_to_path(program_path, platform):
|
|
486
512
|
with open(path_to_rc_file, "a+") as f: # Open bashrc file
|
487
513
|
if program_path not in f.read():
|
488
514
|
f.write("\n# Added by Outerbounds\n")
|
489
|
-
f.write(program_path)
|
515
|
+
f.write(f"export PATH=$PATH:{program_path}")
|
490
516
|
|
491
517
|
|
492
518
|
def to_windows_path(path):
|
493
519
|
return os.path.normpath(path).replace(os.sep, "\\")
|
520
|
+
|
521
|
+
|
522
|
+
@cli.command(help="Show relevant links for a deployment & perimeter", hidden=True)
|
523
|
+
@click.option(
|
524
|
+
"-d",
|
525
|
+
"--config-dir",
|
526
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
527
|
+
help="Path to Metaflow configuration directory",
|
528
|
+
show_default=True,
|
529
|
+
)
|
530
|
+
@click.option(
|
531
|
+
"-p",
|
532
|
+
"--profile",
|
533
|
+
default="",
|
534
|
+
help="The named metaflow profile in which your workstation exists",
|
535
|
+
)
|
536
|
+
@click.option(
|
537
|
+
"--perimeter-id",
|
538
|
+
default="",
|
539
|
+
help="The id of the perimeter to use",
|
540
|
+
)
|
541
|
+
@click.option(
|
542
|
+
"-o",
|
543
|
+
"--output",
|
544
|
+
default="",
|
545
|
+
help="Show output in the specified format.",
|
546
|
+
type=click.Choice(["json", ""]),
|
547
|
+
)
|
548
|
+
def show_relevant_links(config_dir=None, profile=None, perimeter_id="", output=""):
|
549
|
+
show_links_response = OuterboundsCommandResponse()
|
550
|
+
show_links_step = CommandStatus(
|
551
|
+
"showRelevantLinks",
|
552
|
+
OuterboundsCommandStatus.OK,
|
553
|
+
"Relevant links successfully fetched!",
|
554
|
+
)
|
555
|
+
show_links_response.add_or_update_data("links", [])
|
556
|
+
links = []
|
557
|
+
try:
|
558
|
+
if not perimeter_id:
|
559
|
+
metaflow_config = metaflowconfig.init_config(config_dir, profile)
|
560
|
+
else:
|
561
|
+
perimeters_dict = get_perimeters_from_api_or_fail_command(
|
562
|
+
config_dir, profile, output, show_links_response, show_links_step
|
563
|
+
)
|
564
|
+
confirm_user_has_access_to_perimeter_or_fail(
|
565
|
+
perimeter_id,
|
566
|
+
perimeters_dict,
|
567
|
+
output,
|
568
|
+
show_links_response,
|
569
|
+
show_links_step,
|
570
|
+
)
|
571
|
+
|
572
|
+
metaflow_config = metaflowconfig.init_config_from_url(
|
573
|
+
config_dir, profile, perimeters_dict[perimeter_id]["remote_config_url"]
|
574
|
+
)
|
575
|
+
|
576
|
+
links.append(
|
577
|
+
{
|
578
|
+
"id": "metaflow-ui-url",
|
579
|
+
"url": metaflow_config["METAFLOW_UI_URL"],
|
580
|
+
"label": "Metaflow UI URL",
|
581
|
+
}
|
582
|
+
)
|
583
|
+
show_links_response.add_or_update_data("links", links)
|
584
|
+
if output == "json":
|
585
|
+
click.echo(json.dumps(show_links_response.as_dict(), indent=4))
|
586
|
+
except Exception as e:
|
587
|
+
show_links_step.update(
|
588
|
+
OuterboundsCommandStatus.FAIL, "Failed to show relevant links", ""
|
589
|
+
)
|
590
|
+
show_links_response.add_step(show_links_step)
|
591
|
+
if output == "json":
|
592
|
+
show_links_response.add_or_update_data("error", str(e))
|
593
|
+
click.echo(json.dumps(show_links_response.as_dict(), indent=4))
|
594
|
+
else:
|
595
|
+
click.secho("Failed to show relevant links", fg="red", err=True)
|
596
|
+
click.secho("Error: {}".format(str(e)), fg="red", err=True)
|
@@ -1,15 +1,45 @@
|
|
1
|
+
import click
|
1
2
|
import json
|
2
3
|
import os
|
3
4
|
import requests
|
5
|
+
from os import path
|
6
|
+
import requests
|
7
|
+
from typing import Dict
|
8
|
+
import sys
|
9
|
+
|
4
10
|
|
11
|
+
def init_config(config_dir, profile) -> Dict[str, str]:
|
12
|
+
config = read_metaflow_config_from_filesystem(config_dir, profile)
|
5
13
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
14
|
+
# This is new remote-metaflow config; fetch it from the URL
|
15
|
+
if "OBP_METAFLOW_CONFIG_URL" in config:
|
16
|
+
remote_config = init_config_from_url(
|
17
|
+
config_dir, profile, config["OBP_METAFLOW_CONFIG_URL"]
|
18
|
+
)
|
19
|
+
remote_config["OBP_METAFLOW_CONFIG_URL"] = config["OBP_METAFLOW_CONFIG_URL"]
|
20
|
+
return remote_config
|
21
|
+
# Legacy config, use from filesystem
|
22
|
+
return config
|
23
|
+
|
24
|
+
|
25
|
+
def init_config_from_url(config_dir, profile, url) -> Dict[str, str]:
|
26
|
+
config = read_metaflow_config_from_filesystem(config_dir, profile)
|
27
|
+
|
28
|
+
if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
29
|
+
raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
|
30
|
+
|
31
|
+
config_response = requests.get(
|
32
|
+
url,
|
33
|
+
headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
|
10
34
|
)
|
35
|
+
config_response.raise_for_status()
|
36
|
+
remote_config = config_response.json()["config"]
|
37
|
+
return remote_config
|
11
38
|
|
39
|
+
|
40
|
+
def read_metaflow_config_from_filesystem(config_dir, profile) -> Dict[str, str]:
|
12
41
|
config_filename = f"config_{profile}.json" if profile else "config.json"
|
42
|
+
|
13
43
|
path_to_config = os.path.join(config_dir, config_filename)
|
14
44
|
|
15
45
|
if os.path.exists(path_to_config):
|
@@ -17,22 +47,6 @@ def init_config() -> dict:
|
|
17
47
|
config = json.load(json_file)
|
18
48
|
else:
|
19
49
|
raise Exception("Unable to locate metaflow config at '%s')" % (path_to_config))
|
20
|
-
|
21
|
-
# This is new remote-metaflow config; fetch it from the URL
|
22
|
-
if "OBP_METAFLOW_CONFIG_URL" in config:
|
23
|
-
if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
24
|
-
raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
|
25
|
-
|
26
|
-
config_response = requests.get(
|
27
|
-
config["OBP_METAFLOW_CONFIG_URL"],
|
28
|
-
headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
|
29
|
-
)
|
30
|
-
config_response.raise_for_status()
|
31
|
-
remote_config = config_response.json()["config"]
|
32
|
-
remote_config["METAFLOW_SERVICE_AUTH_KEY"] = config["METAFLOW_SERVICE_AUTH_KEY"]
|
33
|
-
return remote_config
|
34
|
-
|
35
|
-
# Legacy config, use from filesystem
|
36
50
|
return config
|
37
51
|
|
38
52
|
|
@@ -44,13 +58,10 @@ def get_metaflow_token_from_config(config_dir: str, profile: str) -> str:
|
|
44
58
|
config_dir (str): Path to the config directory
|
45
59
|
profile (str): The named metaflow profile
|
46
60
|
"""
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
52
|
-
raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
|
53
|
-
return config["METAFLOW_SERVICE_AUTH_KEY"]
|
61
|
+
config = init_config(config_dir, profile)
|
62
|
+
if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
63
|
+
raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
|
64
|
+
return config["METAFLOW_SERVICE_AUTH_KEY"]
|
54
65
|
|
55
66
|
|
56
67
|
def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> str:
|
@@ -62,16 +73,32 @@ def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> st
|
|
62
73
|
profile (str): The named metaflow profile
|
63
74
|
key (str): The key to look up in the config file
|
64
75
|
"""
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
76
|
+
config = init_config(config_dir, profile)
|
77
|
+
if key not in config:
|
78
|
+
raise Exception(f"Key {key} not found in config")
|
79
|
+
url_in_config = config[key]
|
80
|
+
if not url_in_config.startswith("https://"):
|
81
|
+
url_in_config = f"https://{url_in_config}"
|
82
|
+
|
83
|
+
url_in_config = url_in_config.rstrip("/")
|
84
|
+
return url_in_config
|
85
|
+
|
86
|
+
|
87
|
+
def get_remote_metaflow_config_for_perimeter(
|
88
|
+
origin_token: str, perimeter: str, api_server: str
|
89
|
+
):
|
90
|
+
try:
|
91
|
+
response = requests.get(
|
92
|
+
f"{api_server}/v1/perimeters/{perimeter}/metaflowconfigs/default",
|
93
|
+
headers={"x-api-key": origin_token},
|
94
|
+
)
|
95
|
+
response.raise_for_status()
|
96
|
+
config = response.json()["config"]
|
97
|
+
config["METAFLOW_SERVICE_AUTH_KEY"] = origin_token
|
98
|
+
return config
|
99
|
+
except Exception as e:
|
100
|
+
click.secho(
|
101
|
+
f"Failed to get metaflow config from {api_server}. Error: {str(e)}",
|
102
|
+
fg="red",
|
103
|
+
)
|
104
|
+
sys.exit(1)
|
outerbounds/utils/schema.py
CHANGED
@@ -5,6 +5,7 @@ class OuterboundsCommandStatus(Enum):
|
|
5
5
|
OK = "OK"
|
6
6
|
FAIL = "FAIL"
|
7
7
|
WARN = "WARN"
|
8
|
+
NOT_SUPPORTED = "NOT_SUPPORTED"
|
8
9
|
|
9
10
|
|
10
11
|
class CommandStatus:
|
@@ -37,10 +38,19 @@ class OuterboundsCommandResponse:
|
|
37
38
|
self._message = ""
|
38
39
|
self._steps = []
|
39
40
|
self.metadata = {}
|
41
|
+
self._data = {}
|
42
|
+
|
43
|
+
def update(self, status, code, message):
|
44
|
+
self.status = status
|
45
|
+
self._code = code
|
46
|
+
self._message = message
|
40
47
|
|
41
48
|
def add_or_update_metadata(self, key, value):
|
42
49
|
self.metadata[key] = value
|
43
50
|
|
51
|
+
def add_or_update_data(self, key, value):
|
52
|
+
self._data[key] = value
|
53
|
+
|
44
54
|
def add_step(self, step: CommandStatus):
|
45
55
|
self._steps.append(step)
|
46
56
|
self._process_step_status(step)
|
@@ -59,10 +69,11 @@ class OuterboundsCommandResponse:
|
|
59
69
|
self._message = "We found one or more warnings with your installation."
|
60
70
|
|
61
71
|
def as_dict(self):
|
72
|
+
self._data["steps"] = [step.as_dict() for step in self._steps]
|
62
73
|
return {
|
63
74
|
"status": self.status.value,
|
64
75
|
"code": self._code,
|
65
76
|
"message": self._message,
|
66
|
-
"steps": [step.as_dict() for step in self._steps],
|
67
77
|
"metadata": self.metadata,
|
78
|
+
"data": self._data,
|
68
79
|
}
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: outerbounds
|
3
|
-
Version: 0.3.
|
3
|
+
Version: 0.3.60
|
4
4
|
Summary: More Data Science, Less Administration
|
5
5
|
License: Proprietary
|
6
6
|
Keywords: data science,machine learning,MLOps
|
@@ -23,9 +23,9 @@ Requires-Dist: click (>=8.1.3,<9.0.0)
|
|
23
23
|
Requires-Dist: google-api-core (>=2.16.1,<3.0.0) ; extra == "gcp"
|
24
24
|
Requires-Dist: google-auth (>=2.27.0,<3.0.0) ; extra == "gcp"
|
25
25
|
Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
|
26
|
-
Requires-Dist: ob-metaflow (==2.11.
|
27
|
-
Requires-Dist: ob-metaflow-extensions (==1.1.
|
28
|
-
Requires-Dist: ob-metaflow-stubs (==2.11.
|
26
|
+
Requires-Dist: ob-metaflow (==2.11.9.1)
|
27
|
+
Requires-Dist: ob-metaflow-extensions (==1.1.51)
|
28
|
+
Requires-Dist: ob-metaflow-stubs (==2.11.9.1)
|
29
29
|
Requires-Dist: opentelemetry-distro (==0.41b0)
|
30
30
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http (==1.20.0)
|
31
31
|
Requires-Dist: opentelemetry-instrumentation-requests (==0.41b0)
|
@@ -0,0 +1,15 @@
|
|
1
|
+
outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
|
2
|
+
outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
|
3
|
+
outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
|
4
|
+
outerbounds/command_groups/cli.py,sha256=H4LxcYTmsY9DQUrReSRLjvbg9s9Ro7s-eUrcMqEJ_9A,261
|
5
|
+
outerbounds/command_groups/local_setup_cli.py,sha256=cqdZ_Jg6CFlaIFwI-LRb_13LQqN0MUQx8wBFi-okG28,35982
|
6
|
+
outerbounds/command_groups/perimeters_cli.py,sha256=ICH-StHHYXVAAYvVT8NfMxCDDtKnULnP_vCXrqKOZ48,12770
|
7
|
+
outerbounds/command_groups/workstations_cli.py,sha256=b5lt8_g2B0zCoUoNriTRv32IPB6E4mI2sUhubDT7Yjo,21966
|
8
|
+
outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
+
outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
|
10
|
+
outerbounds/utils/metaflowconfig.py,sha256=cQWD7zoVkOPXd6q2tqmqACjL0IN-0RgiQ45ojxXBYSM,3529
|
11
|
+
outerbounds/utils/schema.py,sha256=cNlgjmteLPbDzSEUSQDsq8txdhMGyezSmM83jU3aa0w,2329
|
12
|
+
outerbounds-0.3.60.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
|
13
|
+
outerbounds-0.3.60.dist-info/METADATA,sha256=51YozDJl2MV1cLz5uR0oceAyqNdyom6oSPwVFWw56jA,1407
|
14
|
+
outerbounds-0.3.60.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
|
15
|
+
outerbounds-0.3.60.dist-info/RECORD,,
|
@@ -1,14 +0,0 @@
|
|
1
|
-
outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
|
2
|
-
outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
|
3
|
-
outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
|
4
|
-
outerbounds/command_groups/cli.py,sha256=61VsBlPG2ykP_786eCyllqeM8DMhPAOfj2FhktrSd7k,207
|
5
|
-
outerbounds/command_groups/local_setup_cli.py,sha256=g_kkrlDGzYvZTm184pW6QwotpkcqBamB14kH_Kv8TbM,28685
|
6
|
-
outerbounds/command_groups/workstations_cli.py,sha256=VgydQzCas3mlAFyzZuanjl1E8Zh7pBrbKbbP6t6N2WU,18237
|
7
|
-
outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
-
outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
|
9
|
-
outerbounds/utils/metaflowconfig.py,sha256=6u9D4x-pQVCPKnmGkTg9uSSHrq4mGnWQl7TurwyV2e8,2945
|
10
|
-
outerbounds/utils/schema.py,sha256=nBuarFbdZu0LGhG0YkJ6pEIvdglfM_TO_W_Db2vksb0,2017
|
11
|
-
outerbounds-0.3.58.dist-info/METADATA,sha256=ADmikJlmX_lGTzMKbqEBXBSi_X_pBFTE6iZiZYLCEUw,1407
|
12
|
-
outerbounds-0.3.58.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
|
13
|
-
outerbounds-0.3.58.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
|
14
|
-
outerbounds-0.3.58.dist-info/RECORD,,
|
File without changes
|
File without changes
|