outerbounds 0.3.55rc4__py3-none-any.whl → 0.3.68__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerbounds/command_groups/local_setup_cli.py +249 -29
- outerbounds/command_groups/perimeters_cli.py +33 -8
- outerbounds/command_groups/workstations_cli.py +27 -14
- outerbounds/utils/metaflowconfig.py +52 -21
- outerbounds/utils/schema.py +6 -0
- {outerbounds-0.3.55rc4.dist-info → outerbounds-0.3.68.dist-info}/METADATA +5 -3
- outerbounds-0.3.68.dist-info/RECORD +15 -0
- outerbounds-0.3.55rc4.dist-info/RECORD +0 -15
- {outerbounds-0.3.55rc4.dist-info → outerbounds-0.3.68.dist-info}/WHEEL +0 -0
- {outerbounds-0.3.55rc4.dist-info → outerbounds-0.3.68.dist-info}/entry_points.txt +0 -0
@@ -12,7 +12,6 @@ from os import path
|
|
12
12
|
from pathlib import Path
|
13
13
|
from typing import Any, Callable, Dict, List
|
14
14
|
|
15
|
-
import boto3
|
16
15
|
import click
|
17
16
|
import requests
|
18
17
|
from requests.exceptions import HTTPError
|
@@ -31,7 +30,8 @@ the Outerbounds platform.
|
|
31
30
|
To remove that package, please try `python -m pip uninstall metaflow -y` or reach out to Outerbounds support.
|
32
31
|
After uninstalling the Metaflow package, please reinstall the Outerbounds package using `python -m pip
|
33
32
|
install outerbounds --force`.
|
34
|
-
As always, please reach out to Outerbounds support for any questions.
|
33
|
+
As always, please reach out to Outerbounds support for any questions.
|
34
|
+
"""
|
35
35
|
|
36
36
|
MISSING_EXTENSIONS_MESSAGE = (
|
37
37
|
"The Outerbounds Platform extensions for Metaflow was not found."
|
@@ -43,6 +43,8 @@ BAD_EXTENSION_MESSAGE = (
|
|
43
43
|
"Mis-installation of the Outerbounds Platform extension package has been detected."
|
44
44
|
)
|
45
45
|
|
46
|
+
PERIMETER_CONFIG_URL_KEY = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
|
47
|
+
|
46
48
|
|
47
49
|
class Narrator:
|
48
50
|
def __init__(self, verbose):
|
@@ -54,11 +56,11 @@ class Narrator:
|
|
54
56
|
|
55
57
|
def section_ok(self):
|
56
58
|
if not self.verbose:
|
57
|
-
click.secho("\
|
59
|
+
click.secho("\U00002705", err=True)
|
58
60
|
|
59
61
|
def section_not_ok(self):
|
60
62
|
if not self.verbose:
|
61
|
-
click.secho("\
|
63
|
+
click.secho("\U0000274C", err=True)
|
62
64
|
|
63
65
|
def announce_check(self, name):
|
64
66
|
if self.verbose:
|
@@ -232,25 +234,33 @@ class ConfigEntrySpec:
|
|
232
234
|
self.expected = expected
|
233
235
|
|
234
236
|
|
235
|
-
def get_config_specs():
|
236
|
-
|
237
|
-
ConfigEntrySpec(
|
238
|
-
"METAFLOW_DATASTORE_SYSROOT_S3", "s3://[a-z0-9\-]+/metaflow[/]?"
|
239
|
-
),
|
240
|
-
ConfigEntrySpec("METAFLOW_DATATOOLS_S3ROOT", "s3://[a-z0-9\-]+/data[/]?"),
|
237
|
+
def get_config_specs(default_datastore: str):
|
238
|
+
spec = [
|
241
239
|
ConfigEntrySpec("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "obp", expected="obp"),
|
242
|
-
ConfigEntrySpec("METAFLOW_DEFAULT_DATASTORE", "s3", expected="s3"),
|
243
240
|
ConfigEntrySpec("METAFLOW_DEFAULT_METADATA", "service", expected="service"),
|
244
|
-
ConfigEntrySpec(
|
245
|
-
|
246
|
-
),
|
247
|
-
ConfigEntrySpec("
|
248
|
-
ConfigEntrySpec("
|
249
|
-
ConfigEntrySpec("
|
250
|
-
ConfigEntrySpec("METAFLOW_UI_URL", "https://ui\..*"),
|
251
|
-
ConfigEntrySpec("OBP_AUTH_SERVER", "auth\..*"),
|
241
|
+
ConfigEntrySpec("METAFLOW_KUBERNETES_NAMESPACE", r"jobs-.*"),
|
242
|
+
ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", r"eval \$\(.*"),
|
243
|
+
ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", r"[a-zA-Z0-9!_\-\.]+"),
|
244
|
+
ConfigEntrySpec("METAFLOW_SERVICE_URL", r"https://metadata\..*"),
|
245
|
+
ConfigEntrySpec("METAFLOW_UI_URL", r"https://ui\..*"),
|
246
|
+
ConfigEntrySpec("OBP_AUTH_SERVER", r"auth\..*"),
|
252
247
|
]
|
253
248
|
|
249
|
+
if default_datastore == "s3":
|
250
|
+
spec.extend(
|
251
|
+
[
|
252
|
+
ConfigEntrySpec(
|
253
|
+
"METAFLOW_DATASTORE_SYSROOT_S3",
|
254
|
+
r"s3://[a-z0-9\-]+/metaflow(-[a-z0-9\-]+)?[/]?",
|
255
|
+
),
|
256
|
+
ConfigEntrySpec(
|
257
|
+
"METAFLOW_DATATOOLS_S3ROOT",
|
258
|
+
r"s3://[a-z0-9\-]+/data(-[a-z0-9\-]+)?[/]?",
|
259
|
+
),
|
260
|
+
]
|
261
|
+
)
|
262
|
+
return spec
|
263
|
+
|
254
264
|
|
255
265
|
def check_metaflow_config(narrator: Narrator) -> CommandStatus:
|
256
266
|
narrator.announce_section("local Metaflow config")
|
@@ -261,8 +271,20 @@ def check_metaflow_config(narrator: Narrator) -> CommandStatus:
|
|
261
271
|
mitigation="",
|
262
272
|
)
|
263
273
|
|
264
|
-
|
265
|
-
|
274
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
275
|
+
config_dir = os.path.expanduser(
|
276
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
277
|
+
)
|
278
|
+
|
279
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
280
|
+
|
281
|
+
if "OBP_METAFLOW_CONFIG_URL" in config:
|
282
|
+
# If the config is fetched from a remote source, not much to check
|
283
|
+
narrator.announce_check("config entry OBP_METAFLOW_CONFIG_URL")
|
284
|
+
narrator.ok()
|
285
|
+
return check_status
|
286
|
+
|
287
|
+
for spec in get_config_specs(config.get("METAFLOW_DEFAULT_DATASTORE", "")):
|
266
288
|
narrator.announce_check("config entry " + spec.name)
|
267
289
|
if spec.name not in config:
|
268
290
|
reason = "Missing"
|
@@ -304,7 +326,12 @@ def check_metaflow_token(narrator: Narrator) -> CommandStatus:
|
|
304
326
|
mitigation="",
|
305
327
|
)
|
306
328
|
|
307
|
-
|
329
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
330
|
+
config_dir = os.path.expanduser(
|
331
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
332
|
+
)
|
333
|
+
|
334
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
308
335
|
try:
|
309
336
|
if "OBP_AUTH_SERVER" in config:
|
310
337
|
k8s_response = requests.get(
|
@@ -363,7 +390,13 @@ def check_workstation_api_accessible(narrator: Narrator) -> CommandStatus:
|
|
363
390
|
)
|
364
391
|
|
365
392
|
try:
|
366
|
-
|
393
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
394
|
+
config_dir = os.path.expanduser(
|
395
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
396
|
+
)
|
397
|
+
|
398
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
399
|
+
|
367
400
|
missing_keys = []
|
368
401
|
if "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
369
402
|
missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
|
@@ -422,7 +455,13 @@ def check_kubeconfig_valid_for_workstations(narrator: Narrator) -> CommandStatus
|
|
422
455
|
)
|
423
456
|
|
424
457
|
try:
|
425
|
-
|
458
|
+
profile = os.environ.get("METAFLOW_PROFILE")
|
459
|
+
config_dir = os.path.expanduser(
|
460
|
+
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
461
|
+
)
|
462
|
+
|
463
|
+
config = metaflowconfig.init_config(config_dir, profile)
|
464
|
+
|
426
465
|
missing_keys = []
|
427
466
|
if "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
428
467
|
missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
|
@@ -585,6 +624,7 @@ class ConfigurationWriter:
|
|
585
624
|
self.decoded_config = None
|
586
625
|
self.out_dir = out_dir
|
587
626
|
self.profile = profile
|
627
|
+
self.selected_perimeter = None
|
588
628
|
|
589
629
|
ob_config_dir = path.expanduser(os.getenv("OBP_CONFIG_DIR", out_dir))
|
590
630
|
self.ob_config_path = path.join(
|
@@ -596,8 +636,12 @@ class ConfigurationWriter:
|
|
596
636
|
self.decoded_config = deserialize(self.encoded_config)
|
597
637
|
|
598
638
|
def process_decoded_config(self):
|
639
|
+
assert self.decoded_config is not None
|
599
640
|
config_type = self.decoded_config.get("OB_CONFIG_TYPE", "inline")
|
600
641
|
if config_type == "inline":
|
642
|
+
if "OBP_PERIMETER" in self.decoded_config:
|
643
|
+
self.selected_perimeter = self.decoded_config["OBP_PERIMETER"]
|
644
|
+
|
601
645
|
if "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
|
602
646
|
self.decoded_config = {
|
603
647
|
"OBP_METAFLOW_CONFIG_URL": self.decoded_config[
|
@@ -616,6 +660,8 @@ class ConfigurationWriter:
|
|
616
660
|
f"{str(e)} key is required for aws-ref config type"
|
617
661
|
)
|
618
662
|
try:
|
663
|
+
import boto3
|
664
|
+
|
619
665
|
client = boto3.client("secretsmanager", region_name=region)
|
620
666
|
response = client.get_secret_value(SecretId=secret_arn)
|
621
667
|
self.decoded_config = json.loads(response["SecretBinary"])
|
@@ -633,6 +679,7 @@ class ConfigurationWriter:
|
|
633
679
|
return path.join(self.out_dir, "config_{}.json".format(self.profile))
|
634
680
|
|
635
681
|
def display(self):
|
682
|
+
assert self.decoded_config is not None
|
636
683
|
# Create a copy so we can use the real config later, possibly
|
637
684
|
display_config = dict()
|
638
685
|
for k in self.decoded_config.keys():
|
@@ -647,6 +694,7 @@ class ConfigurationWriter:
|
|
647
694
|
return self.confirm_overwrite_config(self.path())
|
648
695
|
|
649
696
|
def write_config(self):
|
697
|
+
assert self.decoded_config is not None
|
650
698
|
config_path = self.path()
|
651
699
|
# TODO config contains auth token - restrict file/dir modes
|
652
700
|
os.makedirs(os.path.dirname(config_path), exist_ok=True)
|
@@ -656,11 +704,13 @@ class ConfigurationWriter:
|
|
656
704
|
|
657
705
|
# Every time a config is initialized, we should also reset the corresponding ob_config[_profile].json
|
658
706
|
remote_config = metaflowconfig.init_config(self.out_dir, self.profile)
|
659
|
-
if
|
707
|
+
if self.selected_perimeter and "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
|
660
708
|
with open(self.ob_config_path, "w") as fd:
|
661
709
|
ob_config_dict = {
|
662
|
-
"OB_CURRENT_PERIMETER":
|
663
|
-
|
710
|
+
"OB_CURRENT_PERIMETER": self.selected_perimeter,
|
711
|
+
PERIMETER_CONFIG_URL_KEY: self.decoded_config[
|
712
|
+
"OBP_METAFLOW_CONFIG_URL"
|
713
|
+
],
|
664
714
|
}
|
665
715
|
json.dump(ob_config_dict, fd, indent=4)
|
666
716
|
|
@@ -686,6 +736,64 @@ class ConfigurationWriter:
|
|
686
736
|
return True
|
687
737
|
|
688
738
|
|
739
|
+
def get_gha_jwt(audience: str):
|
740
|
+
# These are specific environment variables that are set by GitHub Actions.
|
741
|
+
if (
|
742
|
+
"ACTIONS_ID_TOKEN_REQUEST_TOKEN" in os.environ
|
743
|
+
and "ACTIONS_ID_TOKEN_REQUEST_URL" in os.environ
|
744
|
+
):
|
745
|
+
try:
|
746
|
+
response = requests.get(
|
747
|
+
url=os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"],
|
748
|
+
headers={
|
749
|
+
"Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"
|
750
|
+
},
|
751
|
+
params={"audience": audience},
|
752
|
+
)
|
753
|
+
response.raise_for_status()
|
754
|
+
return response.json()["value"]
|
755
|
+
except Exception as e:
|
756
|
+
click.secho(
|
757
|
+
"Failed to fetch JWT token from GitHub Actions. Please make sure you are permission 'id-token: write' is set on the GHA jobs level.",
|
758
|
+
fg="red",
|
759
|
+
)
|
760
|
+
sys.exit(1)
|
761
|
+
|
762
|
+
click.secho(
|
763
|
+
"The --github-actions flag was set, but we didn't not find '$ACTIONS_ID_TOKEN_REQUEST_TOKEN' and '$ACTIONS_ID_TOKEN_REQUEST_URL' environment variables. Please make sure you are running this command in a GitHub Actions environment and with correct permissions as per https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers",
|
764
|
+
fg="red",
|
765
|
+
)
|
766
|
+
sys.exit(1)
|
767
|
+
|
768
|
+
|
769
|
+
def get_origin_token(
|
770
|
+
service_principal_name: str,
|
771
|
+
deployment: str,
|
772
|
+
perimeter: str,
|
773
|
+
token: str,
|
774
|
+
auth_server: str,
|
775
|
+
):
|
776
|
+
try:
|
777
|
+
response = requests.get(
|
778
|
+
f"{auth_server}/generate/service-principal",
|
779
|
+
headers={"x-api-key": token},
|
780
|
+
data=json.dumps(
|
781
|
+
{
|
782
|
+
"servicePrincipalName": service_principal_name,
|
783
|
+
"deploymentName": deployment,
|
784
|
+
"perimeter": perimeter,
|
785
|
+
}
|
786
|
+
),
|
787
|
+
)
|
788
|
+
response.raise_for_status()
|
789
|
+
return response.json()["token"]
|
790
|
+
except Exception as e:
|
791
|
+
click.secho(
|
792
|
+
f"Failed to get origin token from {auth_server}. Error: {str(e)}", fg="red"
|
793
|
+
)
|
794
|
+
sys.exit(1)
|
795
|
+
|
796
|
+
|
689
797
|
@click.group(help="The Outerbounds Platform CLI", no_args_is_help=True)
|
690
798
|
def cli(**kwargs):
|
691
799
|
pass
|
@@ -727,7 +835,6 @@ def check(no_config, verbose, output, workstation=False):
|
|
727
835
|
]
|
728
836
|
else:
|
729
837
|
check_names = [
|
730
|
-
"metaflow_config",
|
731
838
|
"metaflow_token",
|
732
839
|
"kubeconfig",
|
733
840
|
"api_connectivity",
|
@@ -772,7 +879,7 @@ def check(no_config, verbose, output, workstation=False):
|
|
772
879
|
)
|
773
880
|
@click.argument("encoded_config", required=True)
|
774
881
|
def configure(
|
775
|
-
encoded_config
|
882
|
+
encoded_config: str, config_dir=None, profile=None, echo=None, force=False
|
776
883
|
):
|
777
884
|
writer = ConfigurationWriter(encoded_config, config_dir, profile)
|
778
885
|
try:
|
@@ -794,3 +901,116 @@ def configure(
|
|
794
901
|
except Exception as e:
|
795
902
|
click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
|
796
903
|
click.secho("Error: {}".format(str(e)))
|
904
|
+
|
905
|
+
|
906
|
+
@cli.command(
|
907
|
+
help="Authenticate service principals using JWT minted by their IDPs and configure Metaflow"
|
908
|
+
)
|
909
|
+
@click.option(
|
910
|
+
"-n",
|
911
|
+
"--name",
|
912
|
+
default="",
|
913
|
+
help="The name of service principals to authenticate",
|
914
|
+
required=True,
|
915
|
+
)
|
916
|
+
@click.option(
|
917
|
+
"--deployment-domain",
|
918
|
+
default="",
|
919
|
+
help="The full domain of the target Outerbounds Platform deployment (eg. 'foo.obp.outerbounds.com')",
|
920
|
+
required=True,
|
921
|
+
)
|
922
|
+
@click.option(
|
923
|
+
"-p",
|
924
|
+
"--perimeter",
|
925
|
+
default="default",
|
926
|
+
help="The name of the perimeter to authenticate the service principal in",
|
927
|
+
)
|
928
|
+
@click.option(
|
929
|
+
"-t",
|
930
|
+
"--jwt-token",
|
931
|
+
default="",
|
932
|
+
help="The JWT token that will be used to authenticate against the OBP Auth Server.",
|
933
|
+
)
|
934
|
+
@click.option(
|
935
|
+
"--github-actions",
|
936
|
+
is_flag=True,
|
937
|
+
help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
|
938
|
+
)
|
939
|
+
@click.option(
|
940
|
+
"-d",
|
941
|
+
"--config-dir",
|
942
|
+
default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
|
943
|
+
help="Path to Metaflow configuration directory",
|
944
|
+
show_default=True,
|
945
|
+
)
|
946
|
+
@click.option(
|
947
|
+
"--profile",
|
948
|
+
default="",
|
949
|
+
help="Configure a named profile. Activate the profile by setting "
|
950
|
+
"`METAFLOW_PROFILE` environment variable.",
|
951
|
+
)
|
952
|
+
@click.option(
|
953
|
+
"-e",
|
954
|
+
"--echo",
|
955
|
+
is_flag=True,
|
956
|
+
help="Print decoded configuration to stdout",
|
957
|
+
)
|
958
|
+
@click.option(
|
959
|
+
"-f",
|
960
|
+
"--force",
|
961
|
+
is_flag=True,
|
962
|
+
help="Force overwrite of existing configuration",
|
963
|
+
)
|
964
|
+
def service_principal_configure(
|
965
|
+
name: str,
|
966
|
+
deployment_domain: str,
|
967
|
+
perimeter: str,
|
968
|
+
jwt_token="",
|
969
|
+
github_actions=False,
|
970
|
+
config_dir=None,
|
971
|
+
profile=None,
|
972
|
+
echo=None,
|
973
|
+
force=False,
|
974
|
+
):
|
975
|
+
audience = f"https://{deployment_domain}"
|
976
|
+
if jwt_token == "" and github_actions:
|
977
|
+
jwt_token = get_gha_jwt(audience)
|
978
|
+
|
979
|
+
if jwt_token == "":
|
980
|
+
click.secho(
|
981
|
+
"No JWT token provided. Please provider either a valid jwt token or set --github-actions",
|
982
|
+
fg="red",
|
983
|
+
)
|
984
|
+
sys.exit(1)
|
985
|
+
|
986
|
+
auth_server = f"https://auth.{deployment_domain}"
|
987
|
+
deployment_name = deployment_domain.split(".")[0]
|
988
|
+
origin_token = get_origin_token(
|
989
|
+
name, deployment_name, perimeter, jwt_token, auth_server
|
990
|
+
)
|
991
|
+
|
992
|
+
api_server = f"https://api.{deployment_domain}"
|
993
|
+
metaflow_config = metaflowconfig.get_remote_metaflow_config_for_perimeter(
|
994
|
+
origin_token, perimeter, api_server
|
995
|
+
)
|
996
|
+
|
997
|
+
writer = ConfigurationWriter(serialize(metaflow_config), config_dir, profile)
|
998
|
+
try:
|
999
|
+
writer.decode()
|
1000
|
+
except:
|
1001
|
+
click.secho("Decoding the configuration text failed.", fg="red")
|
1002
|
+
sys.exit(1)
|
1003
|
+
try:
|
1004
|
+
writer.process_decoded_config()
|
1005
|
+
except DecodedConfigProcessingError as e:
|
1006
|
+
click.secho("Resolving the configuration remotely failed.", fg="red")
|
1007
|
+
click.secho(str(e), fg="magenta")
|
1008
|
+
sys.exit(1)
|
1009
|
+
try:
|
1010
|
+
if echo == True:
|
1011
|
+
writer.display()
|
1012
|
+
if force or writer.confirm_overwrite():
|
1013
|
+
writer.write_config()
|
1014
|
+
except Exception as e:
|
1015
|
+
click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
|
1016
|
+
click.secho("Error: {}".format(str(e)))
|
@@ -12,7 +12,6 @@ from os import path
|
|
12
12
|
from pathlib import Path
|
13
13
|
from typing import Any, Callable, Dict, List
|
14
14
|
|
15
|
-
import boto3
|
16
15
|
import click
|
17
16
|
import requests
|
18
17
|
from requests.exceptions import HTTPError
|
@@ -24,13 +23,20 @@ from ..utils.schema import (
|
|
24
23
|
OuterboundsCommandStatus,
|
25
24
|
)
|
26
25
|
|
26
|
+
from .local_setup_cli import PERIMETER_CONFIG_URL_KEY
|
27
|
+
|
27
28
|
|
28
29
|
@click.group()
|
29
30
|
def cli(**kwargs):
|
30
31
|
pass
|
31
32
|
|
32
33
|
|
33
|
-
@
|
34
|
+
@click.group(help="Manage perimeters")
|
35
|
+
def perimeter(**kwargs):
|
36
|
+
pass
|
37
|
+
|
38
|
+
|
39
|
+
@perimeter.command(help="Switch current perimeter")
|
34
40
|
@click.option(
|
35
41
|
"-d",
|
36
42
|
"--config-dir",
|
@@ -59,7 +65,7 @@ def cli(**kwargs):
|
|
59
65
|
help="Force change the existing perimeter",
|
60
66
|
default=False,
|
61
67
|
)
|
62
|
-
def
|
68
|
+
def switch(config_dir=None, profile=None, output="", id=None, force=False):
|
63
69
|
switch_perimeter_response = OuterboundsCommandResponse()
|
64
70
|
|
65
71
|
switch_perimeter_step = CommandStatus(
|
@@ -94,7 +100,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
|
|
94
100
|
|
95
101
|
ob_config_dict = {
|
96
102
|
"OB_CURRENT_PERIMETER": str(id),
|
97
|
-
|
103
|
+
PERIMETER_CONFIG_URL_KEY: perimeters[id]["remote_config_url"],
|
98
104
|
}
|
99
105
|
|
100
106
|
# Now that we have the lock, we can safely write to the file
|
@@ -124,7 +130,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
|
|
124
130
|
return
|
125
131
|
|
126
132
|
|
127
|
-
@
|
133
|
+
@perimeter.command(help="Show current perimeter")
|
128
134
|
@click.option(
|
129
135
|
"-d",
|
130
136
|
"--config-dir",
|
@@ -146,7 +152,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
|
|
146
152
|
help="Show output in the specified format.",
|
147
153
|
type=click.Choice(["json", ""]),
|
148
154
|
)
|
149
|
-
def
|
155
|
+
def show_current(config_dir=None, profile=None, output=""):
|
150
156
|
show_current_perimeter_response = OuterboundsCommandResponse()
|
151
157
|
|
152
158
|
show_current_perimeter_step = CommandStatus(
|
@@ -192,7 +198,7 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
|
|
192
198
|
click.echo(json.dumps(show_current_perimeter_response.as_dict(), indent=4))
|
193
199
|
|
194
200
|
|
195
|
-
@
|
201
|
+
@perimeter.command(help="List all available perimeters")
|
196
202
|
@click.option(
|
197
203
|
"-d",
|
198
204
|
"--config-dir",
|
@@ -213,13 +219,29 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
|
|
213
219
|
help="Show output in the specified format.",
|
214
220
|
type=click.Choice(["json", ""]),
|
215
221
|
)
|
216
|
-
def
|
222
|
+
def list(config_dir=None, profile=None, output=""):
|
217
223
|
list_perimeters_response = OuterboundsCommandResponse()
|
218
224
|
|
219
225
|
list_perimeters_step = CommandStatus(
|
220
226
|
"ListPerimeters", OuterboundsCommandStatus.OK, "Perimeter Fetch Successful."
|
221
227
|
)
|
222
228
|
|
229
|
+
if "WORKSTATION_ID" in os.environ and (
|
230
|
+
"OBP_DEFAULT_PERIMETER" not in os.environ
|
231
|
+
or "OBP_DEFAULT_PERIMETER_URL" not in os.environ
|
232
|
+
):
|
233
|
+
list_perimeters_response.update(
|
234
|
+
OuterboundsCommandStatus.NOT_SUPPORTED,
|
235
|
+
500,
|
236
|
+
"Perimeters are not supported on old workstations.",
|
237
|
+
)
|
238
|
+
click.secho(
|
239
|
+
"Perimeters are not supported on old workstations.", err=True, fg="red"
|
240
|
+
)
|
241
|
+
if output == "json":
|
242
|
+
click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
|
243
|
+
return
|
244
|
+
|
223
245
|
ob_config_dict = get_ob_config_or_fail_command(
|
224
246
|
config_dir, profile, output, list_perimeters_response, list_perimeters_step
|
225
247
|
)
|
@@ -372,3 +394,6 @@ def confirm_user_has_access_to_perimeter_or_fail(
|
|
372
394
|
if output == "json":
|
373
395
|
click.echo(json.dumps(command_response.as_dict(), indent=4))
|
374
396
|
sys.exit(1)
|
397
|
+
|
398
|
+
|
399
|
+
cli.add_command(perimeter, name="perimeter")
|
@@ -19,6 +19,10 @@ from ..utils.schema import (
|
|
19
19
|
OuterboundsCommandStatus,
|
20
20
|
)
|
21
21
|
from tempfile import NamedTemporaryFile
|
22
|
+
from .perimeters_cli import (
|
23
|
+
get_perimeters_from_api_or_fail_command,
|
24
|
+
confirm_user_has_access_to_perimeter_or_fail,
|
25
|
+
)
|
22
26
|
|
23
27
|
KUBECTL_INSTALL_MITIGATION = "Please install kubectl manually from https://kubernetes.io/docs/tasks/tools/#kubectl"
|
24
28
|
|
@@ -89,7 +93,7 @@ def generate_workstation_token(config_dir=None, profile=None):
|
|
89
93
|
@click.option(
|
90
94
|
"-p",
|
91
95
|
"--profile",
|
92
|
-
default="",
|
96
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
93
97
|
help="The named metaflow profile in which your workstation exists",
|
94
98
|
)
|
95
99
|
@click.option(
|
@@ -111,9 +115,6 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
|
|
111
115
|
"ConfigureKubeConfig", OuterboundsCommandStatus.OK, "Kubeconfig is configured"
|
112
116
|
)
|
113
117
|
try:
|
114
|
-
if not profile:
|
115
|
-
profile = metaflowconfig.get_metaflow_profile()
|
116
|
-
|
117
118
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
118
119
|
config_dir, profile
|
119
120
|
)
|
@@ -193,7 +194,7 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
|
|
193
194
|
@click.option(
|
194
195
|
"-p",
|
195
196
|
"--profile",
|
196
|
-
default="",
|
197
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
197
198
|
help="The named metaflow profile in which your workstation exists",
|
198
199
|
)
|
199
200
|
@click.option(
|
@@ -213,8 +214,6 @@ def list_workstations(config_dir=None, profile=None, output="json"):
|
|
213
214
|
list_response.add_or_update_data("workstations", [])
|
214
215
|
|
215
216
|
try:
|
216
|
-
if not profile:
|
217
|
-
profile = metaflowconfig.get_metaflow_profile()
|
218
217
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
219
218
|
config_dir, profile
|
220
219
|
)
|
@@ -260,7 +259,7 @@ def list_workstations(config_dir=None, profile=None, output="json"):
|
|
260
259
|
@click.option(
|
261
260
|
"-w",
|
262
261
|
"--workstation",
|
263
|
-
default="",
|
262
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
264
263
|
help="The ID of the workstation to hibernate",
|
265
264
|
)
|
266
265
|
def hibernate_workstation(config_dir=None, profile=None, workstation=None):
|
@@ -308,7 +307,7 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
|
|
308
307
|
@click.option(
|
309
308
|
"-p",
|
310
309
|
"--profile",
|
311
|
-
default="",
|
310
|
+
default=os.environ.get("METAFLOW_PROFILE", ""),
|
312
311
|
help="The named metaflow profile in which your workstation exists",
|
313
312
|
)
|
314
313
|
@click.option(
|
@@ -322,9 +321,6 @@ def restart_workstation(config_dir=None, profile=None, workstation=None):
|
|
322
321
|
click.secho("Please specify a workstation ID", fg="red")
|
323
322
|
return
|
324
323
|
try:
|
325
|
-
if not profile:
|
326
|
-
profile = metaflowconfig.get_metaflow_profile()
|
327
|
-
|
328
324
|
metaflow_token = metaflowconfig.get_metaflow_token_from_config(
|
329
325
|
config_dir, profile
|
330
326
|
)
|
@@ -516,7 +512,7 @@ def add_to_path(program_path, platform):
|
|
516
512
|
with open(path_to_rc_file, "a+") as f: # Open bashrc file
|
517
513
|
if program_path not in f.read():
|
518
514
|
f.write("\n# Added by Outerbounds\n")
|
519
|
-
f.write(program_path)
|
515
|
+
f.write(f"export PATH=$PATH:{program_path}")
|
520
516
|
|
521
517
|
|
522
518
|
def to_windows_path(path):
|
@@ -559,7 +555,24 @@ def show_relevant_links(config_dir=None, profile=None, perimeter_id="", output="
|
|
559
555
|
show_links_response.add_or_update_data("links", [])
|
560
556
|
links = []
|
561
557
|
try:
|
562
|
-
|
558
|
+
if not perimeter_id:
|
559
|
+
metaflow_config = metaflowconfig.init_config(config_dir, profile)
|
560
|
+
else:
|
561
|
+
perimeters_dict = get_perimeters_from_api_or_fail_command(
|
562
|
+
config_dir, profile, output, show_links_response, show_links_step
|
563
|
+
)
|
564
|
+
confirm_user_has_access_to_perimeter_or_fail(
|
565
|
+
perimeter_id,
|
566
|
+
perimeters_dict,
|
567
|
+
output,
|
568
|
+
show_links_response,
|
569
|
+
show_links_step,
|
570
|
+
)
|
571
|
+
|
572
|
+
metaflow_config = metaflowconfig.init_config_from_url(
|
573
|
+
config_dir, profile, perimeters_dict[perimeter_id]["remote_config_url"]
|
574
|
+
)
|
575
|
+
|
563
576
|
links.append(
|
564
577
|
{
|
565
578
|
"id": "metaflow-ui-url",
|
@@ -1,17 +1,44 @@
|
|
1
|
+
import click
|
1
2
|
import json
|
2
3
|
import os
|
3
4
|
import requests
|
4
5
|
from os import path
|
5
|
-
import
|
6
|
+
from typing import Dict
|
7
|
+
import sys
|
8
|
+
|
9
|
+
|
10
|
+
def init_config(config_dir, profile) -> Dict[str, str]:
|
11
|
+
config = read_metaflow_config_from_filesystem(config_dir, profile)
|
12
|
+
|
13
|
+
# This is new remote-metaflow config; fetch it from the URL
|
14
|
+
if "OBP_METAFLOW_CONFIG_URL" in config:
|
15
|
+
remote_config = init_config_from_url(
|
16
|
+
config_dir, profile, config["OBP_METAFLOW_CONFIG_URL"]
|
17
|
+
)
|
18
|
+
remote_config["OBP_METAFLOW_CONFIG_URL"] = config["OBP_METAFLOW_CONFIG_URL"]
|
19
|
+
return remote_config
|
20
|
+
# Legacy config, use from filesystem
|
21
|
+
return config
|
22
|
+
|
6
23
|
|
24
|
+
def init_config_from_url(config_dir, profile, url) -> Dict[str, str]:
|
25
|
+
config = read_metaflow_config_from_filesystem(config_dir, profile)
|
26
|
+
|
27
|
+
if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
28
|
+
raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
|
7
29
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
|
30
|
+
config_response = requests.get(
|
31
|
+
url,
|
32
|
+
headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
|
12
33
|
)
|
34
|
+
config_response.raise_for_status()
|
35
|
+
remote_config = config_response.json()["config"]
|
36
|
+
return remote_config
|
13
37
|
|
38
|
+
|
39
|
+
def read_metaflow_config_from_filesystem(config_dir, profile) -> Dict[str, str]:
|
14
40
|
config_filename = f"config_{profile}.json" if profile else "config.json"
|
41
|
+
|
15
42
|
path_to_config = os.path.join(config_dir, config_filename)
|
16
43
|
|
17
44
|
if os.path.exists(path_to_config):
|
@@ -19,22 +46,6 @@ def init_config(config_dir="", profile="") -> dict:
|
|
19
46
|
config = json.load(json_file)
|
20
47
|
else:
|
21
48
|
raise Exception("Unable to locate metaflow config at '%s')" % (path_to_config))
|
22
|
-
|
23
|
-
# This is new remote-metaflow config; fetch it from the URL
|
24
|
-
if "OBP_METAFLOW_CONFIG_URL" in config:
|
25
|
-
if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
|
26
|
-
raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
|
27
|
-
|
28
|
-
config_response = requests.get(
|
29
|
-
config["OBP_METAFLOW_CONFIG_URL"],
|
30
|
-
headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
|
31
|
-
)
|
32
|
-
config_response.raise_for_status()
|
33
|
-
remote_config = config_response.json()["config"]
|
34
|
-
remote_config["METAFLOW_SERVICE_AUTH_KEY"] = config["METAFLOW_SERVICE_AUTH_KEY"]
|
35
|
-
return remote_config
|
36
|
-
|
37
|
-
# Legacy config, use from filesystem
|
38
49
|
return config
|
39
50
|
|
40
51
|
|
@@ -70,3 +81,23 @@ def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> st
|
|
70
81
|
|
71
82
|
url_in_config = url_in_config.rstrip("/")
|
72
83
|
return url_in_config
|
84
|
+
|
85
|
+
|
86
|
+
def get_remote_metaflow_config_for_perimeter(
|
87
|
+
origin_token: str, perimeter: str, api_server: str
|
88
|
+
):
|
89
|
+
try:
|
90
|
+
response = requests.get(
|
91
|
+
f"{api_server}/v1/perimeters/{perimeter}/metaflowconfigs/default",
|
92
|
+
headers={"x-api-key": origin_token},
|
93
|
+
)
|
94
|
+
response.raise_for_status()
|
95
|
+
config = response.json()["config"]
|
96
|
+
config["METAFLOW_SERVICE_AUTH_KEY"] = origin_token
|
97
|
+
return config
|
98
|
+
except Exception as e:
|
99
|
+
click.secho(
|
100
|
+
f"Failed to get metaflow config from {api_server}. Error: {str(e)}",
|
101
|
+
fg="red",
|
102
|
+
)
|
103
|
+
sys.exit(1)
|
outerbounds/utils/schema.py
CHANGED
@@ -5,6 +5,7 @@ class OuterboundsCommandStatus(Enum):
|
|
5
5
|
OK = "OK"
|
6
6
|
FAIL = "FAIL"
|
7
7
|
WARN = "WARN"
|
8
|
+
NOT_SUPPORTED = "NOT_SUPPORTED"
|
8
9
|
|
9
10
|
|
10
11
|
class CommandStatus:
|
@@ -39,6 +40,11 @@ class OuterboundsCommandResponse:
|
|
39
40
|
self.metadata = {}
|
40
41
|
self._data = {}
|
41
42
|
|
43
|
+
def update(self, status, code, message):
|
44
|
+
self.status = status
|
45
|
+
self._code = code
|
46
|
+
self._message = message
|
47
|
+
|
42
48
|
def add_or_update_metadata(self, key, value):
|
43
49
|
self.metadata[key] = value
|
44
50
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: outerbounds
|
3
|
-
Version: 0.3.
|
3
|
+
Version: 0.3.68
|
4
4
|
Summary: More Data Science, Less Administration
|
5
5
|
License: Proprietary
|
6
6
|
Keywords: data science,machine learning,MLOps
|
@@ -17,14 +17,16 @@ Provides-Extra: azure
|
|
17
17
|
Provides-Extra: gcp
|
18
18
|
Requires-Dist: PyYAML (>=6.0,<7.0)
|
19
19
|
Requires-Dist: azure-identity (>=1.15.0,<2.0.0) ; extra == "azure"
|
20
|
+
Requires-Dist: azure-keyvault-secrets (>=4.7.0,<5.0.0) ; extra == "azure"
|
20
21
|
Requires-Dist: azure-storage-blob (>=12.9.0,<13.0.0) ; extra == "azure"
|
21
22
|
Requires-Dist: boto3
|
22
23
|
Requires-Dist: click (>=8.1.3,<9.0.0)
|
23
24
|
Requires-Dist: google-api-core (>=2.16.1,<3.0.0) ; extra == "gcp"
|
24
25
|
Requires-Dist: google-auth (>=2.27.0,<3.0.0) ; extra == "gcp"
|
25
26
|
Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
|
26
|
-
Requires-Dist: ob-metaflow (==2.11.
|
27
|
-
Requires-Dist: ob-metaflow-extensions (==1.1.
|
27
|
+
Requires-Dist: ob-metaflow (==2.11.16.1)
|
28
|
+
Requires-Dist: ob-metaflow-extensions (==1.1.59)
|
29
|
+
Requires-Dist: ob-metaflow-stubs (==3.6)
|
28
30
|
Requires-Dist: opentelemetry-distro (==0.41b0)
|
29
31
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http (==1.20.0)
|
30
32
|
Requires-Dist: opentelemetry-instrumentation-requests (==0.41b0)
|
@@ -0,0 +1,15 @@
|
|
1
|
+
outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
|
2
|
+
outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
|
3
|
+
outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
|
4
|
+
outerbounds/command_groups/cli.py,sha256=H4LxcYTmsY9DQUrReSRLjvbg9s9Ro7s-eUrcMqEJ_9A,261
|
5
|
+
outerbounds/command_groups/local_setup_cli.py,sha256=Xqb-tsAkYgc90duC_6COSR9MsDpMNiKigQxlXUUYfN0,36530
|
6
|
+
outerbounds/command_groups/perimeters_cli.py,sha256=OxbxYQnHZDLRb3SFaVpD2mjp8W8s1fvK1Wc4htyRuGw,12757
|
7
|
+
outerbounds/command_groups/workstations_cli.py,sha256=b5lt8_g2B0zCoUoNriTRv32IPB6E4mI2sUhubDT7Yjo,21966
|
8
|
+
outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
+
outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
|
10
|
+
outerbounds/utils/metaflowconfig.py,sha256=JkhT2yOGpN7t2R2p9uaUJRDJU9fqFPwn4DcojjVnJMI,3513
|
11
|
+
outerbounds/utils/schema.py,sha256=cNlgjmteLPbDzSEUSQDsq8txdhMGyezSmM83jU3aa0w,2329
|
12
|
+
outerbounds-0.3.68.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
|
13
|
+
outerbounds-0.3.68.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
|
14
|
+
outerbounds-0.3.68.dist-info/METADATA,sha256=IkAr4Uwjo_zdK1OW1BSMlNZZ7n7-MgOyy39ZL6Y_Raw,1477
|
15
|
+
outerbounds-0.3.68.dist-info/RECORD,,
|
@@ -1,15 +0,0 @@
|
|
1
|
-
outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
|
2
|
-
outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
|
3
|
-
outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
|
4
|
-
outerbounds/command_groups/cli.py,sha256=H4LxcYTmsY9DQUrReSRLjvbg9s9Ro7s-eUrcMqEJ_9A,261
|
5
|
-
outerbounds/command_groups/local_setup_cli.py,sha256=0sEi3V0sqoCW0RI2z3xMLNnit0pCsigWQ07m1mhDBso,29524
|
6
|
-
outerbounds/command_groups/perimeters_cli.py,sha256=9tOql42d00KfHpZYkLLGEAOiy8iRbIzsknldCyICwU0,12063
|
7
|
-
outerbounds/command_groups/workstations_cli.py,sha256=xHgETjWfUjiIkKODR-4qIS5pSc4ww3VS1ouZM9no8CY,21312
|
8
|
-
outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
-
outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
|
10
|
-
outerbounds/utils/metaflowconfig.py,sha256=HgaDmK3F97rppfGUdysS1Zppe28ERTLV_HcB5IuPpV4,2631
|
11
|
-
outerbounds/utils/schema.py,sha256=Ht_Yf5uoKO0m36WXHZLSPmWPH6EFWXfZDQsiAUquc5k,2160
|
12
|
-
outerbounds-0.3.55rc4.dist-info/METADATA,sha256=6dAqYfSf245Yt-UAvKyh3BWE53-cYUbcaQMi9Zphd98,1367
|
13
|
-
outerbounds-0.3.55rc4.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
|
14
|
-
outerbounds-0.3.55rc4.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
|
15
|
-
outerbounds-0.3.55rc4.dist-info/RECORD,,
|
File without changes
|
File without changes
|