outerbounds 0.3.55rc4__py3-none-any.whl → 0.3.68__py3-none-any.whl

outerbounds/command_groups/local_setup_cli.py

@@ -12,7 +12,6 @@ from os import path
 from pathlib import Path
 from typing import Any, Callable, Dict, List
 
-import boto3
 import click
 import requests
 from requests.exceptions import HTTPError
@@ -31,7 +30,8 @@ the Outerbounds platform.
 To remove that package, please try `python -m pip uninstall metaflow -y` or reach out to Outerbounds support.
 After uninstalling the Metaflow package, please reinstall the Outerbounds package using `python -m pip
 install outerbounds --force`.
-As always, please reach out to Outerbounds support for any questions."""
+As always, please reach out to Outerbounds support for any questions.
+"""
 
 MISSING_EXTENSIONS_MESSAGE = (
     "The Outerbounds Platform extensions for Metaflow was not found."
@@ -43,6 +43,8 @@ BAD_EXTENSION_MESSAGE = (
     "Mis-installation of the Outerbounds Platform extension package has been detected."
 )
 
+PERIMETER_CONFIG_URL_KEY = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
+
 
 class Narrator:
     def __init__(self, verbose):
@@ -54,11 +56,11 @@ class Narrator:
 
     def section_ok(self):
         if not self.verbose:
-            click.secho("\U0001F600", err=True)
+            click.secho("\U00002705", err=True)
 
     def section_not_ok(self):
         if not self.verbose:
-            click.secho("\U0001F641", err=True)
+            click.secho("\U0000274C", err=True)
 
     def announce_check(self, name):
         if self.verbose:
@@ -232,25 +234,33 @@ class ConfigEntrySpec:
         self.expected = expected
 
 
-def get_config_specs():
-    return [
-        ConfigEntrySpec(
-            "METAFLOW_DATASTORE_SYSROOT_S3", "s3://[a-z0-9\-]+/metaflow[/]?"
-        ),
-        ConfigEntrySpec("METAFLOW_DATATOOLS_S3ROOT", "s3://[a-z0-9\-]+/data[/]?"),
+def get_config_specs(default_datastore: str):
+    spec = [
         ConfigEntrySpec("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "obp", expected="obp"),
-        ConfigEntrySpec("METAFLOW_DEFAULT_DATASTORE", "s3", expected="s3"),
         ConfigEntrySpec("METAFLOW_DEFAULT_METADATA", "service", expected="service"),
-        ConfigEntrySpec(
-            "METAFLOW_KUBERNETES_NAMESPACE", "jobs\-default", expected="jobs-default"
-        ),
-        ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", "eval \$\(.*"),
-        ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", "[a-zA-Z0-9!_\-\.]+"),
-        ConfigEntrySpec("METAFLOW_SERVICE_URL", "https://metadata\..*"),
-        ConfigEntrySpec("METAFLOW_UI_URL", "https://ui\..*"),
-        ConfigEntrySpec("OBP_AUTH_SERVER", "auth\..*"),
+        ConfigEntrySpec("METAFLOW_KUBERNETES_NAMESPACE", r"jobs-.*"),
+        ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", r"eval \$\(.*"),
+        ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", r"[a-zA-Z0-9!_\-\.]+"),
+        ConfigEntrySpec("METAFLOW_SERVICE_URL", r"https://metadata\..*"),
+        ConfigEntrySpec("METAFLOW_UI_URL", r"https://ui\..*"),
+        ConfigEntrySpec("OBP_AUTH_SERVER", r"auth\..*"),
     ]
 
+    if default_datastore == "s3":
+        spec.extend(
+            [
+                ConfigEntrySpec(
+                    "METAFLOW_DATASTORE_SYSROOT_S3",
+                    r"s3://[a-z0-9\-]+/metaflow(-[a-z0-9\-]+)?[/]?",
+                ),
+                ConfigEntrySpec(
+                    "METAFLOW_DATATOOLS_S3ROOT",
+                    r"s3://[a-z0-9\-]+/data(-[a-z0-9\-]+)?[/]?",
+                ),
+            ]
+        )
+    return spec
+
 
 def check_metaflow_config(narrator: Narrator) -> CommandStatus:
     narrator.announce_section("local Metaflow config")
@@ -261,8 +271,20 @@ def check_metaflow_config(narrator: Narrator) -> CommandStatus:
         mitigation="",
     )
 
-    config = metaflowconfig.init_config()
-    for spec in get_config_specs():
+    profile = os.environ.get("METAFLOW_PROFILE")
+    config_dir = os.path.expanduser(
+        os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+    )
+
+    config = metaflowconfig.init_config(config_dir, profile)
+
+    if "OBP_METAFLOW_CONFIG_URL" in config:
+        # If the config is fetched from a remote source, not much to check
+        narrator.announce_check("config entry OBP_METAFLOW_CONFIG_URL")
+        narrator.ok()
+        return check_status
+
+    for spec in get_config_specs(config.get("METAFLOW_DEFAULT_DATASTORE", "")):
         narrator.announce_check("config entry " + spec.name)
         if spec.name not in config:
             reason = "Missing"
@@ -304,7 +326,12 @@ def check_metaflow_token(narrator: Narrator) -> CommandStatus:
         mitigation="",
     )
 
-    config = metaflowconfig.init_config()
+    profile = os.environ.get("METAFLOW_PROFILE")
+    config_dir = os.path.expanduser(
+        os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+    )
+
+    config = metaflowconfig.init_config(config_dir, profile)
     try:
         if "OBP_AUTH_SERVER" in config:
             k8s_response = requests.get(
@@ -363,7 +390,13 @@ def check_workstation_api_accessible(narrator: Narrator) -> CommandStatus:
     )
 
     try:
-        config = metaflowconfig.init_config()
+        profile = os.environ.get("METAFLOW_PROFILE")
+        config_dir = os.path.expanduser(
+            os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+        )
+
+        config = metaflowconfig.init_config(config_dir, profile)
+
         missing_keys = []
         if "METAFLOW_SERVICE_AUTH_KEY" not in config:
             missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -422,7 +455,13 @@ def check_kubeconfig_valid_for_workstations(narrator: Narrator) -> CommandStatus
     )
 
     try:
-        config = metaflowconfig.init_config()
+        profile = os.environ.get("METAFLOW_PROFILE")
+        config_dir = os.path.expanduser(
+            os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+        )
+
+        config = metaflowconfig.init_config(config_dir, profile)
+
         missing_keys = []
         if "METAFLOW_SERVICE_AUTH_KEY" not in config:
             missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -585,6 +624,7 @@ class ConfigurationWriter:
         self.decoded_config = None
         self.out_dir = out_dir
         self.profile = profile
+        self.selected_perimeter = None
 
         ob_config_dir = path.expanduser(os.getenv("OBP_CONFIG_DIR", out_dir))
         self.ob_config_path = path.join(
@@ -596,8 +636,12 @@ class ConfigurationWriter:
         self.decoded_config = deserialize(self.encoded_config)
 
     def process_decoded_config(self):
+        assert self.decoded_config is not None
         config_type = self.decoded_config.get("OB_CONFIG_TYPE", "inline")
         if config_type == "inline":
+            if "OBP_PERIMETER" in self.decoded_config:
+                self.selected_perimeter = self.decoded_config["OBP_PERIMETER"]
+
             if "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
                 self.decoded_config = {
                     "OBP_METAFLOW_CONFIG_URL": self.decoded_config[
@@ -616,6 +660,8 @@ class ConfigurationWriter:
                     f"{str(e)} key is required for aws-ref config type"
                 )
             try:
+                import boto3
+
                 client = boto3.client("secretsmanager", region_name=region)
                 response = client.get_secret_value(SecretId=secret_arn)
                 self.decoded_config = json.loads(response["SecretBinary"])
@@ -633,6 +679,7 @@ class ConfigurationWriter:
             return path.join(self.out_dir, "config_{}.json".format(self.profile))
 
     def display(self):
+        assert self.decoded_config is not None
         # Create a copy so we can use the real config later, possibly
         display_config = dict()
         for k in self.decoded_config.keys():
@@ -647,6 +694,7 @@ class ConfigurationWriter:
         return self.confirm_overwrite_config(self.path())
 
     def write_config(self):
+        assert self.decoded_config is not None
         config_path = self.path()
         # TODO config contains auth token - restrict file/dir modes
         os.makedirs(os.path.dirname(config_path), exist_ok=True)
@@ -656,11 +704,13 @@ class ConfigurationWriter:
 
         # Every time a config is initialized, we should also reset the corresponding ob_config[_profile].json
         remote_config = metaflowconfig.init_config(self.out_dir, self.profile)
-        if "OBP_PERIMETER" in remote_config and "OBP_PERIMETER_URL" in remote_config:
+        if self.selected_perimeter and "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
             with open(self.ob_config_path, "w") as fd:
                 ob_config_dict = {
-                    "OB_CURRENT_PERIMETER": remote_config["OBP_PERIMETER"],
-                    "OB_CURRENT_PERIMETER_URL": remote_config["OBP_PERIMETER_URL"],
+                    "OB_CURRENT_PERIMETER": self.selected_perimeter,
+                    PERIMETER_CONFIG_URL_KEY: self.decoded_config[
+                        "OBP_METAFLOW_CONFIG_URL"
+                    ],
                 }
                 json.dump(ob_config_dict, fd, indent=4)
 
@@ -686,6 +736,64 @@ class ConfigurationWriter:
         return True
 
 
+def get_gha_jwt(audience: str):
+    # These are specific environment variables that are set by GitHub Actions.
+    if (
+        "ACTIONS_ID_TOKEN_REQUEST_TOKEN" in os.environ
+        and "ACTIONS_ID_TOKEN_REQUEST_URL" in os.environ
+    ):
+        try:
+            response = requests.get(
+                url=os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"],
+                headers={
+                    "Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"
+                },
+                params={"audience": audience},
+            )
+            response.raise_for_status()
+            return response.json()["value"]
+        except Exception as e:
+            click.secho(
+                "Failed to fetch JWT token from GitHub Actions. Please make sure you are permission 'id-token: write' is set on the GHA jobs level.",
+                fg="red",
+            )
+            sys.exit(1)
+
+    click.secho(
+        "The --github-actions flag was set, but we didn't not find '$ACTIONS_ID_TOKEN_REQUEST_TOKEN' and '$ACTIONS_ID_TOKEN_REQUEST_URL' environment variables. Please make sure you are running this command in a GitHub Actions environment and with correct permissions as per https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers",
+        fg="red",
+    )
+    sys.exit(1)
+
+
+def get_origin_token(
+    service_principal_name: str,
+    deployment: str,
+    perimeter: str,
+    token: str,
+    auth_server: str,
+):
+    try:
+        response = requests.get(
+            f"{auth_server}/generate/service-principal",
+            headers={"x-api-key": token},
+            data=json.dumps(
+                {
+                    "servicePrincipalName": service_principal_name,
+                    "deploymentName": deployment,
+                    "perimeter": perimeter,
+                }
+            ),
+        )
+        response.raise_for_status()
+        return response.json()["token"]
+    except Exception as e:
+        click.secho(
+            f"Failed to get origin token from {auth_server}. Error: {str(e)}", fg="red"
+        )
+        sys.exit(1)
+
+
 @click.group(help="The Outerbounds Platform CLI", no_args_is_help=True)
 def cli(**kwargs):
     pass
@@ -727,7 +835,6 @@ def check(no_config, verbose, output, workstation=False):
         ]
     else:
         check_names = [
-            "metaflow_config",
             "metaflow_token",
             "kubeconfig",
             "api_connectivity",
@@ -772,7 +879,7 @@ def check(no_config, verbose, output, workstation=False):
 )
 @click.argument("encoded_config", required=True)
 def configure(
-    encoded_config=None, config_dir=None, profile=None, echo=None, force=False
+    encoded_config: str, config_dir=None, profile=None, echo=None, force=False
 ):
     writer = ConfigurationWriter(encoded_config, config_dir, profile)
     try:
@@ -794,3 +901,116 @@ def configure(
     except Exception as e:
         click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
         click.secho("Error: {}".format(str(e)))
+
+
+@cli.command(
+    help="Authenticate service principals using JWT minted by their IDPs and configure Metaflow"
+)
+@click.option(
+    "-n",
+    "--name",
+    default="",
+    help="The name of service principals to authenticate",
+    required=True,
+)
+@click.option(
+    "--deployment-domain",
+    default="",
+    help="The full domain of the target Outerbounds Platform deployment (eg. 'foo.obp.outerbounds.com')",
+    required=True,
+)
+@click.option(
+    "-p",
+    "--perimeter",
+    default="default",
+    help="The name of the perimeter to authenticate the service principal in",
+)
+@click.option(
+    "-t",
+    "--jwt-token",
+    default="",
+    help="The JWT token that will be used to authenticate against the OBP Auth Server.",
+)
+@click.option(
+    "--github-actions",
+    is_flag=True,
+    help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
+)
+@click.option(
+    "-d",
+    "--config-dir",
+    default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
+    help="Path to Metaflow configuration directory",
+    show_default=True,
+)
+@click.option(
+    "--profile",
+    default="",
+    help="Configure a named profile. Activate the profile by setting "
+    "`METAFLOW_PROFILE` environment variable.",
+)
+@click.option(
+    "-e",
+    "--echo",
+    is_flag=True,
+    help="Print decoded configuration to stdout",
+)
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    help="Force overwrite of existing configuration",
+)
+def service_principal_configure(
+    name: str,
+    deployment_domain: str,
+    perimeter: str,
+    jwt_token="",
+    github_actions=False,
+    config_dir=None,
+    profile=None,
+    echo=None,
+    force=False,
+):
+    audience = f"https://{deployment_domain}"
+    if jwt_token == "" and github_actions:
+        jwt_token = get_gha_jwt(audience)
+
+    if jwt_token == "":
+        click.secho(
+            "No JWT token provided. Please provider either a valid jwt token or set --github-actions",
+            fg="red",
+        )
+        sys.exit(1)
+
+    auth_server = f"https://auth.{deployment_domain}"
+    deployment_name = deployment_domain.split(".")[0]
+    origin_token = get_origin_token(
+        name, deployment_name, perimeter, jwt_token, auth_server
+    )
+
+    api_server = f"https://api.{deployment_domain}"
+    metaflow_config = metaflowconfig.get_remote_metaflow_config_for_perimeter(
+        origin_token, perimeter, api_server
+    )
+
+    writer = ConfigurationWriter(serialize(metaflow_config), config_dir, profile)
+    try:
+        writer.decode()
+    except:
+        click.secho("Decoding the configuration text failed.", fg="red")
+        sys.exit(1)
+    try:
+        writer.process_decoded_config()
+    except DecodedConfigProcessingError as e:
+        click.secho("Resolving the configuration remotely failed.", fg="red")
+        click.secho(str(e), fg="magenta")
+        sys.exit(1)
+    try:
+        if echo == True:
+            writer.display()
+        if force or writer.confirm_overwrite():
+            writer.write_config()
+    except Exception as e:
+        click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
+        click.secho("Error: {}".format(str(e)))

outerbounds/command_groups/perimeters_cli.py

@@ -12,7 +12,6 @@ from os import path
 from pathlib import Path
 from typing import Any, Callable, Dict, List
 
-import boto3
 import click
 import requests
 from requests.exceptions import HTTPError
@@ -24,13 +23,20 @@ from ..utils.schema import (
     OuterboundsCommandStatus,
 )
 
+from .local_setup_cli import PERIMETER_CONFIG_URL_KEY
+
 
 @click.group()
 def cli(**kwargs):
     pass
 
 
-@cli.command(help="Switch current perimeter", hidden=True)
+@click.group(help="Manage perimeters")
+def perimeter(**kwargs):
+    pass
+
+
+@perimeter.command(help="Switch current perimeter")
 @click.option(
     "-d",
     "--config-dir",
@@ -59,7 +65,7 @@ def cli(**kwargs):
     help="Force change the existing perimeter",
     default=False,
 )
-def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=False):
+def switch(config_dir=None, profile=None, output="", id=None, force=False):
     switch_perimeter_response = OuterboundsCommandResponse()
 
     switch_perimeter_step = CommandStatus(
@@ -94,7 +100,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
 
         ob_config_dict = {
             "OB_CURRENT_PERIMETER": str(id),
-            "OB_CURRENT_PERIMETER_URL": perimeters[id]["remote_config_url"],
+            PERIMETER_CONFIG_URL_KEY: perimeters[id]["remote_config_url"],
         }
 
         # Now that we have the lock, we can safely write to the file
@@ -124,7 +130,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
         return
 
 
-@cli.command(help="Show current perimeter", hidden=True)
+@perimeter.command(help="Show current perimeter")
 @click.option(
     "-d",
     "--config-dir",
@@ -146,7 +152,7 @@ def switch_perimeter(config_dir=None, profile=None, output="", id=None, force=Fa
     help="Show output in the specified format.",
     type=click.Choice(["json", ""]),
 )
-def show_current_perimeter(config_dir=None, profile=None, output=""):
+def show_current(config_dir=None, profile=None, output=""):
     show_current_perimeter_response = OuterboundsCommandResponse()
 
     show_current_perimeter_step = CommandStatus(
@@ -192,7 +198,7 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
         click.echo(json.dumps(show_current_perimeter_response.as_dict(), indent=4))
 
 
-@cli.command(help="List all available perimeters", hidden=True)
+@perimeter.command(help="List all available perimeters")
 @click.option(
     "-d",
     "--config-dir",
@@ -213,13 +219,29 @@ def show_current_perimeter(config_dir=None, profile=None, output=""):
     help="Show output in the specified format.",
     type=click.Choice(["json", ""]),
 )
-def list_perimeters(config_dir=None, profile=None, output=""):
+def list(config_dir=None, profile=None, output=""):
     list_perimeters_response = OuterboundsCommandResponse()
 
     list_perimeters_step = CommandStatus(
         "ListPerimeters", OuterboundsCommandStatus.OK, "Perimeter Fetch Successful."
     )
 
+    if "WORKSTATION_ID" in os.environ and (
+        "OBP_DEFAULT_PERIMETER" not in os.environ
+        or "OBP_DEFAULT_PERIMETER_URL" not in os.environ
+    ):
+        list_perimeters_response.update(
+            OuterboundsCommandStatus.NOT_SUPPORTED,
+            500,
+            "Perimeters are not supported on old workstations.",
+        )
+        click.secho(
+            "Perimeters are not supported on old workstations.", err=True, fg="red"
+        )
+        if output == "json":
+            click.echo(json.dumps(list_perimeters_response.as_dict(), indent=4))
+        return
+
     ob_config_dict = get_ob_config_or_fail_command(
         config_dir, profile, output, list_perimeters_response, list_perimeters_step
     )
@@ -372,3 +394,6 @@ def confirm_user_has_access_to_perimeter_or_fail(
         if output == "json":
             click.echo(json.dumps(command_response.as_dict(), indent=4))
         sys.exit(1)
+
+
+cli.add_command(perimeter, name="perimeter")

outerbounds/command_groups/workstations_cli.py

@@ -19,6 +19,10 @@ from ..utils.schema import (
     OuterboundsCommandStatus,
 )
 from tempfile import NamedTemporaryFile
+from .perimeters_cli import (
+    get_perimeters_from_api_or_fail_command,
+    confirm_user_has_access_to_perimeter_or_fail,
+)
 
 KUBECTL_INSTALL_MITIGATION = "Please install kubectl manually from https://kubernetes.io/docs/tasks/tools/#kubectl"
 
@@ -89,7 +93,7 @@ def generate_workstation_token(config_dir=None, profile=None):
 @click.option(
     "-p",
     "--profile",
-    default="",
+    default=os.environ.get("METAFLOW_PROFILE", ""),
     help="The named metaflow profile in which your workstation exists",
 )
 @click.option(
@@ -111,9 +115,6 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
         "ConfigureKubeConfig", OuterboundsCommandStatus.OK, "Kubeconfig is configured"
     )
     try:
-        if not profile:
-            profile = metaflowconfig.get_metaflow_profile()
-
         metaflow_token = metaflowconfig.get_metaflow_token_from_config(
             config_dir, profile
         )
@@ -193,7 +194,7 @@ def configure_cloud_workstation(config_dir=None, profile=None, binary=None, outp
 @click.option(
     "-p",
     "--profile",
-    default="",
+    default=os.environ.get("METAFLOW_PROFILE", ""),
     help="The named metaflow profile in which your workstation exists",
 )
 @click.option(
@@ -213,8 +214,6 @@ def list_workstations(config_dir=None, profile=None, output="json"):
     list_response.add_or_update_data("workstations", [])
 
     try:
-        if not profile:
-            profile = metaflowconfig.get_metaflow_profile()
         metaflow_token = metaflowconfig.get_metaflow_token_from_config(
             config_dir, profile
         )
@@ -260,7 +259,7 @@ def list_workstations(config_dir=None, profile=None, output="json"):
 @click.option(
     "-w",
     "--workstation",
-    default="",
+    default=os.environ.get("METAFLOW_PROFILE", ""),
     help="The ID of the workstation to hibernate",
 )
 def hibernate_workstation(config_dir=None, profile=None, workstation=None):
@@ -308,7 +307,7 @@ def hibernate_workstation(config_dir=None, profile=None, workstation=None):
 @click.option(
     "-p",
     "--profile",
-    default="",
+    default=os.environ.get("METAFLOW_PROFILE", ""),
     help="The named metaflow profile in which your workstation exists",
 )
 @click.option(
@@ -322,9 +321,6 @@ def restart_workstation(config_dir=None, profile=None, workstation=None):
         click.secho("Please specify a workstation ID", fg="red")
         return
     try:
-        if not profile:
-            profile = metaflowconfig.get_metaflow_profile()
-
         metaflow_token = metaflowconfig.get_metaflow_token_from_config(
             config_dir, profile
         )
@@ -516,7 +512,7 @@ def add_to_path(program_path, platform):
         with open(path_to_rc_file, "a+") as f:  # Open bashrc file
             if program_path not in f.read():
                 f.write("\n# Added by Outerbounds\n")
-                f.write(program_path)
+                f.write(f"export PATH=$PATH:{program_path}")
 
 
 def to_windows_path(path):
@@ -559,7 +555,24 @@ def show_relevant_links(config_dir=None, profile=None, perimeter_id="", output="
     show_links_response.add_or_update_data("links", [])
     links = []
     try:
-        metaflow_config = metaflowconfig.init_config(config_dir, profile)
+        if not perimeter_id:
+            metaflow_config = metaflowconfig.init_config(config_dir, profile)
+        else:
+            perimeters_dict = get_perimeters_from_api_or_fail_command(
+                config_dir, profile, output, show_links_response, show_links_step
+            )
+            confirm_user_has_access_to_perimeter_or_fail(
+                perimeter_id,
+                perimeters_dict,
+                output,
+                show_links_response,
+                show_links_step,
+            )
+
+            metaflow_config = metaflowconfig.init_config_from_url(
+                config_dir, profile, perimeters_dict[perimeter_id]["remote_config_url"]
+            )
+
         links.append(
             {
                 "id": "metaflow-ui-url",

outerbounds/utils/metaflowconfig.py

@@ -1,17 +1,44 @@
+import click
 import json
 import os
 import requests
 from os import path
-import requests
+from typing import Dict
+import sys
+
+
+def init_config(config_dir, profile) -> Dict[str, str]:
+    config = read_metaflow_config_from_filesystem(config_dir, profile)
+
+    # This is new remote-metaflow config; fetch it from the URL
+    if "OBP_METAFLOW_CONFIG_URL" in config:
+        remote_config = init_config_from_url(
+            config_dir, profile, config["OBP_METAFLOW_CONFIG_URL"]
+        )
+        remote_config["OBP_METAFLOW_CONFIG_URL"] = config["OBP_METAFLOW_CONFIG_URL"]
+        return remote_config
+    # Legacy config, use from filesystem
+    return config
+
 
+def init_config_from_url(config_dir, profile, url) -> Dict[str, str]:
+    config = read_metaflow_config_from_filesystem(config_dir, profile)
+
+    if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
+        raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
 
-def init_config(config_dir="", profile="") -> dict:
-    profile = profile or os.environ.get("METAFLOW_PROFILE")
-    config_dir = config_dir or os.path.expanduser(
-        os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+    config_response = requests.get(
+        url,
+        headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
     )
+    config_response.raise_for_status()
+    remote_config = config_response.json()["config"]
+    return remote_config
 
+
+def read_metaflow_config_from_filesystem(config_dir, profile) -> Dict[str, str]:
     config_filename = f"config_{profile}.json" if profile else "config.json"
+
     path_to_config = os.path.join(config_dir, config_filename)
 
     if os.path.exists(path_to_config):
@@ -19,22 +46,6 @@ def init_config(config_dir="", profile="") -> dict:
             config = json.load(json_file)
     else:
         raise Exception("Unable to locate metaflow config at '%s')" % (path_to_config))
-
-    # This is new remote-metaflow config; fetch it from the URL
-    if "OBP_METAFLOW_CONFIG_URL" in config:
-        if config is None or "METAFLOW_SERVICE_AUTH_KEY" not in config:
-            raise Exception("METAFLOW_SERVICE_AUTH_KEY not found in config file")
-
-        config_response = requests.get(
-            config["OBP_METAFLOW_CONFIG_URL"],
-            headers={"x-api-key": f'{config["METAFLOW_SERVICE_AUTH_KEY"]}'},
-        )
-        config_response.raise_for_status()
-        remote_config = config_response.json()["config"]
-        remote_config["METAFLOW_SERVICE_AUTH_KEY"] = config["METAFLOW_SERVICE_AUTH_KEY"]
-        return remote_config
-
-    # Legacy config, use from filesystem
     return config
 
 
@@ -70,3 +81,23 @@ def get_sanitized_url_from_config(config_dir: str, profile: str, key: str) -> st
 
     url_in_config = url_in_config.rstrip("/")
     return url_in_config
+
+
+def get_remote_metaflow_config_for_perimeter(
+    origin_token: str, perimeter: str, api_server: str
+):
+    try:
+        response = requests.get(
+            f"{api_server}/v1/perimeters/{perimeter}/metaflowconfigs/default",
+            headers={"x-api-key": origin_token},
+        )
+        response.raise_for_status()
+        config = response.json()["config"]
+        config["METAFLOW_SERVICE_AUTH_KEY"] = origin_token
+        return config
+    except Exception as e:
+        click.secho(
+            f"Failed to get metaflow config from {api_server}. Error: {str(e)}",
+            fg="red",
+        )
+        sys.exit(1)

outerbounds/utils/schema.py

@@ -5,6 +5,7 @@ class OuterboundsCommandStatus(Enum):
     OK = "OK"
     FAIL = "FAIL"
     WARN = "WARN"
+    NOT_SUPPORTED = "NOT_SUPPORTED"
 
 
 class CommandStatus:
@@ -39,6 +40,11 @@ class OuterboundsCommandResponse:
         self.metadata = {}
         self._data = {}
 
+    def update(self, status, code, message):
+        self.status = status
+        self._code = code
+        self._message = message
+
     def add_or_update_metadata(self, key, value):
         self.metadata[key] = value
 

{outerbounds-0.3.55rc4.dist-info → outerbounds-0.3.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: outerbounds
-Version: 0.3.55rc4
+Version: 0.3.68
 Summary: More Data Science, Less Administration
 License: Proprietary
 Keywords: data science,machine learning,MLOps
@@ -17,14 +17,16 @@ Provides-Extra: azure
 Provides-Extra: gcp
 Requires-Dist: PyYAML (>=6.0,<7.0)
 Requires-Dist: azure-identity (>=1.15.0,<2.0.0) ; extra == "azure"
+Requires-Dist: azure-keyvault-secrets (>=4.7.0,<5.0.0) ; extra == "azure"
 Requires-Dist: azure-storage-blob (>=12.9.0,<13.0.0) ; extra == "azure"
 Requires-Dist: boto3
 Requires-Dist: click (>=8.1.3,<9.0.0)
 Requires-Dist: google-api-core (>=2.16.1,<3.0.0) ; extra == "gcp"
 Requires-Dist: google-auth (>=2.27.0,<3.0.0) ; extra == "gcp"
 Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
-Requires-Dist: ob-metaflow (==2.11.0.4)
-Requires-Dist: ob-metaflow-extensions (==1.1.45rc2)
+Requires-Dist: ob-metaflow (==2.11.16.1)
+Requires-Dist: ob-metaflow-extensions (==1.1.59)
+Requires-Dist: ob-metaflow-stubs (==3.6)
 Requires-Dist: opentelemetry-distro (==0.41b0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (==1.20.0)
 Requires-Dist: opentelemetry-instrumentation-requests (==0.41b0)

outerbounds-0.3.68.dist-info/RECORD

@@ -0,0 +1,15 @@
+outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
+outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
+outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
+outerbounds/command_groups/cli.py,sha256=H4LxcYTmsY9DQUrReSRLjvbg9s9Ro7s-eUrcMqEJ_9A,261
+outerbounds/command_groups/local_setup_cli.py,sha256=Xqb-tsAkYgc90duC_6COSR9MsDpMNiKigQxlXUUYfN0,36530
+outerbounds/command_groups/perimeters_cli.py,sha256=OxbxYQnHZDLRb3SFaVpD2mjp8W8s1fvK1Wc4htyRuGw,12757
+outerbounds/command_groups/workstations_cli.py,sha256=b5lt8_g2B0zCoUoNriTRv32IPB6E4mI2sUhubDT7Yjo,21966
+outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
+outerbounds/utils/metaflowconfig.py,sha256=JkhT2yOGpN7t2R2p9uaUJRDJU9fqFPwn4DcojjVnJMI,3513
+outerbounds/utils/schema.py,sha256=cNlgjmteLPbDzSEUSQDsq8txdhMGyezSmM83jU3aa0w,2329
+outerbounds-0.3.68.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
+outerbounds-0.3.68.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
+outerbounds-0.3.68.dist-info/METADATA,sha256=IkAr4Uwjo_zdK1OW1BSMlNZZ7n7-MgOyy39ZL6Y_Raw,1477
+outerbounds-0.3.68.dist-info/RECORD,,

outerbounds-0.3.55rc4.dist-info/RECORD

@@ -1,15 +0,0 @@
-outerbounds/__init__.py,sha256=GPdaubvAYF8pOFWJ3b-sPMKCpyfpteWVMZWkmaYhxRw,32
-outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
-outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
-outerbounds/command_groups/cli.py,sha256=H4LxcYTmsY9DQUrReSRLjvbg9s9Ro7s-eUrcMqEJ_9A,261
-outerbounds/command_groups/local_setup_cli.py,sha256=0sEi3V0sqoCW0RI2z3xMLNnit0pCsigWQ07m1mhDBso,29524
-outerbounds/command_groups/perimeters_cli.py,sha256=9tOql42d00KfHpZYkLLGEAOiy8iRbIzsknldCyICwU0,12063
-outerbounds/command_groups/workstations_cli.py,sha256=xHgETjWfUjiIkKODR-4qIS5pSc4ww3VS1ouZM9no8CY,21312
-outerbounds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-outerbounds/utils/kubeconfig.py,sha256=l1mUP1j9VIq3fsffi5bJ1Nk-hYlwd1dIqkpj7DvVS1E,7936
-outerbounds/utils/metaflowconfig.py,sha256=HgaDmK3F97rppfGUdysS1Zppe28ERTLV_HcB5IuPpV4,2631
-outerbounds/utils/schema.py,sha256=Ht_Yf5uoKO0m36WXHZLSPmWPH6EFWXfZDQsiAUquc5k,2160
-outerbounds-0.3.55rc4.dist-info/METADATA,sha256=6dAqYfSf245Yt-UAvKyh3BWE53-cYUbcaQMi9Zphd98,1367
-outerbounds-0.3.55rc4.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
-outerbounds-0.3.55rc4.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
-outerbounds-0.3.55rc4.dist-info/RECORD,,