outerbounds 0.3.55rc3__py3-none-any.whl → 0.3.133__py3-none-any.whl

outerbounds/command_groups/cli.py

@@ -1,12 +1,16 @@
-import click
-from . import local_setup_cli
-from . import workstations_cli
-from . import perimeters_cli
+from outerbounds._vendor import click
+from . import local_setup_cli, workstations_cli, perimeters_cli, apps_cli, tutorials_cli
 
 
 @click.command(
     cls=click.CommandCollection,
-    sources=[local_setup_cli.cli, workstations_cli.cli, perimeters_cli.cli],
+    sources=[
+        local_setup_cli.cli,
+        workstations_cli.cli,
+        perimeters_cli.cli,
+        apps_cli.cli,
+        tutorials_cli.cli,
+    ],
 )
 def cli(**kwargs):
     pass

outerbounds/command_groups/local_setup_cli.py

@@ -11,9 +11,7 @@ from importlib.machinery import PathFinder
 from os import path
 from pathlib import Path
 from typing import Any, Callable, Dict, List
-
-import boto3
-import click
+from outerbounds._vendor import click
 import requests
 from requests.exceptions import HTTPError
 
@@ -31,7 +29,8 @@ the Outerbounds platform.
 To remove that package, please try `python -m pip uninstall metaflow -y` or reach out to Outerbounds support.
 After uninstalling the Metaflow package, please reinstall the Outerbounds package using `python -m pip
 install outerbounds --force`.
-As always, please reach out to Outerbounds support for any questions."""
+As always, please reach out to Outerbounds support for any questions.
+"""
 
 MISSING_EXTENSIONS_MESSAGE = (
     "The Outerbounds Platform extensions for Metaflow was not found."
@@ -43,6 +42,8 @@ BAD_EXTENSION_MESSAGE = (
     "Mis-installation of the Outerbounds Platform extension package has been detected."
 )
 
+PERIMETER_CONFIG_URL_KEY = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
+
 
 class Narrator:
     def __init__(self, verbose):
@@ -54,11 +55,11 @@ class Narrator:
 
     def section_ok(self):
         if not self.verbose:
-            click.secho("\U0001F600", err=True)
+            click.secho("\U00002705", err=True)
 
     def section_not_ok(self):
         if not self.verbose:
-            click.secho("\U0001F641", err=True)
+            click.secho("\U0000274C", err=True)
 
     def announce_check(self, name):
         if self.verbose:
@@ -232,25 +233,33 @@ class ConfigEntrySpec:
         self.expected = expected
 
 
-def get_config_specs():
-    return [
-        ConfigEntrySpec(
-            "METAFLOW_DATASTORE_SYSROOT_S3", "s3://[a-z0-9\-]+/metaflow[/]?"
-        ),
-        ConfigEntrySpec("METAFLOW_DATATOOLS_S3ROOT", "s3://[a-z0-9\-]+/data[/]?"),
+def get_config_specs(default_datastore: str):
+    spec = [
         ConfigEntrySpec("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "obp", expected="obp"),
-        ConfigEntrySpec("METAFLOW_DEFAULT_DATASTORE", "s3", expected="s3"),
         ConfigEntrySpec("METAFLOW_DEFAULT_METADATA", "service", expected="service"),
-        ConfigEntrySpec(
-            "METAFLOW_KUBERNETES_NAMESPACE", "jobs\-default", expected="jobs-default"
-        ),
-        ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", "eval \$\(.*"),
-        ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", "[a-zA-Z0-9!_\-\.]+"),
-        ConfigEntrySpec("METAFLOW_SERVICE_URL", "https://metadata\..*"),
-        ConfigEntrySpec("METAFLOW_UI_URL", "https://ui\..*"),
-        ConfigEntrySpec("OBP_AUTH_SERVER", "auth\..*"),
+        ConfigEntrySpec("METAFLOW_KUBERNETES_NAMESPACE", r"jobs-.*"),
+        ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", r"eval \$\(.*"),
+        ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", r"[a-zA-Z0-9!_\-\.]+"),
+        ConfigEntrySpec("METAFLOW_SERVICE_URL", r"https://metadata\..*"),
+        ConfigEntrySpec("METAFLOW_UI_URL", r"https://ui\..*"),
+        ConfigEntrySpec("OBP_AUTH_SERVER", r"auth\..*"),
     ]
 
+    if default_datastore == "s3":
+        spec.extend(
+            [
+                ConfigEntrySpec(
+                    "METAFLOW_DATASTORE_SYSROOT_S3",
+                    r"s3://[a-z0-9\-]+/metaflow(-[a-z0-9\-]+)?[/]?",
+                ),
+                ConfigEntrySpec(
+                    "METAFLOW_DATATOOLS_S3ROOT",
+                    r"s3://[a-z0-9\-]+/data(-[a-z0-9\-]+)?[/]?",
+                ),
+            ]
+        )
+    return spec
+
 
 def check_metaflow_config(narrator: Narrator) -> CommandStatus:
     narrator.announce_section("local Metaflow config")
@@ -261,8 +270,20 @@ def check_metaflow_config(narrator: Narrator) -> CommandStatus:
         mitigation="",
     )
 
-    config = metaflowconfig.init_config()
-    for spec in get_config_specs():
+    profile = os.environ.get("METAFLOW_PROFILE")
+    config_dir = os.path.expanduser(
+        os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+    )
+
+    config = metaflowconfig.init_config(config_dir, profile)
+
+    if "OBP_METAFLOW_CONFIG_URL" in config:
+        # If the config is fetched from a remote source, not much to check
+        narrator.announce_check("config entry OBP_METAFLOW_CONFIG_URL")
+        narrator.ok()
+        return check_status
+
+    for spec in get_config_specs(config.get("METAFLOW_DEFAULT_DATASTORE", "")):
         narrator.announce_check("config entry " + spec.name)
         if spec.name not in config:
             reason = "Missing"
@@ -304,7 +325,12 @@ def check_metaflow_token(narrator: Narrator) -> CommandStatus:
         mitigation="",
     )
 
-    config = metaflowconfig.init_config()
+    profile = os.environ.get("METAFLOW_PROFILE")
+    config_dir = os.path.expanduser(
+        os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+    )
+
+    config = metaflowconfig.init_config(config_dir, profile)
     try:
         if "OBP_AUTH_SERVER" in config:
             k8s_response = requests.get(
@@ -363,7 +389,13 @@ def check_workstation_api_accessible(narrator: Narrator) -> CommandStatus:
     )
 
     try:
-        config = metaflowconfig.init_config()
+        profile = os.environ.get("METAFLOW_PROFILE")
+        config_dir = os.path.expanduser(
+            os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+        )
+
+        config = metaflowconfig.init_config(config_dir, profile)
+
         missing_keys = []
         if "METAFLOW_SERVICE_AUTH_KEY" not in config:
             missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -422,7 +454,13 @@ def check_kubeconfig_valid_for_workstations(narrator: Narrator) -> CommandStatus
     )
 
     try:
-        config = metaflowconfig.init_config()
+        profile = os.environ.get("METAFLOW_PROFILE")
+        config_dir = os.path.expanduser(
+            os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+        )
+
+        config = metaflowconfig.init_config(config_dir, profile)
+
         missing_keys = []
         if "METAFLOW_SERVICE_AUTH_KEY" not in config:
             missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -585,6 +623,7 @@ class ConfigurationWriter:
         self.decoded_config = None
         self.out_dir = out_dir
         self.profile = profile
+        self.selected_perimeter = None
 
         ob_config_dir = path.expanduser(os.getenv("OBP_CONFIG_DIR", out_dir))
         self.ob_config_path = path.join(
@@ -596,8 +635,11 @@ class ConfigurationWriter:
         self.decoded_config = deserialize(self.encoded_config)
 
     def process_decoded_config(self):
+        assert self.decoded_config is not None
         config_type = self.decoded_config.get("OB_CONFIG_TYPE", "inline")
         if config_type == "inline":
+            if "OBP_PERIMETER" in self.decoded_config:
+                self.selected_perimeter = self.decoded_config["OBP_PERIMETER"]
             if "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
                 self.decoded_config = {
                     "OBP_METAFLOW_CONFIG_URL": self.decoded_config[
@@ -616,6 +658,8 @@ class ConfigurationWriter:
                     f"{str(e)} key is required for aws-ref config type"
                 )
             try:
+                import boto3
+
                 client = boto3.client("secretsmanager", region_name=region)
                 response = client.get_secret_value(SecretId=secret_arn)
                 self.decoded_config = json.loads(response["SecretBinary"])
@@ -633,6 +677,7 @@ class ConfigurationWriter:
             return path.join(self.out_dir, "config_{}.json".format(self.profile))
 
     def display(self):
+        assert self.decoded_config is not None
         # Create a copy so we can use the real config later, possibly
         display_config = dict()
         for k in self.decoded_config.keys():
@@ -647,6 +692,7 @@ class ConfigurationWriter:
         return self.confirm_overwrite_config(self.path())
 
     def write_config(self):
+        assert self.decoded_config is not None
         config_path = self.path()
         # TODO config contains auth token - restrict file/dir modes
         os.makedirs(os.path.dirname(config_path), exist_ok=True)
@@ -654,13 +700,13 @@ class ConfigurationWriter:
         with open(config_path, "w") as fd:
             json.dump(self.existing, fd, indent=4)
 
-        # Every time a config is initialized, we should also reset the corresponding ob_config[_profile].json
-        remote_config = metaflowconfig.init_config(self.out_dir, self.profile)
-        if "OBP_PERIMETER" in remote_config and "OBP_PERIMETER_URL" in remote_config:
+        if self.selected_perimeter and "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
             with open(self.ob_config_path, "w") as fd:
                 ob_config_dict = {
-                    "OB_CURRENT_PERIMETER": remote_config["OBP_PERIMETER"],
-                    "OB_CURRENT_PERIMETER_URL": remote_config["OBP_PERIMETER_URL"],
+                    "OB_CURRENT_PERIMETER": self.selected_perimeter,
+                    PERIMETER_CONFIG_URL_KEY: self.decoded_config[
+                        "OBP_METAFLOW_CONFIG_URL"
+                    ],
                 }
                 json.dump(ob_config_dict, fd, indent=4)
 
@@ -686,6 +732,64 @@ class ConfigurationWriter:
         return True
 
 
+def get_gha_jwt(audience: str):
+    # These are specific environment variables that are set by GitHub Actions.
+    if (
+        "ACTIONS_ID_TOKEN_REQUEST_TOKEN" in os.environ
+        and "ACTIONS_ID_TOKEN_REQUEST_URL" in os.environ
+    ):
+        try:
+            response = requests.get(
+                url=os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"],
+                headers={
+                    "Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"
+                },
+                params={"audience": audience},
+            )
+            response.raise_for_status()
+            return response.json()["value"]
+        except Exception as e:
+            click.secho(
+                "Failed to fetch JWT token from GitHub Actions. Please make sure you are permission 'id-token: write' is set on the GHA jobs level.",
+                fg="red",
+            )
+            sys.exit(1)
+
+    click.secho(
+        "The --github-actions flag was set, but we didn't not find '$ACTIONS_ID_TOKEN_REQUEST_TOKEN' and '$ACTIONS_ID_TOKEN_REQUEST_URL' environment variables. Please make sure you are running this command in a GitHub Actions environment and with correct permissions as per https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers",
+        fg="red",
+    )
+    sys.exit(1)
+
+
+def get_origin_token(
+    service_principal_name: str,
+    deployment: str,
+    perimeter: str,
+    token: str,
+    auth_server: str,
+):
+    try:
+        response = requests.get(
+            f"{auth_server}/generate/service-principal",
+            headers={"x-api-key": token},
+            data=json.dumps(
+                {
+                    "servicePrincipalName": service_principal_name,
+                    "deploymentName": deployment,
+                    "perimeter": perimeter,
+                }
+            ),
+        )
+        response.raise_for_status()
+        return response.json()["token"]
+    except Exception as e:
+        click.secho(
+            f"Failed to get origin token from {auth_server}. Error: {str(e)}", fg="red"
+        )
+        sys.exit(1)
+
+
 @click.group(help="The Outerbounds Platform CLI", no_args_is_help=True)
 def cli(**kwargs):
     pass
@@ -727,7 +831,6 @@ def check(no_config, verbose, output, workstation=False):
         ]
     else:
         check_names = [
-            "metaflow_config",
             "metaflow_token",
             "kubeconfig",
             "api_connectivity",
@@ -772,7 +875,7 @@ def check(no_config, verbose, output, workstation=False):
 )
 @click.argument("encoded_config", required=True)
 def configure(
-    encoded_config=None, config_dir=None, profile=None, echo=None, force=False
+    encoded_config: str, config_dir=None, profile=None, echo=None, force=False
 ):
     writer = ConfigurationWriter(encoded_config, config_dir, profile)
     try:
@@ -794,3 +897,116 @@ def configure(
     except Exception as e:
         click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
         click.secho("Error: {}".format(str(e)))
+
+
+@cli.command(
+    help="Authenticate service principals using JWT minted by their IDPs and configure Metaflow"
+)
+@click.option(
+    "-n",
+    "--name",
+    default="",
+    help="The name of service principals to authenticate",
+    required=True,
+)
+@click.option(
+    "--deployment-domain",
+    default="",
+    help="The full domain of the target Outerbounds Platform deployment (eg. 'foo.obp.outerbounds.com')",
+    required=True,
+)
+@click.option(
+    "-p",
+    "--perimeter",
+    default="default",
+    help="The name of the perimeter to authenticate the service principal in",
+)
+@click.option(
+    "-t",
+    "--jwt-token",
+    default="",
+    help="The JWT token that will be used to authenticate against the OBP Auth Server.",
+)
+@click.option(
+    "--github-actions",
+    is_flag=True,
+    help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
+)
+@click.option(
+    "-d",
+    "--config-dir",
+    default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
+    help="Path to Metaflow configuration directory",
+    show_default=True,
+)
+@click.option(
+    "--profile",
+    default="",
+    help="Configure a named profile. Activate the profile by setting "
+    "`METAFLOW_PROFILE` environment variable.",
+)
+@click.option(
+    "-e",
+    "--echo",
+    is_flag=True,
+    help="Print decoded configuration to stdout",
+)
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    help="Force overwrite of existing configuration",
+)
+def service_principal_configure(
+    name: str,
+    deployment_domain: str,
+    perimeter: str,
+    jwt_token="",
+    github_actions=False,
+    config_dir=None,
+    profile=None,
+    echo=None,
+    force=False,
+):
+    audience = f"https://{deployment_domain}"
+    if jwt_token == "" and github_actions:
+        jwt_token = get_gha_jwt(audience)
+
+    if jwt_token == "":
+        click.secho(
+            "No JWT token provided. Please provider either a valid jwt token or set --github-actions",
+            fg="red",
+        )
+        sys.exit(1)
+
+    auth_server = f"https://auth.{deployment_domain}"
+    deployment_name = deployment_domain.split(".")[0]
+    origin_token = get_origin_token(
+        name, deployment_name, perimeter, jwt_token, auth_server
+    )
+
+    api_server = f"https://api.{deployment_domain}"
+    metaflow_config = metaflowconfig.get_remote_metaflow_config_for_perimeter(
+        origin_token, perimeter, api_server
+    )
+
+    writer = ConfigurationWriter(serialize(metaflow_config), config_dir, profile)
+    try:
+        writer.decode()
+    except:
+        click.secho("Decoding the configuration text failed.", fg="red")
+        sys.exit(1)
+    try:
+        writer.process_decoded_config()
+    except DecodedConfigProcessingError as e:
+        click.secho("Resolving the configuration remotely failed.", fg="red")
+        click.secho(str(e), fg="magenta")
+        sys.exit(1)
+    try:
+        if echo == True:
+            writer.display()
+        if force or writer.confirm_overwrite():
+            writer.write_config()
+    except Exception as e:
+        click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
+        click.secho("Error: {}".format(str(e)))