outerbounds 0.3.55rc4__py3-none-any.whl → 0.3.89__py3-none-any.whl

outerbounds/_vendor/yaml/serializer.py

@@ -0,0 +1,127 @@
+__all__ = ["Serializer", "SerializerError"]
+
+from .error import YAMLError
+from .events import *
+from .nodes import *
+
+
+class SerializerError(YAMLError):
+    pass
+
+
+class Serializer:
+
+    ANCHOR_TEMPLATE = "id%03d"
+
+    def __init__(
+        self,
+        encoding=None,
+        explicit_start=None,
+        explicit_end=None,
+        version=None,
+        tags=None,
+    ):
+        self.use_encoding = encoding
+        self.use_explicit_start = explicit_start
+        self.use_explicit_end = explicit_end
+        self.use_version = version
+        self.use_tags = tags
+        self.serialized_nodes = {}
+        self.anchors = {}
+        self.last_anchor_id = 0
+        self.closed = None
+
+    def open(self):
+        if self.closed is None:
+            self.emit(StreamStartEvent(encoding=self.use_encoding))
+            self.closed = False
+        elif self.closed:
+            raise SerializerError("serializer is closed")
+        else:
+            raise SerializerError("serializer is already opened")
+
+    def close(self):
+        if self.closed is None:
+            raise SerializerError("serializer is not opened")
+        elif not self.closed:
+            self.emit(StreamEndEvent())
+            self.closed = True
+
+    # def __del__(self):
+    #    self.close()
+
+    def serialize(self, node):
+        if self.closed is None:
+            raise SerializerError("serializer is not opened")
+        elif self.closed:
+            raise SerializerError("serializer is closed")
+        self.emit(
+            DocumentStartEvent(
+                explicit=self.use_explicit_start,
+                version=self.use_version,
+                tags=self.use_tags,
+            )
+        )
+        self.anchor_node(node)
+        self.serialize_node(node, None, None)
+        self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
+        self.serialized_nodes = {}
+        self.anchors = {}
+        self.last_anchor_id = 0
+
+    def anchor_node(self, node):
+        if node in self.anchors:
+            if self.anchors[node] is None:
+                self.anchors[node] = self.generate_anchor(node)
+        else:
+            self.anchors[node] = None
+            if isinstance(node, SequenceNode):
+                for item in node.value:
+                    self.anchor_node(item)
+            elif isinstance(node, MappingNode):
+                for key, value in node.value:
+                    self.anchor_node(key)
+                    self.anchor_node(value)
+
+    def generate_anchor(self, node):
+        self.last_anchor_id += 1
+        return self.ANCHOR_TEMPLATE % self.last_anchor_id
+
+    def serialize_node(self, node, parent, index):
+        alias = self.anchors[node]
+        if node in self.serialized_nodes:
+            self.emit(AliasEvent(alias))
+        else:
+            self.serialized_nodes[node] = True
+            self.descend_resolver(parent, index)
+            if isinstance(node, ScalarNode):
+                detected_tag = self.resolve(ScalarNode, node.value, (True, False))
+                default_tag = self.resolve(ScalarNode, node.value, (False, True))
+                implicit = (node.tag == detected_tag), (node.tag == default_tag)
+                self.emit(
+                    ScalarEvent(alias, node.tag, implicit, node.value, style=node.style)
+                )
+            elif isinstance(node, SequenceNode):
+                implicit = node.tag == self.resolve(SequenceNode, node.value, True)
+                self.emit(
+                    SequenceStartEvent(
+                        alias, node.tag, implicit, flow_style=node.flow_style
+                    )
+                )
+                index = 0
+                for item in node.value:
+                    self.serialize_node(item, node, index)
+                    index += 1
+                self.emit(SequenceEndEvent())
+            elif isinstance(node, MappingNode):
+                implicit = node.tag == self.resolve(MappingNode, node.value, True)
+                self.emit(
+                    MappingStartEvent(
+                        alias, node.tag, implicit, flow_style=node.flow_style
+                    )
+                )
+                for key, value in node.value:
+                    self.serialize_node(key, node, None)
+                    self.serialize_node(value, node, key)
+                self.emit(MappingEndEvent())
+            self.ascend_resolver()

outerbounds/_vendor/yaml/tokens.py

@@ -0,0 +1,129 @@
+class Token(object):
+    def __init__(self, start_mark, end_mark):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+    def __repr__(self):
+        attributes = [key for key in self.__dict__ if not key.endswith("_mark")]
+        attributes.sort()
+        arguments = ", ".join(
+            ["%s=%r" % (key, getattr(self, key)) for key in attributes]
+        )
+        return "%s(%s)" % (self.__class__.__name__, arguments)
+
+
+# class BOMToken(Token):
+#    id = '<byte order mark>'
+
+
+class DirectiveToken(Token):
+    id = "<directive>"
+
+    def __init__(self, name, value, start_mark, end_mark):
+        self.name = name
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+
+class DocumentStartToken(Token):
+    id = "<document start>"
+
+
+class DocumentEndToken(Token):
+    id = "<document end>"
+
+
+class StreamStartToken(Token):
+    id = "<stream start>"
+
+    def __init__(self, start_mark=None, end_mark=None, encoding=None):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.encoding = encoding
+
+
+class StreamEndToken(Token):
+    id = "<stream end>"
+
+
+class BlockSequenceStartToken(Token):
+    id = "<block sequence start>"
+
+
+class BlockMappingStartToken(Token):
+    id = "<block mapping start>"
+
+
+class BlockEndToken(Token):
+    id = "<block end>"
+
+
+class FlowSequenceStartToken(Token):
+    id = "["
+
+
+class FlowMappingStartToken(Token):
+    id = "{"
+
+
+class FlowSequenceEndToken(Token):
+    id = "]"
+
+
+class FlowMappingEndToken(Token):
+    id = "}"
+
+
+class KeyToken(Token):
+    id = "?"
+
+
+class ValueToken(Token):
+    id = ":"
+
+
+class BlockEntryToken(Token):
+    id = "-"
+
+
+class FlowEntryToken(Token):
+    id = ","
+
+
+class AliasToken(Token):
+    id = "<alias>"
+
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+
+class AnchorToken(Token):
+    id = "<anchor>"
+
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+
+class TagToken(Token):
+    id = "<tag>"
+
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+
+class ScalarToken(Token):
+    id = "<scalar>"
+
+    def __init__(self, value, plain, start_mark, end_mark, style=None):
+        self.value = value
+        self.plain = plain
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.style = style

outerbounds/command_groups/cli.py

@@ -1,4 +1,4 @@
-import click
+from outerbounds._vendor import click
 from . import local_setup_cli
 from . import workstations_cli
 from . import perimeters_cli

outerbounds/command_groups/local_setup_cli.py

@@ -11,9 +11,7 @@ from importlib.machinery import PathFinder
 from os import path
 from pathlib import Path
 from typing import Any, Callable, Dict, List
-
-import boto3
-import click
+from outerbounds._vendor import click
 import requests
 from requests.exceptions import HTTPError
 
@@ -31,7 +29,8 @@ the Outerbounds platform.
 To remove that package, please try `python -m pip uninstall metaflow -y` or reach out to Outerbounds support.
 After uninstalling the Metaflow package, please reinstall the Outerbounds package using `python -m pip
 install outerbounds --force`.
-As always, please reach out to Outerbounds support for any questions."""
+As always, please reach out to Outerbounds support for any questions.
+"""
 
 MISSING_EXTENSIONS_MESSAGE = (
     "The Outerbounds Platform extensions for Metaflow was not found."
@@ -43,6 +42,8 @@ BAD_EXTENSION_MESSAGE = (
     "Mis-installation of the Outerbounds Platform extension package has been detected."
 )
 
+PERIMETER_CONFIG_URL_KEY = "OB_CURRENT_PERIMETER_MF_CONFIG_URL"
+
 
 class Narrator:
     def __init__(self, verbose):
@@ -54,11 +55,11 @@ class Narrator:
 
     def section_ok(self):
         if not self.verbose:
-            click.secho("\U0001F600", err=True)
+            click.secho("\U00002705", err=True)
 
     def section_not_ok(self):
         if not self.verbose:
-            click.secho("\U0001F641", err=True)
+            click.secho("\U0000274C", err=True)
 
     def announce_check(self, name):
         if self.verbose:
@@ -232,25 +233,33 @@ class ConfigEntrySpec:
         self.expected = expected
 
 
-def get_config_specs():
-    return [
-        ConfigEntrySpec(
-            "METAFLOW_DATASTORE_SYSROOT_S3", "s3://[a-z0-9\-]+/metaflow[/]?"
-        ),
-        ConfigEntrySpec("METAFLOW_DATATOOLS_S3ROOT", "s3://[a-z0-9\-]+/data[/]?"),
+def get_config_specs(default_datastore: str):
+    spec = [
         ConfigEntrySpec("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "obp", expected="obp"),
-        ConfigEntrySpec("METAFLOW_DEFAULT_DATASTORE", "s3", expected="s3"),
         ConfigEntrySpec("METAFLOW_DEFAULT_METADATA", "service", expected="service"),
-        ConfigEntrySpec(
-            "METAFLOW_KUBERNETES_NAMESPACE", "jobs\-default", expected="jobs-default"
-        ),
-        ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", "eval \$\(.*"),
-        ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", "[a-zA-Z0-9!_\-\.]+"),
-        ConfigEntrySpec("METAFLOW_SERVICE_URL", "https://metadata\..*"),
-        ConfigEntrySpec("METAFLOW_UI_URL", "https://ui\..*"),
-        ConfigEntrySpec("OBP_AUTH_SERVER", "auth\..*"),
+        ConfigEntrySpec("METAFLOW_KUBERNETES_NAMESPACE", r"jobs-.*"),
+        ConfigEntrySpec("METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT", r"eval \$\(.*"),
+        ConfigEntrySpec("METAFLOW_SERVICE_AUTH_KEY", r"[a-zA-Z0-9!_\-\.]+"),
+        ConfigEntrySpec("METAFLOW_SERVICE_URL", r"https://metadata\..*"),
+        ConfigEntrySpec("METAFLOW_UI_URL", r"https://ui\..*"),
+        ConfigEntrySpec("OBP_AUTH_SERVER", r"auth\..*"),
     ]
 
+    if default_datastore == "s3":
+        spec.extend(
+            [
+                ConfigEntrySpec(
+                    "METAFLOW_DATASTORE_SYSROOT_S3",
+                    r"s3://[a-z0-9\-]+/metaflow(-[a-z0-9\-]+)?[/]?",
+                ),
+                ConfigEntrySpec(
+                    "METAFLOW_DATATOOLS_S3ROOT",
+                    r"s3://[a-z0-9\-]+/data(-[a-z0-9\-]+)?[/]?",
+                ),
+            ]
+        )
+    return spec
+
 
 def check_metaflow_config(narrator: Narrator) -> CommandStatus:
     narrator.announce_section("local Metaflow config")
@@ -261,8 +270,20 @@ def check_metaflow_config(narrator: Narrator) -> CommandStatus:
         mitigation="",
     )
 
-    config = metaflowconfig.init_config()
-    for spec in get_config_specs():
+    profile = os.environ.get("METAFLOW_PROFILE")
+    config_dir = os.path.expanduser(
+        os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+    )
+
+    config = metaflowconfig.init_config(config_dir, profile)
+
+    if "OBP_METAFLOW_CONFIG_URL" in config:
+        # If the config is fetched from a remote source, not much to check
+        narrator.announce_check("config entry OBP_METAFLOW_CONFIG_URL")
+        narrator.ok()
+        return check_status
+
+    for spec in get_config_specs(config.get("METAFLOW_DEFAULT_DATASTORE", "")):
         narrator.announce_check("config entry " + spec.name)
         if spec.name not in config:
             reason = "Missing"
@@ -304,7 +325,12 @@ def check_metaflow_token(narrator: Narrator) -> CommandStatus:
         mitigation="",
     )
 
-    config = metaflowconfig.init_config()
+    profile = os.environ.get("METAFLOW_PROFILE")
+    config_dir = os.path.expanduser(
+        os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+    )
+
+    config = metaflowconfig.init_config(config_dir, profile)
     try:
         if "OBP_AUTH_SERVER" in config:
             k8s_response = requests.get(
@@ -363,7 +389,13 @@ def check_workstation_api_accessible(narrator: Narrator) -> CommandStatus:
     )
 
     try:
-        config = metaflowconfig.init_config()
+        profile = os.environ.get("METAFLOW_PROFILE")
+        config_dir = os.path.expanduser(
+            os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+        )
+
+        config = metaflowconfig.init_config(config_dir, profile)
+
         missing_keys = []
         if "METAFLOW_SERVICE_AUTH_KEY" not in config:
             missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -422,7 +454,13 @@ def check_kubeconfig_valid_for_workstations(narrator: Narrator) -> CommandStatus
     )
 
     try:
-        config = metaflowconfig.init_config()
+        profile = os.environ.get("METAFLOW_PROFILE")
+        config_dir = os.path.expanduser(
+            os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")
+        )
+
+        config = metaflowconfig.init_config(config_dir, profile)
+
         missing_keys = []
         if "METAFLOW_SERVICE_AUTH_KEY" not in config:
             missing_keys.append("METAFLOW_SERVICE_AUTH_KEY")
@@ -585,6 +623,7 @@ class ConfigurationWriter:
         self.decoded_config = None
         self.out_dir = out_dir
         self.profile = profile
+        self.selected_perimeter = None
 
         ob_config_dir = path.expanduser(os.getenv("OBP_CONFIG_DIR", out_dir))
         self.ob_config_path = path.join(
@@ -596,8 +635,11 @@ class ConfigurationWriter:
         self.decoded_config = deserialize(self.encoded_config)
 
     def process_decoded_config(self):
+        assert self.decoded_config is not None
         config_type = self.decoded_config.get("OB_CONFIG_TYPE", "inline")
         if config_type == "inline":
+            if "OBP_PERIMETER" in self.decoded_config:
+                self.selected_perimeter = self.decoded_config["OBP_PERIMETER"]
             if "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
                 self.decoded_config = {
                     "OBP_METAFLOW_CONFIG_URL": self.decoded_config[
@@ -616,6 +658,8 @@ class ConfigurationWriter:
                     f"{str(e)} key is required for aws-ref config type"
                 )
             try:
+                import boto3
+
                 client = boto3.client("secretsmanager", region_name=region)
                 response = client.get_secret_value(SecretId=secret_arn)
                 self.decoded_config = json.loads(response["SecretBinary"])
@@ -633,6 +677,7 @@ class ConfigurationWriter:
             return path.join(self.out_dir, "config_{}.json".format(self.profile))
 
     def display(self):
+        assert self.decoded_config is not None
         # Create a copy so we can use the real config later, possibly
         display_config = dict()
         for k in self.decoded_config.keys():
@@ -647,6 +692,7 @@ class ConfigurationWriter:
         return self.confirm_overwrite_config(self.path())
 
     def write_config(self):
+        assert self.decoded_config is not None
         config_path = self.path()
         # TODO config contains auth token - restrict file/dir modes
         os.makedirs(os.path.dirname(config_path), exist_ok=True)
@@ -654,13 +700,13 @@ class ConfigurationWriter:
         with open(config_path, "w") as fd:
             json.dump(self.existing, fd, indent=4)
 
-        # Every time a config is initialized, we should also reset the corresponding ob_config[_profile].json
-        remote_config = metaflowconfig.init_config(self.out_dir, self.profile)
-        if "OBP_PERIMETER" in remote_config and "OBP_PERIMETER_URL" in remote_config:
+        if self.selected_perimeter and "OBP_METAFLOW_CONFIG_URL" in self.decoded_config:
             with open(self.ob_config_path, "w") as fd:
                 ob_config_dict = {
-                    "OB_CURRENT_PERIMETER": remote_config["OBP_PERIMETER"],
-                    "OB_CURRENT_PERIMETER_URL": remote_config["OBP_PERIMETER_URL"],
+                    "OB_CURRENT_PERIMETER": self.selected_perimeter,
+                    PERIMETER_CONFIG_URL_KEY: self.decoded_config[
+                        "OBP_METAFLOW_CONFIG_URL"
+                    ],
                 }
                 json.dump(ob_config_dict, fd, indent=4)
 
@@ -686,6 +732,64 @@ class ConfigurationWriter:
         return True
 
 
+def get_gha_jwt(audience: str):
+    # These are specific environment variables that are set by GitHub Actions.
+    if (
+        "ACTIONS_ID_TOKEN_REQUEST_TOKEN" in os.environ
+        and "ACTIONS_ID_TOKEN_REQUEST_URL" in os.environ
+    ):
+        try:
+            response = requests.get(
+                url=os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"],
+                headers={
+                    "Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"
+                },
+                params={"audience": audience},
+            )
+            response.raise_for_status()
+            return response.json()["value"]
+        except Exception as e:
+            click.secho(
+                "Failed to fetch JWT token from GitHub Actions. Please make sure you are permission 'id-token: write' is set on the GHA jobs level.",
+                fg="red",
+            )
+            sys.exit(1)
+
+    click.secho(
+        "The --github-actions flag was set, but we didn't not find '$ACTIONS_ID_TOKEN_REQUEST_TOKEN' and '$ACTIONS_ID_TOKEN_REQUEST_URL' environment variables. Please make sure you are running this command in a GitHub Actions environment and with correct permissions as per https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers",
+        fg="red",
+    )
+    sys.exit(1)
+
+
+def get_origin_token(
+    service_principal_name: str,
+    deployment: str,
+    perimeter: str,
+    token: str,
+    auth_server: str,
+):
+    try:
+        response = requests.get(
+            f"{auth_server}/generate/service-principal",
+            headers={"x-api-key": token},
+            data=json.dumps(
+                {
+                    "servicePrincipalName": service_principal_name,
+                    "deploymentName": deployment,
+                    "perimeter": perimeter,
+                }
+            ),
+        )
+        response.raise_for_status()
+        return response.json()["token"]
+    except Exception as e:
+        click.secho(
+            f"Failed to get origin token from {auth_server}. Error: {str(e)}", fg="red"
+        )
+        sys.exit(1)
+
+
 @click.group(help="The Outerbounds Platform CLI", no_args_is_help=True)
 def cli(**kwargs):
     pass
@@ -727,7 +831,6 @@ def check(no_config, verbose, output, workstation=False):
         ]
     else:
         check_names = [
-            "metaflow_config",
             "metaflow_token",
             "kubeconfig",
             "api_connectivity",
@@ -772,7 +875,7 @@ def check(no_config, verbose, output, workstation=False):
 )
 @click.argument("encoded_config", required=True)
 def configure(
-    encoded_config=None, config_dir=None, profile=None, echo=None, force=False
+    encoded_config: str, config_dir=None, profile=None, echo=None, force=False
 ):
     writer = ConfigurationWriter(encoded_config, config_dir, profile)
     try:
@@ -794,3 +897,116 @@ def configure(
     except Exception as e:
         click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
         click.secho("Error: {}".format(str(e)))
+
+
+@cli.command(
+    help="Authenticate service principals using JWT minted by their IDPs and configure Metaflow"
+)
+@click.option(
+    "-n",
+    "--name",
+    default="",
+    help="The name of service principals to authenticate",
+    required=True,
+)
+@click.option(
+    "--deployment-domain",
+    default="",
+    help="The full domain of the target Outerbounds Platform deployment (eg. 'foo.obp.outerbounds.com')",
+    required=True,
+)
+@click.option(
+    "-p",
+    "--perimeter",
+    default="default",
+    help="The name of the perimeter to authenticate the service principal in",
+)
+@click.option(
+    "-t",
+    "--jwt-token",
+    default="",
+    help="The JWT token that will be used to authenticate against the OBP Auth Server.",
+)
+@click.option(
+    "--github-actions",
+    is_flag=True,
+    help="Set if the command is being run in a GitHub Actions environment. If both --jwt-token and --github-actions are specified the --github-actions flag will be ignored.",
+)
+@click.option(
+    "-d",
+    "--config-dir",
+    default=path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
+    help="Path to Metaflow configuration directory",
+    show_default=True,
+)
+@click.option(
+    "--profile",
+    default="",
+    help="Configure a named profile. Activate the profile by setting "
+    "`METAFLOW_PROFILE` environment variable.",
+)
+@click.option(
+    "-e",
+    "--echo",
+    is_flag=True,
+    help="Print decoded configuration to stdout",
+)
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    help="Force overwrite of existing configuration",
+)
+def service_principal_configure(
+    name: str,
+    deployment_domain: str,
+    perimeter: str,
+    jwt_token="",
+    github_actions=False,
+    config_dir=None,
+    profile=None,
+    echo=None,
+    force=False,
+):
+    audience = f"https://{deployment_domain}"
+    if jwt_token == "" and github_actions:
+        jwt_token = get_gha_jwt(audience)
+
+    if jwt_token == "":
+        click.secho(
+            "No JWT token provided. Please provider either a valid jwt token or set --github-actions",
+            fg="red",
+        )
+        sys.exit(1)
+
+    auth_server = f"https://auth.{deployment_domain}"
+    deployment_name = deployment_domain.split(".")[0]
+    origin_token = get_origin_token(
+        name, deployment_name, perimeter, jwt_token, auth_server
+    )
+
+    api_server = f"https://api.{deployment_domain}"
+    metaflow_config = metaflowconfig.get_remote_metaflow_config_for_perimeter(
+        origin_token, perimeter, api_server
+    )
+
+    writer = ConfigurationWriter(serialize(metaflow_config), config_dir, profile)
+    try:
+        writer.decode()
+    except:
+        click.secho("Decoding the configuration text failed.", fg="red")
+        sys.exit(1)
+    try:
+        writer.process_decoded_config()
+    except DecodedConfigProcessingError as e:
+        click.secho("Resolving the configuration remotely failed.", fg="red")
+        click.secho(str(e), fg="magenta")
+        sys.exit(1)
+    try:
+        if echo == True:
+            writer.display()
+        if force or writer.confirm_overwrite():
+            writer.write_config()
+    except Exception as e:
+        click.secho("Writing the configuration file '{}' failed.".format(writer.path()))
+        click.secho("Error: {}".format(str(e)))