outerbounds 0.3.176rc6__py3-none-any.whl → 0.3.178__py3-none-any.whl

outerbounds/apps/secrets.py

@@ -1,164 +0,0 @@
-from typing import Dict
-
-import base64
-import json
-import requests
-import random
-import time
-import sys
-
-from .utils import safe_requests_wrapper, TODOException
-
-
-class OuterboundsSecretsException(Exception):
-    pass
-
-
-class SecretNotFound(OuterboundsSecretsException):
-    pass
-
-
-class OuterboundsSecretsApiResponse:
-    def __init__(self, response):
-        self.response = response
-
-    @property
-    def secret_resource_id(self):
-        return self.response["secret_resource_id"]
-
-    @property
-    def secret_backend_type(self):
-        return self.response["secret_backend_type"]
-
-
-class SecretRetriever:
-    def get_secret_as_dict(self, secret_id, options={}, role=None):
-        """
-        Supports a special way of specifying secrets sources in outerbounds using the format:
-            @secrets(sources=["outerbounds.<integrations_name>"])
-
-        When invoked it makes a requests to the integrations secrets metadata endpoint on the
-        keywest server to get the cloud resource id for a secret. It then uses that to invoke
-        secrets manager on the core oss and returns the secrets.
-        """
-        headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
-        perimeter, integrations_url = self._get_secret_configs()
-        integration_name = secret_id
-        request_payload = {
-            "perimeter_name": perimeter,
-            "integration_name": integration_name,
-        }
-        response = self._make_request(integrations_url, headers, request_payload)
-        secret_resource_id = response.secret_resource_id
-        secret_backend_type = response.secret_backend_type
-
-        from metaflow.plugins.secrets.secrets_decorator import (
-            get_secrets_backend_provider,
-        )
-
-        secrets_provider = get_secrets_backend_provider(secret_backend_type)
-        secret_dict = secrets_provider.get_secret_as_dict(
-            secret_resource_id, options={}, role=role
-        )
-
-        # Outerbounds stores secrets as binaries. Hence we expect the returned secret to be
-        # {<cloud-secret-name>: <base64 encoded full secret>}. We decode the secret here like:
-        # 1. decode the base64 encoded full secret
-        # 2. load the decoded secret as a json
-        # 3. decode the base64 encoded values in the dict
-        # 4. return the decoded dict
-        binary_secret = next(iter(secret_dict.values()))
-        return self._decode_secret(binary_secret)
-
-    def _is_base64_encoded(self, data):
-        try:
-            if isinstance(data, str):
-                # Check if the string can be base64 decoded
-                base64.b64decode(data).decode("utf-8")
-                return True
-            return False
-        except Exception:
-            return False
-
-    def _decode_secret(self, secret):
-        try:
-            result = {}
-            secret_str = secret
-            if self._is_base64_encoded(secret):
-                # we check if the secret string is base64 encoded because the returned secret from
-                # AWS secret manager is base64 encoded while the secret from GCP is not
-                secret_str = base64.b64decode(secret).decode("utf-8")
-
-            secret_dict = json.loads(secret_str)
-            for key, value in secret_dict.items():
-                result[key] = base64.b64decode(value).decode("utf-8")
-
-            return result
-        except Exception as e:
-            raise OuterboundsSecretsException(f"Error decoding secret: {e}")
-
-    def _get_secret_configs(self):
-        from metaflow_extensions.outerbounds.remote_config import init_config
-        from os import environ
-
-        conf = init_config()
-        if "OBP_PERIMETER" in conf:
-            perimeter = conf["OBP_PERIMETER"]
-        else:
-            # if the perimeter is not in metaflow config, try to get it from the environment
-            perimeter = environ.get("OBP_PERIMETER", "")
-
-        if "OBP_INTEGRATIONS_URL" in conf:
-            integrations_url = conf["OBP_INTEGRATIONS_URL"]
-        else:
-            # if the integrations is not in metaflow config, try to get it from the environment
-            integrations_url = environ.get("OBP_INTEGRATIONS_URL", "")
-
-        if not perimeter:
-            raise OuterboundsSecretsException(
-                "No perimeter set. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
-            )
-
-        if not integrations_url:
-            raise OuterboundsSecretsException(
-                "No integrations url set. Please notify your Outerbounds support team about this issue."
-            )
-
-        integrations_secrets_metadata_url = f"{integrations_url}/secrets/metadata"
-        return perimeter, integrations_secrets_metadata_url
-
-    def _make_request(self, url, headers: Dict, payload: Dict):
-        try:
-            from metaflow.metaflow_config import SERVICE_HEADERS
-
-            request_headers = {**headers, **(SERVICE_HEADERS or {})}
-        except ImportError:
-            raise TODOException(
-                "Failed to create capsule: No Metaflow service headers found"
-            )
-
-        response = safe_requests_wrapper(
-            requests.get,
-            url,
-            data=json.dumps(payload),
-            headers=request_headers,
-            conn_error_retries=5,
-            retryable_status_codes=[409],
-        )
-        self._handle_error_response(response)
-        return OuterboundsSecretsApiResponse(response.json())
-
-    @staticmethod
-    def _handle_error_response(response: requests.Response):
-        if response.status_code >= 500:
-            raise OuterboundsSecretsException(
-                f"Server error: {response.text}. Please reach out to your Outerbounds support team."
-            )
-        status_code = response.status_code
-        if status_code == 404:
-            raise SecretNotFound(f"Secret not found: {response.text}")
-
-        if status_code >= 400:
-            raise OuterboundsSecretsException(
-                f"status_code={status_code}\t\n\t\t{response.text}"
-            )

outerbounds/apps/utils.py

@@ -1,254 +0,0 @@
-import random
-import time
-import sys
-import json
-import requests
-from outerbounds._vendor import click
-from .app_config import CAPSULE_DEBUG
-
-
-class MaximumRetriesExceeded(Exception):
-    def __init__(self, url, method, status_code, text):
-        self.url = url
-        self.method = method
-        self.status_code = status_code
-        self.text = text
-
-    def __str__(self):
-        return f"Maximum retries exceeded for {self.url}[{self.method}] {self.status_code} {self.text}"
-
-
-class KeyValueDictPair(click.ParamType):
-    name = "KV-DICT-PAIR"
-
-    def convert(self, value, param, ctx):
-        # Parse a string of the form KEY=VALUE into a dict {KEY: VALUE}
-        if len(value.split("=", 1)) != 2:
-            self.fail(
-                f"Invalid format for {value}. Expected format: KEY=VALUE", param, ctx
-            )
-
-        key, _value = value.split("=", 1)
-        try:
-            return {"key": key, "value": json.loads(_value)}
-        except json.JSONDecodeError:
-            return {"key": key, "value": _value}
-        except Exception as e:
-            self.fail(f"Invalid value for {value}. Error: {e}", param, ctx)
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "KV-PAIR"
-
-
-class KeyValuePair(click.ParamType):
-    name = "KV-PAIR"
-
-    def convert(self, value, param, ctx):
-        # Parse a string of the form KEY=VALUE into a dict {KEY: VALUE}
-        if len(value.split("=", 1)) != 2:
-            self.fail(
-                f"Invalid format for {value}. Expected format: KEY=VALUE", param, ctx
-            )
-
-        key, _value = value.split("=", 1)
-        try:
-            return {key: json.loads(_value)}
-        except json.JSONDecodeError:
-            return {key: _value}
-        except Exception as e:
-            self.fail(f"Invalid value for {value}. Error: {e}", param, ctx)
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "KV-PAIR"
-
-
-class MountMetaflowArtifact(click.ParamType):
-    name = "MOUNT-METAFLOW-ARTIFACT"
-
-    def convert(self, value, param, ctx):
-        """
-        Convert a string like "flow=MyFlow,artifact=my_model,path=/tmp/abc" or
-        "pathspec=MyFlow/123/foo/345/my_model,path=/tmp/abc" to a dict.
-        """
-        artifact_dict = {}
-        parts = value.split(",")
-
-        for part in parts:
-            if "=" not in part:
-                self.fail(
-                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
-                )
-
-            key, val = part.split("=", 1)
-            artifact_dict[key.strip()] = val.strip()
-
-        # Validate required fields
-        if "pathspec" in artifact_dict:
-            if "path" not in artifact_dict:
-                self.fail(
-                    "When using 'pathspec', you must also specify 'path'", param, ctx
-                )
-
-            # Return as pathspec format
-            return {
-                "pathspec": artifact_dict["pathspec"],
-                "path": artifact_dict["path"],
-            }
-        elif (
-            "flow" in artifact_dict
-            and "artifact" in artifact_dict
-            and "path" in artifact_dict
-        ):
-            # Return as flow/artifact format
-            result = {
-                "flow": artifact_dict["flow"],
-                "artifact": artifact_dict["artifact"],
-                "path": artifact_dict["path"],
-            }
-
-            # Add optional namespace if provided
-            if "namespace" in artifact_dict:
-                result["namespace"] = artifact_dict["namespace"]
-
-            return result
-        else:
-            self.fail(
-                "Invalid format. Must be either 'flow=X,artifact=Y,path=Z' or 'pathspec=X,path=Z'",
-                param,
-                ctx,
-            )
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "MOUNT-METAFLOW-ARTIFACT"
-
-
-class MountSecret(click.ParamType):
-    name = "MOUNT-SECRET"
-
-    def convert(self, value, param, ctx):
-        """
-        Convert a string like "id=my_secret,path=/tmp/secret" to a dict.
-        """
-        secret_dict = {}
-        parts = value.split(",")
-
-        for part in parts:
-            if "=" not in part:
-                self.fail(
-                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
-                )
-
-            key, val = part.split("=", 1)
-            secret_dict[key.strip()] = val.strip()
-
-        # Validate required fields
-        if "id" in secret_dict and "path" in secret_dict:
-            return {"id": secret_dict["id"], "path": secret_dict["path"]}
-        else:
-            self.fail("Invalid format. Must be 'key=X,path=Y'", param, ctx)
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "MOUNT-SECRET"
-
-
-class CommaSeparatedList(click.ParamType):
-    name = "COMMA-SEPARATED-LIST"
-
-    def convert(self, value, param, ctx):
-        return value.split(",")
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "COMMA-SEPARATED-LIST"
-
-
-KVPairType = KeyValuePair()
-MetaflowArtifactType = MountMetaflowArtifact()
-SecretMountType = MountSecret()
-CommaSeparatedListType = CommaSeparatedList()
-KVDictType = KeyValueDictPair()
-
-
-class TODOException(Exception):
-    pass
-
-
-requests_funcs = [
-    requests.get,
-    requests.post,
-    requests.put,
-    requests.delete,
-    requests.patch,
-    requests.head,
-    requests.options,
-]
-
-
-def safe_requests_wrapper(
-    requests_module_fn,
-    *args,
-    conn_error_retries=2,
-    retryable_status_codes=[409],
-    **kwargs,
-):
-    """
-    There are two categories of errors that we need to handle when dealing with any API server.
-    1. HTTP errors. These are are errors that are returned from the API server.
-        - How to handle retries for this case will be application specific.
-    2. Errors when the API server may not be reachable (DNS resolution / network issues)
-        - In this scenario, we know that something external to the API server is going wrong causing the issue.
-        - Failing pre-maturely in the case might not be the best course of action since critical user jobs might crash on intermittent issues.
-        - So in this case, we can just planely retry the request.
-
-    This function handles the second case. It's a simple wrapper to handle the retry logic for connection errors.
-    If this function is provided a `conn_error_retries` of 5, then the last retry will have waited 32 seconds.
-    Generally this is a safe enough number of retries after which we can assume that something is really broken. Until then,
-    there can be intermittent issues that would resolve themselves if we retry gracefully.
-    """
-    if requests_module_fn not in requests_funcs:
-        raise TODOException(
-            f"safe_requests_wrapper doesn't support {requests_module_fn.__name__}. You can only use the following functions: {requests_funcs}"
-        )
-
-    _num_retries = 0
-    noise = random.uniform(-0.5, 0.5)
-    response = None
-    while _num_retries < conn_error_retries:
-        try:
-            response = requests_module_fn(*args, **kwargs)
-            if response.status_code not in retryable_status_codes:
-                return response
-            if CAPSULE_DEBUG:
-                print(
-                    f"[outerbounds-debug] safe_requests_wrapper: {response.url}[{requests_module_fn.__name__}] {response.status_code} {response.text}",
-                    file=sys.stderr,
-                )
-            _num_retries += 1
-            time.sleep((2 ** (_num_retries + 1)) + noise)
-        except requests.exceptions.ConnectionError:
-            if _num_retries <= conn_error_retries - 1:
-                # Exponential backoff with 2^(_num_retries+1) seconds
-                time.sleep((2 ** (_num_retries + 1)) + noise)
-                _num_retries += 1
-            else:
-                raise
-    raise MaximumRetriesExceeded(
-        response.url,
-        requests_module_fn.__name__,
-        response.status_code,
-        response.text,
-    )

outerbounds/apps/validations.py

@@ -1,34 +0,0 @@
-import os
-from .app_config import AppConfig, AppConfigError
-from .secrets import SecretRetriever, SecretNotFound
-from .dependencies import bake_deployment_image
-
-
-def deploy_validations(app_config: AppConfig, cache_dir: str, logger):
-
-    # First check if the secrets for the app exist.
-    app_secrets = app_config.get("secrets", [])
-    secret_retriever = SecretRetriever()
-    for secret in app_secrets:
-        try:
-            secret_retriever.get_secret_as_dict(secret)
-        except SecretNotFound:
-            raise AppConfigError(f"Secret not found: {secret}")
-
-    # TODO: Next check if the compute pool exists.
-    logger("🍞 Baking Docker Image")
-    baking_status = bake_deployment_image(
-        app_config=app_config,
-        cache_file_path=os.path.join(cache_dir, "image_cache"),
-        logger=logger,
-    )
-    app_config.set_state(
-        "image",
-        baking_status.resolved_image,
-    )
-    app_config.set_state("python_path", baking_status.python_path)
-    logger("🐳 Using The Docker Image : %s" % app_config.get_state("image"))
-
-
-def run_validations(app_config: AppConfig):
-    pass

outerbounds/command_groups/flowprojects_cli.py

@@ -1,137 +0,0 @@
-import json
-import os
-import sys
-import requests
-
-from ..utils import metaflowconfig
-from outerbounds._vendor import click
-
-
-@click.group()
-def cli(**kwargs):
-    pass
-
-
-@cli.group(help="Commands for pushing Deployments metadata.", hidden=True)
-def flowproject(**kwargs):
-    pass
-
-
-@flowproject.command()
-@click.option(
-    "-d",
-    "--config-dir",
-    default=os.path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
-    help="Path to Metaflow configuration directory",
-    show_default=True,
-)
-@click.option(
-    "-p",
-    "--profile",
-    default=os.environ.get("METAFLOW_PROFILE", ""),
-    help="The named metaflow profile in which your workstation exists",
-)
-@click.option("--id", help="The ID for this deployment")
-def get_metadata(config_dir, profile, id):
-    api_url = metaflowconfig.get_sanitized_url_from_config(
-        config_dir, profile, "OBP_API_SERVER"
-    )
-    perimeter = _get_perimeter()
-    headers = _get_request_headers()
-
-    project, branch = _parse_id(id)
-
-    # GET the latest flowproject config in order to modify it
-    # /v1/perimeters/:perimeter/:project/:branch/flowprojects/latest
-    response = requests.get(
-        url=f"{api_url}/v1/perimeters/{perimeter}/projects/{project}/branches/{branch}/latestflowproject",
-        headers=headers,
-    )
-    if response.status_code >= 500:
-        raise Exception("API request failed.")
-
-    body = response.json()
-    if response.status_code >= 400:
-        raise Exception("request failed: %s" % body)
-
-    out = json.dumps(body)
-
-    print(out, file=sys.stdout)
-
-
-@flowproject.command()
-@click.option(
-    "-d",
-    "--config-dir",
-    default=os.path.expanduser(os.environ.get("METAFLOW_HOME", "~/.metaflowconfig")),
-    help="Path to Metaflow configuration directory",
-    show_default=True,
-)
-@click.option(
-    "-p",
-    "--profile",
-    default=os.environ.get("METAFLOW_PROFILE", ""),
-    help="The named metaflow profile in which your workstation exists",
-)
-@click.argument("json_str")
-def set_metadata(config_dir, profile, json_str):
-    api_url = metaflowconfig.get_sanitized_url_from_config(
-        config_dir, profile, "OBP_API_SERVER"
-    )
-
-    perimeter = _get_perimeter()
-    headers = _get_request_headers()
-    payload = json.loads(json_str)
-
-    # POST the updated flowproject config
-    # /v1/perimeters/:perimeter/flowprojects
-    response = requests.post(
-        url=f"{api_url}/v1/perimeters/{perimeter}/flowprojects",
-        json=payload,
-        headers=headers,
-    )
-    if response.status_code >= 500:
-        raise Exception("API request failed. %s" % response.text)
-
-    if response.status_code >= 400:
-        raise Exception("request failed: %s" % response.text)
-    body = response.json()
-
-    print(body, file=sys.stdout)
-
-
-def _get_request_headers():
-    headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
-    try:
-        from metaflow.metaflow_config import SERVICE_HEADERS
-
-        headers = {**headers, **(SERVICE_HEADERS or {})}
-    except ImportError:
-        headers = headers
-
-    return headers
-
-
-def _get_perimeter():
-    # Get current perimeter
-    from metaflow_extensions.outerbounds.remote_config import init_config  # type: ignore
-
-    conf = init_config()
-    if "OBP_PERIMETER" in conf:
-        perimeter = conf["OBP_PERIMETER"]
-    else:
-        # if the perimeter is not in metaflow config, try to get it from the environment
-        perimeter = os.environ.get("OBP_PERIMETER", None)
-    if perimeter is None:
-        raise Exception("Perimeter not found in config, but is required.")
-
-    return perimeter
-
-
-def _parse_id(id: str):
-    parts = id.split("/")
-    if len(parts) != 2:
-        raise Exception("ID should consist of two parts: project/branch")
-
-    project, branch = parts
-    return project, branch