outerbounds 0.3.174__py3-none-any.whl → 0.3.175rc0__py3-none-any.whl

outerbounds/apps/secrets.py

@@ -0,0 +1,164 @@
+from typing import Dict
+
+import base64
+import json
+import requests
+import random
+import time
+import sys
+
+from .utils import safe_requests_wrapper, TODOException
+
+
+class OuterboundsSecretsException(Exception):
+    pass
+
+
+class SecretNotFound(OuterboundsSecretsException):
+    pass
+
+
+class OuterboundsSecretsApiResponse:
+    def __init__(self, response):
+        self.response = response
+
+    @property
+    def secret_resource_id(self):
+        return self.response["secret_resource_id"]
+
+    @property
+    def secret_backend_type(self):
+        return self.response["secret_backend_type"]
+
+
+class SecretRetriever:
+    def get_secret_as_dict(self, secret_id, options={}, role=None):
+        """
+        Supports a special way of specifying secrets sources in outerbounds using the format:
+            @secrets(sources=["outerbounds.<integrations_name>"])
+
+        When invoked it makes a requests to the integrations secrets metadata endpoint on the
+        keywest server to get the cloud resource id for a secret. It then uses that to invoke
+        secrets manager on the core oss and returns the secrets.
+        """
+        headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
+        perimeter, integrations_url = self._get_secret_configs()
+        integration_name = secret_id
+        request_payload = {
+            "perimeter_name": perimeter,
+            "integration_name": integration_name,
+        }
+        response = self._make_request(integrations_url, headers, request_payload)
+        secret_resource_id = response.secret_resource_id
+        secret_backend_type = response.secret_backend_type
+
+        from metaflow.plugins.secrets.secrets_decorator import (
+            get_secrets_backend_provider,
+        )
+
+        secrets_provider = get_secrets_backend_provider(secret_backend_type)
+        secret_dict = secrets_provider.get_secret_as_dict(
+            secret_resource_id, options={}, role=role
+        )
+
+        # Outerbounds stores secrets as binaries. Hence we expect the returned secret to be
+        # {<cloud-secret-name>: <base64 encoded full secret>}. We decode the secret here like:
+        # 1. decode the base64 encoded full secret
+        # 2. load the decoded secret as a json
+        # 3. decode the base64 encoded values in the dict
+        # 4. return the decoded dict
+        binary_secret = next(iter(secret_dict.values()))
+        return self._decode_secret(binary_secret)
+
+    def _is_base64_encoded(self, data):
+        try:
+            if isinstance(data, str):
+                # Check if the string can be base64 decoded
+                base64.b64decode(data).decode("utf-8")
+                return True
+            return False
+        except Exception:
+            return False
+
+    def _decode_secret(self, secret):
+        try:
+            result = {}
+            secret_str = secret
+            if self._is_base64_encoded(secret):
+                # we check if the secret string is base64 encoded because the returned secret from
+                # AWS secret manager is base64 encoded while the secret from GCP is not
+                secret_str = base64.b64decode(secret).decode("utf-8")
+
+            secret_dict = json.loads(secret_str)
+            for key, value in secret_dict.items():
+                result[key] = base64.b64decode(value).decode("utf-8")
+
+            return result
+        except Exception as e:
+            raise OuterboundsSecretsException(f"Error decoding secret: {e}")
+
+    def _get_secret_configs(self):
+        from metaflow_extensions.outerbounds.remote_config import init_config
+        from os import environ
+
+        conf = init_config()
+        if "OBP_PERIMETER" in conf:
+            perimeter = conf["OBP_PERIMETER"]
+        else:
+            # if the perimeter is not in metaflow config, try to get it from the environment
+            perimeter = environ.get("OBP_PERIMETER", "")
+
+        if "OBP_INTEGRATIONS_URL" in conf:
+            integrations_url = conf["OBP_INTEGRATIONS_URL"]
+        else:
+            # if the integrations is not in metaflow config, try to get it from the environment
+            integrations_url = environ.get("OBP_INTEGRATIONS_URL", "")
+
+        if not perimeter:
+            raise OuterboundsSecretsException(
+                "No perimeter set. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
+            )
+
+        if not integrations_url:
+            raise OuterboundsSecretsException(
+                "No integrations url set. Please notify your Outerbounds support team about this issue."
+            )
+
+        integrations_secrets_metadata_url = f"{integrations_url}/secrets/metadata"
+        return perimeter, integrations_secrets_metadata_url
+
+    def _make_request(self, url, headers: Dict, payload: Dict):
+        try:
+            from metaflow.metaflow_config import SERVICE_HEADERS
+
+            request_headers = {**headers, **(SERVICE_HEADERS or {})}
+        except ImportError:
+            raise TODOException(
+                "Failed to create capsule: No Metaflow service headers found"
+            )
+
+        response = safe_requests_wrapper(
+            requests.get,
+            url,
+            data=json.dumps(payload),
+            headers=request_headers,
+            conn_error_retries=5,
+            retryable_status_codes=[409],
+        )
+        self._handle_error_response(response)
+        return OuterboundsSecretsApiResponse(response.json())
+
+    @staticmethod
+    def _handle_error_response(response: requests.Response):
+        if response.status_code >= 500:
+            raise OuterboundsSecretsException(
+                f"Server error: {response.text}. Please reach out to your Outerbounds support team."
+            )
+        status_code = response.status_code
+        if status_code == 404:
+            raise SecretNotFound(f"Secret not found: {response.text}")
+
+        if status_code >= 400:
+            raise OuterboundsSecretsException(
+                f"status_code={status_code}\t\n\t\t{response.text}"
+            )

outerbounds/apps/utils.py

@@ -0,0 +1,228 @@
+import random
+import time
+import sys
+import json
+import requests
+from outerbounds._vendor import click
+from .app_config import CAPSULE_DEBUG
+
+
+class MaximumRetriesExceeded(Exception):
+    def __init__(self, url, method, status_code, text):
+        self.url = url
+        self.method = method
+        self.status_code = status_code
+        self.text = text
+
+    def __str__(self):
+        return f"Maximum retries exceeded for {self.url}[{self.method}] {self.status_code} {self.text}"
+
+
+class KeyValuePair(click.ParamType):
+    name = "KV-PAIR"
+
+    def convert(self, value, param, ctx):
+        # Parse a string of the form KEY=VALUE into a dict {KEY: VALUE}
+        if len(value.split("=", 1)) != 2:
+            self.fail(
+                f"Invalid format for {value}. Expected format: KEY=VALUE", param, ctx
+            )
+
+        key, _value = value.split("=", 1)
+        try:
+            return {key: json.loads(_value)}
+        except json.JSONDecodeError:
+            return {key: _value}
+        except Exception as e:
+            self.fail(f"Invalid value for {value}. Error: {e}", param, ctx)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "KV-PAIR"
+
+
+class MountMetaflowArtifact(click.ParamType):
+    name = "MOUNT-METAFLOW-ARTIFACT"
+
+    def convert(self, value, param, ctx):
+        """
+        Convert a string like "flow=MyFlow,artifact=my_model,path=/tmp/abc" or
+        "pathspec=MyFlow/123/foo/345/my_model,path=/tmp/abc" to a dict.
+        """
+        artifact_dict = {}
+        parts = value.split(",")
+
+        for part in parts:
+            if "=" not in part:
+                self.fail(
+                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
+                )
+
+            key, val = part.split("=", 1)
+            artifact_dict[key.strip()] = val.strip()
+
+        # Validate required fields
+        if "pathspec" in artifact_dict:
+            if "path" not in artifact_dict:
+                self.fail(
+                    "When using 'pathspec', you must also specify 'path'", param, ctx
+                )
+
+            # Return as pathspec format
+            return {
+                "pathspec": artifact_dict["pathspec"],
+                "path": artifact_dict["path"],
+            }
+        elif (
+            "flow" in artifact_dict
+            and "artifact" in artifact_dict
+            and "path" in artifact_dict
+        ):
+            # Return as flow/artifact format
+            result = {
+                "flow": artifact_dict["flow"],
+                "artifact": artifact_dict["artifact"],
+                "path": artifact_dict["path"],
+            }
+
+            # Add optional namespace if provided
+            if "namespace" in artifact_dict:
+                result["namespace"] = artifact_dict["namespace"]
+
+            return result
+        else:
+            self.fail(
+                "Invalid format. Must be either 'flow=X,artifact=Y,path=Z' or 'pathspec=X,path=Z'",
+                param,
+                ctx,
+            )
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "MOUNT-METAFLOW-ARTIFACT"
+
+
+class MountSecret(click.ParamType):
+    name = "MOUNT-SECRET"
+
+    def convert(self, value, param, ctx):
+        """
+        Convert a string like "id=my_secret,path=/tmp/secret" to a dict.
+        """
+        secret_dict = {}
+        parts = value.split(",")
+
+        for part in parts:
+            if "=" not in part:
+                self.fail(
+                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
+                )
+
+            key, val = part.split("=", 1)
+            secret_dict[key.strip()] = val.strip()
+
+        # Validate required fields
+        if "id" in secret_dict and "path" in secret_dict:
+            return {"id": secret_dict["id"], "path": secret_dict["path"]}
+        else:
+            self.fail("Invalid format. Must be 'key=X,path=Y'", param, ctx)
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "MOUNT-SECRET"
+
+
+class CommaSeparatedList(click.ParamType):
+    name = "COMMA-SEPARATED-LIST"
+
+    def convert(self, value, param, ctx):
+        return value.split(",")
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "COMMA-SEPARATED-LIST"
+
+
+KVPairType = KeyValuePair()
+MetaflowArtifactType = MountMetaflowArtifact()
+SecretMountType = MountSecret()
+CommaSeparatedListType = CommaSeparatedList()
+
+
+class TODOException(Exception):
+    pass
+
+
+requests_funcs = [
+    requests.get,
+    requests.post,
+    requests.put,
+    requests.delete,
+    requests.patch,
+    requests.head,
+    requests.options,
+]
+
+
+def safe_requests_wrapper(
+    requests_module_fn,
+    *args,
+    conn_error_retries=2,
+    retryable_status_codes=[409],
+    **kwargs,
+):
+    """
+    There are two categories of errors that we need to handle when dealing with any API server.
+    1. HTTP errors. These are are errors that are returned from the API server.
+        - How to handle retries for this case will be application specific.
+    2. Errors when the API server may not be reachable (DNS resolution / network issues)
+        - In this scenario, we know that something external to the API server is going wrong causing the issue.
+        - Failing pre-maturely in the case might not be the best course of action since critical user jobs might crash on intermittent issues.
+        - So in this case, we can just planely retry the request.
+
+    This function handles the second case. It's a simple wrapper to handle the retry logic for connection errors.
+    If this function is provided a `conn_error_retries` of 5, then the last retry will have waited 32 seconds.
+    Generally this is a safe enough number of retries after which we can assume that something is really broken. Until then,
+    there can be intermittent issues that would resolve themselves if we retry gracefully.
+    """
+    if requests_module_fn not in requests_funcs:
+        raise TODOException(
+            f"safe_requests_wrapper doesn't support {requests_module_fn.__name__}. You can only use the following functions: {requests_funcs}"
+        )
+
+    _num_retries = 0
+    noise = random.uniform(-0.5, 0.5)
+    response = None
+    while _num_retries < conn_error_retries:
+        try:
+            response = requests_module_fn(*args, **kwargs)
+            if response.status_code not in retryable_status_codes:
+                return response
+            if CAPSULE_DEBUG:
+                print(
+                    f"[outerbounds-debug] safe_requests_wrapper: {response.url}[{requests_module_fn.__name__}] {response.status_code} {response.text}",
+                    file=sys.stderr,
+                )
+            _num_retries += 1
+            time.sleep((2 ** (_num_retries + 1)) + noise)
+        except requests.exceptions.ConnectionError:
+            if _num_retries <= conn_error_retries - 1:
+                # Exponential backoff with 2^(_num_retries+1) seconds
+                time.sleep((2 ** (_num_retries + 1)) + noise)
+                _num_retries += 1
+            else:
+                raise
+    raise MaximumRetriesExceeded(
+        response.url,
+        requests_module_fn.__name__,
+        response.status_code,
+        response.text,
+    )

outerbounds/apps/validations.py

@@ -0,0 +1,34 @@
+import os
+from .app_config import AppConfig, AppConfigError
+from .secrets import SecretRetriever, SecretNotFound
+from .dependencies import bake_deployment_image
+
+
+def deploy_validations(app_config: AppConfig, cache_dir: str, logger):
+
+    # First check if the secrets for the app exist.
+    app_secrets = app_config.get("secrets", [])
+    secret_retriever = SecretRetriever()
+    for secret in app_secrets:
+        try:
+            secret_retriever.get_secret_as_dict(secret)
+        except SecretNotFound:
+            raise AppConfigError(f"Secret not found: {secret}")
+
+    # TODO: Next check if the compute pool exists.
+    logger("🍞 Baking Docker Image")
+    baking_status = bake_deployment_image(
+        app_config=app_config,
+        cache_file_path=os.path.join(cache_dir, "image_cache"),
+        logger=logger,
+    )
+    app_config.set_state(
+        "image",
+        baking_status.resolved_image,
+    )
+    app_config.set_state("python_path", baking_status.python_path)
+    logger("🐳 Using The Docker Image : %s" % app_config.get_state("image"))
+
+
+def run_validations(app_config: AppConfig):
+    pass

outerbounds/command_groups/apps_cli.py

@@ -8,6 +8,7 @@ import shutil
 import subprocess
 
 from ..utils import metaflowconfig
+from ..apps.app_cli import app
 
 APP_READY_POLL_TIMEOUT_SECONDS = 300
 # Even after our backend validates that the app routes are ready, it takes a few seconds for
@@ -20,11 +21,6 @@ def cli(**kwargs):
     pass
 
 
-@click.group(help="Manage apps")
-def app(**kwargs):
-    pass
-
-
 @app.command(help="Start an app using a port and a name")
 @click.option(
     "-d",

{outerbounds-0.3.174.dist-info → outerbounds-0.3.175rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: outerbounds
-Version: 0.3.174
+Version: 0.3.175rc0
 Summary: More Data Science, Less Administration
 License: Proprietary
 Keywords: data science,machine learning,MLOps
@@ -29,8 +29,8 @@ Requires-Dist: google-cloud-secret-manager (>=2.20.0,<3.0.0) ; extra == "gcp"
 Requires-Dist: google-cloud-storage (>=2.14.0,<3.0.0) ; extra == "gcp"
 Requires-Dist: metaflow-checkpoint (==0.2.1)
 Requires-Dist: ob-metaflow (==2.15.14.1)
-Requires-Dist: ob-metaflow-extensions (==1.1.161)
-Requires-Dist: ob-metaflow-stubs (==6.0.3.174)
+Requires-Dist: ob-metaflow-extensions (==1.1.162rc0)
+Requires-Dist: ob-metaflow-stubs (==6.0.3.175rc0)
 Requires-Dist: opentelemetry-distro (>=0.41b0) ; extra == "otel"
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.20.0) ; extra == "otel"
 Requires-Dist: opentelemetry-instrumentation-requests (>=0.41b0) ; extra == "otel"

{outerbounds-0.3.174.dist-info → outerbounds-0.3.175rc0.dist-info}/RECORD

@@ -39,9 +39,25 @@ outerbounds/_vendor/yaml/resolver.py,sha256=dPhU1d7G1JCMktPFvNhyqwj2oNvx1yf_Jfa3
 outerbounds/_vendor/yaml/scanner.py,sha256=ZcI8IngR56PaQ0m27WU2vxCqmDCuRjz-hr7pirbMPuw,52982
 outerbounds/_vendor/yaml/serializer.py,sha256=8wFZRy9SsQSktF_f9OOroroqsh4qVUe53ry07P9UgCc,4368
 outerbounds/_vendor/yaml/tokens.py,sha256=JBSu38wihGr4l73JwbfMA7Ks1-X84g8-NskTz7KwPmA,2578
+outerbounds/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+outerbounds/apps/app_cli.py,sha256=oUsPGDr3jnW5l7h4Mp1I4WnRZAanXN1se9_aQ1ybhNQ,17559
+outerbounds/apps/app_config.py,sha256=KBmW9grhiuG9XZG-R0GZkM-024cjj6ztGzOX_2wZW34,11291
+outerbounds/apps/artifacts.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+outerbounds/apps/capsule.py,sha256=n23uJi9877IVnArgqdq_ES_c2vAql8rzVm7jlTYUr1M,13351
+outerbounds/apps/cli_to_config.py,sha256=IjPHeH1lC7l9WPkNLQIWU7obRS3G0G8aTNVZemUvmuw,3168
+outerbounds/apps/code_package/__init__.py,sha256=8McF7pgx8ghvjRnazp2Qktlxi9yYwNiwESSQrk-2oW8,68
+outerbounds/apps/code_package/code_packager.py,sha256=SQDBXKwizzpag5GpwoZpvvkyPOodRSQwk2ecAAfO0HI,23316
+outerbounds/apps/code_package/examples.py,sha256=aF8qKIJxCVv_ugcShQjqUsXKKKMsm1oMkQIl8w3QKuw,4016
+outerbounds/apps/config_schema.yaml,sha256=D-qopf3mGZusa4n5GIbrssoJHS3v96_yponFEM127b4,8275
+outerbounds/apps/dependencies.py,sha256=SqvdFQdFZZW0wXX_CHMHCrfE0TwaRkTvGCRbQ2Mx3q0,3935
+outerbounds/apps/deployer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+outerbounds/apps/experimental/__init__.py,sha256=12L_FzZyzv162uo4I6cmlrxat7feUtIu_kxbObTJZTA,3059
+outerbounds/apps/secrets.py,sha256=27qf04lOBqRjvcswj0ldHOmntP2T6SEjtMJtkJQ_GUg,6100
+outerbounds/apps/utils.py,sha256=JymjsgpU0osF-eDvstFS9zkM7bqliJdqAEV7kAjqxCM,7298
+outerbounds/apps/validations.py,sha256=AVEw9eCvkzqq1m5ZC8btaWrSR6kWYKzarELfrASuAwQ,1117
 outerbounds/cli_main.py,sha256=e9UMnPysmc7gbrimq2I4KfltggyU7pw59Cn9aEguVcU,74
 outerbounds/command_groups/__init__.py,sha256=QPWtj5wDRTINDxVUL7XPqG3HoxHNvYOg08EnuSZB2Hc,21
-outerbounds/command_groups/apps_cli.py,sha256=v9OlQ1b4BGB-cBZiHB6W5gDocDoMmrQ7zdK11QiJ-B8,20847
+outerbounds/command_groups/apps_cli.py,sha256=weXYgUbTVIxMSweLVdod_C1laiB32YKwYhf22OfqQJE,20815
 outerbounds/command_groups/cli.py,sha256=de4_QY1UeoKX6y-IXIbmklAi6bz0DsdBSmAoCg6lq1o,482
 outerbounds/command_groups/fast_bakery_cli.py,sha256=5kja7v6C651XAY6dsP_IkBPJQgfU4hA4S9yTOiVPhW0,6213
 outerbounds/command_groups/local_setup_cli.py,sha256=tuuqJRXQ_guEwOuQSIf9wkUU0yg8yAs31myGViAK15s,36364
@@ -55,7 +71,7 @@ outerbounds/utils/metaflowconfig.py,sha256=l2vJbgPkLISU-XPGZFaC8ZKmYFyJemlD6bwB-
 outerbounds/utils/schema.py,sha256=lMUr9kNgn9wy-sO_t_Tlxmbt63yLeN4b0xQXbDUDj4A,2331
 outerbounds/utils/utils.py,sha256=4Z8cszNob_8kDYCLNTrP-wWads_S_MdL3Uj3ju4mEsk,501
 outerbounds/vendor.py,sha256=gRLRJNXtZBeUpPEog0LOeIsl6GosaFFbCxUvR4bW6IQ,5093
-outerbounds-0.3.174.dist-info/METADATA,sha256=iZlYVROqXN-KK7dEtyZASVkm53ICK8Jq4M5fNElIrR8,1837
-outerbounds-0.3.174.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-outerbounds-0.3.174.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
-outerbounds-0.3.174.dist-info/RECORD,,
+outerbounds-0.3.175rc0.dist-info/METADATA,sha256=c69bE_dY1O_vbcTLpQ0FWj2xL0rF1sIvuI-e3A8MTXA,1846
+outerbounds-0.3.175rc0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+outerbounds-0.3.175rc0.dist-info/entry_points.txt,sha256=7ye0281PKlvqxu15rjw60zKg2pMsXI49_A8BmGqIqBw,47
+outerbounds-0.3.175rc0.dist-info/RECORD,,