outerbounds 0.3.171__py3-none-any.whl → 0.3.173rc0__py3-none-any.whl

outerbounds/apps/code_package/examples.py

@@ -0,0 +1,125 @@
+"""
+Examples demonstrating how to use the code packager abstraction.
+
+This file provides usage examples for the code packager classes.
+These examples are for documentation purposes and are not meant to be run directly.
+"""
+
+import os
+import sys
+from io import BytesIO
+from typing import List, Dict, Any, Callable, Optional
+
+from .code_packager import CodePackager
+
+
+def basic_usage_example(datastore_type: str = "s3") -> None:
+    """
+    Basic usage example with ContentAddressedStore.
+
+    This example shows how to:
+    1. Define paths to include in a package
+    2. Create a package using CodePackager's default packaging
+    3. Store the package using ContentAddressedStore directly
+    4. Generate a download command
+
+    Parameters
+    ----------
+    datastore_type : str, default "s3"
+        The type of datastore to use: "s3", "azure", "gs", or "local"
+    """
+    # Define the paths to include in the package
+    paths_to_include = ["./"]
+
+    # Define which file suffixes to include
+    file_suffixes = [".py", ".md"]
+
+    # Create metadata for the package
+    metadata = {"example": True, "timestamp": "2023-01-01T00:00:00Z"}
+
+    # Initialize the packager with datastore configuration
+    packager = CodePackager(
+        datastore_type=datastore_type,
+        code_package_prefix="my-custom-prefix",  # Optional
+    )
+
+    # Store the package with packaging parameters
+    package_url, package_key = packager.store(
+        paths_to_include=paths_to_include,
+        file_suffixes=file_suffixes,
+        metadata=metadata,
+    )
+
+    # Generate a download command
+    download_cmd = CodePackager.get_download_cmd(
+        package_url=package_url,
+        datastore_type=datastore_type,
+        target_file="my_package.tar",
+    )
+
+    # Generate complete package commands for downloading and setup
+    package_commands = packager.get_package_commands(
+        code_package_url=package_url,
+        target_file="my_package.tar",
+        working_dir="my_app",
+    )
+
+    # Print some information
+    print(f"Package URL: {package_url}")
+    print(f"Package Key: {package_key}")
+    print(f"Download Command: {download_cmd}")
+    print(f"Complete package commands: {package_commands}")
+
+
+def usage_with_custom_package_function(datastore_type: str = "s3") -> None:
+    """
+    Example of using the CodePackager with a custom package creation function.
+
+    Parameters
+    ----------
+    datastore_type : str, default "s3"
+        The type of datastore to use: "s3", "azure", "gs", or "local"
+    """
+
+    # Define a custom package function
+    def create_custom_package():
+        # This is a simple example - in real use, you might create a more complex package
+        from io import BytesIO
+        import tarfile
+        import time
+
+        buf = BytesIO()
+        with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+            # Add a simple file to the tarball
+            content = b"print('Hello, custom package!')"
+            info = tarfile.TarInfo(name="hello.py")
+            info.size = len(content)
+            info.mtime = int(time.time())
+            file_object = BytesIO(content)
+            tar.addfile(info, file_object)
+
+        return bytearray(buf.getvalue())
+
+    # Initialize the packager with datastore configuration
+    packager = CodePackager(datastore_type=datastore_type)
+
+    # Store the package with the custom package function
+    package_url, package_key = packager.store(package_create_fn=create_custom_package)
+
+    # Generate a download command
+    download_cmd = CodePackager.get_download_cmd(
+        package_url=package_url,
+        datastore_type=datastore_type,
+        target_file="custom_package.tar",
+    )
+
+    # Generate complete package commands
+    package_commands = packager.get_package_commands(
+        code_package_url=package_url,
+    )
+
+    # Print some information
+    print(f"Package URL: {package_url}")
+    print(f"Package Key: {package_key}")
+    print(f"Download Command: {download_cmd}")
+    print(f"Complete commands: {package_commands}")

outerbounds/apps/config_schema.yaml

@@ -0,0 +1,194 @@
+
+title: Outerbounds App Configuration Schema
+description: |
+  Schema for defining Outerbounds Apps configuration. This schema is what we will end up using on the CLI/programmatic interface.
+  All the properties in this schema will then translate into an IR that will have resolved information from the
+version: 1.0.0
+type: object
+required:
+  - name
+  - port
+properties:
+  name: # Only used in `deploy` command
+    allow_union: true # Allow overriding name from the CLI.
+    type: string
+    description: The name of the app to deploy.
+    maxLength: 20 # todo: check if we should allow a larger length.
+    example: "myapp"
+  port: # Only used in `deploy` command
+    allow_union: false
+    type: integer
+    description: Port where the app is hosted. When deployed this will be port on which we will deploy the app.
+    minimum: 1
+    maximum: 65535
+    example: 8000
+  tags: # Only used in `deploy` command
+    allow_union: true
+    type: array
+    description: The tags of the app to deploy.
+    items:
+      type: string
+    example: ["production", "v1.0"]
+  image: # Only used in `deploy` command
+    allow_union: true # We will overrwite the image if specified on the CLI.
+    type: string
+    description: The Docker image to deploy with the App.
+    example: "python:3.10-slim"
+  secrets: # Used in `run` command
+    allow_union: true
+    type: array
+    description: Outerbounds integrations to attach to the app. You can use the value you set in the `@secrets` decorator in your code.
+    items:
+      type: string
+    example: ["outerbounds.hf-token"]
+  environment: # Used in `run` command
+    # Todo: So this part might not be best on the CLI. We should probably have a better way to handle this.
+    # In simplicity, we can just JSON dump anything that looks like a dict/list/
+    allow_union: true
+    type: object
+    description: Environment variables to deploy with the App.
+    additionalProperties:
+      oneOf:
+        - type: string
+        - type: number
+        - type: boolean
+        - type: object
+        - type: array # When users give arrays, or objects, we need to JSON dump them. Users need to be aware of this.
+    example:
+      DEBUG: true
+      DATABASE_CONFIG: {"host": "localhost", "port": 5432}
+      ALLOWED_ORIGINS: ["http://localhost:3000", "https://myapp.com"]
+  dependencies: # Used in `run` command
+    allow_union: false
+    type: object
+    description: |
+      The dependencies to attach to the app. Only one of the properties can be specified.
+    properties:
+      from_requirements_file:
+        type: string
+        description: The path to the requirements.txt file to attach to the app.
+        example: "requirements.txt"
+      from_pyproject_toml:
+        type: string
+        description: The path to the pyproject.toml file to attach to the app.
+        example: "pyproject.toml"
+      python:
+        type: string
+        description: |
+          The Python version to use for the app.
+        example: "3.10"
+      pypi:
+        type: object
+        description: |
+          A dictionary of pypi dependencies to attach to the app.
+          The key is the package name and the value is the version.
+        example:
+          numpy: 1.23.0
+          pandas: ""
+      conda:
+        type: object
+        description: |
+          A dictionary of pypi dependencies to attach to the app.
+          The key is the package name and the value is the version.
+        example:
+          numpy: 1.23.0
+          pandas: ""
+  package:
+    allow_union: false
+    type: object
+    description: |
+      Configurations associated with packaging the app.
+    properties:
+      src_path:
+        type: string
+        description: The path to the source code to deploy with the App.
+        example: "./"
+      suffixes:
+        type: array
+        description: |
+          A list of suffixes to add to the source code to deploy with the App.
+        items:
+          type: string
+        example: [".py", ".ipynb"]
+
+  commands: # Used in `run` command
+    allow_union: false
+    type: array
+    description: A list of commands to run the app with. Cannot be configured from the CLI. Only used in `run` command.
+    items:
+      type: string
+    example: ["python app.py", "python app.py --foo bar"]
+  resources: # Only used in `deploy` command
+    allow_union: true # You can override CPU/Memory/GPU/Storage from the CLI.
+    type: object
+    properties:
+      cpu:
+        type: string
+        description: CPU resource request and limit.
+        example: "500m"
+        default: "1"
+      memory:
+        type: string
+        description: Memory resource request and limit.
+        example: "512Mi"
+        default: "4Gi"
+      gpu:
+        type: string
+        description: GPU resource request and limit.
+        example: "1"
+      storage:
+        type: string
+        description: Storage resource request and limit.
+        example: "1Gi"
+        default: "10Gi"
+  health_check: # Can be used in `run` command
+    type: object
+    # `allow_union` property means that any object in this field will be done a union with the config file if something is provided on commanline.
+    # If it is set to false, then we should throw an exception if the CLI is trying to override something specified in the config file.
+    # We will only allow unions in certains options. The rest will not allow any unions and only need to be specified in one place.
+    allow_union: false
+    properties:
+      enabled:
+        type: boolean
+        description: Whether to enable health checks.
+        example: true
+        default: false
+      path:
+        type: string
+        description: The path for health checks.
+        example: "/health"
+      initial_delay_seconds:
+        type: integer
+        description: Number of seconds to wait before performing the first health check.
+        example: 10
+      period_seconds:
+        type: integer
+        description: How often to perform the health check.
+        example: 30
+  compute_pools: # Only used in `deploy` command
+    allow_union: true
+    type: array
+    description: |
+      A list of compute pools to deploy the app to.
+    items:
+      type: string
+    example: ["default", "large"]
+  auth: # Only used in `deploy` command
+    allow_union: false
+    type: object
+    description: |
+      Auth related configurations.
+    properties:
+      type:
+        type: string
+        description: |
+          The type of authentication to use for the app.
+        enum: [API, SSO]
+      public:
+        type: boolean
+        description: |
+          Whether the app is public or not.
+        default: true
+      # There is an allowed perimeters property
+      # But that needs a little more thought on how
+      # to expose.

outerbounds/apps/dependencies.py

@@ -0,0 +1,115 @@
+import copy
+import json
+import os
+import shutil
+import sys
+import tempfile
+from hashlib import sha256
+from typing import List, Optional, Callable, Any
+from .app_config import AppConfig
+from .utils import TODOException
+from metaflow.metaflow_config import (
+    get_pinned_conda_libs,
+    DEFAULT_DATASTORE,
+    KUBERNETES_CONTAINER_IMAGE,
+)
+from collections import namedtuple
+
+BakingStatus = namedtuple(
+    "BakingStatus", ["image_should_be_baked", "python_path", "resolved_image"]
+)
+
+
+class ImageBakingException(Exception):
+    pass
+
+
+def _safe_open_file(path: str):
+    if not os.path.exists(path):
+        raise TODOException(f"File does not exist: {path}")
+    try:
+        with open(path, "r") as f:
+            return f.read()
+    except Exception as e:
+        raise TODOException(f"Failed to open file: {e}")
+
+
+def bake_deployment_image(
+    app_config: AppConfig,
+    cache_file_path: str,
+    logger: Optional[Callable[[str], Any]] = None,
+) -> BakingStatus:
+    # When do we bake an image?
+    # 1. When the user has specified something like `pypi`/`conda`
+    # 2, When the user has specified something like `from_requirements`/ `from_pyproject`
+    # TODO: add parsers for the pyproject/requirements stuff.
+    from metaflow.ob_internal import bake_image
+    from metaflow.plugins.pypi.parsers import (
+        requirements_txt_parser,
+        pyproject_toml_parser,
+    )
+
+    image = app_config.get("image", KUBERNETES_CONTAINER_IMAGE)
+    python_version = "%d.%d.%d" % sys.version_info[:3]
+
+    dependencies = app_config.get_state("dependencies", {})
+    pypi_packages = {}
+    conda_packages = {}
+
+    parsed_packages = {}
+
+    if dependencies.get("from_requirements_file"):
+        parsed_packages = requirements_txt_parser(
+            _safe_open_file(dependencies.get("from_requirements_file"))
+        )
+        pypi_packages = parsed_packages.get("packages", {})
+        python_version = parsed_packages.get("python_version", python_version)
+
+    elif dependencies.get("from_pyproject_toml"):
+        parsed_packages = pyproject_toml_parser(
+            _safe_open_file(dependencies.get("from_pyproject_toml"))
+        )
+        pypi_packages = parsed_packages.get("packages", {})
+        python_version = parsed_packages.get("python_version", python_version)
+
+    elif "pypi" in dependencies:
+        pypi_packages = dependencies.get("pypi", {})
+
+    if "conda" in dependencies:
+        conda_packages = dependencies.get("conda", {})
+    if "python" in dependencies:
+        python_version = dependencies.get("python", python_version)
+
+    python_packages_exist = len(pypi_packages) > 0 or len(conda_packages) > 0
+    if (not python_packages_exist) or app_config.get_state("skip_dependencies", False):
+        # Inform the user that no dependencies are being used.
+        if app_config.get_state("skip_dependencies", False):
+            logger(
+                "⏭️ Skipping baking dependencies into the image based on the --no-deps flag."
+            )
+        # TODO: Handle this a little more nicely.
+        return BakingStatus(
+            image_should_be_baked=False, resolved_image=image, python_path="python"
+        )
+
+    pinned_conda_libs = get_pinned_conda_libs(python_version, DEFAULT_DATASTORE)
+    pypi_packages.update(pinned_conda_libs)
+    _reference = app_config.get("name", "default")
+    # `image` cannot be None. If by change it is none, FB will fart.
+    fb_response = bake_image(
+        cache_file_path=cache_file_path,
+        pypi_packages=pypi_packages,
+        conda_packages=conda_packages,
+        ref=_reference,
+        python=python_version,
+        base_image=image,
+        logger=logger,
+    )
+    if fb_response.failure:
+        raise ImageBakingException(f"Failed to bake image: {fb_response.response}")
+
+    return BakingStatus(
+        image_should_be_baked=True,
+        resolved_image=fb_response.container_image,
+        python_path=fb_response.python_path,
+    )

outerbounds/apps/secrets.py

@@ -0,0 +1,164 @@
+from typing import Dict
+
+import base64
+import json
+import requests
+import random
+import time
+import sys
+
+from .utils import safe_requests_wrapper, TODOException
+
+
+class OuterboundsSecretsException(Exception):
+    pass
+
+
+class SecretNotFound(OuterboundsSecretsException):
+    pass
+
+
+class OuterboundsSecretsApiResponse:
+    def __init__(self, response):
+        self.response = response
+
+    @property
+    def secret_resource_id(self):
+        return self.response["secret_resource_id"]
+
+    @property
+    def secret_backend_type(self):
+        return self.response["secret_backend_type"]
+
+
+class SecretRetriever:
+    def get_secret_as_dict(self, secret_id, options={}, role=None):
+        """
+        Supports a special way of specifying secrets sources in outerbounds using the format:
+            @secrets(sources=["outerbounds.<integrations_name>"])
+
+        When invoked it makes a requests to the integrations secrets metadata endpoint on the
+        keywest server to get the cloud resource id for a secret. It then uses that to invoke
+        secrets manager on the core oss and returns the secrets.
+        """
+        headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
+        perimeter, integrations_url = self._get_secret_configs()
+        integration_name = secret_id
+        request_payload = {
+            "perimeter_name": perimeter,
+            "integration_name": integration_name,
+        }
+        response = self._make_request(integrations_url, headers, request_payload)
+        secret_resource_id = response.secret_resource_id
+        secret_backend_type = response.secret_backend_type
+
+        from metaflow.plugins.secrets.secrets_decorator import (
+            get_secrets_backend_provider,
+        )
+
+        secrets_provider = get_secrets_backend_provider(secret_backend_type)
+        secret_dict = secrets_provider.get_secret_as_dict(
+            secret_resource_id, options={}, role=role
+        )
+
+        # Outerbounds stores secrets as binaries. Hence we expect the returned secret to be
+        # {<cloud-secret-name>: <base64 encoded full secret>}. We decode the secret here like:
+        # 1. decode the base64 encoded full secret
+        # 2. load the decoded secret as a json
+        # 3. decode the base64 encoded values in the dict
+        # 4. return the decoded dict
+        binary_secret = next(iter(secret_dict.values()))
+        return self._decode_secret(binary_secret)
+
+    def _is_base64_encoded(self, data):
+        try:
+            if isinstance(data, str):
+                # Check if the string can be base64 decoded
+                base64.b64decode(data).decode("utf-8")
+                return True
+            return False
+        except Exception:
+            return False
+
+    def _decode_secret(self, secret):
+        try:
+            result = {}
+            secret_str = secret
+            if self._is_base64_encoded(secret):
+                # we check if the secret string is base64 encoded because the returned secret from
+                # AWS secret manager is base64 encoded while the secret from GCP is not
+                secret_str = base64.b64decode(secret).decode("utf-8")
+
+            secret_dict = json.loads(secret_str)
+            for key, value in secret_dict.items():
+                result[key] = base64.b64decode(value).decode("utf-8")
+
+            return result
+        except Exception as e:
+            raise OuterboundsSecretsException(f"Error decoding secret: {e}")
+
+    def _get_secret_configs(self):
+        from metaflow_extensions.outerbounds.remote_config import init_config
+        from os import environ
+
+        conf = init_config()
+        if "OBP_PERIMETER" in conf:
+            perimeter = conf["OBP_PERIMETER"]
+        else:
+            # if the perimeter is not in metaflow config, try to get it from the environment
+            perimeter = environ.get("OBP_PERIMETER", "")
+
+        if "OBP_INTEGRATIONS_URL" in conf:
+            integrations_url = conf["OBP_INTEGRATIONS_URL"]
+        else:
+            # if the integrations is not in metaflow config, try to get it from the environment
+            integrations_url = environ.get("OBP_INTEGRATIONS_URL", "")
+
+        if not perimeter:
+            raise OuterboundsSecretsException(
+                "No perimeter set. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
+            )
+
+        if not integrations_url:
+            raise OuterboundsSecretsException(
+                "No integrations url set. Please notify your Outerbounds support team about this issue."
+            )
+
+        integrations_secrets_metadata_url = f"{integrations_url}/secrets/metadata"
+        return perimeter, integrations_secrets_metadata_url
+
+    def _make_request(self, url, headers: Dict, payload: Dict):
+        try:
+            from metaflow.metaflow_config import SERVICE_HEADERS
+
+            request_headers = {**headers, **(SERVICE_HEADERS or {})}
+        except ImportError:
+            raise TODOException(
+                "Failed to create capsule: No Metaflow service headers found"
+            )
+
+        response = safe_requests_wrapper(
+            requests.get,
+            url,
+            data=json.dumps(payload),
+            headers=request_headers,
+            conn_error_retries=5,
+            retryable_status_codes=[409],
+        )
+        self._handle_error_response(response)
+        return OuterboundsSecretsApiResponse(response.json())
+
+    @staticmethod
+    def _handle_error_response(response: requests.Response):
+        if response.status_code >= 500:
+            raise OuterboundsSecretsException(
+                f"Server error: {response.text}. Please reach out to your Outerbounds support team."
+            )
+        status_code = response.status_code
+        if status_code == 404:
+            raise SecretNotFound(f"Secret not found: {response.text}")
+
+        if status_code >= 400:
+            raise OuterboundsSecretsException(
+                f"status_code={status_code}\t\n\t\t{response.text}"
+            )