outerbounds 0.3.173rc0__py3-none-any.whl → 0.3.175__py3-none-any.whl

outerbounds/apps/code_package/examples.py

@@ -1,125 +0,0 @@
-"""
-Examples demonstrating how to use the code packager abstraction.
-
-This file provides usage examples for the code packager classes.
-These examples are for documentation purposes and are not meant to be run directly.
-"""
-
-import os
-import sys
-from io import BytesIO
-from typing import List, Dict, Any, Callable, Optional
-
-from .code_packager import CodePackager
-
-
-def basic_usage_example(datastore_type: str = "s3") -> None:
-    """
-    Basic usage example with ContentAddressedStore.
-
-    This example shows how to:
-    1. Define paths to include in a package
-    2. Create a package using CodePackager's default packaging
-    3. Store the package using ContentAddressedStore directly
-    4. Generate a download command
-
-    Parameters
-    ----------
-    datastore_type : str, default "s3"
-        The type of datastore to use: "s3", "azure", "gs", or "local"
-    """
-    # Define the paths to include in the package
-    paths_to_include = ["./"]
-
-    # Define which file suffixes to include
-    file_suffixes = [".py", ".md"]
-
-    # Create metadata for the package
-    metadata = {"example": True, "timestamp": "2023-01-01T00:00:00Z"}
-
-    # Initialize the packager with datastore configuration
-    packager = CodePackager(
-        datastore_type=datastore_type,
-        code_package_prefix="my-custom-prefix",  # Optional
-    )
-
-    # Store the package with packaging parameters
-    package_url, package_key = packager.store(
-        paths_to_include=paths_to_include,
-        file_suffixes=file_suffixes,
-        metadata=metadata,
-    )
-
-    # Generate a download command
-    download_cmd = CodePackager.get_download_cmd(
-        package_url=package_url,
-        datastore_type=datastore_type,
-        target_file="my_package.tar",
-    )
-
-    # Generate complete package commands for downloading and setup
-    package_commands = packager.get_package_commands(
-        code_package_url=package_url,
-        target_file="my_package.tar",
-        working_dir="my_app",
-    )
-
-    # Print some information
-    print(f"Package URL: {package_url}")
-    print(f"Package Key: {package_key}")
-    print(f"Download Command: {download_cmd}")
-    print(f"Complete package commands: {package_commands}")
-
-
-def usage_with_custom_package_function(datastore_type: str = "s3") -> None:
-    """
-    Example of using the CodePackager with a custom package creation function.
-
-    Parameters
-    ----------
-    datastore_type : str, default "s3"
-        The type of datastore to use: "s3", "azure", "gs", or "local"
-    """
-
-    # Define a custom package function
-    def create_custom_package():
-        # This is a simple example - in real use, you might create a more complex package
-        from io import BytesIO
-        import tarfile
-        import time
-
-        buf = BytesIO()
-        with tarfile.open(fileobj=buf, mode="w:gz") as tar:
-            # Add a simple file to the tarball
-            content = b"print('Hello, custom package!')"
-            info = tarfile.TarInfo(name="hello.py")
-            info.size = len(content)
-            info.mtime = int(time.time())
-            file_object = BytesIO(content)
-            tar.addfile(info, file_object)
-
-        return bytearray(buf.getvalue())
-
-    # Initialize the packager with datastore configuration
-    packager = CodePackager(datastore_type=datastore_type)
-
-    # Store the package with the custom package function
-    package_url, package_key = packager.store(package_create_fn=create_custom_package)
-
-    # Generate a download command
-    download_cmd = CodePackager.get_download_cmd(
-        package_url=package_url,
-        datastore_type=datastore_type,
-        target_file="custom_package.tar",
-    )
-
-    # Generate complete package commands
-    package_commands = packager.get_package_commands(
-        code_package_url=package_url,
-    )
-
-    # Print some information
-    print(f"Package URL: {package_url}")
-    print(f"Package Key: {package_key}")
-    print(f"Download Command: {download_cmd}")
-    print(f"Complete commands: {package_commands}")

outerbounds/apps/config_schema.yaml

@@ -1,194 +0,0 @@
-
-title: Outerbounds App Configuration Schema
-description: |
-  Schema for defining Outerbounds Apps configuration. This schema is what we will end up using on the CLI/programmatic interface.
-  All the properties in this schema will then translate into an IR that will have resolved information from the
-version: 1.0.0
-type: object
-required:
-  - name
-  - port
-properties:
-  name: # Only used in `deploy` command
-    allow_union: true # Allow overriding name from the CLI.
-    type: string
-    description: The name of the app to deploy.
-    maxLength: 20 # todo: check if we should allow a larger length.
-    example: "myapp"
-  port: # Only used in `deploy` command
-    allow_union: false
-    type: integer
-    description: Port where the app is hosted. When deployed this will be port on which we will deploy the app.
-    minimum: 1
-    maximum: 65535
-    example: 8000
-  tags: # Only used in `deploy` command
-    allow_union: true
-    type: array
-    description: The tags of the app to deploy.
-    items:
-      type: string
-    example: ["production", "v1.0"]
-  image: # Only used in `deploy` command
-    allow_union: true # We will overrwite the image if specified on the CLI.
-    type: string
-    description: The Docker image to deploy with the App.
-    example: "python:3.10-slim"
-  secrets: # Used in `run` command
-    allow_union: true
-    type: array
-    description: Outerbounds integrations to attach to the app. You can use the value you set in the `@secrets` decorator in your code.
-    items:
-      type: string
-    example: ["outerbounds.hf-token"]
-  environment: # Used in `run` command
-    # Todo: So this part might not be best on the CLI. We should probably have a better way to handle this.
-    # In simplicity, we can just JSON dump anything that looks like a dict/list/
-    allow_union: true
-    type: object
-    description: Environment variables to deploy with the App.
-    additionalProperties:
-      oneOf:
-        - type: string
-        - type: number
-        - type: boolean
-        - type: object
-        - type: array # When users give arrays, or objects, we need to JSON dump them. Users need to be aware of this.
-    example:
-      DEBUG: true
-      DATABASE_CONFIG: {"host": "localhost", "port": 5432}
-      ALLOWED_ORIGINS: ["http://localhost:3000", "https://myapp.com"]
-  dependencies: # Used in `run` command
-    allow_union: false
-    type: object
-    description: |
-      The dependencies to attach to the app. Only one of the properties can be specified.
-    properties:
-      from_requirements_file:
-        type: string
-        description: The path to the requirements.txt file to attach to the app.
-        example: "requirements.txt"
-      from_pyproject_toml:
-        type: string
-        description: The path to the pyproject.toml file to attach to the app.
-        example: "pyproject.toml"
-      python:
-        type: string
-        description: |
-          The Python version to use for the app.
-        example: "3.10"
-      pypi:
-        type: object
-        description: |
-          A dictionary of pypi dependencies to attach to the app.
-          The key is the package name and the value is the version.
-        example:
-          numpy: 1.23.0
-          pandas: ""
-      conda:
-        type: object
-        description: |
-          A dictionary of pypi dependencies to attach to the app.
-          The key is the package name and the value is the version.
-        example:
-          numpy: 1.23.0
-          pandas: ""
-  package:
-    allow_union: false
-    type: object
-    description: |
-      Configurations associated with packaging the app.
-    properties:
-      src_path:
-        type: string
-        description: The path to the source code to deploy with the App.
-        example: "./"
-      suffixes:
-        type: array
-        description: |
-          A list of suffixes to add to the source code to deploy with the App.
-        items:
-          type: string
-        example: [".py", ".ipynb"]
-
-  commands: # Used in `run` command
-    allow_union: false
-    type: array
-    description: A list of commands to run the app with. Cannot be configured from the CLI. Only used in `run` command.
-    items:
-      type: string
-    example: ["python app.py", "python app.py --foo bar"]
-  resources: # Only used in `deploy` command
-    allow_union: true # You can override CPU/Memory/GPU/Storage from the CLI.
-    type: object
-    properties:
-      cpu:
-        type: string
-        description: CPU resource request and limit.
-        example: "500m"
-        default: "1"
-      memory:
-        type: string
-        description: Memory resource request and limit.
-        example: "512Mi"
-        default: "4Gi"
-      gpu:
-        type: string
-        description: GPU resource request and limit.
-        example: "1"
-      storage:
-        type: string
-        description: Storage resource request and limit.
-        example: "1Gi"
-        default: "10Gi"
-  health_check: # Can be used in `run` command
-    type: object
-    # `allow_union` property means that any object in this field will be done a union with the config file if something is provided on commanline.
-    # If it is set to false, then we should throw an exception if the CLI is trying to override something specified in the config file.
-    # We will only allow unions in certains options. The rest will not allow any unions and only need to be specified in one place.
-    allow_union: false
-    properties:
-      enabled:
-        type: boolean
-        description: Whether to enable health checks.
-        example: true
-        default: false
-      path:
-        type: string
-        description: The path for health checks.
-        example: "/health"
-      initial_delay_seconds:
-        type: integer
-        description: Number of seconds to wait before performing the first health check.
-        example: 10
-      period_seconds:
-        type: integer
-        description: How often to perform the health check.
-        example: 30
-  compute_pools: # Only used in `deploy` command
-    allow_union: true
-    type: array
-    description: |
-      A list of compute pools to deploy the app to.
-    items:
-      type: string
-    example: ["default", "large"]
-  auth: # Only used in `deploy` command
-    allow_union: false
-    type: object
-    description: |
-      Auth related configurations.
-    properties:
-      type:
-        type: string
-        description: |
-          The type of authentication to use for the app.
-        enum: [API, SSO]
-      public:
-        type: boolean
-        description: |
-          Whether the app is public or not.
-        default: true
-      # There is an allowed perimeters property
-      # But that needs a little more thought on how
-      # to expose.

outerbounds/apps/dependencies.py

@@ -1,115 +0,0 @@
-import copy
-import json
-import os
-import shutil
-import sys
-import tempfile
-from hashlib import sha256
-from typing import List, Optional, Callable, Any
-from .app_config import AppConfig
-from .utils import TODOException
-from metaflow.metaflow_config import (
-    get_pinned_conda_libs,
-    DEFAULT_DATASTORE,
-    KUBERNETES_CONTAINER_IMAGE,
-)
-from collections import namedtuple
-
-BakingStatus = namedtuple(
-    "BakingStatus", ["image_should_be_baked", "python_path", "resolved_image"]
-)
-
-
-class ImageBakingException(Exception):
-    pass
-
-
-def _safe_open_file(path: str):
-    if not os.path.exists(path):
-        raise TODOException(f"File does not exist: {path}")
-    try:
-        with open(path, "r") as f:
-            return f.read()
-    except Exception as e:
-        raise TODOException(f"Failed to open file: {e}")
-
-
-def bake_deployment_image(
-    app_config: AppConfig,
-    cache_file_path: str,
-    logger: Optional[Callable[[str], Any]] = None,
-) -> BakingStatus:
-    # When do we bake an image?
-    # 1. When the user has specified something like `pypi`/`conda`
-    # 2, When the user has specified something like `from_requirements`/ `from_pyproject`
-    # TODO: add parsers for the pyproject/requirements stuff.
-    from metaflow.ob_internal import bake_image
-    from metaflow.plugins.pypi.parsers import (
-        requirements_txt_parser,
-        pyproject_toml_parser,
-    )
-
-    image = app_config.get("image", KUBERNETES_CONTAINER_IMAGE)
-    python_version = "%d.%d.%d" % sys.version_info[:3]
-
-    dependencies = app_config.get_state("dependencies", {})
-    pypi_packages = {}
-    conda_packages = {}
-
-    parsed_packages = {}
-
-    if dependencies.get("from_requirements_file"):
-        parsed_packages = requirements_txt_parser(
-            _safe_open_file(dependencies.get("from_requirements_file"))
-        )
-        pypi_packages = parsed_packages.get("packages", {})
-        python_version = parsed_packages.get("python_version", python_version)
-
-    elif dependencies.get("from_pyproject_toml"):
-        parsed_packages = pyproject_toml_parser(
-            _safe_open_file(dependencies.get("from_pyproject_toml"))
-        )
-        pypi_packages = parsed_packages.get("packages", {})
-        python_version = parsed_packages.get("python_version", python_version)
-
-    elif "pypi" in dependencies:
-        pypi_packages = dependencies.get("pypi", {})
-
-    if "conda" in dependencies:
-        conda_packages = dependencies.get("conda", {})
-    if "python" in dependencies:
-        python_version = dependencies.get("python", python_version)
-
-    python_packages_exist = len(pypi_packages) > 0 or len(conda_packages) > 0
-    if (not python_packages_exist) or app_config.get_state("skip_dependencies", False):
-        # Inform the user that no dependencies are being used.
-        if app_config.get_state("skip_dependencies", False):
-            logger(
-                "⏭️ Skipping baking dependencies into the image based on the --no-deps flag."
-            )
-        # TODO: Handle this a little more nicely.
-        return BakingStatus(
-            image_should_be_baked=False, resolved_image=image, python_path="python"
-        )
-
-    pinned_conda_libs = get_pinned_conda_libs(python_version, DEFAULT_DATASTORE)
-    pypi_packages.update(pinned_conda_libs)
-    _reference = app_config.get("name", "default")
-    # `image` cannot be None. If by change it is none, FB will fart.
-    fb_response = bake_image(
-        cache_file_path=cache_file_path,
-        pypi_packages=pypi_packages,
-        conda_packages=conda_packages,
-        ref=_reference,
-        python=python_version,
-        base_image=image,
-        logger=logger,
-    )
-    if fb_response.failure:
-        raise ImageBakingException(f"Failed to bake image: {fb_response.response}")
-
-    return BakingStatus(
-        image_should_be_baked=True,
-        resolved_image=fb_response.container_image,
-        python_path=fb_response.python_path,
-    )

outerbounds/apps/secrets.py

@@ -1,164 +0,0 @@
-from typing import Dict
-
-import base64
-import json
-import requests
-import random
-import time
-import sys
-
-from .utils import safe_requests_wrapper, TODOException
-
-
-class OuterboundsSecretsException(Exception):
-    pass
-
-
-class SecretNotFound(OuterboundsSecretsException):
-    pass
-
-
-class OuterboundsSecretsApiResponse:
-    def __init__(self, response):
-        self.response = response
-
-    @property
-    def secret_resource_id(self):
-        return self.response["secret_resource_id"]
-
-    @property
-    def secret_backend_type(self):
-        return self.response["secret_backend_type"]
-
-
-class SecretRetriever:
-    def get_secret_as_dict(self, secret_id, options={}, role=None):
-        """
-        Supports a special way of specifying secrets sources in outerbounds using the format:
-            @secrets(sources=["outerbounds.<integrations_name>"])
-
-        When invoked it makes a requests to the integrations secrets metadata endpoint on the
-        keywest server to get the cloud resource id for a secret. It then uses that to invoke
-        secrets manager on the core oss and returns the secrets.
-        """
-        headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
-        perimeter, integrations_url = self._get_secret_configs()
-        integration_name = secret_id
-        request_payload = {
-            "perimeter_name": perimeter,
-            "integration_name": integration_name,
-        }
-        response = self._make_request(integrations_url, headers, request_payload)
-        secret_resource_id = response.secret_resource_id
-        secret_backend_type = response.secret_backend_type
-
-        from metaflow.plugins.secrets.secrets_decorator import (
-            get_secrets_backend_provider,
-        )
-
-        secrets_provider = get_secrets_backend_provider(secret_backend_type)
-        secret_dict = secrets_provider.get_secret_as_dict(
-            secret_resource_id, options={}, role=role
-        )
-
-        # Outerbounds stores secrets as binaries. Hence we expect the returned secret to be
-        # {<cloud-secret-name>: <base64 encoded full secret>}. We decode the secret here like:
-        # 1. decode the base64 encoded full secret
-        # 2. load the decoded secret as a json
-        # 3. decode the base64 encoded values in the dict
-        # 4. return the decoded dict
-        binary_secret = next(iter(secret_dict.values()))
-        return self._decode_secret(binary_secret)
-
-    def _is_base64_encoded(self, data):
-        try:
-            if isinstance(data, str):
-                # Check if the string can be base64 decoded
-                base64.b64decode(data).decode("utf-8")
-                return True
-            return False
-        except Exception:
-            return False
-
-    def _decode_secret(self, secret):
-        try:
-            result = {}
-            secret_str = secret
-            if self._is_base64_encoded(secret):
-                # we check if the secret string is base64 encoded because the returned secret from
-                # AWS secret manager is base64 encoded while the secret from GCP is not
-                secret_str = base64.b64decode(secret).decode("utf-8")
-
-            secret_dict = json.loads(secret_str)
-            for key, value in secret_dict.items():
-                result[key] = base64.b64decode(value).decode("utf-8")
-
-            return result
-        except Exception as e:
-            raise OuterboundsSecretsException(f"Error decoding secret: {e}")
-
-    def _get_secret_configs(self):
-        from metaflow_extensions.outerbounds.remote_config import init_config
-        from os import environ
-
-        conf = init_config()
-        if "OBP_PERIMETER" in conf:
-            perimeter = conf["OBP_PERIMETER"]
-        else:
-            # if the perimeter is not in metaflow config, try to get it from the environment
-            perimeter = environ.get("OBP_PERIMETER", "")
-
-        if "OBP_INTEGRATIONS_URL" in conf:
-            integrations_url = conf["OBP_INTEGRATIONS_URL"]
-        else:
-            # if the integrations is not in metaflow config, try to get it from the environment
-            integrations_url = environ.get("OBP_INTEGRATIONS_URL", "")
-
-        if not perimeter:
-            raise OuterboundsSecretsException(
-                "No perimeter set. Please make sure to run `outerbounds configure <...>` command which can be found on the Ourebounds UI or reach out to your Outerbounds support team."
-            )
-
-        if not integrations_url:
-            raise OuterboundsSecretsException(
-                "No integrations url set. Please notify your Outerbounds support team about this issue."
-            )
-
-        integrations_secrets_metadata_url = f"{integrations_url}/secrets/metadata"
-        return perimeter, integrations_secrets_metadata_url
-
-    def _make_request(self, url, headers: Dict, payload: Dict):
-        try:
-            from metaflow.metaflow_config import SERVICE_HEADERS
-
-            request_headers = {**headers, **(SERVICE_HEADERS or {})}
-        except ImportError:
-            raise TODOException(
-                "Failed to create capsule: No Metaflow service headers found"
-            )
-
-        response = safe_requests_wrapper(
-            requests.get,
-            url,
-            data=json.dumps(payload),
-            headers=request_headers,
-            conn_error_retries=5,
-            retryable_status_codes=[409],
-        )
-        self._handle_error_response(response)
-        return OuterboundsSecretsApiResponse(response.json())
-
-    @staticmethod
-    def _handle_error_response(response: requests.Response):
-        if response.status_code >= 500:
-            raise OuterboundsSecretsException(
-                f"Server error: {response.text}. Please reach out to your Outerbounds support team."
-            )
-        status_code = response.status_code
-        if status_code == 404:
-            raise SecretNotFound(f"Secret not found: {response.text}")
-
-        if status_code >= 400:
-            raise OuterboundsSecretsException(
-                f"status_code={status_code}\t\n\t\t{response.text}"
-            )