outerbounds 0.3.174__py3-none-any.whl → 0.3.175rc0__py3-none-any.whl

outerbounds/apps/code_package/examples.py

@@ -0,0 +1,125 @@
+"""
+Examples demonstrating how to use the code packager abstraction.
+
+This file provides usage examples for the code packager classes.
+These examples are for documentation purposes and are not meant to be run directly.
+"""
+
+import os
+import sys
+from io import BytesIO
+from typing import List, Dict, Any, Callable, Optional
+
+from .code_packager import CodePackager
+
+
+def basic_usage_example(datastore_type: str = "s3") -> None:
+    """
+    Basic usage example with ContentAddressedStore.
+
+    This example shows how to:
+    1. Define paths to include in a package
+    2. Create a package using CodePackager's default packaging
+    3. Store the package using ContentAddressedStore directly
+    4. Generate a download command
+
+    Parameters
+    ----------
+    datastore_type : str, default "s3"
+        The type of datastore to use: "s3", "azure", "gs", or "local"
+    """
+    # Define the paths to include in the package
+    paths_to_include = ["./"]
+
+    # Define which file suffixes to include
+    file_suffixes = [".py", ".md"]
+
+    # Create metadata for the package
+    metadata = {"example": True, "timestamp": "2023-01-01T00:00:00Z"}
+
+    # Initialize the packager with datastore configuration
+    packager = CodePackager(
+        datastore_type=datastore_type,
+        code_package_prefix="my-custom-prefix",  # Optional
+    )
+
+    # Store the package with packaging parameters
+    package_url, package_key = packager.store(
+        paths_to_include=paths_to_include,
+        file_suffixes=file_suffixes,
+        metadata=metadata,
+    )
+
+    # Generate a download command
+    download_cmd = CodePackager.get_download_cmd(
+        package_url=package_url,
+        datastore_type=datastore_type,
+        target_file="my_package.tar",
+    )
+
+    # Generate complete package commands for downloading and setup
+    package_commands = packager.get_package_commands(
+        code_package_url=package_url,
+        target_file="my_package.tar",
+        working_dir="my_app",
+    )
+
+    # Print some information
+    print(f"Package URL: {package_url}")
+    print(f"Package Key: {package_key}")
+    print(f"Download Command: {download_cmd}")
+    print(f"Complete package commands: {package_commands}")
+
+
+def usage_with_custom_package_function(datastore_type: str = "s3") -> None:
+    """
+    Example of using the CodePackager with a custom package creation function.
+
+    Parameters
+    ----------
+    datastore_type : str, default "s3"
+        The type of datastore to use: "s3", "azure", "gs", or "local"
+    """
+
+    # Define a custom package function
+    def create_custom_package():
+        # This is a simple example - in real use, you might create a more complex package
+        from io import BytesIO
+        import tarfile
+        import time
+
+        buf = BytesIO()
+        with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+            # Add a simple file to the tarball
+            content = b"print('Hello, custom package!')"
+            info = tarfile.TarInfo(name="hello.py")
+            info.size = len(content)
+            info.mtime = int(time.time())
+            file_object = BytesIO(content)
+            tar.addfile(info, file_object)
+
+        return bytearray(buf.getvalue())
+
+    # Initialize the packager with datastore configuration
+    packager = CodePackager(datastore_type=datastore_type)
+
+    # Store the package with the custom package function
+    package_url, package_key = packager.store(package_create_fn=create_custom_package)
+
+    # Generate a download command
+    download_cmd = CodePackager.get_download_cmd(
+        package_url=package_url,
+        datastore_type=datastore_type,
+        target_file="custom_package.tar",
+    )
+
+    # Generate complete package commands
+    package_commands = packager.get_package_commands(
+        code_package_url=package_url,
+    )
+
+    # Print some information
+    print(f"Package URL: {package_url}")
+    print(f"Package Key: {package_key}")
+    print(f"Download Command: {download_cmd}")
+    print(f"Complete commands: {package_commands}")

outerbounds/apps/config_schema.yaml

@@ -0,0 +1,250 @@
+
+title: Outerbounds App Configuration Schema
+description: |
+  Schema for defining Outerbounds Apps configuration. This schema is what we will end up using on the CLI/programmatic interface.
+  How to read this schema:
+    1. If the a property has `allow_union`:true then it will allow overrides from the cli.
+    2. If a property has `experimental` set to true then a lot its validations may-be skipped and parsing handled somewhere else.
+version: 1.0.0
+type: object
+required:
+  - name
+  - port
+properties:
+  name: # Only used in `deploy` command
+    allow_union: true # Allow overriding name from the CLI.
+    type: string
+    description: The name of the app to deploy.
+    maxLength: 20 # todo: check if we should allow a larger length.
+    example: "myapp"
+  port: # Only used in `deploy` command
+    allow_union: false
+    type: integer
+    description: Port where the app is hosted. When deployed this will be port on which we will deploy the app.
+    minimum: 1
+    maximum: 65535
+    example: 8000
+  tags: # Only used in `deploy` command
+    allow_union: true
+    type: array
+    description: The tags of the app to deploy.
+    items:
+      type: object
+    example:
+      - foo: bar
+      - x: y
+  image: # Only used in `deploy` command
+    allow_union: true # We will overrwite the image if specified on the CLI.
+    type: string
+    description: The Docker image to deploy with the App.
+    example: "python:3.10-slim"
+  secrets: # Used in `run` command
+    allow_union: true
+    type: array
+    description: Outerbounds integrations to attach to the app. You can use the value you set in the `@secrets` decorator in your code.
+    items:
+      type: string
+    example: ["outerbounds.hf-token"]
+  environment: # Used in `run` command
+    # Todo: So this part might not be best on the CLI. We should probably have a better way to handle this.
+    # In simplicity, we can just JSON dump anything that looks like a dict/list/
+    allow_union: true
+    type: object
+    description: Environment variables to deploy with the App.
+    additionalProperties:
+      oneOf:
+        - type: string
+        - type: number
+        - type: boolean
+        - type: object
+        - type: array # When users give arrays, or objects, we need to JSON dump them. Users need to be aware of this.
+    example:
+      DEBUG: true
+      DATABASE_CONFIG: {"host": "localhost", "port": 5432}
+      ALLOWED_ORIGINS: ["http://localhost:3000", "https://myapp.com"]
+  dependencies: # Used in `run` command
+    allow_union: false
+    type: object
+    description: |
+      The dependencies to attach to the app. Only one of the properties can be specified.
+    properties:
+      from_requirements_file:
+        type: string
+        description: The path to the requirements.txt file to attach to the app.
+        example: "requirements.txt"
+      from_pyproject_toml:
+        type: string
+        description: The path to the pyproject.toml file to attach to the app.
+        example: "pyproject.toml"
+      python:
+        type: string
+        description: |
+          The Python version to use for the app.
+        example: "3.10"
+      pypi:
+        type: object
+        description: |
+          A dictionary of pypi dependencies to attach to the app.
+          The key is the package name and the value is the version.
+        example:
+          numpy: 1.23.0
+          pandas: ""
+      conda:
+        type: object
+        description: |
+          A dictionary of pypi dependencies to attach to the app.
+          The key is the package name and the value is the version.
+        example:
+          numpy: 1.23.0
+          pandas: ""
+  package:
+    allow_union: false
+    type: object
+    description: |
+      Configurations associated with packaging the app.
+    properties:
+      src_path:
+        type: string
+        description: The path to the source code to deploy with the App.
+        example: "./"
+      suffixes:
+        type: array
+        description: |
+          A list of suffixes to add to the source code to deploy with the App.
+        items:
+          type: string
+        example: [".py", ".ipynb"]
+
+  commands: # Used in `run` command
+    allow_union: false
+    type: array
+    description: A list of commands to run the app with. Cannot be configured from the CLI. Only used in `run` command.
+    items:
+      type: string
+    example: ["python app.py", "python app.py --foo bar"]
+  resources: # Only used in `deploy` command
+    allow_union: true # You can override CPU/Memory/GPU/Storage from the CLI.
+    type: object
+    properties:
+      cpu:
+        type: string
+        description: CPU resource request and limit.
+        example: "500m"
+        default: "1"
+      memory:
+        type: string
+        description: Memory resource request and limit.
+        example: "512Mi"
+        default: "4Gi"
+      gpu:
+        type: string
+        description: GPU resource request and limit.
+        example: "1"
+      storage:
+        type: string
+        description: Storage resource request and limit.
+        example: "1Gi"
+        default: "10Gi"
+  replicas:
+    allow_union: true
+    type: object
+    description: |
+      The number of replicas to deploy the app with.
+    properties:
+      min:
+        type: integer
+        description: The minimum number of replicas to deploy the app with.
+        example: 1
+      max:
+        type: integer
+        description: The maximum number of replicas to deploy the app with.
+        example: 10
+  health_check: # Can be used in `run` command
+    type: object
+    # `allow_union` property means that any object in this field will be done a union with the config file if something is provided on commanline.
+    # If it is set to false, then we should throw an exception if the CLI is trying to override something specified in the config file.
+    # We will only allow unions in certains options. The rest will not allow any unions and only need to be specified in one place.
+    allow_union: false
+    properties:
+      enabled:
+        type: boolean
+        description: Whether to enable health checks.
+        example: true
+        default: false
+      path:
+        type: string
+        description: The path for health checks.
+        example: "/health"
+      initial_delay_seconds:
+        type: integer
+        description: Number of seconds to wait before performing the first health check.
+        example: 10
+      period_seconds:
+        type: integer
+        description: How often to perform the health check.
+        example: 30
+  compute_pools: # Only used in `deploy` command
+    allow_union: true
+    type: array
+    description: |
+      A list of compute pools to deploy the app to.
+    items:
+      type: string
+    example: ["default", "large"]
+  auth: # Only used in `deploy` command
+    allow_union: false
+    type: object
+    description: |
+      Auth related configurations.
+    properties:
+      type:
+        type: string
+        description: |
+          The type of authentication to use for the app.
+        enum: [API, Browser]
+      public:
+        type: boolean
+        description: |
+          Whether the app is public or not.
+        default: true
+      # There is an allowed perimeters property
+      # But that needs a little more thought on how
+      # to expose.
+
+  # ------------------------------------ EXPERIMENTAL ------------------------------------
+  project:
+    type: string
+    description: The project name to deploy the app to.
+    experimental: true
+    allow-union: true
+  branch:
+    type: string
+    description: The branch name to deploy the app to.
+    experimental: true
+    allow-union: true
+
+  models: #
+    type: array
+    description: model asset ids to include with the deployment. NO CLI Option for this Now.
+    experimental: true
+    allow-union: true
+    items:
+      type: object
+      properties:
+        asset_id:
+          type: string
+        asset_instance_id:
+          type: string
+  data: #
+    type: array
+    description: data asset ids to include with the deployment.
+    experimental: true
+    allow-union: true
+    items:
+      type: object
+      properties:
+        asset_id:
+          type: string
+        asset_instance_id:
+          type: string
+# ------------------------------------ EXPERIMENTAL ------------------------------------

outerbounds/apps/dependencies.py

@@ -0,0 +1,115 @@
+import copy
+import json
+import os
+import shutil
+import sys
+import tempfile
+from hashlib import sha256
+from typing import List, Optional, Callable, Any
+from .app_config import AppConfig
+from .utils import TODOException
+from metaflow.metaflow_config import (
+    get_pinned_conda_libs,
+    DEFAULT_DATASTORE,
+    KUBERNETES_CONTAINER_IMAGE,
+)
+from collections import namedtuple
+
+BakingStatus = namedtuple(
+    "BakingStatus", ["image_should_be_baked", "python_path", "resolved_image"]
+)
+
+
+class ImageBakingException(Exception):
+    pass
+
+
+def _safe_open_file(path: str):
+    if not os.path.exists(path):
+        raise TODOException(f"File does not exist: {path}")
+    try:
+        with open(path, "r") as f:
+            return f.read()
+    except Exception as e:
+        raise TODOException(f"Failed to open file: {e}")
+
+
+def bake_deployment_image(
+    app_config: AppConfig,
+    cache_file_path: str,
+    logger: Optional[Callable[[str], Any]] = None,
+) -> BakingStatus:
+    # When do we bake an image?
+    # 1. When the user has specified something like `pypi`/`conda`
+    # 2, When the user has specified something like `from_requirements`/ `from_pyproject`
+    # TODO: add parsers for the pyproject/requirements stuff.
+    from metaflow.ob_internal import bake_image
+    from metaflow.plugins.pypi.parsers import (
+        requirements_txt_parser,
+        pyproject_toml_parser,
+    )
+
+    image = app_config.get("image", KUBERNETES_CONTAINER_IMAGE)
+    python_version = "%d.%d.%d" % sys.version_info[:3]
+
+    dependencies = app_config.get_state("dependencies", {})
+    pypi_packages = {}
+    conda_packages = {}
+
+    parsed_packages = {}
+
+    if dependencies.get("from_requirements_file"):
+        parsed_packages = requirements_txt_parser(
+            _safe_open_file(dependencies.get("from_requirements_file"))
+        )
+        pypi_packages = parsed_packages.get("packages", {})
+        python_version = parsed_packages.get("python_version", python_version)
+
+    elif dependencies.get("from_pyproject_toml"):
+        parsed_packages = pyproject_toml_parser(
+            _safe_open_file(dependencies.get("from_pyproject_toml"))
+        )
+        pypi_packages = parsed_packages.get("packages", {})
+        python_version = parsed_packages.get("python_version", python_version)
+
+    elif "pypi" in dependencies:
+        pypi_packages = dependencies.get("pypi", {})
+
+    if "conda" in dependencies:
+        conda_packages = dependencies.get("conda", {})
+    if "python" in dependencies:
+        python_version = dependencies.get("python", python_version)
+
+    python_packages_exist = len(pypi_packages) > 0 or len(conda_packages) > 0
+    if (not python_packages_exist) or app_config.get_state("skip_dependencies", False):
+        # Inform the user that no dependencies are being used.
+        if app_config.get_state("skip_dependencies", False):
+            logger(
+                "⏭️ Skipping baking dependencies into the image based on the --no-deps flag."
+            )
+        # TODO: Handle this a little more nicely.
+        return BakingStatus(
+            image_should_be_baked=False, resolved_image=image, python_path="python"
+        )
+
+    pinned_conda_libs = get_pinned_conda_libs(python_version, DEFAULT_DATASTORE)
+    pypi_packages.update(pinned_conda_libs)
+    _reference = app_config.get("name", "default")
+    # `image` cannot be None. If by change it is none, FB will fart.
+    fb_response = bake_image(
+        cache_file_path=cache_file_path,
+        pypi_packages=pypi_packages,
+        conda_packages=conda_packages,
+        ref=_reference,
+        python=python_version,
+        base_image=image,
+        logger=logger,
+    )
+    if fb_response.failure:
+        raise ImageBakingException(f"Failed to bake image: {fb_response.response}")
+
+    return BakingStatus(
+        image_should_be_baked=True,
+        resolved_image=fb_response.container_image,
+        python_path=fb_response.python_path,
+    )

outerbounds/apps/experimental/__init__.py

@@ -0,0 +1,103 @@
+from functools import wraps
+from outerbounds._vendor import click
+import os
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..app_config import AppConfig
+
+DEFAULT_BRANCH = "test"
+
+
+def wrapping_cli_options(func):
+    @click.option(
+        "--project",
+        type=str,
+        help="The flow project the app/endpoint belongs to",
+        default=None,
+    )
+    @click.option(
+        "--branch",
+        type=str,
+        help="The branch the app/endpoint belongs to",
+        default=None,
+    )
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def build_config_from_options(options):
+    """Build an app configuration from CLI options."""
+    keys = [
+        "project",
+        "branch",
+    ]
+    config = {}
+    for key in keys:
+        if options.get(key):
+            config[key] = options.get(key)
+
+    return config
+
+
+# Account for project / branch and the capsule input.
+def capsule_input_overrides(app_config: "AppConfig", capsule_input: dict):
+    project = app_config.get_state("project", None)
+    # Update the project/branch related configurations.
+    if project is not None:
+        branch = app_config.get_state("branch", DEFAULT_BRANCH)
+        capsule_input["tags"].extend(
+            [dict(key="project", value=project), dict(key="branch", value=branch)]
+        )
+
+    model_asset_conf = app_config.get_state("models", None)
+    data_asset_conf = app_config.get_state("data", None)
+    code_info = _code_info(app_config)
+    # todo:fix me
+    _objects_key = "associatedObjects"
+    if model_asset_conf or data_asset_conf or code_info:
+        capsule_input[_objects_key] = {}
+
+        if model_asset_conf:
+            capsule_input[_objects_key]["models"] = [
+                {"assetId": x["asset_id"], "assetInstanceId": x["asset_instance_id"]}
+                for x in model_asset_conf
+            ]
+        if data_asset_conf:
+            capsule_input[_objects_key]["data"] = [
+                {"assetId": x["asset_id"], "assetInstanceId": x["asset_instance_id"]}
+                for x in data_asset_conf
+            ]
+        if code_info:
+            capsule_input[_objects_key]["code"] = code_info
+
+    return capsule_input
+
+
+def _code_info(app_config: "AppConfig"):
+    from metaflow.metaflow_git import get_repository_info, _call_git
+
+    repo_info = get_repository_info(app_config.get_state("packaging_directory", None))
+    if len(repo_info) == 0:
+        return None
+
+    git_log_info, returncode, failed = _call_git(
+        ["log", "-1", "--pretty=%B"],
+        path=app_config.get_state("packaging_directory", None),
+    )
+    _url = (
+        repo_info["repo_url"]
+        if not repo_info["repo_url"].endswith(".git")
+        else repo_info["repo_url"].rstrip(".git")
+    )
+    _code_info = {
+        "commitId": repo_info["commit_sha"],
+        "commitLink": os.path.join(_url, "commit", repo_info["commit_sha"]),
+    }
+    if not failed and returncode == 0:
+        _code_info["commitMessage"] = git_log_info.strip()
+
+    return _code_info